1 /*
   2  * Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "c1/c1_CFGPrinter.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_FrameMap.hpp"
  30 #include "c1/c1_IR.hpp"
  31 #include "c1/c1_LIRGenerator.hpp"
  32 #include "c1/c1_LinearScan.hpp"
  33 #include "c1/c1_ValueStack.hpp"
  34 #include "code/vmreg.inline.hpp"
  35 #include "runtime/timerTrace.hpp"
  36 #include "utilities/bitMap.inline.hpp"
  37 
  38 #ifndef PRODUCT
  39 
  40   static LinearScanStatistic _stat_before_alloc;
  41   static LinearScanStatistic _stat_after_asign;
  42   static LinearScanStatistic _stat_final;
  43 
  44   static LinearScanTimers _total_timer;
  45 
  46   // helper macro for short definition of timer
  47   #define TIME_LINEAR_SCAN(timer_name)  TraceTime _block_timer("", _total_timer.timer(LinearScanTimers::timer_name), TimeLinearScan || TimeEachLinearScan, Verbose);
  48 
  49   // helper macro for short definition of trace-output inside code
  50   #define TRACE_LINEAR_SCAN(level, code)       \
  51     if (TraceLinearScanLevel >= level) {       \
  52       code;                                    \
  53     }
  54 
  55 #else
  56 
  57   #define TIME_LINEAR_SCAN(timer_name)
  58   #define TRACE_LINEAR_SCAN(level, code)
  59 
  60 #endif
  61 
  62 // Map BasicType to spill size in 32-bit words, matching VMReg's notion of words
  63 #ifdef _LP64
  64 static int type2spill_size[T_CONFLICT+1]={ -1, 0, 0, 0, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 0, 2,  1, 2, 1, -1};
  65 #else
  66 static int type2spill_size[T_CONFLICT+1]={ -1, 0, 0, 0, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, -1, 1, 1, -1};
  67 #endif
  68 
  69 
  70 // Implementation of LinearScan
  71 
  72 LinearScan::LinearScan(IR* ir, LIRGenerator* gen, FrameMap* frame_map)
  73  : _compilation(ir->compilation())
  74  , _ir(ir)
  75  , _gen(gen)
  76  , _frame_map(frame_map)
  77  , _num_virtual_regs(gen->max_virtual_register_number())
  78  , _has_fpu_registers(false)
  79  , _num_calls(-1)
  80  , _max_spills(0)
  81  , _unused_spill_slot(-1)
  82  , _intervals(0)   // initialized later with correct length
  83  , _new_intervals_from_allocation(new IntervalList())
  84  , _sorted_intervals(NULL)
  85  , _needs_full_resort(false)
  86  , _lir_ops(0)     // initialized later with correct length
  87  , _block_of_op(0) // initialized later with correct length
  88  , _has_info(0)
  89  , _has_call(0)
  90  , _scope_value_cache(0) // initialized later with correct length
  91  , _interval_in_loop(0, 0) // initialized later with correct length
  92  , _cached_blocks(*ir->linear_scan_order())
  93 #ifdef X86
  94  , _fpu_stack_allocator(NULL)
  95 #endif
  96 {
  97   assert(this->ir() != NULL,          "check if valid");
  98   assert(this->compilation() != NULL, "check if valid");
  99   assert(this->gen() != NULL,         "check if valid");
 100   assert(this->frame_map() != NULL,   "check if valid");
 101 }
 102 
 103 
 104 // ********** functions for converting LIR-Operands to register numbers
 105 //
 106 // Emulate a flat register file comprising physical integer registers,
 107 // physical floating-point registers and virtual registers, in that order.
 108 // Virtual registers already have appropriate numbers, since V0 is
 109 // the number of physical registers.
 110 // Returns -1 for hi word if opr is a single word operand.
 111 //
 112 // Note: the inverse operation (calculating an operand for register numbers)
 113 //       is done in calc_operand_for_interval()
 114 
 115 int LinearScan::reg_num(LIR_Opr opr) {
 116   assert(opr->is_register(), "should not call this otherwise");
 117 
 118   if (opr->is_virtual_register()) {
 119     assert(opr->vreg_number() >= nof_regs, "found a virtual register with a fixed-register number");
 120     return opr->vreg_number();
 121   } else if (opr->is_single_cpu()) {
 122     return opr->cpu_regnr();
 123   } else if (opr->is_double_cpu()) {
 124     return opr->cpu_regnrLo();
 125 #ifdef X86
 126   } else if (opr->is_single_xmm()) {
 127     return opr->fpu_regnr() + pd_first_xmm_reg;
 128   } else if (opr->is_double_xmm()) {
 129     return opr->fpu_regnrLo() + pd_first_xmm_reg;
 130 #endif
 131   } else if (opr->is_single_fpu()) {
 132     return opr->fpu_regnr() + pd_first_fpu_reg;
 133   } else if (opr->is_double_fpu()) {
 134     return opr->fpu_regnrLo() + pd_first_fpu_reg;
 135   } else {
 136     ShouldNotReachHere();
 137     return -1;
 138   }
 139 }
 140 
 141 int LinearScan::reg_numHi(LIR_Opr opr) {
 142   assert(opr->is_register(), "should not call this otherwise");
 143 
 144   if (opr->is_virtual_register()) {
 145     return -1;
 146   } else if (opr->is_single_cpu()) {
 147     return -1;
 148   } else if (opr->is_double_cpu()) {
 149     return opr->cpu_regnrHi();
 150 #ifdef X86
 151   } else if (opr->is_single_xmm()) {
 152     return -1;
 153   } else if (opr->is_double_xmm()) {
 154     return -1;
 155 #endif
 156   } else if (opr->is_single_fpu()) {
 157     return -1;
 158   } else if (opr->is_double_fpu()) {
 159     return opr->fpu_regnrHi() + pd_first_fpu_reg;
 160   } else {
 161     ShouldNotReachHere();
 162     return -1;
 163   }
 164 }
 165 
 166 
 167 // ********** functions for classification of intervals
 168 
 169 bool LinearScan::is_precolored_interval(const Interval* i) {
 170   return i->reg_num() < LinearScan::nof_regs;
 171 }
 172 
 173 bool LinearScan::is_virtual_interval(const Interval* i) {
 174   return i->reg_num() >= LIR_OprDesc::vreg_base;
 175 }
 176 
 177 bool LinearScan::is_precolored_cpu_interval(const Interval* i) {
 178   return i->reg_num() < LinearScan::nof_cpu_regs;
 179 }
 180 
 181 bool LinearScan::is_virtual_cpu_interval(const Interval* i) {
 182 #if defined(__SOFTFP__) || defined(E500V2)
 183   return i->reg_num() >= LIR_OprDesc::vreg_base;
 184 #else
 185   return i->reg_num() >= LIR_OprDesc::vreg_base && (i->type() != T_FLOAT && i->type() != T_DOUBLE);
 186 #endif // __SOFTFP__ or E500V2
 187 }
 188 
 189 bool LinearScan::is_precolored_fpu_interval(const Interval* i) {
 190   return i->reg_num() >= LinearScan::nof_cpu_regs && i->reg_num() < LinearScan::nof_regs;
 191 }
 192 
 193 bool LinearScan::is_virtual_fpu_interval(const Interval* i) {
 194 #if defined(__SOFTFP__) || defined(E500V2)
 195   return false;
 196 #else
 197   return i->reg_num() >= LIR_OprDesc::vreg_base && (i->type() == T_FLOAT || i->type() == T_DOUBLE);
 198 #endif // __SOFTFP__ or E500V2
 199 }
 200 
 201 bool LinearScan::is_in_fpu_register(const Interval* i) {
 202   // fixed intervals not needed for FPU stack allocation
 203   return i->reg_num() >= nof_regs && pd_first_fpu_reg <= i->assigned_reg() && i->assigned_reg() <= pd_last_fpu_reg;
 204 }
 205 
 206 bool LinearScan::is_oop_interval(const Interval* i) {
 207   // fixed intervals never contain oops
 208   return i->reg_num() >= nof_regs && i->type() == T_OBJECT;
 209 }
 210 
 211 
 212 // ********** General helper functions
 213 
 214 // compute next unused stack index that can be used for spilling
 215 int LinearScan::allocate_spill_slot(bool double_word) {
 216   int spill_slot;
 217   if (double_word) {
 218     if ((_max_spills & 1) == 1) {
 219       // alignment of double-word values
 220       // the hole because of the alignment is filled with the next single-word value
 221       assert(_unused_spill_slot == -1, "wasting a spill slot");
 222       _unused_spill_slot = _max_spills;
 223       _max_spills++;
 224     }
 225     spill_slot = _max_spills;
 226     _max_spills += 2;
 227 
 228   } else if (_unused_spill_slot != -1) {
 229     // re-use hole that was the result of a previous double-word alignment
 230     spill_slot = _unused_spill_slot;
 231     _unused_spill_slot = -1;
 232 
 233   } else {
 234     spill_slot = _max_spills;
 235     _max_spills++;
 236   }
 237 
 238   int result = spill_slot + LinearScan::nof_regs + frame_map()->argcount();
 239 
 240   // the class OopMapValue uses only 11 bits for storing the name of the
 241   // oop location. So a stack slot bigger than 2^11 leads to an overflow
 242   // that is not reported in product builds. Prevent this by checking the
 243   // spill slot here (altough this value and the later used location name
 244   // are slightly different)
 245   if (result > 2000) {
 246     bailout("too many stack slots used");
 247   }
 248 
 249   return result;
 250 }
 251 
 252 void LinearScan::assign_spill_slot(Interval* it) {
 253   // assign the canonical spill slot of the parent (if a part of the interval
 254   // is already spilled) or allocate a new spill slot
 255   if (it->canonical_spill_slot() >= 0) {
 256     it->assign_reg(it->canonical_spill_slot());
 257   } else {
 258     int spill = allocate_spill_slot(type2spill_size[it->type()] == 2);
 259     it->set_canonical_spill_slot(spill);
 260     it->assign_reg(spill);
 261   }
 262 }
 263 
 264 void LinearScan::propagate_spill_slots() {
 265   if (!frame_map()->finalize_frame(max_spills())) {
 266     bailout("frame too large");
 267   }
 268 }
 269 
 270 // create a new interval with a predefined reg_num
 271 // (only used for parent intervals that are created during the building phase)
 272 Interval* LinearScan::create_interval(int reg_num) {
 273   assert(_intervals.at(reg_num) == NULL, "overwriting exisiting interval");
 274 
 275   Interval* interval = new Interval(reg_num);
 276   _intervals.at_put(reg_num, interval);
 277 
 278   // assign register number for precolored intervals
 279   if (reg_num < LIR_OprDesc::vreg_base) {
 280     interval->assign_reg(reg_num);
 281   }
 282   return interval;
 283 }
 284 
 285 // assign a new reg_num to the interval and append it to the list of intervals
 286 // (only used for child intervals that are created during register allocation)
 287 void LinearScan::append_interval(Interval* it) {
 288   it->set_reg_num(_intervals.length());
 289   _intervals.append(it);
 290   _new_intervals_from_allocation->append(it);
 291 }
 292 
 293 // copy the vreg-flags if an interval is split
 294 void LinearScan::copy_register_flags(Interval* from, Interval* to) {
 295   if (gen()->is_vreg_flag_set(from->reg_num(), LIRGenerator::byte_reg)) {
 296     gen()->set_vreg_flag(to->reg_num(), LIRGenerator::byte_reg);
 297   }
 298   if (gen()->is_vreg_flag_set(from->reg_num(), LIRGenerator::callee_saved)) {
 299     gen()->set_vreg_flag(to->reg_num(), LIRGenerator::callee_saved);
 300   }
 301 
 302   // Note: do not copy the must_start_in_memory flag because it is not necessary for child
 303   //       intervals (only the very beginning of the interval must be in memory)
 304 }
 305 
 306 
 307 // ********** spill move optimization
 308 // eliminate moves from register to stack if stack slot is known to be correct
 309 
 310 // called during building of intervals
 311 void LinearScan::change_spill_definition_pos(Interval* interval, int def_pos) {
 312   assert(interval->is_split_parent(), "can only be called for split parents");
 313 
 314   switch (interval->spill_state()) {
 315     case noDefinitionFound:
 316       assert(interval->spill_definition_pos() == -1, "must no be set before");
 317       interval->set_spill_definition_pos(def_pos);
 318       interval->set_spill_state(oneDefinitionFound);
 319       break;
 320 
 321     case oneDefinitionFound:
 322       assert(def_pos <= interval->spill_definition_pos(), "positions are processed in reverse order when intervals are created");
 323       if (def_pos < interval->spill_definition_pos() - 2) {
 324         // second definition found, so no spill optimization possible for this interval
 325         interval->set_spill_state(noOptimization);
 326       } else {
 327         // two consecutive definitions (because of two-operand LIR form)
 328         assert(block_of_op_with_id(def_pos) == block_of_op_with_id(interval->spill_definition_pos()), "block must be equal");
 329       }
 330       break;
 331 
 332     case noOptimization:
 333       // nothing to do
 334       break;
 335 
 336     default:
 337       assert(false, "other states not allowed at this time");
 338   }
 339 }
 340 
 341 // called during register allocation
 342 void LinearScan::change_spill_state(Interval* interval, int spill_pos) {
 343   switch (interval->spill_state()) {
 344     case oneDefinitionFound: {
 345       int def_loop_depth = block_of_op_with_id(interval->spill_definition_pos())->loop_depth();
 346       int spill_loop_depth = block_of_op_with_id(spill_pos)->loop_depth();
 347 
 348       if (def_loop_depth < spill_loop_depth) {
 349         // the loop depth of the spilling position is higher then the loop depth
 350         // at the definition of the interval -> move write to memory out of loop
 351         // by storing at definitin of the interval
 352         interval->set_spill_state(storeAtDefinition);
 353       } else {
 354         // the interval is currently spilled only once, so for now there is no
 355         // reason to store the interval at the definition
 356         interval->set_spill_state(oneMoveInserted);
 357       }
 358       break;
 359     }
 360 
 361     case oneMoveInserted: {
 362       // the interval is spilled more then once, so it is better to store it to
 363       // memory at the definition
 364       interval->set_spill_state(storeAtDefinition);
 365       break;
 366     }
 367 
 368     case storeAtDefinition:
 369     case startInMemory:
 370     case noOptimization:
 371     case noDefinitionFound:
 372       // nothing to do
 373       break;
 374 
 375     default:
 376       assert(false, "other states not allowed at this time");
 377   }
 378 }
 379 
 380 
 381 bool LinearScan::must_store_at_definition(const Interval* i) {
 382   return i->is_split_parent() && i->spill_state() == storeAtDefinition;
 383 }
 384 
 385 // called once before asignment of register numbers
 386 void LinearScan::eliminate_spill_moves() {
 387   TIME_LINEAR_SCAN(timer_eliminate_spill_moves);
 388   TRACE_LINEAR_SCAN(3, tty->print_cr("***** Eliminating unnecessary spill moves"));
 389 
 390   // collect all intervals that must be stored after their definion.
 391   // the list is sorted by Interval::spill_definition_pos
 392   Interval* interval;
 393   Interval* temp_list;
 394   create_unhandled_lists(&interval, &temp_list, must_store_at_definition, NULL);
 395 
 396 #ifdef ASSERT
 397   Interval* prev = NULL;
 398   Interval* temp = interval;
 399   while (temp != Interval::end()) {
 400     assert(temp->spill_definition_pos() > 0, "invalid spill definition pos");
 401     if (prev != NULL) {
 402       assert(temp->from() >= prev->from(), "intervals not sorted");
 403       assert(temp->spill_definition_pos() >= prev->spill_definition_pos(), "when intervals are sorted by from, then they must also be sorted by spill_definition_pos");
 404     }
 405 
 406     assert(temp->canonical_spill_slot() >= LinearScan::nof_regs, "interval has no spill slot assigned");
 407     assert(temp->spill_definition_pos() >= temp->from(), "invalid order");
 408     assert(temp->spill_definition_pos() <= temp->from() + 2, "only intervals defined once at their start-pos can be optimized");
 409 
 410     TRACE_LINEAR_SCAN(4, tty->print_cr("interval %d (from %d to %d) must be stored at %d", temp->reg_num(), temp->from(), temp->to(), temp->spill_definition_pos()));
 411 
 412     temp = temp->next();
 413   }
 414 #endif
 415 
 416   LIR_InsertionBuffer insertion_buffer;
 417   int num_blocks = block_count();
 418   for (int i = 0; i < num_blocks; i++) {
 419     BlockBegin* block = block_at(i);
 420     LIR_OpList* instructions = block->lir()->instructions_list();
 421     int         num_inst = instructions->length();
 422     bool        has_new = false;
 423 
 424     // iterate all instructions of the block. skip the first because it is always a label
 425     for (int j = 1; j < num_inst; j++) {
 426       LIR_Op* op = instructions->at(j);
 427       int op_id = op->id();
 428 
 429       if (op_id == -1) {
 430         // remove move from register to stack if the stack slot is guaranteed to be correct.
 431         // only moves that have been inserted by LinearScan can be removed.
 432         assert(op->code() == lir_move, "only moves can have a op_id of -1");
 433         assert(op->as_Op1() != NULL, "move must be LIR_Op1");
 434         assert(op->as_Op1()->result_opr()->is_virtual(), "LinearScan inserts only moves to virtual registers");
 435 
 436         LIR_Op1* op1 = (LIR_Op1*)op;
 437         Interval* interval = interval_at(op1->result_opr()->vreg_number());
 438 
 439         if (interval->assigned_reg() >= LinearScan::nof_regs && interval->always_in_memory()) {
 440           // move target is a stack slot that is always correct, so eliminate instruction
 441           TRACE_LINEAR_SCAN(4, tty->print_cr("eliminating move from interval %d to %d", op1->in_opr()->vreg_number(), op1->result_opr()->vreg_number()));
 442           instructions->at_put(j, NULL); // NULL-instructions are deleted by assign_reg_num
 443         }
 444 
 445       } else {
 446         // insert move from register to stack just after the beginning of the interval
 447         assert(interval == Interval::end() || interval->spill_definition_pos() >= op_id, "invalid order");
 448         assert(interval == Interval::end() || (interval->is_split_parent() && interval->spill_state() == storeAtDefinition), "invalid interval");
 449 
 450         while (interval != Interval::end() && interval->spill_definition_pos() == op_id) {
 451           if (!has_new) {
 452             // prepare insertion buffer (appended when all instructions of the block are processed)
 453             insertion_buffer.init(block->lir());
 454             has_new = true;
 455           }
 456 
 457           LIR_Opr from_opr = operand_for_interval(interval);
 458           LIR_Opr to_opr = canonical_spill_opr(interval);
 459           assert(from_opr->is_fixed_cpu() || from_opr->is_fixed_fpu(), "from operand must be a register");
 460           assert(to_opr->is_stack(), "to operand must be a stack slot");
 461 
 462           insertion_buffer.move(j, from_opr, to_opr);
 463           TRACE_LINEAR_SCAN(4, tty->print_cr("inserting move after definition of interval %d to stack slot %d at op_id %d", interval->reg_num(), interval->canonical_spill_slot() - LinearScan::nof_regs, op_id));
 464 
 465           interval = interval->next();
 466         }
 467       }
 468     } // end of instruction iteration
 469 
 470     if (has_new) {
 471       block->lir()->append(&insertion_buffer);
 472     }
 473   } // end of block iteration
 474 
 475   assert(interval == Interval::end(), "missed an interval");
 476 }
 477 
 478 
 479 // ********** Phase 1: number all instructions in all blocks
 480 // Compute depth-first and linear scan block orders, and number LIR_Op nodes for linear scan.
 481 
 482 void LinearScan::number_instructions() {
 483   {
 484     // dummy-timer to measure the cost of the timer itself
 485     // (this time is then subtracted from all other timers to get the real value)
 486     TIME_LINEAR_SCAN(timer_do_nothing);
 487   }
 488   TIME_LINEAR_SCAN(timer_number_instructions);
 489 
 490   // Assign IDs to LIR nodes and build a mapping, lir_ops, from ID to LIR_Op node.
 491   int num_blocks = block_count();
 492   int num_instructions = 0;
 493   int i;
 494   for (i = 0; i < num_blocks; i++) {
 495     num_instructions += block_at(i)->lir()->instructions_list()->length();
 496   }
 497 
 498   // initialize with correct length
 499   _lir_ops = LIR_OpArray(num_instructions, num_instructions, NULL);
 500   _block_of_op = BlockBeginArray(num_instructions, num_instructions, NULL);
 501 
 502   int op_id = 0;
 503   int idx = 0;
 504 
 505   for (i = 0; i < num_blocks; i++) {
 506     BlockBegin* block = block_at(i);
 507     block->set_first_lir_instruction_id(op_id);
 508     LIR_OpList* instructions = block->lir()->instructions_list();
 509 
 510     int num_inst = instructions->length();
 511     for (int j = 0; j < num_inst; j++) {
 512       LIR_Op* op = instructions->at(j);
 513       op->set_id(op_id);
 514 
 515       _lir_ops.at_put(idx, op);
 516       _block_of_op.at_put(idx, block);
 517       assert(lir_op_with_id(op_id) == op, "must match");
 518 
 519       idx++;
 520       op_id += 2; // numbering of lir_ops by two
 521     }
 522     block->set_last_lir_instruction_id(op_id - 2);
 523   }
 524   assert(idx == num_instructions, "must match");
 525   assert(idx * 2 == op_id, "must match");
 526 
 527   _has_call = BitMap(num_instructions); _has_call.clear();
 528   _has_info = BitMap(num_instructions); _has_info.clear();
 529 }
 530 
 531 
 532 // ********** Phase 2: compute local live sets separately for each block
 533 // (sets live_gen and live_kill for each block)
 534 
 535 void LinearScan::set_live_gen_kill(Value value, LIR_Op* op, BitMap& live_gen, BitMap& live_kill) {
 536   LIR_Opr opr = value->operand();
 537   Constant* con = value->as_Constant();
 538 
 539   // check some asumptions about debug information
 540   assert(!value->type()->is_illegal(), "if this local is used by the interpreter it shouldn't be of indeterminate type");
 541   assert(con == NULL || opr->is_virtual() || opr->is_constant() || opr->is_illegal(), "asumption: Constant instructions have only constant operands");
 542   assert(con != NULL || opr->is_virtual(), "asumption: non-Constant instructions have only virtual operands");
 543 
 544   if ((con == NULL || con->is_pinned()) && opr->is_register()) {
 545     assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 546     int reg = opr->vreg_number();
 547     if (!live_kill.at(reg)) {
 548       live_gen.set_bit(reg);
 549       TRACE_LINEAR_SCAN(4, tty->print_cr("  Setting live_gen for value %c%d, LIR op_id %d, register number %d", value->type()->tchar(), value->id(), op->id(), reg));
 550     }
 551   }
 552 }
 553 
 554 
 555 void LinearScan::compute_local_live_sets() {
 556   TIME_LINEAR_SCAN(timer_compute_local_live_sets);
 557 
 558   int  num_blocks = block_count();
 559   int  live_size = live_set_size();
 560   bool local_has_fpu_registers = false;
 561   int  local_num_calls = 0;
 562   LIR_OpVisitState visitor;
 563 
 564   BitMap2D local_interval_in_loop = BitMap2D(_num_virtual_regs, num_loops());
 565   local_interval_in_loop.clear();
 566 
 567   // iterate all blocks
 568   for (int i = 0; i < num_blocks; i++) {
 569     BlockBegin* block = block_at(i);
 570 
 571     BitMap live_gen(live_size);  live_gen.clear();
 572     BitMap live_kill(live_size); live_kill.clear();
 573 
 574     if (block->is_set(BlockBegin::exception_entry_flag)) {
 575       // Phi functions at the begin of an exception handler are
 576       // implicitly defined (= killed) at the beginning of the block.
 577       for_each_phi_fun(block, phi,
 578         live_kill.set_bit(phi->operand()->vreg_number())
 579       );
 580     }
 581 
 582     LIR_OpList* instructions = block->lir()->instructions_list();
 583     int num_inst = instructions->length();
 584 
 585     // iterate all instructions of the block. skip the first because it is always a label
 586     assert(visitor.no_operands(instructions->at(0)), "first operation must always be a label");
 587     for (int j = 1; j < num_inst; j++) {
 588       LIR_Op* op = instructions->at(j);
 589 
 590       // visit operation to collect all operands
 591       visitor.visit(op);
 592 
 593       if (visitor.has_call()) {
 594         _has_call.set_bit(op->id() >> 1);
 595         local_num_calls++;
 596       }
 597       if (visitor.info_count() > 0) {
 598         _has_info.set_bit(op->id() >> 1);
 599       }
 600 
 601       // iterate input operands of instruction
 602       int k, n, reg;
 603       n = visitor.opr_count(LIR_OpVisitState::inputMode);
 604       for (k = 0; k < n; k++) {
 605         LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, k);
 606         assert(opr->is_register(), "visitor should only return register operands");
 607 
 608         if (opr->is_virtual_register()) {
 609           assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 610           reg = opr->vreg_number();
 611           if (!live_kill.at(reg)) {
 612             live_gen.set_bit(reg);
 613             TRACE_LINEAR_SCAN(4, tty->print_cr("  Setting live_gen for register %d at instruction %d", reg, op->id()));
 614           }
 615           if (block->loop_index() >= 0) {
 616             local_interval_in_loop.set_bit(reg, block->loop_index());
 617           }
 618           local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu();
 619         }
 620 
 621 #ifdef ASSERT
 622         // fixed intervals are never live at block boundaries, so
 623         // they need not be processed in live sets.
 624         // this is checked by these assertions to be sure about it.
 625         // the entry block may have incoming values in registers, which is ok.
 626         if (!opr->is_virtual_register() && block != ir()->start()) {
 627           reg = reg_num(opr);
 628           if (is_processed_reg_num(reg)) {
 629             assert(live_kill.at(reg), "using fixed register that is not defined in this block");
 630           }
 631           reg = reg_numHi(opr);
 632           if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) {
 633             assert(live_kill.at(reg), "using fixed register that is not defined in this block");
 634           }
 635         }
 636 #endif
 637       }
 638 
 639       // Add uses of live locals from interpreter's point of view for proper debug information generation
 640       n = visitor.info_count();
 641       for (k = 0; k < n; k++) {
 642         CodeEmitInfo* info = visitor.info_at(k);
 643         ValueStack* stack = info->stack();
 644         for_each_state_value(stack, value,
 645           set_live_gen_kill(value, op, live_gen, live_kill)
 646         );
 647       }
 648 
 649       // iterate temp operands of instruction
 650       n = visitor.opr_count(LIR_OpVisitState::tempMode);
 651       for (k = 0; k < n; k++) {
 652         LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, k);
 653         assert(opr->is_register(), "visitor should only return register operands");
 654 
 655         if (opr->is_virtual_register()) {
 656           assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 657           reg = opr->vreg_number();
 658           live_kill.set_bit(reg);
 659           if (block->loop_index() >= 0) {
 660             local_interval_in_loop.set_bit(reg, block->loop_index());
 661           }
 662           local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu();
 663         }
 664 
 665 #ifdef ASSERT
 666         // fixed intervals are never live at block boundaries, so
 667         // they need not be processed in live sets
 668         // process them only in debug mode so that this can be checked
 669         if (!opr->is_virtual_register()) {
 670           reg = reg_num(opr);
 671           if (is_processed_reg_num(reg)) {
 672             live_kill.set_bit(reg_num(opr));
 673           }
 674           reg = reg_numHi(opr);
 675           if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) {
 676             live_kill.set_bit(reg);
 677           }
 678         }
 679 #endif
 680       }
 681 
 682       // iterate output operands of instruction
 683       n = visitor.opr_count(LIR_OpVisitState::outputMode);
 684       for (k = 0; k < n; k++) {
 685         LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, k);
 686         assert(opr->is_register(), "visitor should only return register operands");
 687 
 688         if (opr->is_virtual_register()) {
 689           assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 690           reg = opr->vreg_number();
 691           live_kill.set_bit(reg);
 692           if (block->loop_index() >= 0) {
 693             local_interval_in_loop.set_bit(reg, block->loop_index());
 694           }
 695           local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu();
 696         }
 697 
 698 #ifdef ASSERT
 699         // fixed intervals are never live at block boundaries, so
 700         // they need not be processed in live sets
 701         // process them only in debug mode so that this can be checked
 702         if (!opr->is_virtual_register()) {
 703           reg = reg_num(opr);
 704           if (is_processed_reg_num(reg)) {
 705             live_kill.set_bit(reg_num(opr));
 706           }
 707           reg = reg_numHi(opr);
 708           if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) {
 709             live_kill.set_bit(reg);
 710           }
 711         }
 712 #endif
 713       }
 714     } // end of instruction iteration
 715 
 716     block->set_live_gen (live_gen);
 717     block->set_live_kill(live_kill);
 718     block->set_live_in  (BitMap(live_size)); block->live_in().clear();
 719     block->set_live_out (BitMap(live_size)); block->live_out().clear();
 720 
 721     TRACE_LINEAR_SCAN(4, tty->print("live_gen  B%d ", block->block_id()); print_bitmap(block->live_gen()));
 722     TRACE_LINEAR_SCAN(4, tty->print("live_kill B%d ", block->block_id()); print_bitmap(block->live_kill()));
 723   } // end of block iteration
 724 
 725   // propagate local calculated information into LinearScan object
 726   _has_fpu_registers = local_has_fpu_registers;
 727   compilation()->set_has_fpu_code(local_has_fpu_registers);
 728 
 729   _num_calls = local_num_calls;
 730   _interval_in_loop = local_interval_in_loop;
 731 }
 732 
 733 
 734 // ********** Phase 3: perform a backward dataflow analysis to compute global live sets
 735 // (sets live_in and live_out for each block)
 736 
 737 void LinearScan::compute_global_live_sets() {
 738   TIME_LINEAR_SCAN(timer_compute_global_live_sets);
 739 
 740   int  num_blocks = block_count();
 741   bool change_occurred;
 742   bool change_occurred_in_block;
 743   int  iteration_count = 0;
 744   BitMap live_out(live_set_size()); live_out.clear(); // scratch set for calculations
 745 
 746   // Perform a backward dataflow analysis to compute live_out and live_in for each block.
 747   // The loop is executed until a fixpoint is reached (no changes in an iteration)
 748   // Exception handlers must be processed because not all live values are
 749   // present in the state array, e.g. because of global value numbering
 750   do {
 751     change_occurred = false;
 752 
 753     // iterate all blocks in reverse order
 754     for (int i = num_blocks - 1; i >= 0; i--) {
 755       BlockBegin* block = block_at(i);
 756 
 757       change_occurred_in_block = false;
 758 
 759       // live_out(block) is the union of live_in(sux), for successors sux of block
 760       int n = block->number_of_sux();
 761       int e = block->number_of_exception_handlers();
 762       if (n + e > 0) {
 763         // block has successors
 764         if (n > 0) {
 765           live_out.set_from(block->sux_at(0)->live_in());
 766           for (int j = 1; j < n; j++) {
 767             live_out.set_union(block->sux_at(j)->live_in());
 768           }
 769         } else {
 770           live_out.clear();
 771         }
 772         for (int j = 0; j < e; j++) {
 773           live_out.set_union(block->exception_handler_at(j)->live_in());
 774         }
 775 
 776         if (!block->live_out().is_same(live_out)) {
 777           // A change occurred.  Swap the old and new live out sets to avoid copying.
 778           BitMap temp = block->live_out();
 779           block->set_live_out(live_out);
 780           live_out = temp;
 781 
 782           change_occurred = true;
 783           change_occurred_in_block = true;
 784         }
 785       }
 786 
 787       if (iteration_count == 0 || change_occurred_in_block) {
 788         // live_in(block) is the union of live_gen(block) with (live_out(block) & !live_kill(block))
 789         // note: live_in has to be computed only in first iteration or if live_out has changed!
 790         BitMap live_in = block->live_in();
 791         live_in.set_from(block->live_out());
 792         live_in.set_difference(block->live_kill());
 793         live_in.set_union(block->live_gen());
 794       }
 795 
 796 #ifndef PRODUCT
 797       if (TraceLinearScanLevel >= 4) {
 798         char c = ' ';
 799         if (iteration_count == 0 || change_occurred_in_block) {
 800           c = '*';
 801         }
 802         tty->print("(%d) live_in%c  B%d ", iteration_count, c, block->block_id()); print_bitmap(block->live_in());
 803         tty->print("(%d) live_out%c B%d ", iteration_count, c, block->block_id()); print_bitmap(block->live_out());
 804       }
 805 #endif
 806     }
 807     iteration_count++;
 808 
 809     if (change_occurred && iteration_count > 50) {
 810       BAILOUT("too many iterations in compute_global_live_sets");
 811     }
 812   } while (change_occurred);
 813 
 814 
 815 #ifdef ASSERT
 816   // check that fixed intervals are not live at block boundaries
 817   // (live set must be empty at fixed intervals)
 818   for (int i = 0; i < num_blocks; i++) {
 819     BlockBegin* block = block_at(i);
 820     for (int j = 0; j < LIR_OprDesc::vreg_base; j++) {
 821       assert(block->live_in().at(j)  == false, "live_in  set of fixed register must be empty");
 822       assert(block->live_out().at(j) == false, "live_out set of fixed register must be empty");
 823       assert(block->live_gen().at(j) == false, "live_gen set of fixed register must be empty");
 824     }
 825   }
 826 #endif
 827 
 828   // check that the live_in set of the first block is empty
 829   BitMap live_in_args(ir()->start()->live_in().size());
 830   live_in_args.clear();
 831   if (!ir()->start()->live_in().is_same(live_in_args)) {
 832 #ifdef ASSERT
 833     tty->print_cr("Error: live_in set of first block must be empty (when this fails, virtual registers are used before they are defined)");
 834     tty->print_cr("affected registers:");
 835     print_bitmap(ir()->start()->live_in());
 836 
 837     // print some additional information to simplify debugging
 838     for (unsigned int i = 0; i < ir()->start()->live_in().size(); i++) {
 839       if (ir()->start()->live_in().at(i)) {
 840         Instruction* instr = gen()->instruction_for_vreg(i);
 841         tty->print_cr("* vreg %d (HIR instruction %c%d)", i, instr == NULL ? ' ' : instr->type()->tchar(), instr == NULL ? 0 : instr->id());
 842 
 843         for (int j = 0; j < num_blocks; j++) {
 844           BlockBegin* block = block_at(j);
 845           if (block->live_gen().at(i)) {
 846             tty->print_cr("  used in block B%d", block->block_id());
 847           }
 848           if (block->live_kill().at(i)) {
 849             tty->print_cr("  defined in block B%d", block->block_id());
 850           }
 851         }
 852       }
 853     }
 854 
 855 #endif
 856     // when this fails, virtual registers are used before they are defined.
 857     assert(false, "live_in set of first block must be empty");
 858     // bailout of if this occurs in product mode.
 859     bailout("live_in set of first block not empty");
 860   }
 861 }
 862 
 863 
 864 // ********** Phase 4: build intervals
 865 // (fills the list _intervals)
 866 
 867 void LinearScan::add_use(Value value, int from, int to, IntervalUseKind use_kind) {
 868   assert(!value->type()->is_illegal(), "if this value is used by the interpreter it shouldn't be of indeterminate type");
 869   LIR_Opr opr = value->operand();
 870   Constant* con = value->as_Constant();
 871 
 872   if ((con == NULL || con->is_pinned()) && opr->is_register()) {
 873     assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 874     add_use(opr, from, to, use_kind);
 875   }
 876 }
 877 
 878 
 879 void LinearScan::add_def(LIR_Opr opr, int def_pos, IntervalUseKind use_kind) {
 880   TRACE_LINEAR_SCAN(2, tty->print(" def "); opr->print(tty); tty->print_cr(" def_pos %d (%d)", def_pos, use_kind));
 881   assert(opr->is_register(), "should not be called otherwise");
 882 
 883   if (opr->is_virtual_register()) {
 884     assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 885     add_def(opr->vreg_number(), def_pos, use_kind, opr->type_register());
 886 
 887   } else {
 888     int reg = reg_num(opr);
 889     if (is_processed_reg_num(reg)) {
 890       add_def(reg, def_pos, use_kind, opr->type_register());
 891     }
 892     reg = reg_numHi(opr);
 893     if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) {
 894       add_def(reg, def_pos, use_kind, opr->type_register());
 895     }
 896   }
 897 }
 898 
 899 void LinearScan::add_use(LIR_Opr opr, int from, int to, IntervalUseKind use_kind) {
 900   TRACE_LINEAR_SCAN(2, tty->print(" use "); opr->print(tty); tty->print_cr(" from %d to %d (%d)", from, to, use_kind));
 901   assert(opr->is_register(), "should not be called otherwise");
 902 
 903   if (opr->is_virtual_register()) {
 904     assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 905     add_use(opr->vreg_number(), from, to, use_kind, opr->type_register());
 906 
 907   } else {
 908     int reg = reg_num(opr);
 909     if (is_processed_reg_num(reg)) {
 910       add_use(reg, from, to, use_kind, opr->type_register());
 911     }
 912     reg = reg_numHi(opr);
 913     if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) {
 914       add_use(reg, from, to, use_kind, opr->type_register());
 915     }
 916   }
 917 }
 918 
 919 void LinearScan::add_temp(LIR_Opr opr, int temp_pos, IntervalUseKind use_kind) {
 920   TRACE_LINEAR_SCAN(2, tty->print(" temp "); opr->print(tty); tty->print_cr(" temp_pos %d (%d)", temp_pos, use_kind));
 921   assert(opr->is_register(), "should not be called otherwise");
 922 
 923   if (opr->is_virtual_register()) {
 924     assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below");
 925     add_temp(opr->vreg_number(), temp_pos, use_kind, opr->type_register());
 926 
 927   } else {
 928     int reg = reg_num(opr);
 929     if (is_processed_reg_num(reg)) {
 930       add_temp(reg, temp_pos, use_kind, opr->type_register());
 931     }
 932     reg = reg_numHi(opr);
 933     if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) {
 934       add_temp(reg, temp_pos, use_kind, opr->type_register());
 935     }
 936   }
 937 }
 938 
 939 
 940 void LinearScan::add_def(int reg_num, int def_pos, IntervalUseKind use_kind, BasicType type) {
 941   Interval* interval = interval_at(reg_num);
 942   if (interval != NULL) {
 943     assert(interval->reg_num() == reg_num, "wrong interval");
 944 
 945     if (type != T_ILLEGAL) {
 946       interval->set_type(type);
 947     }
 948 
 949     Range* r = interval->first();
 950     if (r->from() <= def_pos) {
 951       // Update the starting point (when a range is first created for a use, its
 952       // start is the beginning of the current block until a def is encountered.)
 953       r->set_from(def_pos);
 954       interval->add_use_pos(def_pos, use_kind);
 955 
 956     } else {
 957       // Dead value - make vacuous interval
 958       // also add use_kind for dead intervals
 959       interval->add_range(def_pos, def_pos + 1);
 960       interval->add_use_pos(def_pos, use_kind);
 961       TRACE_LINEAR_SCAN(2, tty->print_cr("Warning: def of reg %d at %d occurs without use", reg_num, def_pos));
 962     }
 963 
 964   } else {
 965     // Dead value - make vacuous interval
 966     // also add use_kind for dead intervals
 967     interval = create_interval(reg_num);
 968     if (type != T_ILLEGAL) {
 969       interval->set_type(type);
 970     }
 971 
 972     interval->add_range(def_pos, def_pos + 1);
 973     interval->add_use_pos(def_pos, use_kind);
 974     TRACE_LINEAR_SCAN(2, tty->print_cr("Warning: dead value %d at %d in live intervals", reg_num, def_pos));
 975   }
 976 
 977   change_spill_definition_pos(interval, def_pos);
 978   if (use_kind == noUse && interval->spill_state() <= startInMemory) {
 979         // detection of method-parameters and roundfp-results
 980         // TODO: move this directly to position where use-kind is computed
 981     interval->set_spill_state(startInMemory);
 982   }
 983 }
 984 
 985 void LinearScan::add_use(int reg_num, int from, int to, IntervalUseKind use_kind, BasicType type) {
 986   Interval* interval = interval_at(reg_num);
 987   if (interval == NULL) {
 988     interval = create_interval(reg_num);
 989   }
 990   assert(interval->reg_num() == reg_num, "wrong interval");
 991 
 992   if (type != T_ILLEGAL) {
 993     interval->set_type(type);
 994   }
 995 
 996   interval->add_range(from, to);
 997   interval->add_use_pos(to, use_kind);
 998 }
 999 
1000 void LinearScan::add_temp(int reg_num, int temp_pos, IntervalUseKind use_kind, BasicType type) {
1001   Interval* interval = interval_at(reg_num);
1002   if (interval == NULL) {
1003     interval = create_interval(reg_num);
1004   }
1005   assert(interval->reg_num() == reg_num, "wrong interval");
1006 
1007   if (type != T_ILLEGAL) {
1008     interval->set_type(type);
1009   }
1010 
1011   interval->add_range(temp_pos, temp_pos + 1);
1012   interval->add_use_pos(temp_pos, use_kind);
1013 }
1014 
1015 
1016 // the results of this functions are used for optimizing spilling and reloading
1017 // if the functions return shouldHaveRegister and the interval is spilled,
1018 // it is not reloaded to a register.
1019 IntervalUseKind LinearScan::use_kind_of_output_operand(LIR_Op* op, LIR_Opr opr) {
1020   if (op->code() == lir_move) {
1021     assert(op->as_Op1() != NULL, "lir_move must be LIR_Op1");
1022     LIR_Op1* move = (LIR_Op1*)op;
1023     LIR_Opr res = move->result_opr();
1024     bool result_in_memory = res->is_virtual() && gen()->is_vreg_flag_set(res->vreg_number(), LIRGenerator::must_start_in_memory);
1025 
1026     if (result_in_memory) {
1027       // Begin of an interval with must_start_in_memory set.
1028       // This interval will always get a stack slot first, so return noUse.
1029       return noUse;
1030 
1031     } else if (move->in_opr()->is_stack()) {
1032       // method argument (condition must be equal to handle_method_arguments)
1033       return noUse;
1034 
1035     } else if (move->in_opr()->is_register() && move->result_opr()->is_register()) {
1036       // Move from register to register
1037       if (block_of_op_with_id(op->id())->is_set(BlockBegin::osr_entry_flag)) {
1038         // special handling of phi-function moves inside osr-entry blocks
1039         // input operand must have a register instead of output operand (leads to better register allocation)
1040         return shouldHaveRegister;
1041       }
1042     }
1043   }
1044 
1045   if (opr->is_virtual() &&
1046       gen()->is_vreg_flag_set(opr->vreg_number(), LIRGenerator::must_start_in_memory)) {
1047     // result is a stack-slot, so prevent immediate reloading
1048     return noUse;
1049   }
1050 
1051   // all other operands require a register
1052   return mustHaveRegister;
1053 }
1054 
1055 IntervalUseKind LinearScan::use_kind_of_input_operand(LIR_Op* op, LIR_Opr opr) {
1056   if (op->code() == lir_move) {
1057     assert(op->as_Op1() != NULL, "lir_move must be LIR_Op1");
1058     LIR_Op1* move = (LIR_Op1*)op;
1059     LIR_Opr res = move->result_opr();
1060     bool result_in_memory = res->is_virtual() && gen()->is_vreg_flag_set(res->vreg_number(), LIRGenerator::must_start_in_memory);
1061 
1062     if (result_in_memory) {
1063       // Move to an interval with must_start_in_memory set.
1064       // To avoid moves from stack to stack (not allowed) force the input operand to a register
1065       return mustHaveRegister;
1066 
1067     } else if (move->in_opr()->is_register() && move->result_opr()->is_register()) {
1068       // Move from register to register
1069       if (block_of_op_with_id(op->id())->is_set(BlockBegin::osr_entry_flag)) {
1070         // special handling of phi-function moves inside osr-entry blocks
1071         // input operand must have a register instead of output operand (leads to better register allocation)
1072         return mustHaveRegister;
1073       }
1074 
1075       // The input operand is not forced to a register (moves from stack to register are allowed),
1076       // but it is faster if the input operand is in a register
1077       return shouldHaveRegister;
1078     }
1079   }
1080 
1081 
1082 #ifdef X86
1083   if (op->code() == lir_cmove) {
1084     // conditional moves can handle stack operands
1085     assert(op->result_opr()->is_register(), "result must always be in a register");
1086     return shouldHaveRegister;
1087   }
1088 
1089   // optimizations for second input operand of arithmehtic operations on Intel
1090   // this operand is allowed to be on the stack in some cases
1091   BasicType opr_type = opr->type_register();
1092   if (opr_type == T_FLOAT || opr_type == T_DOUBLE) {
1093     if ((UseSSE == 1 && opr_type == T_FLOAT) || UseSSE >= 2) {
1094       // SSE float instruction (T_DOUBLE only supported with SSE2)
1095       switch (op->code()) {
1096         case lir_cmp:
1097         case lir_add:
1098         case lir_sub:
1099         case lir_mul:
1100         case lir_div:
1101         {
1102           assert(op->as_Op2() != NULL, "must be LIR_Op2");
1103           LIR_Op2* op2 = (LIR_Op2*)op;
1104           if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) {
1105             assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register");
1106             return shouldHaveRegister;
1107           }
1108         }
1109       }
1110     } else {
1111       // FPU stack float instruction
1112       switch (op->code()) {
1113         case lir_add:
1114         case lir_sub:
1115         case lir_mul:
1116         case lir_div:
1117         {
1118           assert(op->as_Op2() != NULL, "must be LIR_Op2");
1119           LIR_Op2* op2 = (LIR_Op2*)op;
1120           if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) {
1121             assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register");
1122             return shouldHaveRegister;
1123           }
1124         }
1125       }
1126     }
1127     // We want to sometimes use logical operations on pointers, in particular in GC barriers.
1128     // Since 64bit logical operations do not current support operands on stack, we have to make sure
1129     // T_OBJECT doesn't get spilled along with T_LONG.
1130   } else if (opr_type != T_LONG LP64_ONLY(&& opr_type != T_OBJECT)) {
1131     // integer instruction (note: long operands must always be in register)
1132     switch (op->code()) {
1133       case lir_cmp:
1134       case lir_add:
1135       case lir_sub:
1136       case lir_logic_and:
1137       case lir_logic_or:
1138       case lir_logic_xor:
1139       {
1140         assert(op->as_Op2() != NULL, "must be LIR_Op2");
1141         LIR_Op2* op2 = (LIR_Op2*)op;
1142         if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) {
1143           assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register");
1144           return shouldHaveRegister;
1145         }
1146       }
1147     }
1148   }
1149 #endif // X86
1150 
1151   // all other operands require a register
1152   return mustHaveRegister;
1153 }
1154 
1155 
1156 void LinearScan::handle_method_arguments(LIR_Op* op) {
1157   // special handling for method arguments (moves from stack to virtual register):
1158   // the interval gets no register assigned, but the stack slot.
1159   // it is split before the first use by the register allocator.
1160 
1161   if (op->code() == lir_move) {
1162     assert(op->as_Op1() != NULL, "must be LIR_Op1");
1163     LIR_Op1* move = (LIR_Op1*)op;
1164 
1165     if (move->in_opr()->is_stack()) {
1166 #ifdef ASSERT
1167       int arg_size = compilation()->method()->arg_size();
1168       LIR_Opr o = move->in_opr();
1169       if (o->is_single_stack()) {
1170         assert(o->single_stack_ix() >= 0 && o->single_stack_ix() < arg_size, "out of range");
1171       } else if (o->is_double_stack()) {
1172         assert(o->double_stack_ix() >= 0 && o->double_stack_ix() < arg_size, "out of range");
1173       } else {
1174         ShouldNotReachHere();
1175       }
1176 
1177       assert(move->id() > 0, "invalid id");
1178       assert(block_of_op_with_id(move->id())->number_of_preds() == 0, "move from stack must be in first block");
1179       assert(move->result_opr()->is_virtual(), "result of move must be a virtual register");
1180 
1181       TRACE_LINEAR_SCAN(4, tty->print_cr("found move from stack slot %d to vreg %d", o->is_single_stack() ? o->single_stack_ix() : o->double_stack_ix(), reg_num(move->result_opr())));
1182 #endif
1183 
1184       Interval* interval = interval_at(reg_num(move->result_opr()));
1185 
1186       int stack_slot = LinearScan::nof_regs + (move->in_opr()->is_single_stack() ? move->in_opr()->single_stack_ix() : move->in_opr()->double_stack_ix());
1187       interval->set_canonical_spill_slot(stack_slot);
1188       interval->assign_reg(stack_slot);
1189     }
1190   }
1191 }
1192 
1193 void LinearScan::handle_doubleword_moves(LIR_Op* op) {
1194   // special handling for doubleword move from memory to register:
1195   // in this case the registers of the input address and the result
1196   // registers must not overlap -> add a temp range for the input registers
1197   if (op->code() == lir_move) {
1198     assert(op->as_Op1() != NULL, "must be LIR_Op1");
1199     LIR_Op1* move = (LIR_Op1*)op;
1200 
1201     if (move->result_opr()->is_double_cpu() && move->in_opr()->is_pointer()) {
1202       LIR_Address* address = move->in_opr()->as_address_ptr();
1203       if (address != NULL) {
1204         if (address->base()->is_valid()) {
1205           add_temp(address->base(), op->id(), noUse);
1206         }
1207         if (address->index()->is_valid()) {
1208           add_temp(address->index(), op->id(), noUse);
1209         }
1210       }
1211     }
1212   }
1213 }
1214 
1215 void LinearScan::add_register_hints(LIR_Op* op) {
1216   switch (op->code()) {
1217     case lir_move:      // fall through
1218     case lir_convert: {
1219       assert(op->as_Op1() != NULL, "lir_move, lir_convert must be LIR_Op1");
1220       LIR_Op1* move = (LIR_Op1*)op;
1221 
1222       LIR_Opr move_from = move->in_opr();
1223       LIR_Opr move_to = move->result_opr();
1224 
1225       if (move_to->is_register() && move_from->is_register()) {
1226         Interval* from = interval_at(reg_num(move_from));
1227         Interval* to = interval_at(reg_num(move_to));
1228         if (from != NULL && to != NULL) {
1229           to->set_register_hint(from);
1230           TRACE_LINEAR_SCAN(4, tty->print_cr("operation at op_id %d: added hint from interval %d to %d", move->id(), from->reg_num(), to->reg_num()));
1231         }
1232       }
1233       break;
1234     }
1235     case lir_cmove: {
1236       assert(op->as_Op2() != NULL, "lir_cmove must be LIR_Op2");
1237       LIR_Op2* cmove = (LIR_Op2*)op;
1238 
1239       LIR_Opr move_from = cmove->in_opr1();
1240       LIR_Opr move_to = cmove->result_opr();
1241 
1242       if (move_to->is_register() && move_from->is_register()) {
1243         Interval* from = interval_at(reg_num(move_from));
1244         Interval* to = interval_at(reg_num(move_to));
1245         if (from != NULL && to != NULL) {
1246           to->set_register_hint(from);
1247           TRACE_LINEAR_SCAN(4, tty->print_cr("operation at op_id %d: added hint from interval %d to %d", cmove->id(), from->reg_num(), to->reg_num()));
1248         }
1249       }
1250       break;
1251     }
1252   }
1253 }
1254 
1255 
1256 void LinearScan::build_intervals() {
1257   TIME_LINEAR_SCAN(timer_build_intervals);
1258 
1259   // initialize interval list with expected number of intervals
1260   // (32 is added to have some space for split children without having to resize the list)
1261   _intervals = IntervalList(num_virtual_regs() + 32);
1262   // initialize all slots that are used by build_intervals
1263   _intervals.at_put_grow(num_virtual_regs() - 1, NULL, NULL);
1264 
1265   // create a list with all caller-save registers (cpu, fpu, xmm)
1266   // when an instruction is a call, a temp range is created for all these registers
1267   int num_caller_save_registers = 0;
1268   int caller_save_registers[LinearScan::nof_regs];
1269 
1270   int i;
1271   for (i = 0; i < FrameMap::nof_caller_save_cpu_regs(); i++) {
1272     LIR_Opr opr = FrameMap::caller_save_cpu_reg_at(i);
1273     assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands");
1274     assert(reg_numHi(opr) == -1, "missing addition of range for hi-register");
1275     caller_save_registers[num_caller_save_registers++] = reg_num(opr);
1276   }
1277 
1278   // temp ranges for fpu registers are only created when the method has
1279   // virtual fpu operands. Otherwise no allocation for fpu registers is
1280   // perfomed and so the temp ranges would be useless
1281   if (has_fpu_registers()) {
1282 #ifdef X86
1283     if (UseSSE < 2) {
1284 #endif
1285       for (i = 0; i < FrameMap::nof_caller_save_fpu_regs; i++) {
1286         LIR_Opr opr = FrameMap::caller_save_fpu_reg_at(i);
1287         assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands");
1288         assert(reg_numHi(opr) == -1, "missing addition of range for hi-register");
1289         caller_save_registers[num_caller_save_registers++] = reg_num(opr);
1290       }
1291 #ifdef X86
1292     }
1293     if (UseSSE > 0) {
1294       int num_caller_save_xmm_regs = FrameMap::get_num_caller_save_xmms();
1295       for (i = 0; i < num_caller_save_xmm_regs; i ++) {
1296         LIR_Opr opr = FrameMap::caller_save_xmm_reg_at(i);
1297         assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands");
1298         assert(reg_numHi(opr) == -1, "missing addition of range for hi-register");
1299         caller_save_registers[num_caller_save_registers++] = reg_num(opr);
1300       }
1301     }
1302 #endif
1303   }
1304   assert(num_caller_save_registers <= LinearScan::nof_regs, "out of bounds");
1305 
1306 
1307   LIR_OpVisitState visitor;
1308 
1309   // iterate all blocks in reverse order
1310   for (i = block_count() - 1; i >= 0; i--) {
1311     BlockBegin* block = block_at(i);
1312     LIR_OpList* instructions = block->lir()->instructions_list();
1313     int         block_from =   block->first_lir_instruction_id();
1314     int         block_to =     block->last_lir_instruction_id();
1315 
1316     assert(block_from == instructions->at(0)->id(), "must be");
1317     assert(block_to   == instructions->at(instructions->length() - 1)->id(), "must be");
1318 
1319     // Update intervals for registers live at the end of this block;
1320     BitMap live = block->live_out();
1321     int size = (int)live.size();
1322     for (int number = (int)live.get_next_one_offset(0, size); number < size; number = (int)live.get_next_one_offset(number + 1, size)) {
1323       assert(live.at(number), "should not stop here otherwise");
1324       assert(number >= LIR_OprDesc::vreg_base, "fixed intervals must not be live on block bounds");
1325       TRACE_LINEAR_SCAN(2, tty->print_cr("live in %d to %d", number, block_to + 2));
1326 
1327       add_use(number, block_from, block_to + 2, noUse, T_ILLEGAL);
1328 
1329       // add special use positions for loop-end blocks when the
1330       // interval is used anywhere inside this loop.  It's possible
1331       // that the block was part of a non-natural loop, so it might
1332       // have an invalid loop index.
1333       if (block->is_set(BlockBegin::linear_scan_loop_end_flag) &&
1334           block->loop_index() != -1 &&
1335           is_interval_in_loop(number, block->loop_index())) {
1336         interval_at(number)->add_use_pos(block_to + 1, loopEndMarker);
1337       }
1338     }
1339 
1340     // iterate all instructions of the block in reverse order.
1341     // skip the first instruction because it is always a label
1342     // definitions of intervals are processed before uses
1343     assert(visitor.no_operands(instructions->at(0)), "first operation must always be a label");
1344     for (int j = instructions->length() - 1; j >= 1; j--) {
1345       LIR_Op* op = instructions->at(j);
1346       int op_id = op->id();
1347 
1348       // visit operation to collect all operands
1349       visitor.visit(op);
1350 
1351       // add a temp range for each register if operation destroys caller-save registers
1352       if (visitor.has_call()) {
1353         for (int k = 0; k < num_caller_save_registers; k++) {
1354           add_temp(caller_save_registers[k], op_id, noUse, T_ILLEGAL);
1355         }
1356         TRACE_LINEAR_SCAN(4, tty->print_cr("operation destroys all caller-save registers"));
1357       }
1358 
1359       // Add any platform dependent temps
1360       pd_add_temps(op);
1361 
1362       // visit definitions (output and temp operands)
1363       int k, n;
1364       n = visitor.opr_count(LIR_OpVisitState::outputMode);
1365       for (k = 0; k < n; k++) {
1366         LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, k);
1367         assert(opr->is_register(), "visitor should only return register operands");
1368         add_def(opr, op_id, use_kind_of_output_operand(op, opr));
1369       }
1370 
1371       n = visitor.opr_count(LIR_OpVisitState::tempMode);
1372       for (k = 0; k < n; k++) {
1373         LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, k);
1374         assert(opr->is_register(), "visitor should only return register operands");
1375         add_temp(opr, op_id, mustHaveRegister);
1376       }
1377 
1378       // visit uses (input operands)
1379       n = visitor.opr_count(LIR_OpVisitState::inputMode);
1380       for (k = 0; k < n; k++) {
1381         LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, k);
1382         assert(opr->is_register(), "visitor should only return register operands");
1383         add_use(opr, block_from, op_id, use_kind_of_input_operand(op, opr));
1384       }
1385 
1386       // Add uses of live locals from interpreter's point of view for proper
1387       // debug information generation
1388       // Treat these operands as temp values (if the life range is extended
1389       // to a call site, the value would be in a register at the call otherwise)
1390       n = visitor.info_count();
1391       for (k = 0; k < n; k++) {
1392         CodeEmitInfo* info = visitor.info_at(k);
1393         ValueStack* stack = info->stack();
1394         for_each_state_value(stack, value,
1395           add_use(value, block_from, op_id + 1, noUse);
1396         );
1397       }
1398 
1399       // special steps for some instructions (especially moves)
1400       handle_method_arguments(op);
1401       handle_doubleword_moves(op);
1402       add_register_hints(op);
1403 
1404     } // end of instruction iteration
1405   } // end of block iteration
1406 
1407 
1408   // add the range [0, 1[ to all fixed intervals
1409   // -> the register allocator need not handle unhandled fixed intervals
1410   for (int n = 0; n < LinearScan::nof_regs; n++) {
1411     Interval* interval = interval_at(n);
1412     if (interval != NULL) {
1413       interval->add_range(0, 1);
1414     }
1415   }
1416 }
1417 
1418 
1419 // ********** Phase 5: actual register allocation
1420 
1421 int LinearScan::interval_cmp(Interval** a, Interval** b) {
1422   if (*a != NULL) {
1423     if (*b != NULL) {
1424       return (*a)->from() - (*b)->from();
1425     } else {
1426       return -1;
1427     }
1428   } else {
1429     if (*b != NULL) {
1430       return 1;
1431     } else {
1432       return 0;
1433     }
1434   }
1435 }
1436 
1437 #ifndef PRODUCT
1438 int interval_cmp(Interval* const& l, Interval* const& r) {
1439   return l->from() - r->from();
1440 }
1441 
1442 bool find_interval(Interval* interval, IntervalArray* intervals) {
1443   bool found;
1444   int idx = intervals->find_sorted<Interval*, interval_cmp>(interval, found);
1445 
1446   if (!found) {
1447     return false;
1448   }
1449 
1450   int from = interval->from();
1451 
1452   // The index we've found using binary search is pointing to an interval
1453   // that is defined in the same place as the interval we were looking for.
1454   // So now we have to look around that index and find exact interval.
1455   for (int i = idx; i >= 0; i--) {
1456     if (intervals->at(i) == interval) {
1457       return true;
1458     }
1459     if (intervals->at(i)->from() != from) {
1460       break;
1461     }
1462   }
1463 
1464   for (int i = idx + 1; i < intervals->length(); i++) {
1465     if (intervals->at(i) == interval) {
1466       return true;
1467     }
1468     if (intervals->at(i)->from() != from) {
1469       break;
1470     }
1471   }
1472 
1473   return false;
1474 }
1475 
1476 bool LinearScan::is_sorted(IntervalArray* intervals) {
1477   int from = -1;
1478   int null_count = 0;
1479 
1480   for (int i = 0; i < intervals->length(); i++) {
1481     Interval* it = intervals->at(i);
1482     if (it != NULL) {
1483       assert(from <= it->from(), "Intervals are unordered");
1484       from = it->from();
1485     } else {
1486       null_count++;
1487     }
1488   }
1489 
1490   assert(null_count == 0, "Sorted intervals should not contain nulls");
1491 
1492   null_count = 0;
1493 
1494   for (int i = 0; i < interval_count(); i++) {
1495     Interval* interval = interval_at(i);
1496     if (interval != NULL) {
1497       assert(find_interval(interval, intervals), "Lists do not contain same intervals");
1498     } else {
1499       null_count++;
1500     }
1501   }
1502 
1503   assert(interval_count() - null_count == intervals->length(),
1504       "Sorted list should contain the same amount of non-NULL intervals as unsorted list");
1505 
1506   return true;
1507 }
1508 #endif
1509 
1510 void LinearScan::add_to_list(Interval** first, Interval** prev, Interval* interval) {
1511   if (*prev != NULL) {
1512     (*prev)->set_next(interval);
1513   } else {
1514     *first = interval;
1515   }
1516   *prev = interval;
1517 }
1518 
1519 void LinearScan::create_unhandled_lists(Interval** list1, Interval** list2, bool (is_list1)(const Interval* i), bool (is_list2)(const Interval* i)) {
1520   assert(is_sorted(_sorted_intervals), "interval list is not sorted");
1521 
1522   *list1 = *list2 = Interval::end();
1523 
1524   Interval* list1_prev = NULL;
1525   Interval* list2_prev = NULL;
1526   Interval* v;
1527 
1528   const int n = _sorted_intervals->length();
1529   for (int i = 0; i < n; i++) {
1530     v = _sorted_intervals->at(i);
1531     if (v == NULL) continue;
1532 
1533     if (is_list1(v)) {
1534       add_to_list(list1, &list1_prev, v);
1535     } else if (is_list2 == NULL || is_list2(v)) {
1536       add_to_list(list2, &list2_prev, v);
1537     }
1538   }
1539 
1540   if (list1_prev != NULL) list1_prev->set_next(Interval::end());
1541   if (list2_prev != NULL) list2_prev->set_next(Interval::end());
1542 
1543   assert(list1_prev == NULL || list1_prev->next() == Interval::end(), "linear list ends not with sentinel");
1544   assert(list2_prev == NULL || list2_prev->next() == Interval::end(), "linear list ends not with sentinel");
1545 }
1546 
1547 
1548 void LinearScan::sort_intervals_before_allocation() {
1549   TIME_LINEAR_SCAN(timer_sort_intervals_before);
1550 
1551   if (_needs_full_resort) {
1552     // There is no known reason why this should occur but just in case...
1553     assert(false, "should never occur");
1554     // Re-sort existing interval list because an Interval::from() has changed
1555     _sorted_intervals->sort(interval_cmp);
1556     _needs_full_resort = false;
1557   }
1558 
1559   IntervalList* unsorted_list = &_intervals;
1560   int unsorted_len = unsorted_list->length();
1561   int sorted_len = 0;
1562   int unsorted_idx;
1563   int sorted_idx = 0;
1564   int sorted_from_max = -1;
1565 
1566   // calc number of items for sorted list (sorted list must not contain NULL values)
1567   for (unsorted_idx = 0; unsorted_idx < unsorted_len; unsorted_idx++) {
1568     if (unsorted_list->at(unsorted_idx) != NULL) {
1569       sorted_len++;
1570     }
1571   }
1572   IntervalArray* sorted_list = new IntervalArray(sorted_len, sorted_len, NULL);
1573 
1574   // special sorting algorithm: the original interval-list is almost sorted,
1575   // only some intervals are swapped. So this is much faster than a complete QuickSort
1576   for (unsorted_idx = 0; unsorted_idx < unsorted_len; unsorted_idx++) {
1577     Interval* cur_interval = unsorted_list->at(unsorted_idx);
1578 
1579     if (cur_interval != NULL) {
1580       int cur_from = cur_interval->from();
1581 
1582       if (sorted_from_max <= cur_from) {
1583         sorted_list->at_put(sorted_idx++, cur_interval);
1584         sorted_from_max = cur_interval->from();
1585       } else {
1586         // the asumption that the intervals are already sorted failed,
1587         // so this interval must be sorted in manually
1588         int j;
1589         for (j = sorted_idx - 1; j >= 0 && cur_from < sorted_list->at(j)->from(); j--) {
1590           sorted_list->at_put(j + 1, sorted_list->at(j));
1591         }
1592         sorted_list->at_put(j + 1, cur_interval);
1593         sorted_idx++;
1594       }
1595     }
1596   }
1597   _sorted_intervals = sorted_list;
1598   assert(is_sorted(_sorted_intervals), "intervals unsorted");
1599 }
1600 
1601 void LinearScan::sort_intervals_after_allocation() {
1602   TIME_LINEAR_SCAN(timer_sort_intervals_after);
1603 
1604   if (_needs_full_resort) {
1605     // Re-sort existing interval list because an Interval::from() has changed
1606     _sorted_intervals->sort(interval_cmp);
1607     _needs_full_resort = false;
1608   }
1609 
1610   IntervalArray* old_list = _sorted_intervals;
1611   IntervalList* new_list = _new_intervals_from_allocation;
1612   int old_len = old_list->length();
1613   int new_len = new_list->length();
1614 
1615   if (new_len == 0) {
1616     // no intervals have been added during allocation, so sorted list is already up to date
1617     assert(is_sorted(_sorted_intervals), "intervals unsorted");
1618     return;
1619   }
1620 
1621   // conventional sort-algorithm for new intervals
1622   new_list->sort(interval_cmp);
1623 
1624   // merge old and new list (both already sorted) into one combined list
1625   int combined_list_len = old_len + new_len;
1626   IntervalArray* combined_list = new IntervalArray(combined_list_len, combined_list_len, NULL);
1627   int old_idx = 0;
1628   int new_idx = 0;
1629 
1630   while (old_idx + new_idx < old_len + new_len) {
1631     if (new_idx >= new_len || (old_idx < old_len && old_list->at(old_idx)->from() <= new_list->at(new_idx)->from())) {
1632       combined_list->at_put(old_idx + new_idx, old_list->at(old_idx));
1633       old_idx++;
1634     } else {
1635       combined_list->at_put(old_idx + new_idx, new_list->at(new_idx));
1636       new_idx++;
1637     }
1638   }
1639 
1640   _sorted_intervals = combined_list;
1641   assert(is_sorted(_sorted_intervals), "intervals unsorted");
1642 }
1643 
1644 
1645 void LinearScan::allocate_registers() {
1646   TIME_LINEAR_SCAN(timer_allocate_registers);
1647 
1648   Interval* precolored_cpu_intervals, *not_precolored_cpu_intervals;
1649   Interval* precolored_fpu_intervals, *not_precolored_fpu_intervals;
1650 
1651   // allocate cpu registers
1652   create_unhandled_lists(&precolored_cpu_intervals, &not_precolored_cpu_intervals,
1653                          is_precolored_cpu_interval, is_virtual_cpu_interval);
1654 
1655   // allocate fpu registers
1656   create_unhandled_lists(&precolored_fpu_intervals, &not_precolored_fpu_intervals,
1657                          is_precolored_fpu_interval, is_virtual_fpu_interval);
1658 
1659   // the fpu interval allocation cannot be moved down below with the fpu section as
1660   // the cpu_lsw.walk() changes interval positions.
1661 
1662   LinearScanWalker cpu_lsw(this, precolored_cpu_intervals, not_precolored_cpu_intervals);
1663   cpu_lsw.walk();
1664   cpu_lsw.finish_allocation();
1665 
1666   if (has_fpu_registers()) {
1667     LinearScanWalker fpu_lsw(this, precolored_fpu_intervals, not_precolored_fpu_intervals);
1668     fpu_lsw.walk();
1669     fpu_lsw.finish_allocation();
1670   }
1671 }
1672 
1673 
1674 // ********** Phase 6: resolve data flow
1675 // (insert moves at edges between blocks if intervals have been split)
1676 
1677 // wrapper for Interval::split_child_at_op_id that performs a bailout in product mode
1678 // instead of returning NULL
1679 Interval* LinearScan::split_child_at_op_id(Interval* interval, int op_id, LIR_OpVisitState::OprMode mode) {
1680   Interval* result = interval->split_child_at_op_id(op_id, mode);
1681   if (result != NULL) {
1682     return result;
1683   }
1684 
1685   assert(false, "must find an interval, but do a clean bailout in product mode");
1686   result = new Interval(LIR_OprDesc::vreg_base);
1687   result->assign_reg(0);
1688   result->set_type(T_INT);
1689   BAILOUT_("LinearScan: interval is NULL", result);
1690 }
1691 
1692 
1693 Interval* LinearScan::interval_at_block_begin(BlockBegin* block, int reg_num) {
1694   assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds");
1695   assert(interval_at(reg_num) != NULL, "no interval found");
1696 
1697   return split_child_at_op_id(interval_at(reg_num), block->first_lir_instruction_id(), LIR_OpVisitState::outputMode);
1698 }
1699 
1700 Interval* LinearScan::interval_at_block_end(BlockBegin* block, int reg_num) {
1701   assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds");
1702   assert(interval_at(reg_num) != NULL, "no interval found");
1703 
1704   return split_child_at_op_id(interval_at(reg_num), block->last_lir_instruction_id() + 1, LIR_OpVisitState::outputMode);
1705 }
1706 
1707 Interval* LinearScan::interval_at_op_id(int reg_num, int op_id) {
1708   assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds");
1709   assert(interval_at(reg_num) != NULL, "no interval found");
1710 
1711   return split_child_at_op_id(interval_at(reg_num), op_id, LIR_OpVisitState::inputMode);
1712 }
1713 
1714 
1715 void LinearScan::resolve_collect_mappings(BlockBegin* from_block, BlockBegin* to_block, MoveResolver &move_resolver) {
1716   DEBUG_ONLY(move_resolver.check_empty());
1717 
1718   const int num_regs = num_virtual_regs();
1719   const int size = live_set_size();
1720   const BitMap live_at_edge = to_block->live_in();
1721 
1722   // visit all registers where the live_at_edge bit is set
1723   for (int r = (int)live_at_edge.get_next_one_offset(0, size); r < size; r = (int)live_at_edge.get_next_one_offset(r + 1, size)) {
1724     assert(r < num_regs, "live information set for not exisiting interval");
1725     assert(from_block->live_out().at(r) && to_block->live_in().at(r), "interval not live at this edge");
1726 
1727     Interval* from_interval = interval_at_block_end(from_block, r);
1728     Interval* to_interval = interval_at_block_begin(to_block, r);
1729 
1730     if (from_interval != to_interval && (from_interval->assigned_reg() != to_interval->assigned_reg() || from_interval->assigned_regHi() != to_interval->assigned_regHi())) {
1731       // need to insert move instruction
1732       move_resolver.add_mapping(from_interval, to_interval);
1733     }
1734   }
1735 }
1736 
1737 
1738 void LinearScan::resolve_find_insert_pos(BlockBegin* from_block, BlockBegin* to_block, MoveResolver &move_resolver) {
1739   if (from_block->number_of_sux() <= 1) {
1740     TRACE_LINEAR_SCAN(4, tty->print_cr("inserting moves at end of from_block B%d", from_block->block_id()));
1741 
1742     LIR_OpList* instructions = from_block->lir()->instructions_list();
1743     LIR_OpBranch* branch = instructions->last()->as_OpBranch();
1744     if (branch != NULL) {
1745       // insert moves before branch
1746       assert(branch->cond() == lir_cond_always, "block does not end with an unconditional jump");
1747       move_resolver.set_insert_position(from_block->lir(), instructions->length() - 2);
1748     } else {
1749       move_resolver.set_insert_position(from_block->lir(), instructions->length() - 1);
1750     }
1751 
1752   } else {
1753     TRACE_LINEAR_SCAN(4, tty->print_cr("inserting moves at beginning of to_block B%d", to_block->block_id()));
1754 #ifdef ASSERT
1755     assert(from_block->lir()->instructions_list()->at(0)->as_OpLabel() != NULL, "block does not start with a label");
1756 
1757     // because the number of predecessor edges matches the number of
1758     // successor edges, blocks which are reached by switch statements
1759     // may have be more than one predecessor but it will be guaranteed
1760     // that all predecessors will be the same.
1761     for (int i = 0; i < to_block->number_of_preds(); i++) {
1762       assert(from_block == to_block->pred_at(i), "all critical edges must be broken");
1763     }
1764 #endif
1765 
1766     move_resolver.set_insert_position(to_block->lir(), 0);
1767   }
1768 }
1769 
1770 
1771 // insert necessary moves (spilling or reloading) at edges between blocks if interval has been split
1772 void LinearScan::resolve_data_flow() {
1773   TIME_LINEAR_SCAN(timer_resolve_data_flow);
1774 
1775   int num_blocks = block_count();
1776   MoveResolver move_resolver(this);
1777   BitMap block_completed(num_blocks);  block_completed.clear();
1778   BitMap already_resolved(num_blocks); already_resolved.clear();
1779 
1780   int i;
1781   for (i = 0; i < num_blocks; i++) {
1782     BlockBegin* block = block_at(i);
1783 
1784     // check if block has only one predecessor and only one successor
1785     if (block->number_of_preds() == 1 && block->number_of_sux() == 1 && block->number_of_exception_handlers() == 0) {
1786       LIR_OpList* instructions = block->lir()->instructions_list();
1787       assert(instructions->at(0)->code() == lir_label, "block must start with label");
1788       assert(instructions->last()->code() == lir_branch, "block with successors must end with branch");
1789       assert(instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block with successor must end with unconditional branch");
1790 
1791       // check if block is empty (only label and branch)
1792       if (instructions->length() == 2) {
1793         BlockBegin* pred = block->pred_at(0);
1794         BlockBegin* sux = block->sux_at(0);
1795 
1796         // prevent optimization of two consecutive blocks
1797         if (!block_completed.at(pred->linear_scan_number()) && !block_completed.at(sux->linear_scan_number())) {
1798           TRACE_LINEAR_SCAN(3, tty->print_cr("**** optimizing empty block B%d (pred: B%d, sux: B%d)", block->block_id(), pred->block_id(), sux->block_id()));
1799           block_completed.set_bit(block->linear_scan_number());
1800 
1801           // directly resolve between pred and sux (without looking at the empty block between)
1802           resolve_collect_mappings(pred, sux, move_resolver);
1803           if (move_resolver.has_mappings()) {
1804             move_resolver.set_insert_position(block->lir(), 0);
1805             move_resolver.resolve_and_append_moves();
1806           }
1807         }
1808       }
1809     }
1810   }
1811 
1812 
1813   for (i = 0; i < num_blocks; i++) {
1814     if (!block_completed.at(i)) {
1815       BlockBegin* from_block = block_at(i);
1816       already_resolved.set_from(block_completed);
1817 
1818       int num_sux = from_block->number_of_sux();
1819       for (int s = 0; s < num_sux; s++) {
1820         BlockBegin* to_block = from_block->sux_at(s);
1821 
1822         // check for duplicate edges between the same blocks (can happen with switch blocks)
1823         if (!already_resolved.at(to_block->linear_scan_number())) {
1824           TRACE_LINEAR_SCAN(3, tty->print_cr("**** processing edge between B%d and B%d", from_block->block_id(), to_block->block_id()));
1825           already_resolved.set_bit(to_block->linear_scan_number());
1826 
1827           // collect all intervals that have been split between from_block and to_block
1828           resolve_collect_mappings(from_block, to_block, move_resolver);
1829           if (move_resolver.has_mappings()) {
1830             resolve_find_insert_pos(from_block, to_block, move_resolver);
1831             move_resolver.resolve_and_append_moves();
1832           }
1833         }
1834       }
1835     }
1836   }
1837 }
1838 
1839 
1840 void LinearScan::resolve_exception_entry(BlockBegin* block, int reg_num, MoveResolver &move_resolver) {
1841   if (interval_at(reg_num) == NULL) {
1842     // if a phi function is never used, no interval is created -> ignore this
1843     return;
1844   }
1845 
1846   Interval* interval = interval_at_block_begin(block, reg_num);
1847   int reg = interval->assigned_reg();
1848   int regHi = interval->assigned_regHi();
1849 
1850   if ((reg < nof_regs && interval->always_in_memory()) ||
1851       (use_fpu_stack_allocation() && reg >= pd_first_fpu_reg && reg <= pd_last_fpu_reg)) {
1852     // the interval is split to get a short range that is located on the stack
1853     // in the following two cases:
1854     // * the interval started in memory (e.g. method parameter), but is currently in a register
1855     //   this is an optimization for exception handling that reduces the number of moves that
1856     //   are necessary for resolving the states when an exception uses this exception handler
1857     // * the interval would be on the fpu stack at the begin of the exception handler
1858     //   this is not allowed because of the complicated fpu stack handling on Intel
1859 
1860     // range that will be spilled to memory
1861     int from_op_id = block->first_lir_instruction_id();
1862     int to_op_id = from_op_id + 1;  // short live range of length 1
1863     assert(interval->from() <= from_op_id && interval->to() >= to_op_id,
1864            "no split allowed between exception entry and first instruction");
1865 
1866     if (interval->from() != from_op_id) {
1867       // the part before from_op_id is unchanged
1868       interval = interval->split(from_op_id);
1869       interval->assign_reg(reg, regHi);
1870       append_interval(interval);
1871     } else {
1872       _needs_full_resort = true;
1873     }
1874     assert(interval->from() == from_op_id, "must be true now");
1875 
1876     Interval* spilled_part = interval;
1877     if (interval->to() != to_op_id) {
1878       // the part after to_op_id is unchanged
1879       spilled_part = interval->split_from_start(to_op_id);
1880       append_interval(spilled_part);
1881       move_resolver.add_mapping(spilled_part, interval);
1882     }
1883     assign_spill_slot(spilled_part);
1884 
1885     assert(spilled_part->from() == from_op_id && spilled_part->to() == to_op_id, "just checking");
1886   }
1887 }
1888 
1889 void LinearScan::resolve_exception_entry(BlockBegin* block, MoveResolver &move_resolver) {
1890   assert(block->is_set(BlockBegin::exception_entry_flag), "should not call otherwise");
1891   DEBUG_ONLY(move_resolver.check_empty());
1892 
1893   // visit all registers where the live_in bit is set
1894   int size = live_set_size();
1895   for (int r = (int)block->live_in().get_next_one_offset(0, size); r < size; r = (int)block->live_in().get_next_one_offset(r + 1, size)) {
1896     resolve_exception_entry(block, r, move_resolver);
1897   }
1898 
1899   // the live_in bits are not set for phi functions of the xhandler entry, so iterate them separately
1900   for_each_phi_fun(block, phi,
1901     resolve_exception_entry(block, phi->operand()->vreg_number(), move_resolver)
1902   );
1903 
1904   if (move_resolver.has_mappings()) {
1905     // insert moves after first instruction
1906     move_resolver.set_insert_position(block->lir(), 0);
1907     move_resolver.resolve_and_append_moves();
1908   }
1909 }
1910 
1911 
1912 void LinearScan::resolve_exception_edge(XHandler* handler, int throwing_op_id, int reg_num, Phi* phi, MoveResolver &move_resolver) {
1913   if (interval_at(reg_num) == NULL) {
1914     // if a phi function is never used, no interval is created -> ignore this
1915     return;
1916   }
1917 
1918   // the computation of to_interval is equal to resolve_collect_mappings,
1919   // but from_interval is more complicated because of phi functions
1920   BlockBegin* to_block = handler->entry_block();
1921   Interval* to_interval = interval_at_block_begin(to_block, reg_num);
1922 
1923   if (phi != NULL) {
1924     // phi function of the exception entry block
1925     // no moves are created for this phi function in the LIR_Generator, so the
1926     // interval at the throwing instruction must be searched using the operands
1927     // of the phi function
1928     Value from_value = phi->operand_at(handler->phi_operand());
1929 
1930     // with phi functions it can happen that the same from_value is used in
1931     // multiple mappings, so notify move-resolver that this is allowed
1932     move_resolver.set_multiple_reads_allowed();
1933 
1934     Constant* con = from_value->as_Constant();
1935     if (con != NULL && !con->is_pinned()) {
1936       // unpinned constants may have no register, so add mapping from constant to interval
1937       move_resolver.add_mapping(LIR_OprFact::value_type(con->type()), to_interval);
1938     } else {
1939       // search split child at the throwing op_id
1940       Interval* from_interval = interval_at_op_id(from_value->operand()->vreg_number(), throwing_op_id);
1941       move_resolver.add_mapping(from_interval, to_interval);
1942     }
1943 
1944   } else {
1945     // no phi function, so use reg_num also for from_interval
1946     // search split child at the throwing op_id
1947     Interval* from_interval = interval_at_op_id(reg_num, throwing_op_id);
1948     if (from_interval != to_interval) {
1949       // optimization to reduce number of moves: when to_interval is on stack and
1950       // the stack slot is known to be always correct, then no move is necessary
1951       if (!from_interval->always_in_memory() || from_interval->canonical_spill_slot() != to_interval->assigned_reg()) {
1952         move_resolver.add_mapping(from_interval, to_interval);
1953       }
1954     }
1955   }
1956 }
1957 
1958 void LinearScan::resolve_exception_edge(XHandler* handler, int throwing_op_id, MoveResolver &move_resolver) {
1959   TRACE_LINEAR_SCAN(4, tty->print_cr("resolving exception handler B%d: throwing_op_id=%d", handler->entry_block()->block_id(), throwing_op_id));
1960 
1961   DEBUG_ONLY(move_resolver.check_empty());
1962   assert(handler->lir_op_id() == -1, "already processed this xhandler");
1963   DEBUG_ONLY(handler->set_lir_op_id(throwing_op_id));
1964   assert(handler->entry_code() == NULL, "code already present");
1965 
1966   // visit all registers where the live_in bit is set
1967   BlockBegin* block = handler->entry_block();
1968   int size = live_set_size();
1969   for (int r = (int)block->live_in().get_next_one_offset(0, size); r < size; r = (int)block->live_in().get_next_one_offset(r + 1, size)) {
1970     resolve_exception_edge(handler, throwing_op_id, r, NULL, move_resolver);
1971   }
1972 
1973   // the live_in bits are not set for phi functions of the xhandler entry, so iterate them separately
1974   for_each_phi_fun(block, phi,
1975     resolve_exception_edge(handler, throwing_op_id, phi->operand()->vreg_number(), phi, move_resolver)
1976   );
1977 
1978   if (move_resolver.has_mappings()) {
1979     LIR_List* entry_code = new LIR_List(compilation());
1980     move_resolver.set_insert_position(entry_code, 0);
1981     move_resolver.resolve_and_append_moves();
1982 
1983     entry_code->jump(handler->entry_block());
1984     handler->set_entry_code(entry_code);
1985   }
1986 }
1987 
1988 
1989 void LinearScan::resolve_exception_handlers() {
1990   MoveResolver move_resolver(this);
1991   LIR_OpVisitState visitor;
1992   int num_blocks = block_count();
1993 
1994   int i;
1995   for (i = 0; i < num_blocks; i++) {
1996     BlockBegin* block = block_at(i);
1997     if (block->is_set(BlockBegin::exception_entry_flag)) {
1998       resolve_exception_entry(block, move_resolver);
1999     }
2000   }
2001 
2002   for (i = 0; i < num_blocks; i++) {
2003     BlockBegin* block = block_at(i);
2004     LIR_List* ops = block->lir();
2005     int num_ops = ops->length();
2006 
2007     // iterate all instructions of the block. skip the first because it is always a label
2008     assert(visitor.no_operands(ops->at(0)), "first operation must always be a label");
2009     for (int j = 1; j < num_ops; j++) {
2010       LIR_Op* op = ops->at(j);
2011       int op_id = op->id();
2012 
2013       if (op_id != -1 && has_info(op_id)) {
2014         // visit operation to collect all operands
2015         visitor.visit(op);
2016         assert(visitor.info_count() > 0, "should not visit otherwise");
2017 
2018         XHandlers* xhandlers = visitor.all_xhandler();
2019         int n = xhandlers->length();
2020         for (int k = 0; k < n; k++) {
2021           resolve_exception_edge(xhandlers->handler_at(k), op_id, move_resolver);
2022         }
2023 
2024 #ifdef ASSERT
2025       } else {
2026         visitor.visit(op);
2027         assert(visitor.all_xhandler()->length() == 0, "missed exception handler");
2028 #endif
2029       }
2030     }
2031   }
2032 }
2033 
2034 
2035 // ********** Phase 7: assign register numbers back to LIR
2036 // (includes computation of debug information and oop maps)
2037 
2038 VMReg LinearScan::vm_reg_for_interval(Interval* interval) {
2039   VMReg reg = interval->cached_vm_reg();
2040   if (!reg->is_valid() ) {
2041     reg = vm_reg_for_operand(operand_for_interval(interval));
2042     interval->set_cached_vm_reg(reg);
2043   }
2044   assert(reg == vm_reg_for_operand(operand_for_interval(interval)), "wrong cached value");
2045   return reg;
2046 }
2047 
2048 VMReg LinearScan::vm_reg_for_operand(LIR_Opr opr) {
2049   assert(opr->is_oop(), "currently only implemented for oop operands");
2050   return frame_map()->regname(opr);
2051 }
2052 
2053 
2054 LIR_Opr LinearScan::operand_for_interval(Interval* interval) {
2055   LIR_Opr opr = interval->cached_opr();
2056   if (opr->is_illegal()) {
2057     opr = calc_operand_for_interval(interval);
2058     interval->set_cached_opr(opr);
2059   }
2060 
2061   assert(opr == calc_operand_for_interval(interval), "wrong cached value");
2062   return opr;
2063 }
2064 
2065 LIR_Opr LinearScan::calc_operand_for_interval(const Interval* interval) {
2066   int assigned_reg = interval->assigned_reg();
2067   BasicType type = interval->type();
2068 
2069   if (assigned_reg >= nof_regs) {
2070     // stack slot
2071     assert(interval->assigned_regHi() == any_reg, "must not have hi register");
2072     return LIR_OprFact::stack(assigned_reg - nof_regs, type);
2073 
2074   } else {
2075     // register
2076     switch (type) {
2077       case T_OBJECT: {
2078         assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register");
2079         assert(interval->assigned_regHi() == any_reg, "must not have hi register");
2080         return LIR_OprFact::single_cpu_oop(assigned_reg);
2081       }
2082 
2083       case T_ADDRESS: {
2084         assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register");
2085         assert(interval->assigned_regHi() == any_reg, "must not have hi register");
2086         return LIR_OprFact::single_cpu_address(assigned_reg);
2087       }
2088 
2089       case T_METADATA: {
2090         assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register");
2091         assert(interval->assigned_regHi() == any_reg, "must not have hi register");
2092         return LIR_OprFact::single_cpu_metadata(assigned_reg);
2093       }
2094 
2095 #ifdef __SOFTFP__
2096       case T_FLOAT:  // fall through
2097 #endif // __SOFTFP__
2098       case T_INT: {
2099         assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register");
2100         assert(interval->assigned_regHi() == any_reg, "must not have hi register");
2101         return LIR_OprFact::single_cpu(assigned_reg);
2102       }
2103 
2104 #ifdef __SOFTFP__
2105       case T_DOUBLE:  // fall through
2106 #endif // __SOFTFP__
2107       case T_LONG: {
2108         int assigned_regHi = interval->assigned_regHi();
2109         assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register");
2110         assert(num_physical_regs(T_LONG) == 1 ||
2111                (assigned_regHi >= pd_first_cpu_reg && assigned_regHi <= pd_last_cpu_reg), "no cpu register");
2112 
2113         assert(assigned_reg != assigned_regHi, "invalid allocation");
2114         assert(num_physical_regs(T_LONG) == 1 || assigned_reg < assigned_regHi,
2115                "register numbers must be sorted (ensure that e.g. a move from eax,ebx to ebx,eax can not occur)");
2116         assert((assigned_regHi != any_reg) ^ (num_physical_regs(T_LONG) == 1), "must be match");
2117         if (requires_adjacent_regs(T_LONG)) {
2118           assert(assigned_reg % 2 == 0 && assigned_reg + 1 == assigned_regHi, "must be sequential and even");
2119         }
2120 
2121 #ifdef _LP64
2122         return LIR_OprFact::double_cpu(assigned_reg, assigned_reg);
2123 #else
2124 #if defined(SPARC) || defined(PPC32)
2125         return LIR_OprFact::double_cpu(assigned_regHi, assigned_reg);
2126 #else
2127         return LIR_OprFact::double_cpu(assigned_reg, assigned_regHi);
2128 #endif // SPARC
2129 #endif // LP64
2130       }
2131 
2132 #ifndef __SOFTFP__
2133       case T_FLOAT: {
2134 #ifdef X86
2135         if (UseSSE >= 1) {
2136           int last_xmm_reg = pd_last_xmm_reg;
2137 #ifdef _LP64
2138           if (UseAVX < 3) {
2139             last_xmm_reg = pd_first_xmm_reg + (pd_nof_xmm_regs_frame_map / 2) - 1;
2140           }
2141 #endif
2142           assert(assigned_reg >= pd_first_xmm_reg && assigned_reg <= last_xmm_reg, "no xmm register");
2143           assert(interval->assigned_regHi() == any_reg, "must not have hi register");
2144           return LIR_OprFact::single_xmm(assigned_reg - pd_first_xmm_reg);
2145         }
2146 #endif
2147 
2148         assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register");
2149         assert(interval->assigned_regHi() == any_reg, "must not have hi register");
2150         return LIR_OprFact::single_fpu(assigned_reg - pd_first_fpu_reg);
2151       }
2152 
2153       case T_DOUBLE: {
2154 #ifdef X86
2155         if (UseSSE >= 2) {
2156           int last_xmm_reg = pd_last_xmm_reg;
2157 #ifdef _LP64
2158           if (UseAVX < 3) {
2159             last_xmm_reg = pd_first_xmm_reg + (pd_nof_xmm_regs_frame_map / 2) - 1;
2160           }
2161 #endif
2162           assert(assigned_reg >= pd_first_xmm_reg && assigned_reg <= last_xmm_reg, "no xmm register");
2163           assert(interval->assigned_regHi() == any_reg, "must not have hi register (double xmm values are stored in one register)");
2164           return LIR_OprFact::double_xmm(assigned_reg - pd_first_xmm_reg);
2165         }
2166 #endif
2167 
2168 #ifdef SPARC
2169         assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register");
2170         assert(interval->assigned_regHi() >= pd_first_fpu_reg && interval->assigned_regHi() <= pd_last_fpu_reg, "no fpu register");
2171         assert(assigned_reg % 2 == 0 && assigned_reg + 1 == interval->assigned_regHi(), "must be sequential and even");
2172         LIR_Opr result = LIR_OprFact::double_fpu(interval->assigned_regHi() - pd_first_fpu_reg, assigned_reg - pd_first_fpu_reg);
2173 #elif defined(ARM32)
2174         assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register");
2175         assert(interval->assigned_regHi() >= pd_first_fpu_reg && interval->assigned_regHi() <= pd_last_fpu_reg, "no fpu register");
2176         assert(assigned_reg % 2 == 0 && assigned_reg + 1 == interval->assigned_regHi(), "must be sequential and even");
2177         LIR_Opr result = LIR_OprFact::double_fpu(assigned_reg - pd_first_fpu_reg, interval->assigned_regHi() - pd_first_fpu_reg);
2178 #else
2179         assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register");
2180         assert(interval->assigned_regHi() == any_reg, "must not have hi register (double fpu values are stored in one register on Intel)");
2181         LIR_Opr result = LIR_OprFact::double_fpu(assigned_reg - pd_first_fpu_reg);
2182 #endif
2183         return result;
2184       }
2185 #endif // __SOFTFP__
2186 
2187       default: {
2188         ShouldNotReachHere();
2189         return LIR_OprFact::illegalOpr;
2190       }
2191     }
2192   }
2193 }
2194 
2195 LIR_Opr LinearScan::canonical_spill_opr(Interval* interval) {
2196   assert(interval->canonical_spill_slot() >= nof_regs, "canonical spill slot not set");
2197   return LIR_OprFact::stack(interval->canonical_spill_slot() - nof_regs, interval->type());
2198 }
2199 
2200 LIR_Opr LinearScan::color_lir_opr(LIR_Opr opr, int op_id, LIR_OpVisitState::OprMode mode) {
2201   assert(opr->is_virtual(), "should not call this otherwise");
2202 
2203   Interval* interval = interval_at(opr->vreg_number());
2204   assert(interval != NULL, "interval must exist");
2205 
2206   if (op_id != -1) {
2207 #ifdef ASSERT
2208     BlockBegin* block = block_of_op_with_id(op_id);
2209     if (block->number_of_sux() <= 1 && op_id == block->last_lir_instruction_id()) {
2210       // check if spill moves could have been appended at the end of this block, but
2211       // before the branch instruction. So the split child information for this branch would
2212       // be incorrect.
2213       LIR_OpBranch* branch = block->lir()->instructions_list()->last()->as_OpBranch();
2214       if (branch != NULL) {
2215         if (block->live_out().at(opr->vreg_number())) {
2216           assert(branch->cond() == lir_cond_always, "block does not end with an unconditional jump");
2217           assert(false, "can't get split child for the last branch of a block because the information would be incorrect (moves are inserted before the branch in resolve_data_flow)");
2218         }
2219       }
2220     }
2221 #endif
2222 
2223     // operands are not changed when an interval is split during allocation,
2224     // so search the right interval here
2225     interval = split_child_at_op_id(interval, op_id, mode);
2226   }
2227 
2228   LIR_Opr res = operand_for_interval(interval);
2229 
2230 #ifdef X86
2231   // new semantic for is_last_use: not only set on definite end of interval,
2232   // but also before hole
2233   // This may still miss some cases (e.g. for dead values), but it is not necessary that the
2234   // last use information is completely correct
2235   // information is only needed for fpu stack allocation
2236   if (res->is_fpu_register()) {
2237     if (opr->is_last_use() || op_id == interval->to() || (op_id != -1 && interval->has_hole_between(op_id, op_id + 1))) {
2238       assert(op_id == -1 || !is_block_begin(op_id), "holes at begin of block may also result from control flow");
2239       res = res->make_last_use();
2240     }
2241   }
2242 #endif
2243 
2244   assert(!gen()->is_vreg_flag_set(opr->vreg_number(), LIRGenerator::callee_saved) || !FrameMap::is_caller_save_register(res), "bad allocation");
2245 
2246   return res;
2247 }
2248 
2249 
2250 #ifdef ASSERT
2251 // some methods used to check correctness of debug information
2252 
2253 void assert_no_register_values(GrowableArray<ScopeValue*>* values) {
2254   if (values == NULL) {
2255     return;
2256   }
2257 
2258   for (int i = 0; i < values->length(); i++) {
2259     ScopeValue* value = values->at(i);
2260 
2261     if (value->is_location()) {
2262       Location location = ((LocationValue*)value)->location();
2263       assert(location.where() == Location::on_stack, "value is in register");
2264     }
2265   }
2266 }
2267 
2268 void assert_no_register_values(GrowableArray<MonitorValue*>* values) {
2269   if (values == NULL) {
2270     return;
2271   }
2272 
2273   for (int i = 0; i < values->length(); i++) {
2274     MonitorValue* value = values->at(i);
2275 
2276     if (value->owner()->is_location()) {
2277       Location location = ((LocationValue*)value->owner())->location();
2278       assert(location.where() == Location::on_stack, "owner is in register");
2279     }
2280     assert(value->basic_lock().where() == Location::on_stack, "basic_lock is in register");
2281   }
2282 }
2283 
2284 void assert_equal(Location l1, Location l2) {
2285   assert(l1.where() == l2.where() && l1.type() == l2.type() && l1.offset() == l2.offset(), "");
2286 }
2287 
2288 void assert_equal(ScopeValue* v1, ScopeValue* v2) {
2289   if (v1->is_location()) {
2290     assert(v2->is_location(), "");
2291     assert_equal(((LocationValue*)v1)->location(), ((LocationValue*)v2)->location());
2292   } else if (v1->is_constant_int()) {
2293     assert(v2->is_constant_int(), "");
2294     assert(((ConstantIntValue*)v1)->value() == ((ConstantIntValue*)v2)->value(), "");
2295   } else if (v1->is_constant_double()) {
2296     assert(v2->is_constant_double(), "");
2297     assert(((ConstantDoubleValue*)v1)->value() == ((ConstantDoubleValue*)v2)->value(), "");
2298   } else if (v1->is_constant_long()) {
2299     assert(v2->is_constant_long(), "");
2300     assert(((ConstantLongValue*)v1)->value() == ((ConstantLongValue*)v2)->value(), "");
2301   } else if (v1->is_constant_oop()) {
2302     assert(v2->is_constant_oop(), "");
2303     assert(((ConstantOopWriteValue*)v1)->value() == ((ConstantOopWriteValue*)v2)->value(), "");
2304   } else {
2305     ShouldNotReachHere();
2306   }
2307 }
2308 
2309 void assert_equal(MonitorValue* m1, MonitorValue* m2) {
2310   assert_equal(m1->owner(), m2->owner());
2311   assert_equal(m1->basic_lock(), m2->basic_lock());
2312 }
2313 
2314 void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) {
2315   assert(d1->scope() == d2->scope(), "not equal");
2316   assert(d1->bci() == d2->bci(), "not equal");
2317 
2318   if (d1->locals() != NULL) {
2319     assert(d1->locals() != NULL && d2->locals() != NULL, "not equal");
2320     assert(d1->locals()->length() == d2->locals()->length(), "not equal");
2321     for (int i = 0; i < d1->locals()->length(); i++) {
2322       assert_equal(d1->locals()->at(i), d2->locals()->at(i));
2323     }
2324   } else {
2325     assert(d1->locals() == NULL && d2->locals() == NULL, "not equal");
2326   }
2327 
2328   if (d1->expressions() != NULL) {
2329     assert(d1->expressions() != NULL && d2->expressions() != NULL, "not equal");
2330     assert(d1->expressions()->length() == d2->expressions()->length(), "not equal");
2331     for (int i = 0; i < d1->expressions()->length(); i++) {
2332       assert_equal(d1->expressions()->at(i), d2->expressions()->at(i));
2333     }
2334   } else {
2335     assert(d1->expressions() == NULL && d2->expressions() == NULL, "not equal");
2336   }
2337 
2338   if (d1->monitors() != NULL) {
2339     assert(d1->monitors() != NULL && d2->monitors() != NULL, "not equal");
2340     assert(d1->monitors()->length() == d2->monitors()->length(), "not equal");
2341     for (int i = 0; i < d1->monitors()->length(); i++) {
2342       assert_equal(d1->monitors()->at(i), d2->monitors()->at(i));
2343     }
2344   } else {
2345     assert(d1->monitors() == NULL && d2->monitors() == NULL, "not equal");
2346   }
2347 
2348   if (d1->caller() != NULL) {
2349     assert(d1->caller() != NULL && d2->caller() != NULL, "not equal");
2350     assert_equal(d1->caller(), d2->caller());
2351   } else {
2352     assert(d1->caller() == NULL && d2->caller() == NULL, "not equal");
2353   }
2354 }
2355 
2356 void check_stack_depth(CodeEmitInfo* info, int stack_end) {
2357   if (info->stack()->bci() != SynchronizationEntryBCI && !info->scope()->method()->is_native()) {
2358     Bytecodes::Code code = info->scope()->method()->java_code_at_bci(info->stack()->bci());
2359     switch (code) {
2360       case Bytecodes::_ifnull    : // fall through
2361       case Bytecodes::_ifnonnull : // fall through
2362       case Bytecodes::_ifeq      : // fall through
2363       case Bytecodes::_ifne      : // fall through
2364       case Bytecodes::_iflt      : // fall through
2365       case Bytecodes::_ifge      : // fall through
2366       case Bytecodes::_ifgt      : // fall through
2367       case Bytecodes::_ifle      : // fall through
2368       case Bytecodes::_if_icmpeq : // fall through
2369       case Bytecodes::_if_icmpne : // fall through
2370       case Bytecodes::_if_icmplt : // fall through
2371       case Bytecodes::_if_icmpge : // fall through
2372       case Bytecodes::_if_icmpgt : // fall through
2373       case Bytecodes::_if_icmple : // fall through
2374       case Bytecodes::_if_acmpeq : // fall through
2375       case Bytecodes::_if_acmpne :
2376         assert(stack_end >= -Bytecodes::depth(code), "must have non-empty expression stack at if bytecode");
2377         break;
2378     }
2379   }
2380 }
2381 
2382 #endif // ASSERT
2383 
2384 
2385 IntervalWalker* LinearScan::init_compute_oop_maps() {
2386   // setup lists of potential oops for walking
2387   Interval* oop_intervals;
2388   Interval* non_oop_intervals;
2389 
2390   create_unhandled_lists(&oop_intervals, &non_oop_intervals, is_oop_interval, NULL);
2391 
2392   // intervals that have no oops inside need not to be processed
2393   // to ensure a walking until the last instruction id, add a dummy interval
2394   // with a high operation id
2395   non_oop_intervals = new Interval(any_reg);
2396   non_oop_intervals->add_range(max_jint - 2, max_jint - 1);
2397 
2398   return new IntervalWalker(this, oop_intervals, non_oop_intervals);
2399 }
2400 
2401 
2402 OopMap* LinearScan::compute_oop_map(IntervalWalker* iw, LIR_Op* op, CodeEmitInfo* info, bool is_call_site) {
2403   TRACE_LINEAR_SCAN(3, tty->print_cr("creating oop map at op_id %d", op->id()));
2404 
2405   // walk before the current operation -> intervals that start at
2406   // the operation (= output operands of the operation) are not
2407   // included in the oop map
2408   iw->walk_before(op->id());
2409 
2410   int frame_size = frame_map()->framesize();
2411   int arg_count = frame_map()->oop_map_arg_count();
2412   OopMap* map = new OopMap(frame_size, arg_count);
2413 
2414   // Iterate through active intervals
2415   for (Interval* interval = iw->active_first(fixedKind); interval != Interval::end(); interval = interval->next()) {
2416     int assigned_reg = interval->assigned_reg();
2417 
2418     assert(interval->current_from() <= op->id() && op->id() <= interval->current_to(), "interval should not be active otherwise");
2419     assert(interval->assigned_regHi() == any_reg, "oop must be single word");
2420     assert(interval->reg_num() >= LIR_OprDesc::vreg_base, "fixed interval found");
2421 
2422     // Check if this range covers the instruction. Intervals that
2423     // start or end at the current operation are not included in the
2424     // oop map, except in the case of patching moves.  For patching
2425     // moves, any intervals which end at this instruction are included
2426     // in the oop map since we may safepoint while doing the patch
2427     // before we've consumed the inputs.
2428     if (op->is_patching() || op->id() < interval->current_to()) {
2429 
2430       // caller-save registers must not be included into oop-maps at calls
2431       assert(!is_call_site || assigned_reg >= nof_regs || !is_caller_save(assigned_reg), "interval is in a caller-save register at a call -> register will be overwritten");
2432 
2433       VMReg name = vm_reg_for_interval(interval);
2434       set_oop(map, name);
2435 
2436       // Spill optimization: when the stack value is guaranteed to be always correct,
2437       // then it must be added to the oop map even if the interval is currently in a register
2438       if (interval->always_in_memory() &&
2439           op->id() > interval->spill_definition_pos() &&
2440           interval->assigned_reg() != interval->canonical_spill_slot()) {
2441         assert(interval->spill_definition_pos() > 0, "position not set correctly");
2442         assert(interval->canonical_spill_slot() >= LinearScan::nof_regs, "no spill slot assigned");
2443         assert(interval->assigned_reg() < LinearScan::nof_regs, "interval is on stack, so stack slot is registered twice");
2444 
2445         set_oop(map, frame_map()->slot_regname(interval->canonical_spill_slot() - LinearScan::nof_regs));
2446       }
2447     }
2448   }
2449 
2450   // add oops from lock stack
2451   assert(info->stack() != NULL, "CodeEmitInfo must always have a stack");
2452   int locks_count = info->stack()->total_locks_size();
2453   for (int i = 0; i < locks_count; i++) {
2454     set_oop(map, frame_map()->monitor_object_regname(i));
2455   }
2456 
2457   return map;
2458 }
2459 
2460 
2461 void LinearScan::compute_oop_map(IntervalWalker* iw, const LIR_OpVisitState &visitor, LIR_Op* op) {
2462   assert(visitor.info_count() > 0, "no oop map needed");
2463 
2464   // compute oop_map only for first CodeEmitInfo
2465   // because it is (in most cases) equal for all other infos of the same operation
2466   CodeEmitInfo* first_info = visitor.info_at(0);
2467   OopMap* first_oop_map = compute_oop_map(iw, op, first_info, visitor.has_call());
2468 
2469   for (int i = 0; i < visitor.info_count(); i++) {
2470     CodeEmitInfo* info = visitor.info_at(i);
2471     OopMap* oop_map = first_oop_map;
2472 
2473     // compute worst case interpreter size in case of a deoptimization
2474     _compilation->update_interpreter_frame_size(info->interpreter_frame_size());
2475 
2476     if (info->stack()->locks_size() != first_info->stack()->locks_size()) {
2477       // this info has a different number of locks then the precomputed oop map
2478       // (possible for lock and unlock instructions) -> compute oop map with
2479       // correct lock information
2480       oop_map = compute_oop_map(iw, op, info, visitor.has_call());
2481     }
2482 
2483     if (info->_oop_map == NULL) {
2484       info->_oop_map = oop_map;
2485     } else {
2486       // a CodeEmitInfo can not be shared between different LIR-instructions
2487       // because interval splitting can occur anywhere between two instructions
2488       // and so the oop maps must be different
2489       // -> check if the already set oop_map is exactly the one calculated for this operation
2490       assert(info->_oop_map == oop_map, "same CodeEmitInfo used for multiple LIR instructions");
2491     }
2492   }
2493 }
2494 
2495 
2496 // frequently used constants
2497 // Allocate them with new so they are never destroyed (otherwise, a
2498 // forced exit could destroy these objects while they are still in
2499 // use).
2500 ConstantOopWriteValue* LinearScan::_oop_null_scope_value = new (ResourceObj::C_HEAP, mtCompiler) ConstantOopWriteValue(NULL);
2501 ConstantIntValue*      LinearScan::_int_m1_scope_value = new (ResourceObj::C_HEAP, mtCompiler) ConstantIntValue(-1);
2502 ConstantIntValue*      LinearScan::_int_0_scope_value =  new (ResourceObj::C_HEAP, mtCompiler) ConstantIntValue(0);
2503 ConstantIntValue*      LinearScan::_int_1_scope_value =  new (ResourceObj::C_HEAP, mtCompiler) ConstantIntValue(1);
2504 ConstantIntValue*      LinearScan::_int_2_scope_value =  new (ResourceObj::C_HEAP, mtCompiler) ConstantIntValue(2);
2505 LocationValue*         _illegal_value = new (ResourceObj::C_HEAP, mtCompiler) LocationValue(Location());
2506 
2507 void LinearScan::init_compute_debug_info() {
2508   // cache for frequently used scope values
2509   // (cpu registers and stack slots)
2510   int cache_size = (LinearScan::nof_cpu_regs + frame_map()->argcount() + max_spills()) * 2;
2511   _scope_value_cache = ScopeValueArray(cache_size, cache_size, NULL);
2512 }
2513 
2514 MonitorValue* LinearScan::location_for_monitor_index(int monitor_index) {
2515   Location loc;
2516   if (!frame_map()->location_for_monitor_object(monitor_index, &loc)) {
2517     bailout("too large frame");
2518   }
2519   ScopeValue* object_scope_value = new LocationValue(loc);
2520 
2521   if (!frame_map()->location_for_monitor_lock(monitor_index, &loc)) {
2522     bailout("too large frame");
2523   }
2524   return new MonitorValue(object_scope_value, loc);
2525 }
2526 
2527 LocationValue* LinearScan::location_for_name(int name, Location::Type loc_type) {
2528   Location loc;
2529   if (!frame_map()->locations_for_slot(name, loc_type, &loc)) {
2530     bailout("too large frame");
2531   }
2532   return new LocationValue(loc);
2533 }
2534 
2535 
2536 int LinearScan::append_scope_value_for_constant(LIR_Opr opr, GrowableArray<ScopeValue*>* scope_values) {
2537   assert(opr->is_constant(), "should not be called otherwise");
2538 
2539   LIR_Const* c = opr->as_constant_ptr();
2540   BasicType t = c->type();
2541   switch (t) {
2542     case T_OBJECT: {
2543       jobject value = c->as_jobject();
2544       if (value == NULL) {
2545         scope_values->append(_oop_null_scope_value);
2546       } else {
2547         scope_values->append(new ConstantOopWriteValue(c->as_jobject()));
2548       }
2549       return 1;
2550     }
2551 
2552     case T_INT: // fall through
2553     case T_FLOAT: {
2554       int value = c->as_jint_bits();
2555       switch (value) {
2556         case -1: scope_values->append(_int_m1_scope_value); break;
2557         case 0:  scope_values->append(_int_0_scope_value); break;
2558         case 1:  scope_values->append(_int_1_scope_value); break;
2559         case 2:  scope_values->append(_int_2_scope_value); break;
2560         default: scope_values->append(new ConstantIntValue(c->as_jint_bits())); break;
2561       }
2562       return 1;
2563     }
2564 
2565     case T_LONG: // fall through
2566     case T_DOUBLE: {
2567 #ifdef _LP64
2568       scope_values->append(_int_0_scope_value);
2569       scope_values->append(new ConstantLongValue(c->as_jlong_bits()));
2570 #else
2571       if (hi_word_offset_in_bytes > lo_word_offset_in_bytes) {
2572         scope_values->append(new ConstantIntValue(c->as_jint_hi_bits()));
2573         scope_values->append(new ConstantIntValue(c->as_jint_lo_bits()));
2574       } else {
2575         scope_values->append(new ConstantIntValue(c->as_jint_lo_bits()));
2576         scope_values->append(new ConstantIntValue(c->as_jint_hi_bits()));
2577       }
2578 #endif
2579       return 2;
2580     }
2581 
2582     case T_ADDRESS: {
2583 #ifdef _LP64
2584       scope_values->append(new ConstantLongValue(c->as_jint()));
2585 #else
2586       scope_values->append(new ConstantIntValue(c->as_jint()));
2587 #endif
2588       return 1;
2589     }
2590 
2591     default:
2592       ShouldNotReachHere();
2593       return -1;
2594   }
2595 }
2596 
2597 int LinearScan::append_scope_value_for_operand(LIR_Opr opr, GrowableArray<ScopeValue*>* scope_values) {
2598   if (opr->is_single_stack()) {
2599     int stack_idx = opr->single_stack_ix();
2600     bool is_oop = opr->is_oop_register();
2601     int cache_idx = (stack_idx + LinearScan::nof_cpu_regs) * 2 + (is_oop ? 1 : 0);
2602 
2603     ScopeValue* sv = _scope_value_cache.at(cache_idx);
2604     if (sv == NULL) {
2605       Location::Type loc_type = is_oop ? Location::oop : Location::normal;
2606       sv = location_for_name(stack_idx, loc_type);
2607       _scope_value_cache.at_put(cache_idx, sv);
2608     }
2609 
2610     // check if cached value is correct
2611     DEBUG_ONLY(assert_equal(sv, location_for_name(stack_idx, is_oop ? Location::oop : Location::normal)));
2612 
2613     scope_values->append(sv);
2614     return 1;
2615 
2616   } else if (opr->is_single_cpu()) {
2617     bool is_oop = opr->is_oop_register();
2618     int cache_idx = opr->cpu_regnr() * 2 + (is_oop ? 1 : 0);
2619     Location::Type int_loc_type = NOT_LP64(Location::normal) LP64_ONLY(Location::int_in_long);
2620 
2621     ScopeValue* sv = _scope_value_cache.at(cache_idx);
2622     if (sv == NULL) {
2623       Location::Type loc_type = is_oop ? Location::oop : int_loc_type;
2624       VMReg rname = frame_map()->regname(opr);
2625       sv = new LocationValue(Location::new_reg_loc(loc_type, rname));
2626       _scope_value_cache.at_put(cache_idx, sv);
2627     }
2628 
2629     // check if cached value is correct
2630     DEBUG_ONLY(assert_equal(sv, new LocationValue(Location::new_reg_loc(is_oop ? Location::oop : int_loc_type, frame_map()->regname(opr)))));
2631 
2632     scope_values->append(sv);
2633     return 1;
2634 
2635 #ifdef X86
2636   } else if (opr->is_single_xmm()) {
2637     VMReg rname = opr->as_xmm_float_reg()->as_VMReg();
2638     LocationValue* sv = new LocationValue(Location::new_reg_loc(Location::normal, rname));
2639 
2640     scope_values->append(sv);
2641     return 1;
2642 #endif
2643 
2644   } else if (opr->is_single_fpu()) {
2645 #ifdef X86
2646     // the exact location of fpu stack values is only known
2647     // during fpu stack allocation, so the stack allocator object
2648     // must be present
2649     assert(use_fpu_stack_allocation(), "should not have float stack values without fpu stack allocation (all floats must be SSE2)");
2650     assert(_fpu_stack_allocator != NULL, "must be present");
2651     opr = _fpu_stack_allocator->to_fpu_stack(opr);
2652 #endif
2653 
2654     Location::Type loc_type = float_saved_as_double ? Location::float_in_dbl : Location::normal;
2655     VMReg rname = frame_map()->fpu_regname(opr->fpu_regnr());
2656 #ifndef __SOFTFP__
2657 #ifndef VM_LITTLE_ENDIAN
2658     if (! float_saved_as_double) {
2659       // On big endian system, we may have an issue if float registers use only
2660       // the low half of the (same) double registers.
2661       // Both the float and the double could have the same regnr but would correspond
2662       // to two different addresses once saved.
2663 
2664       // get next safely (no assertion checks)
2665       VMReg next = VMRegImpl::as_VMReg(1+rname->value());
2666       if (next->is_reg() &&
2667           (next->as_FloatRegister() == rname->as_FloatRegister())) {
2668         // the back-end does use the same numbering for the double and the float
2669         rname = next; // VMReg for the low bits, e.g. the real VMReg for the float
2670       }
2671     }
2672 #endif
2673 #endif
2674     LocationValue* sv = new LocationValue(Location::new_reg_loc(loc_type, rname));
2675 
2676     scope_values->append(sv);
2677     return 1;
2678 
2679   } else {
2680     // double-size operands
2681 
2682     ScopeValue* first;
2683     ScopeValue* second;
2684 
2685     if (opr->is_double_stack()) {
2686 #ifdef _LP64
2687       Location loc1;
2688       Location::Type loc_type = opr->type() == T_LONG ? Location::lng : Location::dbl;
2689       if (!frame_map()->locations_for_slot(opr->double_stack_ix(), loc_type, &loc1, NULL)) {
2690         bailout("too large frame");
2691       }
2692       // Does this reverse on x86 vs. sparc?
2693       first =  new LocationValue(loc1);
2694       second = _int_0_scope_value;
2695 #else
2696       Location loc1, loc2;
2697       if (!frame_map()->locations_for_slot(opr->double_stack_ix(), Location::normal, &loc1, &loc2)) {
2698         bailout("too large frame");
2699       }
2700       first =  new LocationValue(loc1);
2701       second = new LocationValue(loc2);
2702 #endif // _LP64
2703 
2704     } else if (opr->is_double_cpu()) {
2705 #ifdef _LP64
2706       VMReg rname_first = opr->as_register_lo()->as_VMReg();
2707       first = new LocationValue(Location::new_reg_loc(Location::lng, rname_first));
2708       second = _int_0_scope_value;
2709 #else
2710       VMReg rname_first = opr->as_register_lo()->as_VMReg();
2711       VMReg rname_second = opr->as_register_hi()->as_VMReg();
2712 
2713       if (hi_word_offset_in_bytes < lo_word_offset_in_bytes) {
2714         // lo/hi and swapped relative to first and second, so swap them
2715         VMReg tmp = rname_first;
2716         rname_first = rname_second;
2717         rname_second = tmp;
2718       }
2719 
2720       first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first));
2721       second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second));
2722 #endif //_LP64
2723 
2724 
2725 #ifdef X86
2726     } else if (opr->is_double_xmm()) {
2727       assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation");
2728       VMReg rname_first  = opr->as_xmm_double_reg()->as_VMReg();
2729 #  ifdef _LP64
2730       first = new LocationValue(Location::new_reg_loc(Location::dbl, rname_first));
2731       second = _int_0_scope_value;
2732 #  else
2733       first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first));
2734       // %%% This is probably a waste but we'll keep things as they were for now
2735       if (true) {
2736         VMReg rname_second = rname_first->next();
2737         second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second));
2738       }
2739 #  endif
2740 #endif
2741 
2742     } else if (opr->is_double_fpu()) {
2743       // On SPARC, fpu_regnrLo/fpu_regnrHi represents the two halves of
2744       // the double as float registers in the native ordering. On X86,
2745       // fpu_regnrLo is a FPU stack slot whose VMReg represents
2746       // the low-order word of the double and fpu_regnrLo + 1 is the
2747       // name for the other half.  *first and *second must represent the
2748       // least and most significant words, respectively.
2749 
2750 #ifdef X86
2751       // the exact location of fpu stack values is only known
2752       // during fpu stack allocation, so the stack allocator object
2753       // must be present
2754       assert(use_fpu_stack_allocation(), "should not have float stack values without fpu stack allocation (all floats must be SSE2)");
2755       assert(_fpu_stack_allocator != NULL, "must be present");
2756       opr = _fpu_stack_allocator->to_fpu_stack(opr);
2757 
2758       assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation (only fpu_regnrLo is used)");
2759 #endif
2760 #ifdef SPARC
2761       assert(opr->fpu_regnrLo() == opr->fpu_regnrHi() + 1, "assumed in calculation (only fpu_regnrHi is used)");
2762 #endif
2763 #ifdef ARM32
2764       assert(opr->fpu_regnrHi() == opr->fpu_regnrLo() + 1, "assumed in calculation (only fpu_regnrLo is used)");
2765 #endif
2766 #ifdef PPC32
2767       assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation (only fpu_regnrHi is used)");
2768 #endif
2769 
2770 #ifdef VM_LITTLE_ENDIAN
2771       VMReg rname_first = frame_map()->fpu_regname(opr->fpu_regnrLo());
2772 #else
2773       VMReg rname_first = frame_map()->fpu_regname(opr->fpu_regnrHi());
2774 #endif
2775 
2776 #ifdef _LP64
2777       first = new LocationValue(Location::new_reg_loc(Location::dbl, rname_first));
2778       second = _int_0_scope_value;
2779 #else
2780       first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first));
2781       // %%% This is probably a waste but we'll keep things as they were for now
2782       if (true) {
2783         VMReg rname_second = rname_first->next();
2784         second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second));
2785       }
2786 #endif
2787 
2788     } else {
2789       ShouldNotReachHere();
2790       first = NULL;
2791       second = NULL;
2792     }
2793 
2794     assert(first != NULL && second != NULL, "must be set");
2795     // The convention the interpreter uses is that the second local
2796     // holds the first raw word of the native double representation.
2797     // This is actually reasonable, since locals and stack arrays
2798     // grow downwards in all implementations.
2799     // (If, on some machine, the interpreter's Java locals or stack
2800     // were to grow upwards, the embedded doubles would be word-swapped.)
2801     scope_values->append(second);
2802     scope_values->append(first);
2803     return 2;
2804   }
2805 }
2806 
2807 
2808 int LinearScan::append_scope_value(int op_id, Value value, GrowableArray<ScopeValue*>* scope_values) {
2809   if (value != NULL) {
2810     LIR_Opr opr = value->operand();
2811     Constant* con = value->as_Constant();
2812 
2813     assert(con == NULL || opr->is_virtual() || opr->is_constant() || opr->is_illegal(), "asumption: Constant instructions have only constant operands (or illegal if constant is optimized away)");
2814     assert(con != NULL || opr->is_virtual(), "asumption: non-Constant instructions have only virtual operands");
2815 
2816     if (con != NULL && !con->is_pinned() && !opr->is_constant()) {
2817       // Unpinned constants may have a virtual operand for a part of the lifetime
2818       // or may be illegal when it was optimized away,
2819       // so always use a constant operand
2820       opr = LIR_OprFact::value_type(con->type());
2821     }
2822     assert(opr->is_virtual() || opr->is_constant(), "other cases not allowed here");
2823 
2824     if (opr->is_virtual()) {
2825       LIR_OpVisitState::OprMode mode = LIR_OpVisitState::inputMode;
2826 
2827       BlockBegin* block = block_of_op_with_id(op_id);
2828       if (block->number_of_sux() == 1 && op_id == block->last_lir_instruction_id()) {
2829         // generating debug information for the last instruction of a block.
2830         // if this instruction is a branch, spill moves are inserted before this branch
2831         // and so the wrong operand would be returned (spill moves at block boundaries are not
2832         // considered in the live ranges of intervals)
2833         // Solution: use the first op_id of the branch target block instead.
2834         if (block->lir()->instructions_list()->last()->as_OpBranch() != NULL) {
2835           if (block->live_out().at(opr->vreg_number())) {
2836             op_id = block->sux_at(0)->first_lir_instruction_id();
2837             mode = LIR_OpVisitState::outputMode;
2838           }
2839         }
2840       }
2841 
2842       // Get current location of operand
2843       // The operand must be live because debug information is considered when building the intervals
2844       // if the interval is not live, color_lir_opr will cause an assertion failure
2845       opr = color_lir_opr(opr, op_id, mode);
2846       assert(!has_call(op_id) || opr->is_stack() || !is_caller_save(reg_num(opr)), "can not have caller-save register operands at calls");
2847 
2848       // Append to ScopeValue array
2849       return append_scope_value_for_operand(opr, scope_values);
2850 
2851     } else {
2852       assert(value->as_Constant() != NULL, "all other instructions have only virtual operands");
2853       assert(opr->is_constant(), "operand must be constant");
2854 
2855       return append_scope_value_for_constant(opr, scope_values);
2856     }
2857   } else {
2858     // append a dummy value because real value not needed
2859     scope_values->append(_illegal_value);
2860     return 1;
2861   }
2862 }
2863 
2864 
2865 IRScopeDebugInfo* LinearScan::compute_debug_info_for_scope(int op_id, IRScope* cur_scope, ValueStack* cur_state, ValueStack* innermost_state) {
2866   IRScopeDebugInfo* caller_debug_info = NULL;
2867 
2868   ValueStack* caller_state = cur_state->caller_state();
2869   if (caller_state != NULL) {
2870     // process recursively to compute outermost scope first
2871     caller_debug_info = compute_debug_info_for_scope(op_id, cur_scope->caller(), caller_state, innermost_state);
2872   }
2873 
2874   // initialize these to null.
2875   // If we don't need deopt info or there are no locals, expressions or monitors,
2876   // then these get recorded as no information and avoids the allocation of 0 length arrays.
2877   GrowableArray<ScopeValue*>*   locals      = NULL;
2878   GrowableArray<ScopeValue*>*   expressions = NULL;
2879   GrowableArray<MonitorValue*>* monitors    = NULL;
2880 
2881   // describe local variable values
2882   int nof_locals = cur_state->locals_size();
2883   if (nof_locals > 0) {
2884     locals = new GrowableArray<ScopeValue*>(nof_locals);
2885 
2886     int pos = 0;
2887     while (pos < nof_locals) {
2888       assert(pos < cur_state->locals_size(), "why not?");
2889 
2890       Value local = cur_state->local_at(pos);
2891       pos += append_scope_value(op_id, local, locals);
2892 
2893       assert(locals->length() == pos, "must match");
2894     }
2895     assert(locals->length() == cur_scope->method()->max_locals(), "wrong number of locals");
2896     assert(locals->length() == cur_state->locals_size(), "wrong number of locals");
2897   } else if (cur_scope->method()->max_locals() > 0) {
2898     assert(cur_state->kind() == ValueStack::EmptyExceptionState, "should be");
2899     nof_locals = cur_scope->method()->max_locals();
2900     locals = new GrowableArray<ScopeValue*>(nof_locals);
2901     for(int i = 0; i < nof_locals; i++) {
2902       locals->append(_illegal_value);
2903     }
2904   }
2905 
2906   // describe expression stack
2907   int nof_stack = cur_state->stack_size();
2908   if (nof_stack > 0) {
2909     expressions = new GrowableArray<ScopeValue*>(nof_stack);
2910 
2911     int pos = 0;
2912     while (pos < nof_stack) {
2913       Value expression = cur_state->stack_at_inc(pos);
2914       append_scope_value(op_id, expression, expressions);
2915 
2916       assert(expressions->length() == pos, "must match");
2917     }
2918     assert(expressions->length() == cur_state->stack_size(), "wrong number of stack entries");
2919   }
2920 
2921   // describe monitors
2922   int nof_locks = cur_state->locks_size();
2923   if (nof_locks > 0) {
2924     int lock_offset = cur_state->caller_state() != NULL ? cur_state->caller_state()->total_locks_size() : 0;
2925     monitors = new GrowableArray<MonitorValue*>(nof_locks);
2926     for (int i = 0; i < nof_locks; i++) {
2927       monitors->append(location_for_monitor_index(lock_offset + i));
2928     }
2929   }
2930 
2931   return new IRScopeDebugInfo(cur_scope, cur_state->bci(), locals, expressions, monitors, caller_debug_info);
2932 }
2933 
2934 
2935 void LinearScan::compute_debug_info(CodeEmitInfo* info, int op_id) {
2936   TRACE_LINEAR_SCAN(3, tty->print_cr("creating debug information at op_id %d", op_id));
2937 
2938   IRScope* innermost_scope = info->scope();
2939   ValueStack* innermost_state = info->stack();
2940 
2941   assert(innermost_scope != NULL && innermost_state != NULL, "why is it missing?");
2942 
2943   DEBUG_ONLY(check_stack_depth(info, innermost_state->stack_size()));
2944 
2945   if (info->_scope_debug_info == NULL) {
2946     // compute debug information
2947     info->_scope_debug_info = compute_debug_info_for_scope(op_id, innermost_scope, innermost_state, innermost_state);
2948   } else {
2949     // debug information already set. Check that it is correct from the current point of view
2950     DEBUG_ONLY(assert_equal(info->_scope_debug_info, compute_debug_info_for_scope(op_id, innermost_scope, innermost_state, innermost_state)));
2951   }
2952 }
2953 
2954 
2955 void LinearScan::assign_reg_num(LIR_OpList* instructions, IntervalWalker* iw) {
2956   LIR_OpVisitState visitor;
2957   int num_inst = instructions->length();
2958   bool has_dead = false;
2959 
2960   for (int j = 0; j < num_inst; j++) {
2961     LIR_Op* op = instructions->at(j);
2962     if (op == NULL) {  // this can happen when spill-moves are removed in eliminate_spill_moves
2963       has_dead = true;
2964       continue;
2965     }
2966     int op_id = op->id();
2967 
2968     // visit instruction to get list of operands
2969     visitor.visit(op);
2970 
2971     // iterate all modes of the visitor and process all virtual operands
2972     for_each_visitor_mode(mode) {
2973       int n = visitor.opr_count(mode);
2974       for (int k = 0; k < n; k++) {
2975         LIR_Opr opr = visitor.opr_at(mode, k);
2976         if (opr->is_virtual_register()) {
2977           visitor.set_opr_at(mode, k, color_lir_opr(opr, op_id, mode));
2978         }
2979       }
2980     }
2981 
2982     if (visitor.info_count() > 0) {
2983       // exception handling
2984       if (compilation()->has_exception_handlers()) {
2985         XHandlers* xhandlers = visitor.all_xhandler();
2986         int n = xhandlers->length();
2987         for (int k = 0; k < n; k++) {
2988           XHandler* handler = xhandlers->handler_at(k);
2989           if (handler->entry_code() != NULL) {
2990             assign_reg_num(handler->entry_code()->instructions_list(), NULL);
2991           }
2992         }
2993       } else {
2994         assert(visitor.all_xhandler()->length() == 0, "missed exception handler");
2995       }
2996 
2997       // compute oop map
2998       assert(iw != NULL, "needed for compute_oop_map");
2999       compute_oop_map(iw, visitor, op);
3000 
3001       // compute debug information
3002       if (!use_fpu_stack_allocation()) {
3003         // compute debug information if fpu stack allocation is not needed.
3004         // when fpu stack allocation is needed, the debug information can not
3005         // be computed here because the exact location of fpu operands is not known
3006         // -> debug information is created inside the fpu stack allocator
3007         int n = visitor.info_count();
3008         for (int k = 0; k < n; k++) {
3009           compute_debug_info(visitor.info_at(k), op_id);
3010         }
3011       }
3012     }
3013 
3014 #ifdef ASSERT
3015     // make sure we haven't made the op invalid.
3016     op->verify();
3017 #endif
3018 
3019     // remove useless moves
3020     if (op->code() == lir_move) {
3021       assert(op->as_Op1() != NULL, "move must be LIR_Op1");
3022       LIR_Op1* move = (LIR_Op1*)op;
3023       LIR_Opr src = move->in_opr();
3024       LIR_Opr dst = move->result_opr();
3025       if (dst == src ||
3026           !dst->is_pointer() && !src->is_pointer() &&
3027           src->is_same_register(dst)) {
3028         instructions->at_put(j, NULL);
3029         has_dead = true;
3030       }
3031     }
3032   }
3033 
3034   if (has_dead) {
3035     // iterate all instructions of the block and remove all null-values.
3036     int insert_point = 0;
3037     for (int j = 0; j < num_inst; j++) {
3038       LIR_Op* op = instructions->at(j);
3039       if (op != NULL) {
3040         if (insert_point != j) {
3041           instructions->at_put(insert_point, op);
3042         }
3043         insert_point++;
3044       }
3045     }
3046     instructions->trunc_to(insert_point);
3047   }
3048 }
3049 
3050 void LinearScan::assign_reg_num() {
3051   TIME_LINEAR_SCAN(timer_assign_reg_num);
3052 
3053   init_compute_debug_info();
3054   IntervalWalker* iw = init_compute_oop_maps();
3055 
3056   int num_blocks = block_count();
3057   for (int i = 0; i < num_blocks; i++) {
3058     BlockBegin* block = block_at(i);
3059     assign_reg_num(block->lir()->instructions_list(), iw);
3060   }
3061 }
3062 
3063 
3064 void LinearScan::do_linear_scan() {
3065   NOT_PRODUCT(_total_timer.begin_method());
3066 
3067   number_instructions();
3068 
3069   NOT_PRODUCT(print_lir(1, "Before Register Allocation"));
3070 
3071   compute_local_live_sets();
3072   compute_global_live_sets();
3073   CHECK_BAILOUT();
3074 
3075   build_intervals();
3076   CHECK_BAILOUT();
3077   sort_intervals_before_allocation();
3078 
3079   NOT_PRODUCT(print_intervals("Before Register Allocation"));
3080   NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_before_alloc));
3081 
3082   allocate_registers();
3083   CHECK_BAILOUT();
3084 
3085   resolve_data_flow();
3086   if (compilation()->has_exception_handlers()) {
3087     resolve_exception_handlers();
3088   }
3089   // fill in number of spill slots into frame_map
3090   propagate_spill_slots();
3091   CHECK_BAILOUT();
3092 
3093   NOT_PRODUCT(print_intervals("After Register Allocation"));
3094   NOT_PRODUCT(print_lir(2, "LIR after register allocation:"));
3095 
3096   sort_intervals_after_allocation();
3097 
3098   DEBUG_ONLY(verify());
3099 
3100   eliminate_spill_moves();
3101   assign_reg_num();
3102   CHECK_BAILOUT();
3103 
3104   NOT_PRODUCT(print_lir(2, "LIR after assignment of register numbers:"));
3105   NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_after_asign));
3106 
3107   { TIME_LINEAR_SCAN(timer_allocate_fpu_stack);
3108 
3109     if (use_fpu_stack_allocation()) {
3110       allocate_fpu_stack(); // Only has effect on Intel
3111       NOT_PRODUCT(print_lir(2, "LIR after FPU stack allocation:"));
3112     }
3113   }
3114 
3115   { TIME_LINEAR_SCAN(timer_optimize_lir);
3116 
3117     EdgeMoveOptimizer::optimize(ir()->code());
3118     ControlFlowOptimizer::optimize(ir()->code());
3119     // check that cfg is still correct after optimizations
3120     ir()->verify();
3121   }
3122 
3123   NOT_PRODUCT(print_lir(1, "Before Code Generation", false));
3124   NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_final));
3125   NOT_PRODUCT(_total_timer.end_method(this));
3126 }
3127 
3128 
3129 // ********** Printing functions
3130 
3131 #ifndef PRODUCT
3132 
3133 void LinearScan::print_timers(double total) {
3134   _total_timer.print(total);
3135 }
3136 
3137 void LinearScan::print_statistics() {
3138   _stat_before_alloc.print("before allocation");
3139   _stat_after_asign.print("after assignment of register");
3140   _stat_final.print("after optimization");
3141 }
3142 
3143 void LinearScan::print_bitmap(BitMap& b) {
3144   for (unsigned int i = 0; i < b.size(); i++) {
3145     if (b.at(i)) tty->print("%d ", i);
3146   }
3147   tty->cr();
3148 }
3149 
3150 void LinearScan::print_intervals(const char* label) {
3151   if (TraceLinearScanLevel >= 1) {
3152     int i;
3153     tty->cr();
3154     tty->print_cr("%s", label);
3155 
3156     for (i = 0; i < interval_count(); i++) {
3157       Interval* interval = interval_at(i);
3158       if (interval != NULL) {
3159         interval->print();
3160       }
3161     }
3162 
3163     tty->cr();
3164     tty->print_cr("--- Basic Blocks ---");
3165     for (i = 0; i < block_count(); i++) {
3166       BlockBegin* block = block_at(i);
3167       tty->print("B%d [%d, %d, %d, %d] ", block->block_id(), block->first_lir_instruction_id(), block->last_lir_instruction_id(), block->loop_index(), block->loop_depth());
3168     }
3169     tty->cr();
3170     tty->cr();
3171   }
3172 
3173   if (PrintCFGToFile) {
3174     CFGPrinter::print_intervals(&_intervals, label);
3175   }
3176 }
3177 
3178 void LinearScan::print_lir(int level, const char* label, bool hir_valid) {
3179   if (TraceLinearScanLevel >= level) {
3180     tty->cr();
3181     tty->print_cr("%s", label);
3182     print_LIR(ir()->linear_scan_order());
3183     tty->cr();
3184   }
3185 
3186   if (level == 1 && PrintCFGToFile) {
3187     CFGPrinter::print_cfg(ir()->linear_scan_order(), label, hir_valid, true);
3188   }
3189 }
3190 
3191 #endif //PRODUCT
3192 
3193 
3194 // ********** verification functions for allocation
3195 // (check that all intervals have a correct register and that no registers are overwritten)
3196 #ifdef ASSERT
3197 
3198 void LinearScan::verify() {
3199   TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying intervals ******************************************"));
3200   verify_intervals();
3201 
3202   TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying that no oops are in fixed intervals ****************"));
3203   verify_no_oops_in_fixed_intervals();
3204 
3205   TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying that unpinned constants are not alive across block boundaries"));
3206   verify_constants();
3207 
3208   TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying register allocation ********************************"));
3209   verify_registers();
3210 
3211   TRACE_LINEAR_SCAN(2, tty->print_cr("********* no errors found **********************************************"));
3212 }
3213 
3214 void LinearScan::verify_intervals() {
3215   int len = interval_count();
3216   bool has_error = false;
3217 
3218   for (int i = 0; i < len; i++) {
3219     Interval* i1 = interval_at(i);
3220     if (i1 == NULL) continue;
3221 
3222     i1->check_split_children();
3223 
3224     if (i1->reg_num() != i) {
3225       tty->print_cr("Interval %d is on position %d in list", i1->reg_num(), i); i1->print(); tty->cr();
3226       has_error = true;
3227     }
3228 
3229     if (i1->reg_num() >= LIR_OprDesc::vreg_base && i1->type() == T_ILLEGAL) {
3230       tty->print_cr("Interval %d has no type assigned", i1->reg_num()); i1->print(); tty->cr();
3231       has_error = true;
3232     }
3233 
3234     if (i1->assigned_reg() == any_reg) {
3235       tty->print_cr("Interval %d has no register assigned", i1->reg_num()); i1->print(); tty->cr();
3236       has_error = true;
3237     }
3238 
3239     if (i1->assigned_reg() == i1->assigned_regHi()) {
3240       tty->print_cr("Interval %d: low and high register equal", i1->reg_num()); i1->print(); tty->cr();
3241       has_error = true;
3242     }
3243 
3244     if (!is_processed_reg_num(i1->assigned_reg())) {
3245       tty->print_cr("Can not have an Interval for an ignored register"); i1->print(); tty->cr();
3246       has_error = true;
3247     }
3248 
3249     // special intervals that are created in MoveResolver
3250     // -> ignore them because the range information has no meaning there
3251     if (i1->from() == 1 && i1->to() == 2) continue;
3252 
3253     if (i1->first() == Range::end()) {
3254       tty->print_cr("Interval %d has no Range", i1->reg_num()); i1->print(); tty->cr();
3255       has_error = true;
3256     }
3257 
3258     for (Range* r = i1->first(); r != Range::end(); r = r->next()) {
3259       if (r->from() >= r->to()) {
3260         tty->print_cr("Interval %d has zero length range", i1->reg_num()); i1->print(); tty->cr();
3261         has_error = true;
3262       }
3263     }
3264 
3265     for (int j = i + 1; j < len; j++) {
3266       Interval* i2 = interval_at(j);
3267       if (i2 == NULL || (i2->from() == 1 && i2->to() == 2)) continue;
3268 
3269       int r1 = i1->assigned_reg();
3270       int r1Hi = i1->assigned_regHi();
3271       int r2 = i2->assigned_reg();
3272       int r2Hi = i2->assigned_regHi();
3273       if ((r1 == r2 || r1 == r2Hi || (r1Hi != any_reg && (r1Hi == r2 || r1Hi == r2Hi))) && i1->intersects(i2)) {
3274         tty->print_cr("Intervals %d and %d overlap and have the same register assigned", i1->reg_num(), i2->reg_num());
3275         i1->print(); tty->cr();
3276         i2->print(); tty->cr();
3277         has_error = true;
3278       }
3279     }
3280   }
3281 
3282   assert(has_error == false, "register allocation invalid");
3283 }
3284 
3285 
3286 void LinearScan::verify_no_oops_in_fixed_intervals() {
3287   Interval* fixed_intervals;
3288   Interval* other_intervals;
3289   create_unhandled_lists(&fixed_intervals, &other_intervals, is_precolored_cpu_interval, NULL);
3290 
3291   // to ensure a walking until the last instruction id, add a dummy interval
3292   // with a high operation id
3293   other_intervals = new Interval(any_reg);
3294   other_intervals->add_range(max_jint - 2, max_jint - 1);
3295   IntervalWalker* iw = new IntervalWalker(this, fixed_intervals, other_intervals);
3296 
3297   LIR_OpVisitState visitor;
3298   for (int i = 0; i < block_count(); i++) {
3299     BlockBegin* block = block_at(i);
3300 
3301     LIR_OpList* instructions = block->lir()->instructions_list();
3302 
3303     for (int j = 0; j < instructions->length(); j++) {
3304       LIR_Op* op = instructions->at(j);
3305       int op_id = op->id();
3306 
3307       visitor.visit(op);
3308 
3309       if (visitor.info_count() > 0) {
3310         iw->walk_before(op->id());
3311         bool check_live = true;
3312         if (op->code() == lir_move) {
3313           LIR_Op1* move = (LIR_Op1*)op;
3314           check_live = (move->patch_code() == lir_patch_none);
3315         }
3316         LIR_OpBranch* branch = op->as_OpBranch();
3317         if (branch != NULL && branch->stub() != NULL && branch->stub()->is_exception_throw_stub()) {
3318           // Don't bother checking the stub in this case since the
3319           // exception stub will never return to normal control flow.
3320           check_live = false;
3321         }
3322 
3323         // Make sure none of the fixed registers is live across an
3324         // oopmap since we can't handle that correctly.
3325         if (check_live) {
3326           for (Interval* interval = iw->active_first(fixedKind);
3327                interval != Interval::end();
3328                interval = interval->next()) {
3329             if (interval->current_to() > op->id() + 1) {
3330               // This interval is live out of this op so make sure
3331               // that this interval represents some value that's
3332               // referenced by this op either as an input or output.
3333               bool ok = false;
3334               for_each_visitor_mode(mode) {
3335                 int n = visitor.opr_count(mode);
3336                 for (int k = 0; k < n; k++) {
3337                   LIR_Opr opr = visitor.opr_at(mode, k);
3338                   if (opr->is_fixed_cpu()) {
3339                     if (interval_at(reg_num(opr)) == interval) {
3340                       ok = true;
3341                       break;
3342                     }
3343                     int hi = reg_numHi(opr);
3344                     if (hi != -1 && interval_at(hi) == interval) {
3345                       ok = true;
3346                       break;
3347                     }
3348                   }
3349                 }
3350               }
3351               assert(ok, "fixed intervals should never be live across an oopmap point");
3352             }
3353           }
3354         }
3355       }
3356 
3357       // oop-maps at calls do not contain registers, so check is not needed
3358       if (!visitor.has_call()) {
3359 
3360         for_each_visitor_mode(mode) {
3361           int n = visitor.opr_count(mode);
3362           for (int k = 0; k < n; k++) {
3363             LIR_Opr opr = visitor.opr_at(mode, k);
3364 
3365             if (opr->is_fixed_cpu() && opr->is_oop()) {
3366               // operand is a non-virtual cpu register and contains an oop
3367               TRACE_LINEAR_SCAN(4, op->print_on(tty); tty->print("checking operand "); opr->print(); tty->cr());
3368 
3369               Interval* interval = interval_at(reg_num(opr));
3370               assert(interval != NULL, "no interval");
3371 
3372               if (mode == LIR_OpVisitState::inputMode) {
3373                 if (interval->to() >= op_id + 1) {
3374                   assert(interval->to() < op_id + 2 ||
3375                          interval->has_hole_between(op_id, op_id + 2),
3376                          "oop input operand live after instruction");
3377                 }
3378               } else if (mode == LIR_OpVisitState::outputMode) {
3379                 if (interval->from() <= op_id - 1) {
3380                   assert(interval->has_hole_between(op_id - 1, op_id),
3381                          "oop input operand live after instruction");
3382                 }
3383               }
3384             }
3385           }
3386         }
3387       }
3388     }
3389   }
3390 }
3391 
3392 
3393 void LinearScan::verify_constants() {
3394   int num_regs = num_virtual_regs();
3395   int size = live_set_size();
3396   int num_blocks = block_count();
3397 
3398   for (int i = 0; i < num_blocks; i++) {
3399     BlockBegin* block = block_at(i);
3400     BitMap live_at_edge = block->live_in();
3401 
3402     // visit all registers where the live_at_edge bit is set
3403     for (int r = (int)live_at_edge.get_next_one_offset(0, size); r < size; r = (int)live_at_edge.get_next_one_offset(r + 1, size)) {
3404       TRACE_LINEAR_SCAN(4, tty->print("checking interval %d of block B%d", r, block->block_id()));
3405 
3406       Value value = gen()->instruction_for_vreg(r);
3407 
3408       assert(value != NULL, "all intervals live across block boundaries must have Value");
3409       assert(value->operand()->is_register() && value->operand()->is_virtual(), "value must have virtual operand");
3410       assert(value->operand()->vreg_number() == r, "register number must match");
3411       // TKR assert(value->as_Constant() == NULL || value->is_pinned(), "only pinned constants can be alive accross block boundaries");
3412     }
3413   }
3414 }
3415 
3416 
3417 class RegisterVerifier: public StackObj {
3418  private:
3419   LinearScan*   _allocator;
3420   BlockList     _work_list;      // all blocks that must be processed
3421   IntervalsList _saved_states;   // saved information of previous check
3422 
3423   // simplified access to methods of LinearScan
3424   Compilation*  compilation() const              { return _allocator->compilation(); }
3425   Interval*     interval_at(int reg_num) const   { return _allocator->interval_at(reg_num); }
3426   int           reg_num(LIR_Opr opr) const       { return _allocator->reg_num(opr); }
3427 
3428   // currently, only registers are processed
3429   int           state_size()                     { return LinearScan::nof_regs; }
3430 
3431   // accessors
3432   IntervalList* state_for_block(BlockBegin* block) { return _saved_states.at(block->block_id()); }
3433   void          set_state_for_block(BlockBegin* block, IntervalList* saved_state) { _saved_states.at_put(block->block_id(), saved_state); }
3434   void          add_to_work_list(BlockBegin* block) { if (!_work_list.contains(block)) _work_list.append(block); }
3435 
3436   // helper functions
3437   IntervalList* copy(IntervalList* input_state);
3438   void          state_put(IntervalList* input_state, int reg, Interval* interval);
3439   bool          check_state(IntervalList* input_state, int reg, Interval* interval);
3440 
3441   void process_block(BlockBegin* block);
3442   void process_xhandler(XHandler* xhandler, IntervalList* input_state);
3443   void process_successor(BlockBegin* block, IntervalList* input_state);
3444   void process_operations(LIR_List* ops, IntervalList* input_state);
3445 
3446  public:
3447   RegisterVerifier(LinearScan* allocator)
3448     : _allocator(allocator)
3449     , _work_list(16)
3450     , _saved_states(BlockBegin::number_of_blocks(), BlockBegin::number_of_blocks(), NULL)
3451   { }
3452 
3453   void verify(BlockBegin* start);
3454 };
3455 
3456 
3457 // entry function from LinearScan that starts the verification
3458 void LinearScan::verify_registers() {
3459   RegisterVerifier verifier(this);
3460   verifier.verify(block_at(0));
3461 }
3462 
3463 
3464 void RegisterVerifier::verify(BlockBegin* start) {
3465   // setup input registers (method arguments) for first block
3466   int input_state_len = state_size();
3467   IntervalList* input_state = new IntervalList(input_state_len, input_state_len, NULL);
3468   CallingConvention* args = compilation()->frame_map()->incoming_arguments();
3469   for (int n = 0; n < args->length(); n++) {
3470     LIR_Opr opr = args->at(n);
3471     if (opr->is_register()) {
3472       Interval* interval = interval_at(reg_num(opr));
3473 
3474       if (interval->assigned_reg() < state_size()) {
3475         input_state->at_put(interval->assigned_reg(), interval);
3476       }
3477       if (interval->assigned_regHi() != LinearScan::any_reg && interval->assigned_regHi() < state_size()) {
3478         input_state->at_put(interval->assigned_regHi(), interval);
3479       }
3480     }
3481   }
3482 
3483   set_state_for_block(start, input_state);
3484   add_to_work_list(start);
3485 
3486   // main loop for verification
3487   do {
3488     BlockBegin* block = _work_list.at(0);
3489     _work_list.remove_at(0);
3490 
3491     process_block(block);
3492   } while (!_work_list.is_empty());
3493 }
3494 
3495 void RegisterVerifier::process_block(BlockBegin* block) {
3496   TRACE_LINEAR_SCAN(2, tty->cr(); tty->print_cr("process_block B%d", block->block_id()));
3497 
3498   // must copy state because it is modified
3499   IntervalList* input_state = copy(state_for_block(block));
3500 
3501   if (TraceLinearScanLevel >= 4) {
3502     tty->print_cr("Input-State of intervals:");
3503     tty->print("    ");
3504     for (int i = 0; i < state_size(); i++) {
3505       if (input_state->at(i) != NULL) {
3506         tty->print(" %4d", input_state->at(i)->reg_num());
3507       } else {
3508         tty->print("   __");
3509       }
3510     }
3511     tty->cr();
3512     tty->cr();
3513   }
3514 
3515   // process all operations of the block
3516   process_operations(block->lir(), input_state);
3517 
3518   // iterate all successors
3519   for (int i = 0; i < block->number_of_sux(); i++) {
3520     process_successor(block->sux_at(i), input_state);
3521   }
3522 }
3523 
3524 void RegisterVerifier::process_xhandler(XHandler* xhandler, IntervalList* input_state) {
3525   TRACE_LINEAR_SCAN(2, tty->print_cr("process_xhandler B%d", xhandler->entry_block()->block_id()));
3526 
3527   // must copy state because it is modified
3528   input_state = copy(input_state);
3529 
3530   if (xhandler->entry_code() != NULL) {
3531     process_operations(xhandler->entry_code(), input_state);
3532   }
3533   process_successor(xhandler->entry_block(), input_state);
3534 }
3535 
3536 void RegisterVerifier::process_successor(BlockBegin* block, IntervalList* input_state) {
3537   IntervalList* saved_state = state_for_block(block);
3538 
3539   if (saved_state != NULL) {
3540     // this block was already processed before.
3541     // check if new input_state is consistent with saved_state
3542 
3543     bool saved_state_correct = true;
3544     for (int i = 0; i < state_size(); i++) {
3545       if (input_state->at(i) != saved_state->at(i)) {
3546         // current input_state and previous saved_state assume a different
3547         // interval in this register -> assume that this register is invalid
3548         if (saved_state->at(i) != NULL) {
3549           // invalidate old calculation only if it assumed that
3550           // register was valid. when the register was already invalid,
3551           // then the old calculation was correct.
3552           saved_state_correct = false;
3553           saved_state->at_put(i, NULL);
3554 
3555           TRACE_LINEAR_SCAN(4, tty->print_cr("process_successor B%d: invalidating slot %d", block->block_id(), i));
3556         }
3557       }
3558     }
3559 
3560     if (saved_state_correct) {
3561       // already processed block with correct input_state
3562       TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: previous visit already correct", block->block_id()));
3563     } else {
3564       // must re-visit this block
3565       TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: must re-visit because input state changed", block->block_id()));
3566       add_to_work_list(block);
3567     }
3568 
3569   } else {
3570     // block was not processed before, so set initial input_state
3571     TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: initial visit", block->block_id()));
3572 
3573     set_state_for_block(block, copy(input_state));
3574     add_to_work_list(block);
3575   }
3576 }
3577 
3578 
3579 IntervalList* RegisterVerifier::copy(IntervalList* input_state) {
3580   IntervalList* copy_state = new IntervalList(input_state->length());
3581   copy_state->appendAll(input_state);
3582   return copy_state;
3583 }
3584 
3585 void RegisterVerifier::state_put(IntervalList* input_state, int reg, Interval* interval) {
3586   if (reg != LinearScan::any_reg && reg < state_size()) {
3587     if (interval != NULL) {
3588       TRACE_LINEAR_SCAN(4, tty->print_cr("        reg[%d] = %d", reg, interval->reg_num()));
3589     } else if (input_state->at(reg) != NULL) {
3590       TRACE_LINEAR_SCAN(4, tty->print_cr("        reg[%d] = NULL", reg));
3591     }
3592 
3593     input_state->at_put(reg, interval);
3594   }
3595 }
3596 
3597 bool RegisterVerifier::check_state(IntervalList* input_state, int reg, Interval* interval) {
3598   if (reg != LinearScan::any_reg && reg < state_size()) {
3599     if (input_state->at(reg) != interval) {
3600       tty->print_cr("!! Error in register allocation: register %d does not contain interval %d", reg, interval->reg_num());
3601       return true;
3602     }
3603   }
3604   return false;
3605 }
3606 
3607 void RegisterVerifier::process_operations(LIR_List* ops, IntervalList* input_state) {
3608   // visit all instructions of the block
3609   LIR_OpVisitState visitor;
3610   bool has_error = false;
3611 
3612   for (int i = 0; i < ops->length(); i++) {
3613     LIR_Op* op = ops->at(i);
3614     visitor.visit(op);
3615 
3616     TRACE_LINEAR_SCAN(4, op->print_on(tty));
3617 
3618     // check if input operands are correct
3619     int j;
3620     int n = visitor.opr_count(LIR_OpVisitState::inputMode);
3621     for (j = 0; j < n; j++) {
3622       LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, j);
3623       if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) {
3624         Interval* interval = interval_at(reg_num(opr));
3625         if (op->id() != -1) {
3626           interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::inputMode);
3627         }
3628 
3629         has_error |= check_state(input_state, interval->assigned_reg(),   interval->split_parent());
3630         has_error |= check_state(input_state, interval->assigned_regHi(), interval->split_parent());
3631 
3632         // When an operand is marked with is_last_use, then the fpu stack allocator
3633         // removes the register from the fpu stack -> the register contains no value
3634         if (opr->is_last_use()) {
3635           state_put(input_state, interval->assigned_reg(),   NULL);
3636           state_put(input_state, interval->assigned_regHi(), NULL);
3637         }
3638       }
3639     }
3640 
3641     // invalidate all caller save registers at calls
3642     if (visitor.has_call()) {
3643       for (j = 0; j < FrameMap::nof_caller_save_cpu_regs(); j++) {
3644         state_put(input_state, reg_num(FrameMap::caller_save_cpu_reg_at(j)), NULL);
3645       }
3646       for (j = 0; j < FrameMap::nof_caller_save_fpu_regs; j++) {
3647         state_put(input_state, reg_num(FrameMap::caller_save_fpu_reg_at(j)), NULL);
3648       }
3649 
3650 #ifdef X86
3651       int num_caller_save_xmm_regs = FrameMap::get_num_caller_save_xmms();
3652       for (j = 0; j < num_caller_save_xmm_regs; j++) {
3653         state_put(input_state, reg_num(FrameMap::caller_save_xmm_reg_at(j)), NULL);
3654       }
3655 #endif
3656     }
3657 
3658     // process xhandler before output and temp operands
3659     XHandlers* xhandlers = visitor.all_xhandler();
3660     n = xhandlers->length();
3661     for (int k = 0; k < n; k++) {
3662       process_xhandler(xhandlers->handler_at(k), input_state);
3663     }
3664 
3665     // set temp operands (some operations use temp operands also as output operands, so can't set them NULL)
3666     n = visitor.opr_count(LIR_OpVisitState::tempMode);
3667     for (j = 0; j < n; j++) {
3668       LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, j);
3669       if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) {
3670         Interval* interval = interval_at(reg_num(opr));
3671         if (op->id() != -1) {
3672           interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::tempMode);
3673         }
3674 
3675         state_put(input_state, interval->assigned_reg(),   interval->split_parent());
3676         state_put(input_state, interval->assigned_regHi(), interval->split_parent());
3677       }
3678     }
3679 
3680     // set output operands
3681     n = visitor.opr_count(LIR_OpVisitState::outputMode);
3682     for (j = 0; j < n; j++) {
3683       LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, j);
3684       if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) {
3685         Interval* interval = interval_at(reg_num(opr));
3686         if (op->id() != -1) {
3687           interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::outputMode);
3688         }
3689 
3690         state_put(input_state, interval->assigned_reg(),   interval->split_parent());
3691         state_put(input_state, interval->assigned_regHi(), interval->split_parent());
3692       }
3693     }
3694   }
3695   assert(has_error == false, "Error in register allocation");
3696 }
3697 
3698 #endif // ASSERT
3699 
3700 
3701 
3702 // **** Implementation of MoveResolver ******************************
3703 
3704 MoveResolver::MoveResolver(LinearScan* allocator) :
3705   _allocator(allocator),
3706   _multiple_reads_allowed(false),
3707   _mapping_from(8),
3708   _mapping_from_opr(8),
3709   _mapping_to(8),
3710   _insert_list(NULL),
3711   _insert_idx(-1),
3712   _insertion_buffer()
3713 {
3714   for (int i = 0; i < LinearScan::nof_regs; i++) {
3715     _register_blocked[i] = 0;
3716   }
3717   DEBUG_ONLY(check_empty());
3718 }
3719 
3720 
3721 #ifdef ASSERT
3722 
3723 void MoveResolver::check_empty() {
3724   assert(_mapping_from.length() == 0 && _mapping_from_opr.length() == 0 && _mapping_to.length() == 0, "list must be empty before and after processing");
3725   for (int i = 0; i < LinearScan::nof_regs; i++) {
3726     assert(register_blocked(i) == 0, "register map must be empty before and after processing");
3727   }
3728   assert(_multiple_reads_allowed == false, "must have default value");
3729 }
3730 
3731 void MoveResolver::verify_before_resolve() {
3732   assert(_mapping_from.length() == _mapping_from_opr.length(), "length must be equal");
3733   assert(_mapping_from.length() == _mapping_to.length(), "length must be equal");
3734   assert(_insert_list != NULL && _insert_idx != -1, "insert position not set");
3735 
3736   int i, j;
3737   if (!_multiple_reads_allowed) {
3738     for (i = 0; i < _mapping_from.length(); i++) {
3739       for (j = i + 1; j < _mapping_from.length(); j++) {
3740         assert(_mapping_from.at(i) == NULL || _mapping_from.at(i) != _mapping_from.at(j), "cannot read from same interval twice");
3741       }
3742     }
3743   }
3744 
3745   for (i = 0; i < _mapping_to.length(); i++) {
3746     for (j = i + 1; j < _mapping_to.length(); j++) {
3747       assert(_mapping_to.at(i) != _mapping_to.at(j), "cannot write to same interval twice");
3748     }
3749   }
3750 
3751 
3752   BitMap used_regs(LinearScan::nof_regs + allocator()->frame_map()->argcount() + allocator()->max_spills());
3753   used_regs.clear();
3754   if (!_multiple_reads_allowed) {
3755     for (i = 0; i < _mapping_from.length(); i++) {
3756       Interval* it = _mapping_from.at(i);
3757       if (it != NULL) {
3758         assert(!used_regs.at(it->assigned_reg()), "cannot read from same register twice");
3759         used_regs.set_bit(it->assigned_reg());
3760 
3761         if (it->assigned_regHi() != LinearScan::any_reg) {
3762           assert(!used_regs.at(it->assigned_regHi()), "cannot read from same register twice");
3763           used_regs.set_bit(it->assigned_regHi());
3764         }
3765       }
3766     }
3767   }
3768 
3769   used_regs.clear();
3770   for (i = 0; i < _mapping_to.length(); i++) {
3771     Interval* it = _mapping_to.at(i);
3772     assert(!used_regs.at(it->assigned_reg()), "cannot write to same register twice");
3773     used_regs.set_bit(it->assigned_reg());
3774 
3775     if (it->assigned_regHi() != LinearScan::any_reg) {
3776       assert(!used_regs.at(it->assigned_regHi()), "cannot write to same register twice");
3777       used_regs.set_bit(it->assigned_regHi());
3778     }
3779   }
3780 
3781   used_regs.clear();
3782   for (i = 0; i < _mapping_from.length(); i++) {
3783     Interval* it = _mapping_from.at(i);
3784     if (it != NULL && it->assigned_reg() >= LinearScan::nof_regs) {
3785       used_regs.set_bit(it->assigned_reg());
3786     }
3787   }
3788   for (i = 0; i < _mapping_to.length(); i++) {
3789     Interval* it = _mapping_to.at(i);
3790     assert(!used_regs.at(it->assigned_reg()) || it->assigned_reg() == _mapping_from.at(i)->assigned_reg(), "stack slots used in _mapping_from must be disjoint to _mapping_to");
3791   }
3792 }
3793 
3794 #endif // ASSERT
3795 
3796 
3797 // mark assigned_reg and assigned_regHi of the interval as blocked
3798 void MoveResolver::block_registers(Interval* it) {
3799   int reg = it->assigned_reg();
3800   if (reg < LinearScan::nof_regs) {
3801     assert(_multiple_reads_allowed || register_blocked(reg) == 0, "register already marked as used");
3802     set_register_blocked(reg, 1);
3803   }
3804   reg = it->assigned_regHi();
3805   if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) {
3806     assert(_multiple_reads_allowed || register_blocked(reg) == 0, "register already marked as used");
3807     set_register_blocked(reg, 1);
3808   }
3809 }
3810 
3811 // mark assigned_reg and assigned_regHi of the interval as unblocked
3812 void MoveResolver::unblock_registers(Interval* it) {
3813   int reg = it->assigned_reg();
3814   if (reg < LinearScan::nof_regs) {
3815     assert(register_blocked(reg) > 0, "register already marked as unused");
3816     set_register_blocked(reg, -1);
3817   }
3818   reg = it->assigned_regHi();
3819   if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) {
3820     assert(register_blocked(reg) > 0, "register already marked as unused");
3821     set_register_blocked(reg, -1);
3822   }
3823 }
3824 
3825 // check if assigned_reg and assigned_regHi of the to-interval are not blocked (or only blocked by from)
3826 bool MoveResolver::save_to_process_move(Interval* from, Interval* to) {
3827   int from_reg = -1;
3828   int from_regHi = -1;
3829   if (from != NULL) {
3830     from_reg = from->assigned_reg();
3831     from_regHi = from->assigned_regHi();
3832   }
3833 
3834   int reg = to->assigned_reg();
3835   if (reg < LinearScan::nof_regs) {
3836     if (register_blocked(reg) > 1 || (register_blocked(reg) == 1 && reg != from_reg && reg != from_regHi)) {
3837       return false;
3838     }
3839   }
3840   reg = to->assigned_regHi();
3841   if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) {
3842     if (register_blocked(reg) > 1 || (register_blocked(reg) == 1 && reg != from_reg && reg != from_regHi)) {
3843       return false;
3844     }
3845   }
3846 
3847   return true;
3848 }
3849 
3850 
3851 void MoveResolver::create_insertion_buffer(LIR_List* list) {
3852   assert(!_insertion_buffer.initialized(), "overwriting existing buffer");
3853   _insertion_buffer.init(list);
3854 }
3855 
3856 void MoveResolver::append_insertion_buffer() {
3857   if (_insertion_buffer.initialized()) {
3858     _insertion_buffer.lir_list()->append(&_insertion_buffer);
3859   }
3860   assert(!_insertion_buffer.initialized(), "must be uninitialized now");
3861 
3862   _insert_list = NULL;
3863   _insert_idx = -1;
3864 }
3865 
3866 void MoveResolver::insert_move(Interval* from_interval, Interval* to_interval) {
3867   assert(from_interval->reg_num() != to_interval->reg_num(), "from and to interval equal");
3868   assert(from_interval->type() == to_interval->type(), "move between different types");
3869   assert(_insert_list != NULL && _insert_idx != -1, "must setup insert position first");
3870   assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer");
3871 
3872   LIR_Opr from_opr = LIR_OprFact::virtual_register(from_interval->reg_num(), from_interval->type());
3873   LIR_Opr to_opr = LIR_OprFact::virtual_register(to_interval->reg_num(), to_interval->type());
3874 
3875   if (!_multiple_reads_allowed) {
3876     // the last_use flag is an optimization for FPU stack allocation. When the same
3877     // input interval is used in more than one move, then it is too difficult to determine
3878     // if this move is really the last use.
3879     from_opr = from_opr->make_last_use();
3880   }
3881   _insertion_buffer.move(_insert_idx, from_opr, to_opr);
3882 
3883   TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: inserted move from register %d (%d, %d) to %d (%d, %d)", from_interval->reg_num(), from_interval->assigned_reg(), from_interval->assigned_regHi(), to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi()));
3884 }
3885 
3886 void MoveResolver::insert_move(LIR_Opr from_opr, Interval* to_interval) {
3887   assert(from_opr->type() == to_interval->type(), "move between different types");
3888   assert(_insert_list != NULL && _insert_idx != -1, "must setup insert position first");
3889   assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer");
3890 
3891   LIR_Opr to_opr = LIR_OprFact::virtual_register(to_interval->reg_num(), to_interval->type());
3892   _insertion_buffer.move(_insert_idx, from_opr, to_opr);
3893 
3894   TRACE_LINEAR_SCAN(4, tty->print("MoveResolver: inserted move from constant "); from_opr->print(); tty->print_cr("  to %d (%d, %d)", to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi()));
3895 }
3896 
3897 
3898 void MoveResolver::resolve_mappings() {
3899   TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: resolving mappings for Block B%d, index %d", _insert_list->block() != NULL ? _insert_list->block()->block_id() : -1, _insert_idx));
3900   DEBUG_ONLY(verify_before_resolve());
3901 
3902   // Block all registers that are used as input operands of a move.
3903   // When a register is blocked, no move to this register is emitted.
3904   // This is necessary for detecting cycles in moves.
3905   int i;
3906   for (i = _mapping_from.length() - 1; i >= 0; i--) {
3907     Interval* from_interval = _mapping_from.at(i);
3908     if (from_interval != NULL) {
3909       block_registers(from_interval);
3910     }
3911   }
3912 
3913   int spill_candidate = -1;
3914   while (_mapping_from.length() > 0) {
3915     bool processed_interval = false;
3916 
3917     for (i = _mapping_from.length() - 1; i >= 0; i--) {
3918       Interval* from_interval = _mapping_from.at(i);
3919       Interval* to_interval = _mapping_to.at(i);
3920 
3921       if (save_to_process_move(from_interval, to_interval)) {
3922         // this inverval can be processed because target is free
3923         if (from_interval != NULL) {
3924           insert_move(from_interval, to_interval);
3925           unblock_registers(from_interval);
3926         } else {
3927           insert_move(_mapping_from_opr.at(i), to_interval);
3928         }
3929         _mapping_from.remove_at(i);
3930         _mapping_from_opr.remove_at(i);
3931         _mapping_to.remove_at(i);
3932 
3933         processed_interval = true;
3934       } else if (from_interval != NULL && from_interval->assigned_reg() < LinearScan::nof_regs) {
3935         // this interval cannot be processed now because target is not free
3936         // it starts in a register, so it is a possible candidate for spilling
3937         spill_candidate = i;
3938       }
3939     }
3940 
3941     if (!processed_interval) {
3942       // no move could be processed because there is a cycle in the move list
3943       // (e.g. r1 -> r2, r2 -> r1), so one interval must be spilled to memory
3944       assert(spill_candidate != -1, "no interval in register for spilling found");
3945 
3946       // create a new spill interval and assign a stack slot to it
3947       Interval* from_interval = _mapping_from.at(spill_candidate);
3948       Interval* spill_interval = new Interval(-1);
3949       spill_interval->set_type(from_interval->type());
3950 
3951       // add a dummy range because real position is difficult to calculate
3952       // Note: this range is a special case when the integrity of the allocation is checked
3953       spill_interval->add_range(1, 2);
3954 
3955       //       do not allocate a new spill slot for temporary interval, but
3956       //       use spill slot assigned to from_interval. Otherwise moves from
3957       //       one stack slot to another can happen (not allowed by LIR_Assembler
3958       int spill_slot = from_interval->canonical_spill_slot();
3959       if (spill_slot < 0) {
3960         spill_slot = allocator()->allocate_spill_slot(type2spill_size[spill_interval->type()] == 2);
3961         from_interval->set_canonical_spill_slot(spill_slot);
3962       }
3963       spill_interval->assign_reg(spill_slot);
3964       allocator()->append_interval(spill_interval);
3965 
3966       TRACE_LINEAR_SCAN(4, tty->print_cr("created new Interval %d for spilling", spill_interval->reg_num()));
3967 
3968       // insert a move from register to stack and update the mapping
3969       insert_move(from_interval, spill_interval);
3970       _mapping_from.at_put(spill_candidate, spill_interval);
3971       unblock_registers(from_interval);
3972     }
3973   }
3974 
3975   // reset to default value
3976   _multiple_reads_allowed = false;
3977 
3978   // check that all intervals have been processed
3979   DEBUG_ONLY(check_empty());
3980 }
3981 
3982 
3983 void MoveResolver::set_insert_position(LIR_List* insert_list, int insert_idx) {
3984   TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: setting insert position to Block B%d, index %d", insert_list->block() != NULL ? insert_list->block()->block_id() : -1, insert_idx));
3985   assert(_insert_list == NULL && _insert_idx == -1, "use move_insert_position instead of set_insert_position when data already set");
3986 
3987   create_insertion_buffer(insert_list);
3988   _insert_list = insert_list;
3989   _insert_idx = insert_idx;
3990 }
3991 
3992 void MoveResolver::move_insert_position(LIR_List* insert_list, int insert_idx) {
3993   TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: moving insert position to Block B%d, index %d", insert_list->block() != NULL ? insert_list->block()->block_id() : -1, insert_idx));
3994 
3995   if (_insert_list != NULL && (insert_list != _insert_list || insert_idx != _insert_idx)) {
3996     // insert position changed -> resolve current mappings
3997     resolve_mappings();
3998   }
3999 
4000   if (insert_list != _insert_list) {
4001     // block changed -> append insertion_buffer because it is
4002     // bound to a specific block and create a new insertion_buffer
4003     append_insertion_buffer();
4004     create_insertion_buffer(insert_list);
4005   }
4006 
4007   _insert_list = insert_list;
4008   _insert_idx = insert_idx;
4009 }
4010 
4011 void MoveResolver::add_mapping(Interval* from_interval, Interval* to_interval) {
4012   TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: adding mapping from %d (%d, %d) to %d (%d, %d)", from_interval->reg_num(), from_interval->assigned_reg(), from_interval->assigned_regHi(), to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi()));
4013 
4014   _mapping_from.append(from_interval);
4015   _mapping_from_opr.append(LIR_OprFact::illegalOpr);
4016   _mapping_to.append(to_interval);
4017 }
4018 
4019 
4020 void MoveResolver::add_mapping(LIR_Opr from_opr, Interval* to_interval) {
4021   TRACE_LINEAR_SCAN(4, tty->print("MoveResolver: adding mapping from "); from_opr->print(); tty->print_cr(" to %d (%d, %d)", to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi()));
4022   assert(from_opr->is_constant(), "only for constants");
4023 
4024   _mapping_from.append(NULL);
4025   _mapping_from_opr.append(from_opr);
4026   _mapping_to.append(to_interval);
4027 }
4028 
4029 void MoveResolver::resolve_and_append_moves() {
4030   if (has_mappings()) {
4031     resolve_mappings();
4032   }
4033   append_insertion_buffer();
4034 }
4035 
4036 
4037 
4038 // **** Implementation of Range *************************************
4039 
4040 Range::Range(int from, int to, Range* next) :
4041   _from(from),
4042   _to(to),
4043   _next(next)
4044 {
4045 }
4046 
4047 // initialize sentinel
4048 Range* Range::_end = NULL;
4049 void Range::initialize(Arena* arena) {
4050   _end = new (arena) Range(max_jint, max_jint, NULL);
4051 }
4052 
4053 int Range::intersects_at(Range* r2) const {
4054   const Range* r1 = this;
4055 
4056   assert(r1 != NULL && r2 != NULL, "null ranges not allowed");
4057   assert(r1 != _end && r2 != _end, "empty ranges not allowed");
4058 
4059   do {
4060     if (r1->from() < r2->from()) {
4061       if (r1->to() <= r2->from()) {
4062         r1 = r1->next(); if (r1 == _end) return -1;
4063       } else {
4064         return r2->from();
4065       }
4066     } else if (r2->from() < r1->from()) {
4067       if (r2->to() <= r1->from()) {
4068         r2 = r2->next(); if (r2 == _end) return -1;
4069       } else {
4070         return r1->from();
4071       }
4072     } else { // r1->from() == r2->from()
4073       if (r1->from() == r1->to()) {
4074         r1 = r1->next(); if (r1 == _end) return -1;
4075       } else if (r2->from() == r2->to()) {
4076         r2 = r2->next(); if (r2 == _end) return -1;
4077       } else {
4078         return r1->from();
4079       }
4080     }
4081   } while (true);
4082 }
4083 
4084 #ifndef PRODUCT
4085 void Range::print(outputStream* out) const {
4086   out->print("[%d, %d[ ", _from, _to);
4087 }
4088 #endif
4089 
4090 
4091 
4092 // **** Implementation of Interval **********************************
4093 
4094 // initialize sentinel
4095 Interval* Interval::_end = NULL;
4096 void Interval::initialize(Arena* arena) {
4097   Range::initialize(arena);
4098   _end = new (arena) Interval(-1);
4099 }
4100 
4101 Interval::Interval(int reg_num) :
4102   _reg_num(reg_num),
4103   _type(T_ILLEGAL),
4104   _first(Range::end()),
4105   _use_pos_and_kinds(12),
4106   _current(Range::end()),
4107   _next(_end),
4108   _state(invalidState),
4109   _assigned_reg(LinearScan::any_reg),
4110   _assigned_regHi(LinearScan::any_reg),
4111   _cached_to(-1),
4112   _cached_opr(LIR_OprFact::illegalOpr),
4113   _cached_vm_reg(VMRegImpl::Bad()),
4114   _split_children(0),
4115   _canonical_spill_slot(-1),
4116   _insert_move_when_activated(false),
4117   _register_hint(NULL),
4118   _spill_state(noDefinitionFound),
4119   _spill_definition_pos(-1)
4120 {
4121   _split_parent = this;
4122   _current_split_child = this;
4123 }
4124 
4125 int Interval::calc_to() {
4126   assert(_first != Range::end(), "interval has no range");
4127 
4128   Range* r = _first;
4129   while (r->next() != Range::end()) {
4130     r = r->next();
4131   }
4132   return r->to();
4133 }
4134 
4135 
4136 #ifdef ASSERT
4137 // consistency check of split-children
4138 void Interval::check_split_children() {
4139   if (_split_children.length() > 0) {
4140     assert(is_split_parent(), "only split parents can have children");
4141 
4142     for (int i = 0; i < _split_children.length(); i++) {
4143       Interval* i1 = _split_children.at(i);
4144 
4145       assert(i1->split_parent() == this, "not a split child of this interval");
4146       assert(i1->type() == type(), "must be equal for all split children");
4147       assert(i1->canonical_spill_slot() == canonical_spill_slot(), "must be equal for all split children");
4148 
4149       for (int j = i + 1; j < _split_children.length(); j++) {
4150         Interval* i2 = _split_children.at(j);
4151 
4152         assert(i1->reg_num() != i2->reg_num(), "same register number");
4153 
4154         if (i1->from() < i2->from()) {
4155           assert(i1->to() <= i2->from() && i1->to() < i2->to(), "intervals overlapping");
4156         } else {
4157           assert(i2->from() < i1->from(), "intervals start at same op_id");
4158           assert(i2->to() <= i1->from() && i2->to() < i1->to(), "intervals overlapping");
4159         }
4160       }
4161     }
4162   }
4163 }
4164 #endif // ASSERT
4165 
4166 Interval* Interval::register_hint(bool search_split_child) const {
4167   if (!search_split_child) {
4168     return _register_hint;
4169   }
4170 
4171   if (_register_hint != NULL) {
4172     assert(_register_hint->is_split_parent(), "ony split parents are valid hint registers");
4173 
4174     if (_register_hint->assigned_reg() >= 0 && _register_hint->assigned_reg() < LinearScan::nof_regs) {
4175       return _register_hint;
4176 
4177     } else if (_register_hint->_split_children.length() > 0) {
4178       // search the first split child that has a register assigned
4179       int len = _register_hint->_split_children.length();
4180       for (int i = 0; i < len; i++) {
4181         Interval* cur = _register_hint->_split_children.at(i);
4182 
4183         if (cur->assigned_reg() >= 0 && cur->assigned_reg() < LinearScan::nof_regs) {
4184           return cur;
4185         }
4186       }
4187     }
4188   }
4189 
4190   // no hint interval found that has a register assigned
4191   return NULL;
4192 }
4193 
4194 
4195 Interval* Interval::split_child_at_op_id(int op_id, LIR_OpVisitState::OprMode mode) {
4196   assert(is_split_parent(), "can only be called for split parents");
4197   assert(op_id >= 0, "invalid op_id (method can not be called for spill moves)");
4198 
4199   Interval* result;
4200   if (_split_children.length() == 0) {
4201     result = this;
4202   } else {
4203     result = NULL;
4204     int len = _split_children.length();
4205 
4206     // in outputMode, the end of the interval (op_id == cur->to()) is not valid
4207     int to_offset = (mode == LIR_OpVisitState::outputMode ? 0 : 1);
4208 
4209     int i;
4210     for (i = 0; i < len; i++) {
4211       Interval* cur = _split_children.at(i);
4212       if (cur->from() <= op_id && op_id < cur->to() + to_offset) {
4213         if (i > 0) {
4214           // exchange current split child to start of list (faster access for next call)
4215           _split_children.at_put(i, _split_children.at(0));
4216           _split_children.at_put(0, cur);
4217         }
4218 
4219         // interval found
4220         result = cur;
4221         break;
4222       }
4223     }
4224 
4225 #ifdef ASSERT
4226     for (i = 0; i < len; i++) {
4227       Interval* tmp = _split_children.at(i);
4228       if (tmp != result && tmp->from() <= op_id && op_id < tmp->to() + to_offset) {
4229         tty->print_cr("two valid result intervals found for op_id %d: %d and %d", op_id, result->reg_num(), tmp->reg_num());
4230         result->print();
4231         tmp->print();
4232         assert(false, "two valid result intervals found");
4233       }
4234     }
4235 #endif
4236   }
4237 
4238   assert(result != NULL, "no matching interval found");
4239   assert(result->covers(op_id, mode), "op_id not covered by interval");
4240 
4241   return result;
4242 }
4243 
4244 
4245 // returns the last split child that ends before the given op_id
4246 Interval* Interval::split_child_before_op_id(int op_id) {
4247   assert(op_id >= 0, "invalid op_id");
4248 
4249   Interval* parent = split_parent();
4250   Interval* result = NULL;
4251 
4252   int len = parent->_split_children.length();
4253   assert(len > 0, "no split children available");
4254 
4255   for (int i = len - 1; i >= 0; i--) {
4256     Interval* cur = parent->_split_children.at(i);
4257     if (cur->to() <= op_id && (result == NULL || result->to() < cur->to())) {
4258       result = cur;
4259     }
4260   }
4261 
4262   assert(result != NULL, "no split child found");
4263   return result;
4264 }
4265 
4266 
4267 // checks if op_id is covered by any split child
4268 bool Interval::split_child_covers(int op_id, LIR_OpVisitState::OprMode mode) {
4269   assert(is_split_parent(), "can only be called for split parents");
4270   assert(op_id >= 0, "invalid op_id (method can not be called for spill moves)");
4271 
4272   if (_split_children.length() == 0) {
4273     // simple case if interval was not split
4274     return covers(op_id, mode);
4275 
4276   } else {
4277     // extended case: check all split children
4278     int len = _split_children.length();
4279     for (int i = 0; i < len; i++) {
4280       Interval* cur = _split_children.at(i);
4281       if (cur->covers(op_id, mode)) {
4282         return true;
4283       }
4284     }
4285     return false;
4286   }
4287 }
4288 
4289 
4290 // Note: use positions are sorted descending -> first use has highest index
4291 int Interval::first_usage(IntervalUseKind min_use_kind) const {
4292   assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals");
4293 
4294   for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) {
4295     if (_use_pos_and_kinds.at(i + 1) >= min_use_kind) {
4296       return _use_pos_and_kinds.at(i);
4297     }
4298   }
4299   return max_jint;
4300 }
4301 
4302 int Interval::next_usage(IntervalUseKind min_use_kind, int from) const {
4303   assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals");
4304 
4305   for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) {
4306     if (_use_pos_and_kinds.at(i) >= from && _use_pos_and_kinds.at(i + 1) >= min_use_kind) {
4307       return _use_pos_and_kinds.at(i);
4308     }
4309   }
4310   return max_jint;
4311 }
4312 
4313 int Interval::next_usage_exact(IntervalUseKind exact_use_kind, int from) const {
4314   assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals");
4315 
4316   for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) {
4317     if (_use_pos_and_kinds.at(i) >= from && _use_pos_and_kinds.at(i + 1) == exact_use_kind) {
4318       return _use_pos_and_kinds.at(i);
4319     }
4320   }
4321   return max_jint;
4322 }
4323 
4324 int Interval::previous_usage(IntervalUseKind min_use_kind, int from) const {
4325   assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals");
4326 
4327   int prev = 0;
4328   for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) {
4329     if (_use_pos_and_kinds.at(i) > from) {
4330       return prev;
4331     }
4332     if (_use_pos_and_kinds.at(i + 1) >= min_use_kind) {
4333       prev = _use_pos_and_kinds.at(i);
4334     }
4335   }
4336   return prev;
4337 }
4338 
4339 void Interval::add_use_pos(int pos, IntervalUseKind use_kind) {
4340   assert(covers(pos, LIR_OpVisitState::inputMode), "use position not covered by live range");
4341 
4342   // do not add use positions for precolored intervals because
4343   // they are never used
4344   if (use_kind != noUse && reg_num() >= LIR_OprDesc::vreg_base) {
4345 #ifdef ASSERT
4346     assert(_use_pos_and_kinds.length() % 2 == 0, "must be");
4347     for (int i = 0; i < _use_pos_and_kinds.length(); i += 2) {
4348       assert(pos <= _use_pos_and_kinds.at(i), "already added a use-position with lower position");
4349       assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind");
4350       if (i > 0) {
4351         assert(_use_pos_and_kinds.at(i) < _use_pos_and_kinds.at(i - 2), "not sorted descending");
4352       }
4353     }
4354 #endif
4355 
4356     // Note: add_use is called in descending order, so list gets sorted
4357     //       automatically by just appending new use positions
4358     int len = _use_pos_and_kinds.length();
4359     if (len == 0 || _use_pos_and_kinds.at(len - 2) > pos) {
4360       _use_pos_and_kinds.append(pos);
4361       _use_pos_and_kinds.append(use_kind);
4362     } else if (_use_pos_and_kinds.at(len - 1) < use_kind) {
4363       assert(_use_pos_and_kinds.at(len - 2) == pos, "list not sorted correctly");
4364       _use_pos_and_kinds.at_put(len - 1, use_kind);
4365     }
4366   }
4367 }
4368 
4369 void Interval::add_range(int from, int to) {
4370   assert(from < to, "invalid range");
4371   assert(first() == Range::end() || to < first()->next()->from(), "not inserting at begin of interval");
4372   assert(from <= first()->to(), "not inserting at begin of interval");
4373 
4374   if (first()->from() <= to) {
4375     // join intersecting ranges
4376     first()->set_from(MIN2(from, first()->from()));
4377     first()->set_to  (MAX2(to,   first()->to()));
4378   } else {
4379     // insert new range
4380     _first = new Range(from, to, first());
4381   }
4382 }
4383 
4384 Interval* Interval::new_split_child() {
4385   // allocate new interval
4386   Interval* result = new Interval(-1);
4387   result->set_type(type());
4388 
4389   Interval* parent = split_parent();
4390   result->_split_parent = parent;
4391   result->set_register_hint(parent);
4392 
4393   // insert new interval in children-list of parent
4394   if (parent->_split_children.length() == 0) {
4395     assert(is_split_parent(), "list must be initialized at first split");
4396 
4397     parent->_split_children = IntervalList(4);
4398     parent->_split_children.append(this);
4399   }
4400   parent->_split_children.append(result);
4401 
4402   return result;
4403 }
4404 
4405 // split this interval at the specified position and return
4406 // the remainder as a new interval.
4407 //
4408 // when an interval is split, a bi-directional link is established between the original interval
4409 // (the split parent) and the intervals that are split off this interval (the split children)
4410 // When a split child is split again, the new created interval is also a direct child
4411 // of the original parent (there is no tree of split children stored, but a flat list)
4412 // All split children are spilled to the same stack slot (stored in _canonical_spill_slot)
4413 //
4414 // Note: The new interval has no valid reg_num
4415 Interval* Interval::split(int split_pos) {
4416   assert(LinearScan::is_virtual_interval(this), "cannot split fixed intervals");
4417 
4418   // allocate new interval
4419   Interval* result = new_split_child();
4420 
4421   // split the ranges
4422   Range* prev = NULL;
4423   Range* cur = _first;
4424   while (cur != Range::end() && cur->to() <= split_pos) {
4425     prev = cur;
4426     cur = cur->next();
4427   }
4428   assert(cur != Range::end(), "split interval after end of last range");
4429 
4430   if (cur->from() < split_pos) {
4431     result->_first = new Range(split_pos, cur->to(), cur->next());
4432     cur->set_to(split_pos);
4433     cur->set_next(Range::end());
4434 
4435   } else {
4436     assert(prev != NULL, "split before start of first range");
4437     result->_first = cur;
4438     prev->set_next(Range::end());
4439   }
4440   result->_current = result->_first;
4441   _cached_to = -1; // clear cached value
4442 
4443   // split list of use positions
4444   int total_len = _use_pos_and_kinds.length();
4445   int start_idx = total_len - 2;
4446   while (start_idx >= 0 && _use_pos_and_kinds.at(start_idx) < split_pos) {
4447     start_idx -= 2;
4448   }
4449 
4450   intStack new_use_pos_and_kinds(total_len - start_idx);
4451   int i;
4452   for (i = start_idx + 2; i < total_len; i++) {
4453     new_use_pos_and_kinds.append(_use_pos_and_kinds.at(i));
4454   }
4455 
4456   _use_pos_and_kinds.trunc_to(start_idx + 2);
4457   result->_use_pos_and_kinds = _use_pos_and_kinds;
4458   _use_pos_and_kinds = new_use_pos_and_kinds;
4459 
4460 #ifdef ASSERT
4461   assert(_use_pos_and_kinds.length() % 2 == 0, "must have use kind for each use pos");
4462   assert(result->_use_pos_and_kinds.length() % 2 == 0, "must have use kind for each use pos");
4463   assert(_use_pos_and_kinds.length() + result->_use_pos_and_kinds.length() == total_len, "missed some entries");
4464 
4465   for (i = 0; i < _use_pos_and_kinds.length(); i += 2) {
4466     assert(_use_pos_and_kinds.at(i) < split_pos, "must be");
4467     assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind");
4468   }
4469   for (i = 0; i < result->_use_pos_and_kinds.length(); i += 2) {
4470     assert(result->_use_pos_and_kinds.at(i) >= split_pos, "must be");
4471     assert(result->_use_pos_and_kinds.at(i + 1) >= firstValidKind && result->_use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind");
4472   }
4473 #endif
4474 
4475   return result;
4476 }
4477 
4478 // split this interval at the specified position and return
4479 // the head as a new interval (the original interval is the tail)
4480 //
4481 // Currently, only the first range can be split, and the new interval
4482 // must not have split positions
4483 Interval* Interval::split_from_start(int split_pos) {
4484   assert(LinearScan::is_virtual_interval(this), "cannot split fixed intervals");
4485   assert(split_pos > from() && split_pos < to(), "can only split inside interval");
4486   assert(split_pos > _first->from() && split_pos <= _first->to(), "can only split inside first range");
4487   assert(first_usage(noUse) > split_pos, "can not split when use positions are present");
4488 
4489   // allocate new interval
4490   Interval* result = new_split_child();
4491 
4492   // the new created interval has only one range (checked by assertion above),
4493   // so the splitting of the ranges is very simple
4494   result->add_range(_first->from(), split_pos);
4495 
4496   if (split_pos == _first->to()) {
4497     assert(_first->next() != Range::end(), "must not be at end");
4498     _first = _first->next();
4499   } else {
4500     _first->set_from(split_pos);
4501   }
4502 
4503   return result;
4504 }
4505 
4506 
4507 // returns true if the op_id is inside the interval
4508 bool Interval::covers(int op_id, LIR_OpVisitState::OprMode mode) const {
4509   Range* cur  = _first;
4510 
4511   while (cur != Range::end() && cur->to() < op_id) {
4512     cur = cur->next();
4513   }
4514   if (cur != Range::end()) {
4515     assert(cur->to() != cur->next()->from(), "ranges not separated");
4516 
4517     if (mode == LIR_OpVisitState::outputMode) {
4518       return cur->from() <= op_id && op_id < cur->to();
4519     } else {
4520       return cur->from() <= op_id && op_id <= cur->to();
4521     }
4522   }
4523   return false;
4524 }
4525 
4526 // returns true if the interval has any hole between hole_from and hole_to
4527 // (even if the hole has only the length 1)
4528 bool Interval::has_hole_between(int hole_from, int hole_to) {
4529   assert(hole_from < hole_to, "check");
4530   assert(from() <= hole_from && hole_to <= to(), "index out of interval");
4531 
4532   Range* cur  = _first;
4533   while (cur != Range::end()) {
4534     assert(cur->to() < cur->next()->from(), "no space between ranges");
4535 
4536     // hole-range starts before this range -> hole
4537     if (hole_from < cur->from()) {
4538       return true;
4539 
4540     // hole-range completely inside this range -> no hole
4541     } else if (hole_to <= cur->to()) {
4542       return false;
4543 
4544     // overlapping of hole-range with this range -> hole
4545     } else if (hole_from <= cur->to()) {
4546       return true;
4547     }
4548 
4549     cur = cur->next();
4550   }
4551 
4552   return false;
4553 }
4554 
4555 
4556 #ifndef PRODUCT
4557 void Interval::print(outputStream* out) const {
4558   const char* SpillState2Name[] = { "no definition", "no spill store", "one spill store", "store at definition", "start in memory", "no optimization" };
4559   const char* UseKind2Name[] = { "N", "L", "S", "M" };
4560 
4561   const char* type_name;
4562   LIR_Opr opr = LIR_OprFact::illegal();
4563   if (reg_num() < LIR_OprDesc::vreg_base) {
4564     type_name = "fixed";
4565     // need a temporary operand for fixed intervals because type() cannot be called
4566 #ifdef X86
4567     int last_xmm_reg = pd_last_xmm_reg;
4568 #ifdef _LP64
4569     if (UseAVX < 3) {
4570       last_xmm_reg = pd_first_xmm_reg + (pd_nof_xmm_regs_frame_map / 2) - 1;
4571     }
4572 #endif
4573 #endif
4574     if (assigned_reg() >= pd_first_cpu_reg && assigned_reg() <= pd_last_cpu_reg) {
4575       opr = LIR_OprFact::single_cpu(assigned_reg());
4576     } else if (assigned_reg() >= pd_first_fpu_reg && assigned_reg() <= pd_last_fpu_reg) {
4577       opr = LIR_OprFact::single_fpu(assigned_reg() - pd_first_fpu_reg);
4578 #ifdef X86
4579     } else if (assigned_reg() >= pd_first_xmm_reg && assigned_reg() <= last_xmm_reg) {
4580       opr = LIR_OprFact::single_xmm(assigned_reg() - pd_first_xmm_reg);
4581 #endif
4582     } else {
4583       ShouldNotReachHere();
4584     }
4585   } else {
4586     type_name = type2name(type());
4587     if (assigned_reg() != -1 &&
4588         (LinearScan::num_physical_regs(type()) == 1 || assigned_regHi() != -1)) {
4589       opr = LinearScan::calc_operand_for_interval(this);
4590     }
4591   }
4592 
4593   out->print("%d %s ", reg_num(), type_name);
4594   if (opr->is_valid()) {
4595     out->print("\"");
4596     opr->print(out);
4597     out->print("\" ");
4598   }
4599   out->print("%d %d ", split_parent()->reg_num(), (register_hint(false) != NULL ? register_hint(false)->reg_num() : -1));
4600 
4601   // print ranges
4602   Range* cur = _first;
4603   while (cur != Range::end()) {
4604     cur->print(out);
4605     cur = cur->next();
4606     assert(cur != NULL, "range list not closed with range sentinel");
4607   }
4608 
4609   // print use positions
4610   int prev = 0;
4611   assert(_use_pos_and_kinds.length() % 2 == 0, "must be");
4612   for (int i =_use_pos_and_kinds.length() - 2; i >= 0; i -= 2) {
4613     assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind");
4614     assert(prev < _use_pos_and_kinds.at(i), "use positions not sorted");
4615 
4616     out->print("%d %s ", _use_pos_and_kinds.at(i), UseKind2Name[_use_pos_and_kinds.at(i + 1)]);
4617     prev = _use_pos_and_kinds.at(i);
4618   }
4619 
4620   out->print(" \"%s\"", SpillState2Name[spill_state()]);
4621   out->cr();
4622 }
4623 #endif
4624 
4625 
4626 
4627 // **** Implementation of IntervalWalker ****************************
4628 
4629 IntervalWalker::IntervalWalker(LinearScan* allocator, Interval* unhandled_fixed_first, Interval* unhandled_any_first)
4630  : _compilation(allocator->compilation())
4631  , _allocator(allocator)
4632 {
4633   _unhandled_first[fixedKind] = unhandled_fixed_first;
4634   _unhandled_first[anyKind]   = unhandled_any_first;
4635   _active_first[fixedKind]    = Interval::end();
4636   _inactive_first[fixedKind]  = Interval::end();
4637   _active_first[anyKind]      = Interval::end();
4638   _inactive_first[anyKind]    = Interval::end();
4639   _current_position = -1;
4640   _current = NULL;
4641   next_interval();
4642 }
4643 
4644 
4645 // append interval at top of list
4646 void IntervalWalker::append_unsorted(Interval** list, Interval* interval) {
4647   interval->set_next(*list); *list = interval;
4648 }
4649 
4650 
4651 // append interval in order of current range from()
4652 void IntervalWalker::append_sorted(Interval** list, Interval* interval) {
4653   Interval* prev = NULL;
4654   Interval* cur  = *list;
4655   while (cur->current_from() < interval->current_from()) {
4656     prev = cur; cur = cur->next();
4657   }
4658   if (prev == NULL) {
4659     *list = interval;
4660   } else {
4661     prev->set_next(interval);
4662   }
4663   interval->set_next(cur);
4664 }
4665 
4666 void IntervalWalker::append_to_unhandled(Interval** list, Interval* interval) {
4667   assert(interval->from() >= current()->current_from(), "cannot append new interval before current walk position");
4668 
4669   Interval* prev = NULL;
4670   Interval* cur  = *list;
4671   while (cur->from() < interval->from() || (cur->from() == interval->from() && cur->first_usage(noUse) < interval->first_usage(noUse))) {
4672     prev = cur; cur = cur->next();
4673   }
4674   if (prev == NULL) {
4675     *list = interval;
4676   } else {
4677     prev->set_next(interval);
4678   }
4679   interval->set_next(cur);
4680 }
4681 
4682 
4683 inline bool IntervalWalker::remove_from_list(Interval** list, Interval* i) {
4684   while (*list != Interval::end() && *list != i) {
4685     list = (*list)->next_addr();
4686   }
4687   if (*list != Interval::end()) {
4688     assert(*list == i, "check");
4689     *list = (*list)->next();
4690     return true;
4691   } else {
4692     return false;
4693   }
4694 }
4695 
4696 void IntervalWalker::remove_from_list(Interval* i) {
4697   bool deleted;
4698 
4699   if (i->state() == activeState) {
4700     deleted = remove_from_list(active_first_addr(anyKind), i);
4701   } else {
4702     assert(i->state() == inactiveState, "invalid state");
4703     deleted = remove_from_list(inactive_first_addr(anyKind), i);
4704   }
4705 
4706   assert(deleted, "interval has not been found in list");
4707 }
4708 
4709 
4710 void IntervalWalker::walk_to(IntervalState state, int from) {
4711   assert (state == activeState || state == inactiveState, "wrong state");
4712   for_each_interval_kind(kind) {
4713     Interval** prev = state == activeState ? active_first_addr(kind) : inactive_first_addr(kind);
4714     Interval* next   = *prev;
4715     while (next->current_from() <= from) {
4716       Interval* cur = next;
4717       next = cur->next();
4718 
4719       bool range_has_changed = false;
4720       while (cur->current_to() <= from) {
4721         cur->next_range();
4722         range_has_changed = true;
4723       }
4724 
4725       // also handle move from inactive list to active list
4726       range_has_changed = range_has_changed || (state == inactiveState && cur->current_from() <= from);
4727 
4728       if (range_has_changed) {
4729         // remove cur from list
4730         *prev = next;
4731         if (cur->current_at_end()) {
4732           // move to handled state (not maintained as a list)
4733           cur->set_state(handledState);
4734           interval_moved(cur, kind, state, handledState);
4735         } else if (cur->current_from() <= from){
4736           // sort into active list
4737           append_sorted(active_first_addr(kind), cur);
4738           cur->set_state(activeState);
4739           if (*prev == cur) {
4740             assert(state == activeState, "check");
4741             prev = cur->next_addr();
4742           }
4743           interval_moved(cur, kind, state, activeState);
4744         } else {
4745           // sort into inactive list
4746           append_sorted(inactive_first_addr(kind), cur);
4747           cur->set_state(inactiveState);
4748           if (*prev == cur) {
4749             assert(state == inactiveState, "check");
4750             prev = cur->next_addr();
4751           }
4752           interval_moved(cur, kind, state, inactiveState);
4753         }
4754       } else {
4755         prev = cur->next_addr();
4756         continue;
4757       }
4758     }
4759   }
4760 }
4761 
4762 
4763 void IntervalWalker::next_interval() {
4764   IntervalKind kind;
4765   Interval* any   = _unhandled_first[anyKind];
4766   Interval* fixed = _unhandled_first[fixedKind];
4767 
4768   if (any != Interval::end()) {
4769     // intervals may start at same position -> prefer fixed interval
4770     kind = fixed != Interval::end() && fixed->from() <= any->from() ? fixedKind : anyKind;
4771 
4772     assert (kind == fixedKind && fixed->from() <= any->from() ||
4773             kind == anyKind   && any->from() <= fixed->from(), "wrong interval!!!");
4774     assert(any == Interval::end() || fixed == Interval::end() || any->from() != fixed->from() || kind == fixedKind, "if fixed and any-Interval start at same position, fixed must be processed first");
4775 
4776   } else if (fixed != Interval::end()) {
4777     kind = fixedKind;
4778   } else {
4779     _current = NULL; return;
4780   }
4781   _current_kind = kind;
4782   _current = _unhandled_first[kind];
4783   _unhandled_first[kind] = _current->next();
4784   _current->set_next(Interval::end());
4785   _current->rewind_range();
4786 }
4787 
4788 
4789 void IntervalWalker::walk_to(int lir_op_id) {
4790   assert(_current_position <= lir_op_id, "can not walk backwards");
4791   while (current() != NULL) {
4792     bool is_active = current()->from() <= lir_op_id;
4793     int id = is_active ? current()->from() : lir_op_id;
4794 
4795     TRACE_LINEAR_SCAN(2, if (_current_position < id) { tty->cr(); tty->print_cr("walk_to(%d) **************************************************************", id); })
4796 
4797     // set _current_position prior to call of walk_to
4798     _current_position = id;
4799 
4800     // call walk_to even if _current_position == id
4801     walk_to(activeState, id);
4802     walk_to(inactiveState, id);
4803 
4804     if (is_active) {
4805       current()->set_state(activeState);
4806       if (activate_current()) {
4807         append_sorted(active_first_addr(current_kind()), current());
4808         interval_moved(current(), current_kind(), unhandledState, activeState);
4809       }
4810 
4811       next_interval();
4812     } else {
4813       return;
4814     }
4815   }
4816 }
4817 
4818 void IntervalWalker::interval_moved(Interval* interval, IntervalKind kind, IntervalState from, IntervalState to) {
4819 #ifndef PRODUCT
4820   if (TraceLinearScanLevel >= 4) {
4821     #define print_state(state) \
4822     switch(state) {\
4823       case unhandledState: tty->print("unhandled"); break;\
4824       case activeState: tty->print("active"); break;\
4825       case inactiveState: tty->print("inactive"); break;\
4826       case handledState: tty->print("handled"); break;\
4827       default: ShouldNotReachHere(); \
4828     }
4829 
4830     print_state(from); tty->print(" to "); print_state(to);
4831     tty->fill_to(23);
4832     interval->print();
4833 
4834     #undef print_state
4835   }
4836 #endif
4837 }
4838 
4839 
4840 
4841 // **** Implementation of LinearScanWalker **************************
4842 
4843 LinearScanWalker::LinearScanWalker(LinearScan* allocator, Interval* unhandled_fixed_first, Interval* unhandled_any_first)
4844   : IntervalWalker(allocator, unhandled_fixed_first, unhandled_any_first)
4845   , _move_resolver(allocator)
4846 {
4847   for (int i = 0; i < LinearScan::nof_regs; i++) {
4848     _spill_intervals[i] = new IntervalList(2);
4849   }
4850 }
4851 
4852 
4853 inline void LinearScanWalker::init_use_lists(bool only_process_use_pos) {
4854   for (int i = _first_reg; i <= _last_reg; i++) {
4855     _use_pos[i] = max_jint;
4856 
4857     if (!only_process_use_pos) {
4858       _block_pos[i] = max_jint;
4859       _spill_intervals[i]->clear();
4860     }
4861   }
4862 }
4863 
4864 inline void LinearScanWalker::exclude_from_use(int reg) {
4865   assert(reg < LinearScan::nof_regs, "interval must have a register assigned (stack slots not allowed)");
4866   if (reg >= _first_reg && reg <= _last_reg) {
4867     _use_pos[reg] = 0;
4868   }
4869 }
4870 inline void LinearScanWalker::exclude_from_use(Interval* i) {
4871   assert(i->assigned_reg() != any_reg, "interval has no register assigned");
4872 
4873   exclude_from_use(i->assigned_reg());
4874   exclude_from_use(i->assigned_regHi());
4875 }
4876 
4877 inline void LinearScanWalker::set_use_pos(int reg, Interval* i, int use_pos, bool only_process_use_pos) {
4878   assert(use_pos != 0, "must use exclude_from_use to set use_pos to 0");
4879 
4880   if (reg >= _first_reg && reg <= _last_reg) {
4881     if (_use_pos[reg] > use_pos) {
4882       _use_pos[reg] = use_pos;
4883     }
4884     if (!only_process_use_pos) {
4885       _spill_intervals[reg]->append(i);
4886     }
4887   }
4888 }
4889 inline void LinearScanWalker::set_use_pos(Interval* i, int use_pos, bool only_process_use_pos) {
4890   assert(i->assigned_reg() != any_reg, "interval has no register assigned");
4891   if (use_pos != -1) {
4892     set_use_pos(i->assigned_reg(), i, use_pos, only_process_use_pos);
4893     set_use_pos(i->assigned_regHi(), i, use_pos, only_process_use_pos);
4894   }
4895 }
4896 
4897 inline void LinearScanWalker::set_block_pos(int reg, Interval* i, int block_pos) {
4898   if (reg >= _first_reg && reg <= _last_reg) {
4899     if (_block_pos[reg] > block_pos) {
4900       _block_pos[reg] = block_pos;
4901     }
4902     if (_use_pos[reg] > block_pos) {
4903       _use_pos[reg] = block_pos;
4904     }
4905   }
4906 }
4907 inline void LinearScanWalker::set_block_pos(Interval* i, int block_pos) {
4908   assert(i->assigned_reg() != any_reg, "interval has no register assigned");
4909   if (block_pos != -1) {
4910     set_block_pos(i->assigned_reg(), i, block_pos);
4911     set_block_pos(i->assigned_regHi(), i, block_pos);
4912   }
4913 }
4914 
4915 
4916 void LinearScanWalker::free_exclude_active_fixed() {
4917   Interval* list = active_first(fixedKind);
4918   while (list != Interval::end()) {
4919     assert(list->assigned_reg() < LinearScan::nof_regs, "active interval must have a register assigned");
4920     exclude_from_use(list);
4921     list = list->next();
4922   }
4923 }
4924 
4925 void LinearScanWalker::free_exclude_active_any() {
4926   Interval* list = active_first(anyKind);
4927   while (list != Interval::end()) {
4928     exclude_from_use(list);
4929     list = list->next();
4930   }
4931 }
4932 
4933 void LinearScanWalker::free_collect_inactive_fixed(Interval* cur) {
4934   Interval* list = inactive_first(fixedKind);
4935   while (list != Interval::end()) {
4936     if (cur->to() <= list->current_from()) {
4937       assert(list->current_intersects_at(cur) == -1, "must not intersect");
4938       set_use_pos(list, list->current_from(), true);
4939     } else {
4940       set_use_pos(list, list->current_intersects_at(cur), true);
4941     }
4942     list = list->next();
4943   }
4944 }
4945 
4946 void LinearScanWalker::free_collect_inactive_any(Interval* cur) {
4947   Interval* list = inactive_first(anyKind);
4948   while (list != Interval::end()) {
4949     set_use_pos(list, list->current_intersects_at(cur), true);
4950     list = list->next();
4951   }
4952 }
4953 
4954 void LinearScanWalker::free_collect_unhandled(IntervalKind kind, Interval* cur) {
4955   Interval* list = unhandled_first(kind);
4956   while (list != Interval::end()) {
4957     set_use_pos(list, list->intersects_at(cur), true);
4958     if (kind == fixedKind && cur->to() <= list->from()) {
4959       set_use_pos(list, list->from(), true);
4960     }
4961     list = list->next();
4962   }
4963 }
4964 
4965 void LinearScanWalker::spill_exclude_active_fixed() {
4966   Interval* list = active_first(fixedKind);
4967   while (list != Interval::end()) {
4968     exclude_from_use(list);
4969     list = list->next();
4970   }
4971 }
4972 
4973 void LinearScanWalker::spill_block_unhandled_fixed(Interval* cur) {
4974   Interval* list = unhandled_first(fixedKind);
4975   while (list != Interval::end()) {
4976     set_block_pos(list, list->intersects_at(cur));
4977     list = list->next();
4978   }
4979 }
4980 
4981 void LinearScanWalker::spill_block_inactive_fixed(Interval* cur) {
4982   Interval* list = inactive_first(fixedKind);
4983   while (list != Interval::end()) {
4984     if (cur->to() > list->current_from()) {
4985       set_block_pos(list, list->current_intersects_at(cur));
4986     } else {
4987       assert(list->current_intersects_at(cur) == -1, "invalid optimization: intervals intersect");
4988     }
4989 
4990     list = list->next();
4991   }
4992 }
4993 
4994 void LinearScanWalker::spill_collect_active_any() {
4995   Interval* list = active_first(anyKind);
4996   while (list != Interval::end()) {
4997     set_use_pos(list, MIN2(list->next_usage(loopEndMarker, _current_position), list->to()), false);
4998     list = list->next();
4999   }
5000 }
5001 
5002 void LinearScanWalker::spill_collect_inactive_any(Interval* cur) {
5003   Interval* list = inactive_first(anyKind);
5004   while (list != Interval::end()) {
5005     if (list->current_intersects(cur)) {
5006       set_use_pos(list, MIN2(list->next_usage(loopEndMarker, _current_position), list->to()), false);
5007     }
5008     list = list->next();
5009   }
5010 }
5011 
5012 
5013 void LinearScanWalker::insert_move(int op_id, Interval* src_it, Interval* dst_it) {
5014   // output all moves here. When source and target are equal, the move is
5015   // optimized away later in assign_reg_nums
5016 
5017   op_id = (op_id + 1) & ~1;
5018   BlockBegin* op_block = allocator()->block_of_op_with_id(op_id);
5019   assert(op_id > 0 && allocator()->block_of_op_with_id(op_id - 2) == op_block, "cannot insert move at block boundary");
5020 
5021   // calculate index of instruction inside instruction list of current block
5022   // the minimal index (for a block with no spill moves) can be calculated because the
5023   // numbering of instructions is known.
5024   // When the block already contains spill moves, the index must be increased until the
5025   // correct index is reached.
5026   LIR_OpList* list = op_block->lir()->instructions_list();
5027   int index = (op_id - list->at(0)->id()) / 2;
5028   assert(list->at(index)->id() <= op_id, "error in calculation");
5029 
5030   while (list->at(index)->id() != op_id) {
5031     index++;
5032     assert(0 <= index && index < list->length(), "index out of bounds");
5033   }
5034   assert(1 <= index && index < list->length(), "index out of bounds");
5035   assert(list->at(index)->id() == op_id, "error in calculation");
5036 
5037   // insert new instruction before instruction at position index
5038   _move_resolver.move_insert_position(op_block->lir(), index - 1);
5039   _move_resolver.add_mapping(src_it, dst_it);
5040 }
5041 
5042 
5043 int LinearScanWalker::find_optimal_split_pos(BlockBegin* min_block, BlockBegin* max_block, int max_split_pos) {
5044   int from_block_nr = min_block->linear_scan_number();
5045   int to_block_nr = max_block->linear_scan_number();
5046 
5047   assert(0 <= from_block_nr && from_block_nr < block_count(), "out of range");
5048   assert(0 <= to_block_nr && to_block_nr < block_count(), "out of range");
5049   assert(from_block_nr < to_block_nr, "must cross block boundary");
5050 
5051   // Try to split at end of max_block. If this would be after
5052   // max_split_pos, then use the begin of max_block
5053   int optimal_split_pos = max_block->last_lir_instruction_id() + 2;
5054   if (optimal_split_pos > max_split_pos) {
5055     optimal_split_pos = max_block->first_lir_instruction_id();
5056   }
5057 
5058   int min_loop_depth = max_block->loop_depth();
5059   for (int i = to_block_nr - 1; i >= from_block_nr; i--) {
5060     BlockBegin* cur = block_at(i);
5061 
5062     if (cur->loop_depth() < min_loop_depth) {
5063       // block with lower loop-depth found -> split at the end of this block
5064       min_loop_depth = cur->loop_depth();
5065       optimal_split_pos = cur->last_lir_instruction_id() + 2;
5066     }
5067   }
5068   assert(optimal_split_pos > allocator()->max_lir_op_id() || allocator()->is_block_begin(optimal_split_pos), "algorithm must move split pos to block boundary");
5069 
5070   return optimal_split_pos;
5071 }
5072 
5073 
5074 int LinearScanWalker::find_optimal_split_pos(Interval* it, int min_split_pos, int max_split_pos, bool do_loop_optimization) {
5075   int optimal_split_pos = -1;
5076   if (min_split_pos == max_split_pos) {
5077     // trivial case, no optimization of split position possible
5078     TRACE_LINEAR_SCAN(4, tty->print_cr("      min-pos and max-pos are equal, no optimization possible"));
5079     optimal_split_pos = min_split_pos;
5080 
5081   } else {
5082     assert(min_split_pos < max_split_pos, "must be true then");
5083     assert(min_split_pos > 0, "cannot access min_split_pos - 1 otherwise");
5084 
5085     // reason for using min_split_pos - 1: when the minimal split pos is exactly at the
5086     // beginning of a block, then min_split_pos is also a possible split position.
5087     // Use the block before as min_block, because then min_block->last_lir_instruction_id() + 2 == min_split_pos
5088     BlockBegin* min_block = allocator()->block_of_op_with_id(min_split_pos - 1);
5089 
5090     // reason for using max_split_pos - 1: otherwise there would be an assertion failure
5091     // when an interval ends at the end of the last block of the method
5092     // (in this case, max_split_pos == allocator()->max_lir_op_id() + 2, and there is no
5093     // block at this op_id)
5094     BlockBegin* max_block = allocator()->block_of_op_with_id(max_split_pos - 1);
5095 
5096     assert(min_block->linear_scan_number() <= max_block->linear_scan_number(), "invalid order");
5097     if (min_block == max_block) {
5098       // split position cannot be moved to block boundary, so split as late as possible
5099       TRACE_LINEAR_SCAN(4, tty->print_cr("      cannot move split pos to block boundary because min_pos and max_pos are in same block"));
5100       optimal_split_pos = max_split_pos;
5101 
5102     } else if (it->has_hole_between(max_split_pos - 1, max_split_pos) && !allocator()->is_block_begin(max_split_pos)) {
5103       // Do not move split position if the interval has a hole before max_split_pos.
5104       // Intervals resulting from Phi-Functions have more than one definition (marked
5105       // as mustHaveRegister) with a hole before each definition. When the register is needed
5106       // for the second definition, an earlier reloading is unnecessary.
5107       TRACE_LINEAR_SCAN(4, tty->print_cr("      interval has hole just before max_split_pos, so splitting at max_split_pos"));
5108       optimal_split_pos = max_split_pos;
5109 
5110     } else {
5111       // seach optimal block boundary between min_split_pos and max_split_pos
5112       TRACE_LINEAR_SCAN(4, tty->print_cr("      moving split pos to optimal block boundary between block B%d and B%d", min_block->block_id(), max_block->block_id()));
5113 
5114       if (do_loop_optimization) {
5115         // Loop optimization: if a loop-end marker is found between min- and max-position,
5116         // then split before this loop
5117         int loop_end_pos = it->next_usage_exact(loopEndMarker, min_block->last_lir_instruction_id() + 2);
5118         TRACE_LINEAR_SCAN(4, tty->print_cr("      loop optimization: loop end found at pos %d", loop_end_pos));
5119 
5120         assert(loop_end_pos > min_split_pos, "invalid order");
5121         if (loop_end_pos < max_split_pos) {
5122           // loop-end marker found between min- and max-position
5123           // if it is not the end marker for the same loop as the min-position, then move
5124           // the max-position to this loop block.
5125           // Desired result: uses tagged as shouldHaveRegister inside a loop cause a reloading
5126           // of the interval (normally, only mustHaveRegister causes a reloading)
5127           BlockBegin* loop_block = allocator()->block_of_op_with_id(loop_end_pos);
5128 
5129           TRACE_LINEAR_SCAN(4, tty->print_cr("      interval is used in loop that ends in block B%d, so trying to move max_block back from B%d to B%d", loop_block->block_id(), max_block->block_id(), loop_block->block_id()));
5130           assert(loop_block != min_block, "loop_block and min_block must be different because block boundary is needed between");
5131 
5132           optimal_split_pos = find_optimal_split_pos(min_block, loop_block, loop_block->last_lir_instruction_id() + 2);
5133           if (optimal_split_pos == loop_block->last_lir_instruction_id() + 2) {
5134             optimal_split_pos = -1;
5135             TRACE_LINEAR_SCAN(4, tty->print_cr("      loop optimization not necessary"));
5136           } else {
5137             TRACE_LINEAR_SCAN(4, tty->print_cr("      loop optimization successful"));
5138           }
5139         }
5140       }
5141 
5142       if (optimal_split_pos == -1) {
5143         // not calculated by loop optimization
5144         optimal_split_pos = find_optimal_split_pos(min_block, max_block, max_split_pos);
5145       }
5146     }
5147   }
5148   TRACE_LINEAR_SCAN(4, tty->print_cr("      optimal split position: %d", optimal_split_pos));
5149 
5150   return optimal_split_pos;
5151 }
5152 
5153 
5154 /*
5155   split an interval at the optimal position between min_split_pos and
5156   max_split_pos in two parts:
5157   1) the left part has already a location assigned
5158   2) the right part is sorted into to the unhandled-list
5159 */
5160 void LinearScanWalker::split_before_usage(Interval* it, int min_split_pos, int max_split_pos) {
5161   TRACE_LINEAR_SCAN(2, tty->print   ("----- splitting interval: "); it->print());
5162   TRACE_LINEAR_SCAN(2, tty->print_cr("      between %d and %d", min_split_pos, max_split_pos));
5163 
5164   assert(it->from() < min_split_pos,         "cannot split at start of interval");
5165   assert(current_position() < min_split_pos, "cannot split before current position");
5166   assert(min_split_pos <= max_split_pos,     "invalid order");
5167   assert(max_split_pos <= it->to(),          "cannot split after end of interval");
5168 
5169   int optimal_split_pos = find_optimal_split_pos(it, min_split_pos, max_split_pos, true);
5170 
5171   assert(min_split_pos <= optimal_split_pos && optimal_split_pos <= max_split_pos, "out of range");
5172   assert(optimal_split_pos <= it->to(),  "cannot split after end of interval");
5173   assert(optimal_split_pos > it->from(), "cannot split at start of interval");
5174 
5175   if (optimal_split_pos == it->to() && it->next_usage(mustHaveRegister, min_split_pos) == max_jint) {
5176     // the split position would be just before the end of the interval
5177     // -> no split at all necessary
5178     TRACE_LINEAR_SCAN(4, tty->print_cr("      no split necessary because optimal split position is at end of interval"));
5179     return;
5180   }
5181 
5182   // must calculate this before the actual split is performed and before split position is moved to odd op_id
5183   bool move_necessary = !allocator()->is_block_begin(optimal_split_pos) && !it->has_hole_between(optimal_split_pos - 1, optimal_split_pos);
5184 
5185   if (!allocator()->is_block_begin(optimal_split_pos)) {
5186     // move position before actual instruction (odd op_id)
5187     optimal_split_pos = (optimal_split_pos - 1) | 1;
5188   }
5189 
5190   TRACE_LINEAR_SCAN(4, tty->print_cr("      splitting at position %d", optimal_split_pos));
5191   assert(allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 1), "split pos must be odd when not on block boundary");
5192   assert(!allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 0), "split pos must be even on block boundary");
5193 
5194   Interval* split_part = it->split(optimal_split_pos);
5195 
5196   allocator()->append_interval(split_part);
5197   allocator()->copy_register_flags(it, split_part);
5198   split_part->set_insert_move_when_activated(move_necessary);
5199   append_to_unhandled(unhandled_first_addr(anyKind), split_part);
5200 
5201   TRACE_LINEAR_SCAN(2, tty->print_cr("      split interval in two parts (insert_move_when_activated: %d)", move_necessary));
5202   TRACE_LINEAR_SCAN(2, tty->print   ("      "); it->print());
5203   TRACE_LINEAR_SCAN(2, tty->print   ("      "); split_part->print());
5204 }
5205 
5206 /*
5207   split an interval at the optimal position between min_split_pos and
5208   max_split_pos in two parts:
5209   1) the left part has already a location assigned
5210   2) the right part is always on the stack and therefore ignored in further processing
5211 */
5212 void LinearScanWalker::split_for_spilling(Interval* it) {
5213   // calculate allowed range of splitting position
5214   int max_split_pos = current_position();
5215   int min_split_pos = MAX2(it->previous_usage(shouldHaveRegister, max_split_pos) + 1, it->from());
5216 
5217   TRACE_LINEAR_SCAN(2, tty->print   ("----- splitting and spilling interval: "); it->print());
5218   TRACE_LINEAR_SCAN(2, tty->print_cr("      between %d and %d", min_split_pos, max_split_pos));
5219 
5220   assert(it->state() == activeState,     "why spill interval that is not active?");
5221   assert(it->from() <= min_split_pos,    "cannot split before start of interval");
5222   assert(min_split_pos <= max_split_pos, "invalid order");
5223   assert(max_split_pos < it->to(),       "cannot split at end end of interval");
5224   assert(current_position() < it->to(),  "interval must not end before current position");
5225 
5226   if (min_split_pos == it->from()) {
5227     // the whole interval is never used, so spill it entirely to memory
5228     TRACE_LINEAR_SCAN(2, tty->print_cr("      spilling entire interval because split pos is at beginning of interval"));
5229     assert(it->first_usage(shouldHaveRegister) > current_position(), "interval must not have use position before current_position");
5230 
5231     allocator()->assign_spill_slot(it);
5232     allocator()->change_spill_state(it, min_split_pos);
5233 
5234     // Also kick parent intervals out of register to memory when they have no use
5235     // position. This avoids short interval in register surrounded by intervals in
5236     // memory -> avoid useless moves from memory to register and back
5237     Interval* parent = it;
5238     while (parent != NULL && parent->is_split_child()) {
5239       parent = parent->split_child_before_op_id(parent->from());
5240 
5241       if (parent->assigned_reg() < LinearScan::nof_regs) {
5242         if (parent->first_usage(shouldHaveRegister) == max_jint) {
5243           // parent is never used, so kick it out of its assigned register
5244           TRACE_LINEAR_SCAN(4, tty->print_cr("      kicking out interval %d out of its register because it is never used", parent->reg_num()));
5245           allocator()->assign_spill_slot(parent);
5246         } else {
5247           // do not go further back because the register is actually used by the interval
5248           parent = NULL;
5249         }
5250       }
5251     }
5252 
5253   } else {
5254     // search optimal split pos, split interval and spill only the right hand part
5255     int optimal_split_pos = find_optimal_split_pos(it, min_split_pos, max_split_pos, false);
5256 
5257     assert(min_split_pos <= optimal_split_pos && optimal_split_pos <= max_split_pos, "out of range");
5258     assert(optimal_split_pos < it->to(), "cannot split at end of interval");
5259     assert(optimal_split_pos >= it->from(), "cannot split before start of interval");
5260 
5261     if (!allocator()->is_block_begin(optimal_split_pos)) {
5262       // move position before actual instruction (odd op_id)
5263       optimal_split_pos = (optimal_split_pos - 1) | 1;
5264     }
5265 
5266     TRACE_LINEAR_SCAN(4, tty->print_cr("      splitting at position %d", optimal_split_pos));
5267     assert(allocator()->is_block_begin(optimal_split_pos)  || (optimal_split_pos % 2 == 1), "split pos must be odd when not on block boundary");
5268     assert(!allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 0), "split pos must be even on block boundary");
5269 
5270     Interval* spilled_part = it->split(optimal_split_pos);
5271     allocator()->append_interval(spilled_part);
5272     allocator()->assign_spill_slot(spilled_part);
5273     allocator()->change_spill_state(spilled_part, optimal_split_pos);
5274 
5275     if (!allocator()->is_block_begin(optimal_split_pos)) {
5276       TRACE_LINEAR_SCAN(4, tty->print_cr("      inserting move from interval %d to %d", it->reg_num(), spilled_part->reg_num()));
5277       insert_move(optimal_split_pos, it, spilled_part);
5278     }
5279 
5280     // the current_split_child is needed later when moves are inserted for reloading
5281     assert(spilled_part->current_split_child() == it, "overwriting wrong current_split_child");
5282     spilled_part->make_current_split_child();
5283 
5284     TRACE_LINEAR_SCAN(2, tty->print_cr("      split interval in two parts"));
5285     TRACE_LINEAR_SCAN(2, tty->print   ("      "); it->print());
5286     TRACE_LINEAR_SCAN(2, tty->print   ("      "); spilled_part->print());
5287   }
5288 }
5289 
5290 
5291 void LinearScanWalker::split_stack_interval(Interval* it) {
5292   int min_split_pos = current_position() + 1;
5293   int max_split_pos = MIN2(it->first_usage(shouldHaveRegister), it->to());
5294 
5295   split_before_usage(it, min_split_pos, max_split_pos);
5296 }
5297 
5298 void LinearScanWalker::split_when_partial_register_available(Interval* it, int register_available_until) {
5299   int min_split_pos = MAX2(it->previous_usage(shouldHaveRegister, register_available_until), it->from() + 1);
5300   int max_split_pos = register_available_until;
5301 
5302   split_before_usage(it, min_split_pos, max_split_pos);
5303 }
5304 
5305 void LinearScanWalker::split_and_spill_interval(Interval* it) {
5306   assert(it->state() == activeState || it->state() == inactiveState, "other states not allowed");
5307 
5308   int current_pos = current_position();
5309   if (it->state() == inactiveState) {
5310     // the interval is currently inactive, so no spill slot is needed for now.
5311     // when the split part is activated, the interval has a new chance to get a register,
5312     // so in the best case no stack slot is necessary
5313     assert(it->has_hole_between(current_pos - 1, current_pos + 1), "interval can not be inactive otherwise");
5314     split_before_usage(it, current_pos + 1, current_pos + 1);
5315 
5316   } else {
5317     // search the position where the interval must have a register and split
5318     // at the optimal position before.
5319     // The new created part is added to the unhandled list and will get a register
5320     // when it is activated
5321     int min_split_pos = current_pos + 1;
5322     int max_split_pos = MIN2(it->next_usage(mustHaveRegister, min_split_pos), it->to());
5323 
5324     split_before_usage(it, min_split_pos, max_split_pos);
5325 
5326     assert(it->next_usage(mustHaveRegister, current_pos) == max_jint, "the remaining part is spilled to stack and therefore has no register");
5327     split_for_spilling(it);
5328   }
5329 }
5330 
5331 
5332 int LinearScanWalker::find_free_reg(int reg_needed_until, int interval_to, int hint_reg, int ignore_reg, bool* need_split) {
5333   int min_full_reg = any_reg;
5334   int max_partial_reg = any_reg;
5335 
5336   for (int i = _first_reg; i <= _last_reg; i++) {
5337     if (i == ignore_reg) {
5338       // this register must be ignored
5339 
5340     } else if (_use_pos[i] >= interval_to) {
5341       // this register is free for the full interval
5342       if (min_full_reg == any_reg || i == hint_reg || (_use_pos[i] < _use_pos[min_full_reg] && min_full_reg != hint_reg)) {
5343         min_full_reg = i;
5344       }
5345     } else if (_use_pos[i] > reg_needed_until) {
5346       // this register is at least free until reg_needed_until
5347       if (max_partial_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_partial_reg] && max_partial_reg != hint_reg)) {
5348         max_partial_reg = i;
5349       }
5350     }
5351   }
5352 
5353   if (min_full_reg != any_reg) {
5354     return min_full_reg;
5355   } else if (max_partial_reg != any_reg) {
5356     *need_split = true;
5357     return max_partial_reg;
5358   } else {
5359     return any_reg;
5360   }
5361 }
5362 
5363 int LinearScanWalker::find_free_double_reg(int reg_needed_until, int interval_to, int hint_reg, bool* need_split) {
5364   assert((_last_reg - _first_reg + 1) % 2 == 0, "adjust algorithm");
5365 
5366   int min_full_reg = any_reg;
5367   int max_partial_reg = any_reg;
5368 
5369   for (int i = _first_reg; i < _last_reg; i+=2) {
5370     if (_use_pos[i] >= interval_to && _use_pos[i + 1] >= interval_to) {
5371       // this register is free for the full interval
5372       if (min_full_reg == any_reg || i == hint_reg || (_use_pos[i] < _use_pos[min_full_reg] && min_full_reg != hint_reg)) {
5373         min_full_reg = i;
5374       }
5375     } else if (_use_pos[i] > reg_needed_until && _use_pos[i + 1] > reg_needed_until) {
5376       // this register is at least free until reg_needed_until
5377       if (max_partial_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_partial_reg] && max_partial_reg != hint_reg)) {
5378         max_partial_reg = i;
5379       }
5380     }
5381   }
5382 
5383   if (min_full_reg != any_reg) {
5384     return min_full_reg;
5385   } else if (max_partial_reg != any_reg) {
5386     *need_split = true;
5387     return max_partial_reg;
5388   } else {
5389     return any_reg;
5390   }
5391 }
5392 
5393 
5394 bool LinearScanWalker::alloc_free_reg(Interval* cur) {
5395   TRACE_LINEAR_SCAN(2, tty->print("trying to find free register for "); cur->print());
5396 
5397   init_use_lists(true);
5398   free_exclude_active_fixed();
5399   free_exclude_active_any();
5400   free_collect_inactive_fixed(cur);
5401   free_collect_inactive_any(cur);
5402 //  free_collect_unhandled(fixedKind, cur);
5403   assert(unhandled_first(fixedKind) == Interval::end(), "must not have unhandled fixed intervals because all fixed intervals have a use at position 0");
5404 
5405   // _use_pos contains the start of the next interval that has this register assigned
5406   // (either as a fixed register or a normal allocated register in the past)
5407   // only intervals overlapping with cur are processed, non-overlapping invervals can be ignored safely
5408   TRACE_LINEAR_SCAN(4, tty->print_cr("      state of registers:"));
5409   TRACE_LINEAR_SCAN(4, for (int i = _first_reg; i <= _last_reg; i++) tty->print_cr("      reg %d: use_pos: %d", i, _use_pos[i]));
5410 
5411   int hint_reg, hint_regHi;
5412   Interval* register_hint = cur->register_hint();
5413   if (register_hint != NULL) {
5414     hint_reg = register_hint->assigned_reg();
5415     hint_regHi = register_hint->assigned_regHi();
5416 
5417     if (allocator()->is_precolored_cpu_interval(register_hint)) {
5418       assert(hint_reg != any_reg && hint_regHi == any_reg, "must be for fixed intervals");
5419       hint_regHi = hint_reg + 1;  // connect e.g. eax-edx
5420     }
5421     TRACE_LINEAR_SCAN(4, tty->print("      hint registers %d, %d from interval ", hint_reg, hint_regHi); register_hint->print());
5422 
5423   } else {
5424     hint_reg = any_reg;
5425     hint_regHi = any_reg;
5426   }
5427   assert(hint_reg == any_reg || hint_reg != hint_regHi, "hint reg and regHi equal");
5428   assert(cur->assigned_reg() == any_reg && cur->assigned_regHi() == any_reg, "register already assigned to interval");
5429 
5430   // the register must be free at least until this position
5431   int reg_needed_until = cur->from() + 1;
5432   int interval_to = cur->to();
5433 
5434   bool need_split = false;
5435   int split_pos = -1;
5436   int reg = any_reg;
5437   int regHi = any_reg;
5438 
5439   if (_adjacent_regs) {
5440     reg = find_free_double_reg(reg_needed_until, interval_to, hint_reg, &need_split);
5441     regHi = reg + 1;
5442     if (reg == any_reg) {
5443       return false;
5444     }
5445     split_pos = MIN2(_use_pos[reg], _use_pos[regHi]);
5446 
5447   } else {
5448     reg = find_free_reg(reg_needed_until, interval_to, hint_reg, any_reg, &need_split);
5449     if (reg == any_reg) {
5450       return false;
5451     }
5452     split_pos = _use_pos[reg];
5453 
5454     if (_num_phys_regs == 2) {
5455       regHi = find_free_reg(reg_needed_until, interval_to, hint_regHi, reg, &need_split);
5456 
5457       if (_use_pos[reg] < interval_to && regHi == any_reg) {
5458         // do not split interval if only one register can be assigned until the split pos
5459         // (when one register is found for the whole interval, split&spill is only
5460         // performed for the hi register)
5461         return false;
5462 
5463       } else if (regHi != any_reg) {
5464         split_pos = MIN2(split_pos, _use_pos[regHi]);
5465 
5466         // sort register numbers to prevent e.g. a move from eax,ebx to ebx,eax
5467         if (reg > regHi) {
5468           int temp = reg;
5469           reg = regHi;
5470           regHi = temp;
5471         }
5472       }
5473     }
5474   }
5475 
5476   cur->assign_reg(reg, regHi);
5477   TRACE_LINEAR_SCAN(2, tty->print_cr("selected register %d, %d", reg, regHi));
5478 
5479   assert(split_pos > 0, "invalid split_pos");
5480   if (need_split) {
5481     // register not available for full interval, so split it
5482     split_when_partial_register_available(cur, split_pos);
5483   }
5484 
5485   // only return true if interval is completely assigned
5486   return _num_phys_regs == 1 || regHi != any_reg;
5487 }
5488 
5489 
5490 int LinearScanWalker::find_locked_reg(int reg_needed_until, int interval_to, int hint_reg, int ignore_reg, bool* need_split) {
5491   int max_reg = any_reg;
5492 
5493   for (int i = _first_reg; i <= _last_reg; i++) {
5494     if (i == ignore_reg) {
5495       // this register must be ignored
5496 
5497     } else if (_use_pos[i] > reg_needed_until) {
5498       if (max_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_reg] && max_reg != hint_reg)) {
5499         max_reg = i;
5500       }
5501     }
5502   }
5503 
5504   if (max_reg != any_reg && _block_pos[max_reg] <= interval_to) {
5505     *need_split = true;
5506   }
5507 
5508   return max_reg;
5509 }
5510 
5511 int LinearScanWalker::find_locked_double_reg(int reg_needed_until, int interval_to, int hint_reg, bool* need_split) {
5512   assert((_last_reg - _first_reg + 1) % 2 == 0, "adjust algorithm");
5513 
5514   int max_reg = any_reg;
5515 
5516   for (int i = _first_reg; i < _last_reg; i+=2) {
5517     if (_use_pos[i] > reg_needed_until && _use_pos[i + 1] > reg_needed_until) {
5518       if (max_reg == any_reg || _use_pos[i] > _use_pos[max_reg]) {
5519         max_reg = i;
5520       }
5521     }
5522   }
5523 
5524   if (max_reg != any_reg &&
5525       (_block_pos[max_reg] <= interval_to || _block_pos[max_reg + 1] <= interval_to)) {
5526     *need_split = true;
5527   }
5528 
5529   return max_reg;
5530 }
5531 
5532 void LinearScanWalker::split_and_spill_intersecting_intervals(int reg, int regHi) {
5533   assert(reg != any_reg, "no register assigned");
5534 
5535   for (int i = 0; i < _spill_intervals[reg]->length(); i++) {
5536     Interval* it = _spill_intervals[reg]->at(i);
5537     remove_from_list(it);
5538     split_and_spill_interval(it);
5539   }
5540 
5541   if (regHi != any_reg) {
5542     IntervalList* processed = _spill_intervals[reg];
5543     for (int i = 0; i < _spill_intervals[regHi]->length(); i++) {
5544       Interval* it = _spill_intervals[regHi]->at(i);
5545       if (processed->find(it) == -1) {
5546         remove_from_list(it);
5547         split_and_spill_interval(it);
5548       }
5549     }
5550   }
5551 }
5552 
5553 
5554 // Split an Interval and spill it to memory so that cur can be placed in a register
5555 void LinearScanWalker::alloc_locked_reg(Interval* cur) {
5556   TRACE_LINEAR_SCAN(2, tty->print("need to split and spill to get register for "); cur->print());
5557 
5558   // collect current usage of registers
5559   init_use_lists(false);
5560   spill_exclude_active_fixed();
5561 //  spill_block_unhandled_fixed(cur);
5562   assert(unhandled_first(fixedKind) == Interval::end(), "must not have unhandled fixed intervals because all fixed intervals have a use at position 0");
5563   spill_block_inactive_fixed(cur);
5564   spill_collect_active_any();
5565   spill_collect_inactive_any(cur);
5566 
5567 #ifndef PRODUCT
5568   if (TraceLinearScanLevel >= 4) {
5569     tty->print_cr("      state of registers:");
5570     for (int i = _first_reg; i <= _last_reg; i++) {
5571       tty->print("      reg %d: use_pos: %d, block_pos: %d, intervals: ", i, _use_pos[i], _block_pos[i]);
5572       for (int j = 0; j < _spill_intervals[i]->length(); j++) {
5573         tty->print("%d ", _spill_intervals[i]->at(j)->reg_num());
5574       }
5575       tty->cr();
5576     }
5577   }
5578 #endif
5579 
5580   // the register must be free at least until this position
5581   int reg_needed_until = MIN2(cur->first_usage(mustHaveRegister), cur->from() + 1);
5582   int interval_to = cur->to();
5583   assert (reg_needed_until > 0 && reg_needed_until < max_jint, "interval has no use");
5584 
5585   int split_pos = 0;
5586   int use_pos = 0;
5587   bool need_split = false;
5588   int reg, regHi;
5589 
5590   if (_adjacent_regs) {
5591     reg = find_locked_double_reg(reg_needed_until, interval_to, any_reg, &need_split);
5592     regHi = reg + 1;
5593 
5594     if (reg != any_reg) {
5595       use_pos = MIN2(_use_pos[reg], _use_pos[regHi]);
5596       split_pos = MIN2(_block_pos[reg], _block_pos[regHi]);
5597     }
5598   } else {
5599     reg = find_locked_reg(reg_needed_until, interval_to, any_reg, cur->assigned_reg(), &need_split);
5600     regHi = any_reg;
5601 
5602     if (reg != any_reg) {
5603       use_pos = _use_pos[reg];
5604       split_pos = _block_pos[reg];
5605 
5606       if (_num_phys_regs == 2) {
5607         if (cur->assigned_reg() != any_reg) {
5608           regHi = reg;
5609           reg = cur->assigned_reg();
5610         } else {
5611           regHi = find_locked_reg(reg_needed_until, interval_to, any_reg, reg, &need_split);
5612           if (regHi != any_reg) {
5613             use_pos = MIN2(use_pos, _use_pos[regHi]);
5614             split_pos = MIN2(split_pos, _block_pos[regHi]);
5615           }
5616         }
5617 
5618         if (regHi != any_reg && reg > regHi) {
5619           // sort register numbers to prevent e.g. a move from eax,ebx to ebx,eax
5620           int temp = reg;
5621           reg = regHi;
5622           regHi = temp;
5623         }
5624       }
5625     }
5626   }
5627 
5628   if (reg == any_reg || (_num_phys_regs == 2 && regHi == any_reg) || use_pos <= cur->first_usage(mustHaveRegister)) {
5629     // the first use of cur is later than the spilling position -> spill cur
5630     TRACE_LINEAR_SCAN(4, tty->print_cr("able to spill current interval. first_usage(register): %d, use_pos: %d", cur->first_usage(mustHaveRegister), use_pos));
5631 
5632     if (cur->first_usage(mustHaveRegister) <= cur->from() + 1) {
5633       assert(false, "cannot spill interval that is used in first instruction (possible reason: no register found)");
5634       // assign a reasonable register and do a bailout in product mode to avoid errors
5635       allocator()->assign_spill_slot(cur);
5636       BAILOUT("LinearScan: no register found");
5637     }
5638 
5639     split_and_spill_interval(cur);
5640   } else {
5641     TRACE_LINEAR_SCAN(4, tty->print_cr("decided to use register %d, %d", reg, regHi));
5642     assert(reg != any_reg && (_num_phys_regs == 1 || regHi != any_reg), "no register found");
5643     assert(split_pos > 0, "invalid split_pos");
5644     assert(need_split == false || split_pos > cur->from(), "splitting interval at from");
5645 
5646     cur->assign_reg(reg, regHi);
5647     if (need_split) {
5648       // register not available for full interval, so split it
5649       split_when_partial_register_available(cur, split_pos);
5650     }
5651 
5652     // perform splitting and spilling for all affected intervalls
5653     split_and_spill_intersecting_intervals(reg, regHi);
5654   }
5655 }
5656 
5657 bool LinearScanWalker::no_allocation_possible(Interval* cur) {
5658 #ifdef X86
5659   // fast calculation of intervals that can never get a register because the
5660   // the next instruction is a call that blocks all registers
5661   // Note: this does not work if callee-saved registers are available (e.g. on Sparc)
5662 
5663   // check if this interval is the result of a split operation
5664   // (an interval got a register until this position)
5665   int pos = cur->from();
5666   if ((pos & 1) == 1) {
5667     // the current instruction is a call that blocks all registers
5668     if (pos < allocator()->max_lir_op_id() && allocator()->has_call(pos + 1)) {
5669       TRACE_LINEAR_SCAN(4, tty->print_cr("      free register cannot be available because all registers blocked by following call"));
5670 
5671       // safety check that there is really no register available
5672       assert(alloc_free_reg(cur) == false, "found a register for this interval");
5673       return true;
5674     }
5675 
5676   }
5677 #endif
5678   return false;
5679 }
5680 
5681 void LinearScanWalker::init_vars_for_alloc(Interval* cur) {
5682   BasicType type = cur->type();
5683   _num_phys_regs = LinearScan::num_physical_regs(type);
5684   _adjacent_regs = LinearScan::requires_adjacent_regs(type);
5685 
5686   if (pd_init_regs_for_alloc(cur)) {
5687     // the appropriate register range was selected.
5688   } else if (type == T_FLOAT || type == T_DOUBLE) {
5689     _first_reg = pd_first_fpu_reg;
5690     _last_reg = pd_last_fpu_reg;
5691   } else {
5692     _first_reg = pd_first_cpu_reg;
5693     _last_reg = FrameMap::last_cpu_reg();
5694   }
5695 
5696   assert(0 <= _first_reg && _first_reg < LinearScan::nof_regs, "out of range");
5697   assert(0 <= _last_reg && _last_reg < LinearScan::nof_regs, "out of range");
5698 }
5699 
5700 
5701 bool LinearScanWalker::is_move(LIR_Op* op, Interval* from, Interval* to) {
5702   if (op->code() != lir_move) {
5703     return false;
5704   }
5705   assert(op->as_Op1() != NULL, "move must be LIR_Op1");
5706 
5707   LIR_Opr in = ((LIR_Op1*)op)->in_opr();
5708   LIR_Opr res = ((LIR_Op1*)op)->result_opr();
5709   return in->is_virtual() && res->is_virtual() && in->vreg_number() == from->reg_num() && res->vreg_number() == to->reg_num();
5710 }
5711 
5712 // optimization (especially for phi functions of nested loops):
5713 // assign same spill slot to non-intersecting intervals
5714 void LinearScanWalker::combine_spilled_intervals(Interval* cur) {
5715   if (cur->is_split_child()) {
5716     // optimization is only suitable for split parents
5717     return;
5718   }
5719 
5720   Interval* register_hint = cur->register_hint(false);
5721   if (register_hint == NULL) {
5722     // cur is not the target of a move, otherwise register_hint would be set
5723     return;
5724   }
5725   assert(register_hint->is_split_parent(), "register hint must be split parent");
5726 
5727   if (cur->spill_state() != noOptimization || register_hint->spill_state() != noOptimization) {
5728     // combining the stack slots for intervals where spill move optimization is applied
5729     // is not benefitial and would cause problems
5730     return;
5731   }
5732 
5733   int begin_pos = cur->from();
5734   int end_pos = cur->to();
5735   if (end_pos > allocator()->max_lir_op_id() || (begin_pos & 1) != 0 || (end_pos & 1) != 0) {
5736     // safety check that lir_op_with_id is allowed
5737     return;
5738   }
5739 
5740   if (!is_move(allocator()->lir_op_with_id(begin_pos), register_hint, cur) || !is_move(allocator()->lir_op_with_id(end_pos), cur, register_hint)) {
5741     // cur and register_hint are not connected with two moves
5742     return;
5743   }
5744 
5745   Interval* begin_hint = register_hint->split_child_at_op_id(begin_pos, LIR_OpVisitState::inputMode);
5746   Interval* end_hint = register_hint->split_child_at_op_id(end_pos, LIR_OpVisitState::outputMode);
5747   if (begin_hint == end_hint || begin_hint->to() != begin_pos || end_hint->from() != end_pos) {
5748     // register_hint must be split, otherwise the re-writing of use positions does not work
5749     return;
5750   }
5751 
5752   assert(begin_hint->assigned_reg() != any_reg, "must have register assigned");
5753   assert(end_hint->assigned_reg() == any_reg, "must not have register assigned");
5754   assert(cur->first_usage(mustHaveRegister) == begin_pos, "must have use position at begin of interval because of move");
5755   assert(end_hint->first_usage(mustHaveRegister) == end_pos, "must have use position at begin of interval because of move");
5756 
5757   if (begin_hint->assigned_reg() < LinearScan::nof_regs) {
5758     // register_hint is not spilled at begin_pos, so it would not be benefitial to immediately spill cur
5759     return;
5760   }
5761   assert(register_hint->canonical_spill_slot() != -1, "must be set when part of interval was spilled");
5762 
5763   // modify intervals such that cur gets the same stack slot as register_hint
5764   // delete use positions to prevent the intervals to get a register at beginning
5765   cur->set_canonical_spill_slot(register_hint->canonical_spill_slot());
5766   cur->remove_first_use_pos();
5767   end_hint->remove_first_use_pos();
5768 }
5769 
5770 
5771 // allocate a physical register or memory location to an interval
5772 bool LinearScanWalker::activate_current() {
5773   Interval* cur = current();
5774   bool result = true;
5775 
5776   TRACE_LINEAR_SCAN(2, tty->print   ("+++++ activating interval "); cur->print());
5777   TRACE_LINEAR_SCAN(4, tty->print_cr("      split_parent: %d, insert_move_when_activated: %d", cur->split_parent()->reg_num(), cur->insert_move_when_activated()));
5778 
5779   if (cur->assigned_reg() >= LinearScan::nof_regs) {
5780     // activating an interval that has a stack slot assigned -> split it at first use position
5781     // used for method parameters
5782     TRACE_LINEAR_SCAN(4, tty->print_cr("      interval has spill slot assigned (method parameter) -> split it before first use"));
5783 
5784     split_stack_interval(cur);
5785     result = false;
5786 
5787   } else if (allocator()->gen()->is_vreg_flag_set(cur->reg_num(), LIRGenerator::must_start_in_memory)) {
5788     // activating an interval that must start in a stack slot, but may get a register later
5789     // used for lir_roundfp: rounding is done by store to stack and reload later
5790     TRACE_LINEAR_SCAN(4, tty->print_cr("      interval must start in stack slot -> split it before first use"));
5791     assert(cur->assigned_reg() == any_reg && cur->assigned_regHi() == any_reg, "register already assigned");
5792 
5793     allocator()->assign_spill_slot(cur);
5794     split_stack_interval(cur);
5795     result = false;
5796 
5797   } else if (cur->assigned_reg() == any_reg) {
5798     // interval has not assigned register -> normal allocation
5799     // (this is the normal case for most intervals)
5800     TRACE_LINEAR_SCAN(4, tty->print_cr("      normal allocation of register"));
5801 
5802     // assign same spill slot to non-intersecting intervals
5803     combine_spilled_intervals(cur);
5804 
5805     init_vars_for_alloc(cur);
5806     if (no_allocation_possible(cur) || !alloc_free_reg(cur)) {
5807       // no empty register available.
5808       // split and spill another interval so that this interval gets a register
5809       alloc_locked_reg(cur);
5810     }
5811 
5812     // spilled intervals need not be move to active-list
5813     if (cur->assigned_reg() >= LinearScan::nof_regs) {
5814       result = false;
5815     }
5816   }
5817 
5818   // load spilled values that become active from stack slot to register
5819   if (cur->insert_move_when_activated()) {
5820     assert(cur->is_split_child(), "must be");
5821     assert(cur->current_split_child() != NULL, "must be");
5822     assert(cur->current_split_child()->reg_num() != cur->reg_num(), "cannot insert move between same interval");
5823     TRACE_LINEAR_SCAN(4, tty->print_cr("Inserting move from interval %d to %d because insert_move_when_activated is set", cur->current_split_child()->reg_num(), cur->reg_num()));
5824 
5825     insert_move(cur->from(), cur->current_split_child(), cur);
5826   }
5827   cur->make_current_split_child();
5828 
5829   return result; // true = interval is moved to active list
5830 }
5831 
5832 
5833 // Implementation of EdgeMoveOptimizer
5834 
5835 EdgeMoveOptimizer::EdgeMoveOptimizer() :
5836   _edge_instructions(4),
5837   _edge_instructions_idx(4)
5838 {
5839 }
5840 
5841 void EdgeMoveOptimizer::optimize(BlockList* code) {
5842   EdgeMoveOptimizer optimizer = EdgeMoveOptimizer();
5843 
5844   // ignore the first block in the list (index 0 is not processed)
5845   for (int i = code->length() - 1; i >= 1; i--) {
5846     BlockBegin* block = code->at(i);
5847 
5848     if (block->number_of_preds() > 1 && !block->is_set(BlockBegin::exception_entry_flag)) {
5849       optimizer.optimize_moves_at_block_end(block);
5850     }
5851     if (block->number_of_sux() == 2) {
5852       optimizer.optimize_moves_at_block_begin(block);
5853     }
5854   }
5855 }
5856 
5857 
5858 // clear all internal data structures
5859 void EdgeMoveOptimizer::init_instructions() {
5860   _edge_instructions.clear();
5861   _edge_instructions_idx.clear();
5862 }
5863 
5864 // append a lir-instruction-list and the index of the current operation in to the list
5865 void EdgeMoveOptimizer::append_instructions(LIR_OpList* instructions, int instructions_idx) {
5866   _edge_instructions.append(instructions);
5867   _edge_instructions_idx.append(instructions_idx);
5868 }
5869 
5870 // return the current operation of the given edge (predecessor or successor)
5871 LIR_Op* EdgeMoveOptimizer::instruction_at(int edge) {
5872   LIR_OpList* instructions = _edge_instructions.at(edge);
5873   int idx = _edge_instructions_idx.at(edge);
5874 
5875   if (idx < instructions->length()) {
5876     return instructions->at(idx);
5877   } else {
5878     return NULL;
5879   }
5880 }
5881 
5882 // removes the current operation of the given edge (predecessor or successor)
5883 void EdgeMoveOptimizer::remove_cur_instruction(int edge, bool decrement_index) {
5884   LIR_OpList* instructions = _edge_instructions.at(edge);
5885   int idx = _edge_instructions_idx.at(edge);
5886   instructions->remove_at(idx);
5887 
5888   if (decrement_index) {
5889     _edge_instructions_idx.at_put(edge, idx - 1);
5890   }
5891 }
5892 
5893 
5894 bool EdgeMoveOptimizer::operations_different(LIR_Op* op1, LIR_Op* op2) {
5895   if (op1 == NULL || op2 == NULL) {
5896     // at least one block is already empty -> no optimization possible
5897     return true;
5898   }
5899 
5900   if (op1->code() == lir_move && op2->code() == lir_move) {
5901     assert(op1->as_Op1() != NULL, "move must be LIR_Op1");
5902     assert(op2->as_Op1() != NULL, "move must be LIR_Op1");
5903     LIR_Op1* move1 = (LIR_Op1*)op1;
5904     LIR_Op1* move2 = (LIR_Op1*)op2;
5905     if (move1->info() == move2->info() && move1->in_opr() == move2->in_opr() && move1->result_opr() == move2->result_opr()) {
5906       // these moves are exactly equal and can be optimized
5907       return false;
5908     }
5909 
5910   } else if (op1->code() == lir_fxch && op2->code() == lir_fxch) {
5911     assert(op1->as_Op1() != NULL, "fxch must be LIR_Op1");
5912     assert(op2->as_Op1() != NULL, "fxch must be LIR_Op1");
5913     LIR_Op1* fxch1 = (LIR_Op1*)op1;
5914     LIR_Op1* fxch2 = (LIR_Op1*)op2;
5915     if (fxch1->in_opr()->as_jint() == fxch2->in_opr()->as_jint()) {
5916       // equal FPU stack operations can be optimized
5917       return false;
5918     }
5919 
5920   } else if (op1->code() == lir_fpop_raw && op2->code() == lir_fpop_raw) {
5921     // equal FPU stack operations can be optimized
5922     return false;
5923   }
5924 
5925   // no optimization possible
5926   return true;
5927 }
5928 
5929 void EdgeMoveOptimizer::optimize_moves_at_block_end(BlockBegin* block) {
5930   TRACE_LINEAR_SCAN(4, tty->print_cr("optimizing moves at end of block B%d", block->block_id()));
5931 
5932   if (block->is_predecessor(block)) {
5933     // currently we can't handle this correctly.
5934     return;
5935   }
5936 
5937   init_instructions();
5938   int num_preds = block->number_of_preds();
5939   assert(num_preds > 1, "do not call otherwise");
5940   assert(!block->is_set(BlockBegin::exception_entry_flag), "exception handlers not allowed");
5941 
5942   // setup a list with the lir-instructions of all predecessors
5943   int i;
5944   for (i = 0; i < num_preds; i++) {
5945     BlockBegin* pred = block->pred_at(i);
5946     LIR_OpList* pred_instructions = pred->lir()->instructions_list();
5947 
5948     if (pred->number_of_sux() != 1) {
5949       // this can happen with switch-statements where multiple edges are between
5950       // the same blocks.
5951       return;
5952     }
5953 
5954     assert(pred->number_of_sux() == 1, "can handle only one successor");
5955     assert(pred->sux_at(0) == block, "invalid control flow");
5956     assert(pred_instructions->last()->code() == lir_branch, "block with successor must end with branch");
5957     assert(pred_instructions->last()->as_OpBranch() != NULL, "branch must be LIR_OpBranch");
5958     assert(pred_instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block must end with unconditional branch");
5959 
5960     if (pred_instructions->last()->info() != NULL) {
5961       // can not optimize instructions when debug info is needed
5962       return;
5963     }
5964 
5965     // ignore the unconditional branch at the end of the block
5966     append_instructions(pred_instructions, pred_instructions->length() - 2);
5967   }
5968 
5969 
5970   // process lir-instructions while all predecessors end with the same instruction
5971   while (true) {
5972     LIR_Op* op = instruction_at(0);
5973     for (i = 1; i < num_preds; i++) {
5974       if (operations_different(op, instruction_at(i))) {
5975         // these instructions are different and cannot be optimized ->
5976         // no further optimization possible
5977         return;
5978       }
5979     }
5980 
5981     TRACE_LINEAR_SCAN(4, tty->print("found instruction that is equal in all %d predecessors: ", num_preds); op->print());
5982 
5983     // insert the instruction at the beginning of the current block
5984     block->lir()->insert_before(1, op);
5985 
5986     // delete the instruction at the end of all predecessors
5987     for (i = 0; i < num_preds; i++) {
5988       remove_cur_instruction(i, true);
5989     }
5990   }
5991 }
5992 
5993 
5994 void EdgeMoveOptimizer::optimize_moves_at_block_begin(BlockBegin* block) {
5995   TRACE_LINEAR_SCAN(4, tty->print_cr("optimization moves at begin of block B%d", block->block_id()));
5996 
5997   init_instructions();
5998   int num_sux = block->number_of_sux();
5999 
6000   LIR_OpList* cur_instructions = block->lir()->instructions_list();
6001 
6002   assert(num_sux == 2, "method should not be called otherwise");
6003   assert(cur_instructions->last()->code() == lir_branch, "block with successor must end with branch");
6004   assert(cur_instructions->last()->as_OpBranch() != NULL, "branch must be LIR_OpBranch");
6005   assert(cur_instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block must end with unconditional branch");
6006 
6007   if (cur_instructions->last()->info() != NULL) {
6008     // can no optimize instructions when debug info is needed
6009     return;
6010   }
6011 
6012   LIR_Op* branch = cur_instructions->at(cur_instructions->length() - 2);
6013   if (branch->info() != NULL || (branch->code() != lir_branch && branch->code() != lir_cond_float_branch)) {
6014     // not a valid case for optimization
6015     // currently, only blocks that end with two branches (conditional branch followed
6016     // by unconditional branch) are optimized
6017     return;
6018   }
6019 
6020   // now it is guaranteed that the block ends with two branch instructions.
6021   // the instructions are inserted at the end of the block before these two branches
6022   int insert_idx = cur_instructions->length() - 2;
6023 
6024   int i;
6025 #ifdef ASSERT
6026   for (i = insert_idx - 1; i >= 0; i--) {
6027     LIR_Op* op = cur_instructions->at(i);
6028     if ((op->code() == lir_branch || op->code() == lir_cond_float_branch) && ((LIR_OpBranch*)op)->block() != NULL) {
6029       assert(false, "block with two successors can have only two branch instructions");
6030     }
6031   }
6032 #endif
6033 
6034   // setup a list with the lir-instructions of all successors
6035   for (i = 0; i < num_sux; i++) {
6036     BlockBegin* sux = block->sux_at(i);
6037     LIR_OpList* sux_instructions = sux->lir()->instructions_list();
6038 
6039     assert(sux_instructions->at(0)->code() == lir_label, "block must start with label");
6040 
6041     if (sux->number_of_preds() != 1) {
6042       // this can happen with switch-statements where multiple edges are between
6043       // the same blocks.
6044       return;
6045     }
6046     assert(sux->pred_at(0) == block, "invalid control flow");
6047     assert(!sux->is_set(BlockBegin::exception_entry_flag), "exception handlers not allowed");
6048 
6049     // ignore the label at the beginning of the block
6050     append_instructions(sux_instructions, 1);
6051   }
6052 
6053   // process lir-instructions while all successors begin with the same instruction
6054   while (true) {
6055     LIR_Op* op = instruction_at(0);
6056     for (i = 1; i < num_sux; i++) {
6057       if (operations_different(op, instruction_at(i))) {
6058         // these instructions are different and cannot be optimized ->
6059         // no further optimization possible
6060         return;
6061       }
6062     }
6063 
6064     TRACE_LINEAR_SCAN(4, tty->print("----- found instruction that is equal in all %d successors: ", num_sux); op->print());
6065 
6066     // insert instruction at end of current block
6067     block->lir()->insert_before(insert_idx, op);
6068     insert_idx++;
6069 
6070     // delete the instructions at the beginning of all successors
6071     for (i = 0; i < num_sux; i++) {
6072       remove_cur_instruction(i, false);
6073     }
6074   }
6075 }
6076 
6077 
6078 // Implementation of ControlFlowOptimizer
6079 
6080 ControlFlowOptimizer::ControlFlowOptimizer() :
6081   _original_preds(4)
6082 {
6083 }
6084 
6085 void ControlFlowOptimizer::optimize(BlockList* code) {
6086   ControlFlowOptimizer optimizer = ControlFlowOptimizer();
6087 
6088   // push the OSR entry block to the end so that we're not jumping over it.
6089   BlockBegin* osr_entry = code->at(0)->end()->as_Base()->osr_entry();
6090   if (osr_entry) {
6091     int index = osr_entry->linear_scan_number();
6092     assert(code->at(index) == osr_entry, "wrong index");
6093     code->remove_at(index);
6094     code->append(osr_entry);
6095   }
6096 
6097   optimizer.reorder_short_loops(code);
6098   optimizer.delete_empty_blocks(code);
6099   optimizer.delete_unnecessary_jumps(code);
6100   optimizer.delete_jumps_to_return(code);
6101 }
6102 
6103 void ControlFlowOptimizer::reorder_short_loop(BlockList* code, BlockBegin* header_block, int header_idx) {
6104   int i = header_idx + 1;
6105   int max_end = MIN2(header_idx + ShortLoopSize, code->length());
6106   while (i < max_end && code->at(i)->loop_depth() >= header_block->loop_depth()) {
6107     i++;
6108   }
6109 
6110   if (i == code->length() || code->at(i)->loop_depth() < header_block->loop_depth()) {
6111     int end_idx = i - 1;
6112     BlockBegin* end_block = code->at(end_idx);
6113 
6114     if (end_block->number_of_sux() == 1 && end_block->sux_at(0) == header_block) {
6115       // short loop from header_idx to end_idx found -> reorder blocks such that
6116       // the header_block is the last block instead of the first block of the loop
6117       TRACE_LINEAR_SCAN(1, tty->print_cr("Reordering short loop: length %d, header B%d, end B%d",
6118                                          end_idx - header_idx + 1,
6119                                          header_block->block_id(), end_block->block_id()));
6120 
6121       for (int j = header_idx; j < end_idx; j++) {
6122         code->at_put(j, code->at(j + 1));
6123       }
6124       code->at_put(end_idx, header_block);
6125 
6126       // correct the flags so that any loop alignment occurs in the right place.
6127       assert(code->at(end_idx)->is_set(BlockBegin::backward_branch_target_flag), "must be backward branch target");
6128       code->at(end_idx)->clear(BlockBegin::backward_branch_target_flag);
6129       code->at(header_idx)->set(BlockBegin::backward_branch_target_flag);
6130     }
6131   }
6132 }
6133 
6134 void ControlFlowOptimizer::reorder_short_loops(BlockList* code) {
6135   for (int i = code->length() - 1; i >= 0; i--) {
6136     BlockBegin* block = code->at(i);
6137 
6138     if (block->is_set(BlockBegin::linear_scan_loop_header_flag)) {
6139       reorder_short_loop(code, block, i);
6140     }
6141   }
6142 
6143   DEBUG_ONLY(verify(code));
6144 }
6145 
6146 // only blocks with exactly one successor can be deleted. Such blocks
6147 // must always end with an unconditional branch to this successor
6148 bool ControlFlowOptimizer::can_delete_block(BlockBegin* block) {
6149   if (block->number_of_sux() != 1 || block->number_of_exception_handlers() != 0 || block->is_entry_block()) {
6150     return false;
6151   }
6152 
6153   LIR_OpList* instructions = block->lir()->instructions_list();
6154 
6155   assert(instructions->length() >= 2, "block must have label and branch");
6156   assert(instructions->at(0)->code() == lir_label, "first instruction must always be a label");
6157   assert(instructions->last()->as_OpBranch() != NULL, "last instrcution must always be a branch");
6158   assert(instructions->last()->as_OpBranch()->cond() == lir_cond_always, "branch must be unconditional");
6159   assert(instructions->last()->as_OpBranch()->block() == block->sux_at(0), "branch target must be the successor");
6160 
6161   // block must have exactly one successor
6162 
6163   if (instructions->length() == 2 && instructions->last()->info() == NULL) {
6164     return true;
6165   }
6166   return false;
6167 }
6168 
6169 // substitute branch targets in all branch-instructions of this blocks
6170 void ControlFlowOptimizer::substitute_branch_target(BlockBegin* block, BlockBegin* target_from, BlockBegin* target_to) {
6171   TRACE_LINEAR_SCAN(3, tty->print_cr("Deleting empty block: substituting from B%d to B%d inside B%d", target_from->block_id(), target_to->block_id(), block->block_id()));
6172 
6173   LIR_OpList* instructions = block->lir()->instructions_list();
6174 
6175   assert(instructions->at(0)->code() == lir_label, "first instruction must always be a label");
6176   for (int i = instructions->length() - 1; i >= 1; i--) {
6177     LIR_Op* op = instructions->at(i);
6178 
6179     if (op->code() == lir_branch || op->code() == lir_cond_float_branch) {
6180       assert(op->as_OpBranch() != NULL, "branch must be of type LIR_OpBranch");
6181       LIR_OpBranch* branch = (LIR_OpBranch*)op;
6182 
6183       if (branch->block() == target_from) {
6184         branch->change_block(target_to);
6185       }
6186       if (branch->ublock() == target_from) {
6187         branch->change_ublock(target_to);
6188       }
6189     }
6190   }
6191 }
6192 
6193 void ControlFlowOptimizer::delete_empty_blocks(BlockList* code) {
6194   int old_pos = 0;
6195   int new_pos = 0;
6196   int num_blocks = code->length();
6197 
6198   while (old_pos < num_blocks) {
6199     BlockBegin* block = code->at(old_pos);
6200 
6201     if (can_delete_block(block)) {
6202       BlockBegin* new_target = block->sux_at(0);
6203 
6204       // propagate backward branch target flag for correct code alignment
6205       if (block->is_set(BlockBegin::backward_branch_target_flag)) {
6206         new_target->set(BlockBegin::backward_branch_target_flag);
6207       }
6208 
6209       // collect a list with all predecessors that contains each predecessor only once
6210       // the predecessors of cur are changed during the substitution, so a copy of the
6211       // predecessor list is necessary
6212       int j;
6213       _original_preds.clear();
6214       for (j = block->number_of_preds() - 1; j >= 0; j--) {
6215         BlockBegin* pred = block->pred_at(j);
6216         if (_original_preds.find(pred) == -1) {
6217           _original_preds.append(pred);
6218         }
6219       }
6220 
6221       for (j = _original_preds.length() - 1; j >= 0; j--) {
6222         BlockBegin* pred = _original_preds.at(j);
6223         substitute_branch_target(pred, block, new_target);
6224         pred->substitute_sux(block, new_target);
6225       }
6226     } else {
6227       // adjust position of this block in the block list if blocks before
6228       // have been deleted
6229       if (new_pos != old_pos) {
6230         code->at_put(new_pos, code->at(old_pos));
6231       }
6232       new_pos++;
6233     }
6234     old_pos++;
6235   }
6236   code->trunc_to(new_pos);
6237 
6238   DEBUG_ONLY(verify(code));
6239 }
6240 
6241 void ControlFlowOptimizer::delete_unnecessary_jumps(BlockList* code) {
6242   // skip the last block because there a branch is always necessary
6243   for (int i = code->length() - 2; i >= 0; i--) {
6244     BlockBegin* block = code->at(i);
6245     LIR_OpList* instructions = block->lir()->instructions_list();
6246 
6247     LIR_Op* last_op = instructions->last();
6248     if (last_op->code() == lir_branch) {
6249       assert(last_op->as_OpBranch() != NULL, "branch must be of type LIR_OpBranch");
6250       LIR_OpBranch* last_branch = (LIR_OpBranch*)last_op;
6251 
6252       assert(last_branch->block() != NULL, "last branch must always have a block as target");
6253       assert(last_branch->label() == last_branch->block()->label(), "must be equal");
6254 
6255       if (last_branch->info() == NULL) {
6256         if (last_branch->block() == code->at(i + 1)) {
6257 
6258           TRACE_LINEAR_SCAN(3, tty->print_cr("Deleting unconditional branch at end of block B%d", block->block_id()));
6259 
6260           // delete last branch instruction
6261           instructions->trunc_to(instructions->length() - 1);
6262 
6263         } else {
6264           LIR_Op* prev_op = instructions->at(instructions->length() - 2);
6265           if (prev_op->code() == lir_branch || prev_op->code() == lir_cond_float_branch) {
6266             assert(prev_op->as_OpBranch() != NULL, "branch must be of type LIR_OpBranch");
6267             LIR_OpBranch* prev_branch = (LIR_OpBranch*)prev_op;
6268 
6269             if (prev_branch->stub() == NULL) {
6270 
6271               LIR_Op2* prev_cmp = NULL;
6272               // There might be a cmove inserted for profiling which depends on the same
6273               // compare. If we change the condition of the respective compare, we have
6274               // to take care of this cmove as well.
6275               LIR_Op2* prev_cmove = NULL;
6276 
6277               for(int j = instructions->length() - 3; j >= 0 && prev_cmp == NULL; j--) {
6278                 prev_op = instructions->at(j);
6279                 // check for the cmove
6280                 if (prev_op->code() == lir_cmove) {
6281                   assert(prev_op->as_Op2() != NULL, "cmove must be of type LIR_Op2");
6282                   prev_cmove = (LIR_Op2*)prev_op;
6283                   assert(prev_branch->cond() == prev_cmove->condition(), "should be the same");
6284                 }
6285                 if (prev_op->code() == lir_cmp) {
6286                   assert(prev_op->as_Op2() != NULL, "branch must be of type LIR_Op2");
6287                   prev_cmp = (LIR_Op2*)prev_op;
6288                   assert(prev_branch->cond() == prev_cmp->condition(), "should be the same");
6289                 }
6290               }
6291               assert(prev_cmp != NULL, "should have found comp instruction for branch");
6292               if (prev_branch->block() == code->at(i + 1) && prev_branch->info() == NULL) {
6293 
6294                 TRACE_LINEAR_SCAN(3, tty->print_cr("Negating conditional branch and deleting unconditional branch at end of block B%d", block->block_id()));
6295 
6296                 // eliminate a conditional branch to the immediate successor
6297                 prev_branch->change_block(last_branch->block());
6298                 prev_branch->negate_cond();
6299                 prev_cmp->set_condition(prev_branch->cond());
6300                 instructions->trunc_to(instructions->length() - 1);
6301                 // if we do change the condition, we have to change the cmove as well
6302                 if (prev_cmove != NULL) {
6303                   prev_cmove->set_condition(prev_branch->cond());
6304                   LIR_Opr t = prev_cmove->in_opr1();
6305                   prev_cmove->set_in_opr1(prev_cmove->in_opr2());
6306                   prev_cmove->set_in_opr2(t);
6307                 }
6308               }
6309             }
6310           }
6311         }
6312       }
6313     }
6314   }
6315 
6316   DEBUG_ONLY(verify(code));
6317 }
6318 
6319 void ControlFlowOptimizer::delete_jumps_to_return(BlockList* code) {
6320 #ifdef ASSERT
6321   BitMap return_converted(BlockBegin::number_of_blocks());
6322   return_converted.clear();
6323 #endif
6324 
6325   for (int i = code->length() - 1; i >= 0; i--) {
6326     BlockBegin* block = code->at(i);
6327     LIR_OpList* cur_instructions = block->lir()->instructions_list();
6328     LIR_Op*     cur_last_op = cur_instructions->last();
6329 
6330     assert(cur_instructions->at(0)->code() == lir_label, "first instruction must always be a label");
6331     if (cur_instructions->length() == 2 && cur_last_op->code() == lir_return) {
6332       // the block contains only a label and a return
6333       // if a predecessor ends with an unconditional jump to this block, then the jump
6334       // can be replaced with a return instruction
6335       //
6336       // Note: the original block with only a return statement cannot be deleted completely
6337       //       because the predecessors might have other (conditional) jumps to this block
6338       //       -> this may lead to unnecesary return instructions in the final code
6339 
6340       assert(cur_last_op->info() == NULL, "return instructions do not have debug information");
6341       assert(block->number_of_sux() == 0 ||
6342              (return_converted.at(block->block_id()) && block->number_of_sux() == 1),
6343              "blocks that end with return must not have successors");
6344 
6345       assert(cur_last_op->as_Op1() != NULL, "return must be LIR_Op1");
6346       LIR_Opr return_opr = ((LIR_Op1*)cur_last_op)->in_opr();
6347 
6348       for (int j = block->number_of_preds() - 1; j >= 0; j--) {
6349         BlockBegin* pred = block->pred_at(j);
6350         LIR_OpList* pred_instructions = pred->lir()->instructions_list();
6351         LIR_Op*     pred_last_op = pred_instructions->last();
6352 
6353         if (pred_last_op->code() == lir_branch) {
6354           assert(pred_last_op->as_OpBranch() != NULL, "branch must be LIR_OpBranch");
6355           LIR_OpBranch* pred_last_branch = (LIR_OpBranch*)pred_last_op;
6356 
6357           if (pred_last_branch->block() == block && pred_last_branch->cond() == lir_cond_always && pred_last_branch->info() == NULL) {
6358             // replace the jump to a return with a direct return
6359             // Note: currently the edge between the blocks is not deleted
6360             pred_instructions->at_put(pred_instructions->length() - 1, new LIR_Op1(lir_return, return_opr));
6361 #ifdef ASSERT
6362             return_converted.set_bit(pred->block_id());
6363 #endif
6364           }
6365         }
6366       }
6367     }
6368   }
6369 }
6370 
6371 
6372 #ifdef ASSERT
6373 void ControlFlowOptimizer::verify(BlockList* code) {
6374   for (int i = 0; i < code->length(); i++) {
6375     BlockBegin* block = code->at(i);
6376     LIR_OpList* instructions = block->lir()->instructions_list();
6377 
6378     int j;
6379     for (j = 0; j < instructions->length(); j++) {
6380       LIR_OpBranch* op_branch = instructions->at(j)->as_OpBranch();
6381 
6382       if (op_branch != NULL) {
6383         assert(op_branch->block() == NULL || code->find(op_branch->block()) != -1, "branch target not valid");
6384         assert(op_branch->ublock() == NULL || code->find(op_branch->ublock()) != -1, "branch target not valid");
6385       }
6386     }
6387 
6388     for (j = 0; j < block->number_of_sux() - 1; j++) {
6389       BlockBegin* sux = block->sux_at(j);
6390       assert(code->find(sux) != -1, "successor not valid");
6391     }
6392 
6393     for (j = 0; j < block->number_of_preds() - 1; j++) {
6394       BlockBegin* pred = block->pred_at(j);
6395       assert(code->find(pred) != -1, "successor not valid");
6396     }
6397   }
6398 }
6399 #endif
6400 
6401 
6402 #ifndef PRODUCT
6403 
6404 // Implementation of LinearStatistic
6405 
6406 const char* LinearScanStatistic::counter_name(int counter_idx) {
6407   switch (counter_idx) {
6408     case counter_method:          return "compiled methods";
6409     case counter_fpu_method:      return "methods using fpu";
6410     case counter_loop_method:     return "methods with loops";
6411     case counter_exception_method:return "methods with xhandler";
6412 
6413     case counter_loop:            return "loops";
6414     case counter_block:           return "blocks";
6415     case counter_loop_block:      return "blocks inside loop";
6416     case counter_exception_block: return "exception handler entries";
6417     case counter_interval:        return "intervals";
6418     case counter_fixed_interval:  return "fixed intervals";
6419     case counter_range:           return "ranges";
6420     case counter_fixed_range:     return "fixed ranges";
6421     case counter_use_pos:         return "use positions";
6422     case counter_fixed_use_pos:   return "fixed use positions";
6423     case counter_spill_slots:     return "spill slots";
6424 
6425     // counter for classes of lir instructions
6426     case counter_instruction:     return "total instructions";
6427     case counter_label:           return "labels";
6428     case counter_entry:           return "method entries";
6429     case counter_return:          return "method returns";
6430     case counter_call:            return "method calls";
6431     case counter_move:            return "moves";
6432     case counter_cmp:             return "compare";
6433     case counter_cond_branch:     return "conditional branches";
6434     case counter_uncond_branch:   return "unconditional branches";
6435     case counter_stub_branch:     return "branches to stub";
6436     case counter_alu:             return "artithmetic + logic";
6437     case counter_alloc:           return "allocations";
6438     case counter_sync:            return "synchronisation";
6439     case counter_throw:           return "throw";
6440     case counter_unwind:          return "unwind";
6441     case counter_typecheck:       return "type+null-checks";
6442     case counter_fpu_stack:       return "fpu-stack";
6443     case counter_misc_inst:       return "other instructions";
6444     case counter_other_inst:      return "misc. instructions";
6445 
6446     // counter for different types of moves
6447     case counter_move_total:      return "total moves";
6448     case counter_move_reg_reg:    return "register->register";
6449     case counter_move_reg_stack:  return "register->stack";
6450     case counter_move_stack_reg:  return "stack->register";
6451     case counter_move_stack_stack:return "stack->stack";
6452     case counter_move_reg_mem:    return "register->memory";
6453     case counter_move_mem_reg:    return "memory->register";
6454     case counter_move_const_any:  return "constant->any";
6455 
6456     case blank_line_1:            return "";
6457     case blank_line_2:            return "";
6458 
6459     default: ShouldNotReachHere(); return "";
6460   }
6461 }
6462 
6463 LinearScanStatistic::Counter LinearScanStatistic::base_counter(int counter_idx) {
6464   if (counter_idx == counter_fpu_method || counter_idx == counter_loop_method || counter_idx == counter_exception_method) {
6465     return counter_method;
6466   } else if (counter_idx == counter_loop_block || counter_idx == counter_exception_block) {
6467     return counter_block;
6468   } else if (counter_idx >= counter_instruction && counter_idx <= counter_other_inst) {
6469     return counter_instruction;
6470   } else if (counter_idx >= counter_move_total && counter_idx <= counter_move_const_any) {
6471     return counter_move_total;
6472   }
6473   return invalid_counter;
6474 }
6475 
6476 LinearScanStatistic::LinearScanStatistic() {
6477   for (int i = 0; i < number_of_counters; i++) {
6478     _counters_sum[i] = 0;
6479     _counters_max[i] = -1;
6480   }
6481 
6482 }
6483 
6484 // add the method-local numbers to the total sum
6485 void LinearScanStatistic::sum_up(LinearScanStatistic &method_statistic) {
6486   for (int i = 0; i < number_of_counters; i++) {
6487     _counters_sum[i] += method_statistic._counters_sum[i];
6488     _counters_max[i] = MAX2(_counters_max[i], method_statistic._counters_sum[i]);
6489   }
6490 }
6491 
6492 void LinearScanStatistic::print(const char* title) {
6493   if (CountLinearScan || TraceLinearScanLevel > 0) {
6494     tty->cr();
6495     tty->print_cr("***** LinearScan statistic - %s *****", title);
6496 
6497     for (int i = 0; i < number_of_counters; i++) {
6498       if (_counters_sum[i] > 0 || _counters_max[i] >= 0) {
6499         tty->print("%25s: %8d", counter_name(i), _counters_sum[i]);
6500 
6501         LinearScanStatistic::Counter cntr = base_counter(i);
6502         if (cntr != invalid_counter) {
6503           tty->print("  (%5.1f%%) ", _counters_sum[i] * 100.0 / _counters_sum[cntr]);
6504         } else {
6505           tty->print("           ");
6506         }
6507 
6508         if (_counters_max[i] >= 0) {
6509           tty->print("%8d", _counters_max[i]);
6510         }
6511       }
6512       tty->cr();
6513     }
6514   }
6515 }
6516 
6517 void LinearScanStatistic::collect(LinearScan* allocator) {
6518   inc_counter(counter_method);
6519   if (allocator->has_fpu_registers()) {
6520     inc_counter(counter_fpu_method);
6521   }
6522   if (allocator->num_loops() > 0) {
6523     inc_counter(counter_loop_method);
6524   }
6525   inc_counter(counter_loop, allocator->num_loops());
6526   inc_counter(counter_spill_slots, allocator->max_spills());
6527 
6528   int i;
6529   for (i = 0; i < allocator->interval_count(); i++) {
6530     Interval* cur = allocator->interval_at(i);
6531 
6532     if (cur != NULL) {
6533       inc_counter(counter_interval);
6534       inc_counter(counter_use_pos, cur->num_use_positions());
6535       if (LinearScan::is_precolored_interval(cur)) {
6536         inc_counter(counter_fixed_interval);
6537         inc_counter(counter_fixed_use_pos, cur->num_use_positions());
6538       }
6539 
6540       Range* range = cur->first();
6541       while (range != Range::end()) {
6542         inc_counter(counter_range);
6543         if (LinearScan::is_precolored_interval(cur)) {
6544           inc_counter(counter_fixed_range);
6545         }
6546         range = range->next();
6547       }
6548     }
6549   }
6550 
6551   bool has_xhandlers = false;
6552   // Note: only count blocks that are in code-emit order
6553   for (i = 0; i < allocator->ir()->code()->length(); i++) {
6554     BlockBegin* cur = allocator->ir()->code()->at(i);
6555 
6556     inc_counter(counter_block);
6557     if (cur->loop_depth() > 0) {
6558       inc_counter(counter_loop_block);
6559     }
6560     if (cur->is_set(BlockBegin::exception_entry_flag)) {
6561       inc_counter(counter_exception_block);
6562       has_xhandlers = true;
6563     }
6564 
6565     LIR_OpList* instructions = cur->lir()->instructions_list();
6566     for (int j = 0; j < instructions->length(); j++) {
6567       LIR_Op* op = instructions->at(j);
6568 
6569       inc_counter(counter_instruction);
6570 
6571       switch (op->code()) {
6572         case lir_label:           inc_counter(counter_label); break;
6573         case lir_std_entry:
6574         case lir_osr_entry:       inc_counter(counter_entry); break;
6575         case lir_return:          inc_counter(counter_return); break;
6576 
6577         case lir_rtcall:
6578         case lir_static_call:
6579         case lir_optvirtual_call:
6580         case lir_virtual_call:    inc_counter(counter_call); break;
6581 
6582         case lir_move: {
6583           inc_counter(counter_move);
6584           inc_counter(counter_move_total);
6585 
6586           LIR_Opr in = op->as_Op1()->in_opr();
6587           LIR_Opr res = op->as_Op1()->result_opr();
6588           if (in->is_register()) {
6589             if (res->is_register()) {
6590               inc_counter(counter_move_reg_reg);
6591             } else if (res->is_stack()) {
6592               inc_counter(counter_move_reg_stack);
6593             } else if (res->is_address()) {
6594               inc_counter(counter_move_reg_mem);
6595             } else {
6596               ShouldNotReachHere();
6597             }
6598           } else if (in->is_stack()) {
6599             if (res->is_register()) {
6600               inc_counter(counter_move_stack_reg);
6601             } else {
6602               inc_counter(counter_move_stack_stack);
6603             }
6604           } else if (in->is_address()) {
6605             assert(res->is_register(), "must be");
6606             inc_counter(counter_move_mem_reg);
6607           } else if (in->is_constant()) {
6608             inc_counter(counter_move_const_any);
6609           } else {
6610             ShouldNotReachHere();
6611           }
6612           break;
6613         }
6614 
6615         case lir_cmp:             inc_counter(counter_cmp); break;
6616 
6617         case lir_branch:
6618         case lir_cond_float_branch: {
6619           LIR_OpBranch* branch = op->as_OpBranch();
6620           if (branch->block() == NULL) {
6621             inc_counter(counter_stub_branch);
6622           } else if (branch->cond() == lir_cond_always) {
6623             inc_counter(counter_uncond_branch);
6624           } else {
6625             inc_counter(counter_cond_branch);
6626           }
6627           break;
6628         }
6629 
6630         case lir_neg:
6631         case lir_add:
6632         case lir_sub:
6633         case lir_mul:
6634         case lir_mul_strictfp:
6635         case lir_div:
6636         case lir_div_strictfp:
6637         case lir_rem:
6638         case lir_sqrt:
6639         case lir_abs:
6640         case lir_log10:
6641         case lir_logic_and:
6642         case lir_logic_or:
6643         case lir_logic_xor:
6644         case lir_shl:
6645         case lir_shr:
6646         case lir_ushr:            inc_counter(counter_alu); break;
6647 
6648         case lir_alloc_object:
6649         case lir_alloc_array:     inc_counter(counter_alloc); break;
6650 
6651         case lir_monaddr:
6652         case lir_lock:
6653         case lir_unlock:          inc_counter(counter_sync); break;
6654 
6655         case lir_throw:           inc_counter(counter_throw); break;
6656 
6657         case lir_unwind:          inc_counter(counter_unwind); break;
6658 
6659         case lir_null_check:
6660         case lir_leal:
6661         case lir_instanceof:
6662         case lir_checkcast:
6663         case lir_store_check:     inc_counter(counter_typecheck); break;
6664 
6665         case lir_fpop_raw:
6666         case lir_fxch:
6667         case lir_fld:             inc_counter(counter_fpu_stack); break;
6668 
6669         case lir_nop:
6670         case lir_push:
6671         case lir_pop:
6672         case lir_convert:
6673         case lir_roundfp:
6674         case lir_cmove:           inc_counter(counter_misc_inst); break;
6675 
6676         default:                  inc_counter(counter_other_inst); break;
6677       }
6678     }
6679   }
6680 
6681   if (has_xhandlers) {
6682     inc_counter(counter_exception_method);
6683   }
6684 }
6685 
6686 void LinearScanStatistic::compute(LinearScan* allocator, LinearScanStatistic &global_statistic) {
6687   if (CountLinearScan || TraceLinearScanLevel > 0) {
6688 
6689     LinearScanStatistic local_statistic = LinearScanStatistic();
6690 
6691     local_statistic.collect(allocator);
6692     global_statistic.sum_up(local_statistic);
6693 
6694     if (TraceLinearScanLevel > 2) {
6695       local_statistic.print("current local statistic");
6696     }
6697   }
6698 }
6699 
6700 
6701 // Implementation of LinearTimers
6702 
6703 LinearScanTimers::LinearScanTimers() {
6704   for (int i = 0; i < number_of_timers; i++) {
6705     timer(i)->reset();
6706   }
6707 }
6708 
6709 const char* LinearScanTimers::timer_name(int idx) {
6710   switch (idx) {
6711     case timer_do_nothing:               return "Nothing (Time Check)";
6712     case timer_number_instructions:      return "Number Instructions";
6713     case timer_compute_local_live_sets:  return "Local Live Sets";
6714     case timer_compute_global_live_sets: return "Global Live Sets";
6715     case timer_build_intervals:          return "Build Intervals";
6716     case timer_sort_intervals_before:    return "Sort Intervals Before";
6717     case timer_allocate_registers:       return "Allocate Registers";
6718     case timer_resolve_data_flow:        return "Resolve Data Flow";
6719     case timer_sort_intervals_after:     return "Sort Intervals After";
6720     case timer_eliminate_spill_moves:    return "Spill optimization";
6721     case timer_assign_reg_num:           return "Assign Reg Num";
6722     case timer_allocate_fpu_stack:       return "Allocate FPU Stack";
6723     case timer_optimize_lir:             return "Optimize LIR";
6724     default: ShouldNotReachHere();       return "";
6725   }
6726 }
6727 
6728 void LinearScanTimers::begin_method() {
6729   if (TimeEachLinearScan) {
6730     // reset all timers to measure only current method
6731     for (int i = 0; i < number_of_timers; i++) {
6732       timer(i)->reset();
6733     }
6734   }
6735 }
6736 
6737 void LinearScanTimers::end_method(LinearScan* allocator) {
6738   if (TimeEachLinearScan) {
6739 
6740     double c = timer(timer_do_nothing)->seconds();
6741     double total = 0;
6742     for (int i = 1; i < number_of_timers; i++) {
6743       total += timer(i)->seconds() - c;
6744     }
6745 
6746     if (total >= 0.0005) {
6747       // print all information in one line for automatic processing
6748       tty->print("@"); allocator->compilation()->method()->print_name();
6749 
6750       tty->print("@ %d ", allocator->compilation()->method()->code_size());
6751       tty->print("@ %d ", allocator->block_at(allocator->block_count() - 1)->last_lir_instruction_id() / 2);
6752       tty->print("@ %d ", allocator->block_count());
6753       tty->print("@ %d ", allocator->num_virtual_regs());
6754       tty->print("@ %d ", allocator->interval_count());
6755       tty->print("@ %d ", allocator->_num_calls);
6756       tty->print("@ %d ", allocator->num_loops());
6757 
6758       tty->print("@ %6.6f ", total);
6759       for (int i = 1; i < number_of_timers; i++) {
6760         tty->print("@ %4.1f ", ((timer(i)->seconds() - c) / total) * 100);
6761       }
6762       tty->cr();
6763     }
6764   }
6765 }
6766 
6767 void LinearScanTimers::print(double total_time) {
6768   if (TimeLinearScan) {
6769     // correction value: sum of dummy-timer that only measures the time that
6770     // is necesary to start and stop itself
6771     double c = timer(timer_do_nothing)->seconds();
6772 
6773     for (int i = 0; i < number_of_timers; i++) {
6774       double t = timer(i)->seconds();
6775       tty->print_cr("    %25s: %6.3f s (%4.1f%%)  corrected: %6.3f s (%4.1f%%)", timer_name(i), t, (t / total_time) * 100.0, t - c, (t - c) / (total_time - 2 * number_of_timers * c) * 100);
6776     }
6777   }
6778 }
6779 
6780 #endif // #ifndef PRODUCT