1 /*
   2  * Copyright (c) 1998, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "compiler/compileLog.hpp"
  27 #include "interpreter/linkResolver.hpp"
  28 #include "memory/universe.inline.hpp"
  29 #include "oops/objArrayKlass.hpp"
  30 #include "opto/addnode.hpp"
  31 #include "opto/memnode.hpp"
  32 #include "opto/parse.hpp"
  33 #include "opto/rootnode.hpp"
  34 #include "opto/runtime.hpp"
  35 #include "opto/subnode.hpp"
  36 #include "runtime/deoptimization.hpp"
  37 #include "runtime/handles.inline.hpp"
  38 
  39 //=============================================================================
  40 // Helper methods for _get* and _put* bytecodes
  41 //=============================================================================
  42 bool Parse::static_field_ok_in_clinit(ciField *field, ciMethod *method) {
  43   // Could be the field_holder's <clinit> method, or <clinit> for a subklass.
  44   // Better to check now than to Deoptimize as soon as we execute
  45   assert( field->is_static(), "Only check if field is static");
  46   // is_being_initialized() is too generous.  It allows access to statics
  47   // by threads that are not running the <clinit> before the <clinit> finishes.
  48   // return field->holder()->is_being_initialized();
  49 
  50   // The following restriction is correct but conservative.
  51   // It is also desirable to allow compilation of methods called from <clinit>
  52   // but this generated code will need to be made safe for execution by
  53   // other threads, or the transition from interpreted to compiled code would
  54   // need to be guarded.
  55   ciInstanceKlass *field_holder = field->holder();
  56 
  57   bool access_OK = false;
  58   if (method->holder()->is_subclass_of(field_holder)) {
  59     if (method->is_static()) {
  60       if (method->name() == ciSymbol::class_initializer_name()) {
  61         // OK to access static fields inside initializer
  62         access_OK = true;
  63       }
  64     } else {
  65       if (method->name() == ciSymbol::object_initializer_name()) {
  66         // It's also OK to access static fields inside a constructor,
  67         // because any thread calling the constructor must first have
  68         // synchronized on the class by executing a '_new' bytecode.
  69         access_OK = true;
  70       }
  71     }
  72   }
  73 
  74   return access_OK;
  75 
  76 }
  77 
  78 
  79 void Parse::do_field_access(bool is_get, bool is_field) {
  80   bool will_link;
  81   ciField* field = iter().get_field(will_link);
  82   assert(will_link, "getfield: typeflow responsibility");
  83 
  84   ciInstanceKlass* field_holder = field->holder();
  85 
  86   if (is_field == field->is_static()) {
  87     // Interpreter will throw java_lang_IncompatibleClassChangeError
  88     // Check this before allowing <clinit> methods to access static fields
  89     uncommon_trap(Deoptimization::Reason_unhandled,
  90                   Deoptimization::Action_none);
  91     return;
  92   }
  93 
  94   if (!is_field && !field_holder->is_initialized()) {
  95     if (!static_field_ok_in_clinit(field, method())) {
  96       uncommon_trap(Deoptimization::Reason_uninitialized,
  97                     Deoptimization::Action_reinterpret,
  98                     NULL, "!static_field_ok_in_clinit");
  99       return;
 100     }
 101   }
 102 
 103   // Deoptimize on putfield writes to call site target field.
 104   if (!is_get && field->is_call_site_target()) {
 105     uncommon_trap(Deoptimization::Reason_unhandled,
 106                   Deoptimization::Action_reinterpret,
 107                   NULL, "put to call site target field");
 108     return;
 109   }
 110 
 111   assert(field->will_link(method()->holder(), bc()), "getfield: typeflow responsibility");
 112 
 113   // Note:  We do not check for an unloaded field type here any more.
 114 
 115   // Generate code for the object pointer.
 116   Node* obj;
 117   if (is_field) {
 118     int obj_depth = is_get ? 0 : field->type()->size();
 119     obj = null_check(peek(obj_depth));
 120     // Compile-time detect of null-exception?
 121     if (stopped())  return;
 122 
 123 #ifdef ASSERT
 124     const TypeInstPtr *tjp = TypeInstPtr::make(TypePtr::NotNull, iter().get_declared_field_holder());
 125     assert(_gvn.type(obj)->higher_equal(tjp), "cast_up is no longer needed");
 126 #endif
 127 
 128     if (is_get) {
 129       (void) pop();  // pop receiver before getting
 130       do_get_xxx(obj, field, is_field);
 131     } else {
 132       do_put_xxx(obj, field, is_field);
 133       (void) pop();  // pop receiver after putting
 134     }
 135   } else {
 136     const TypeInstPtr* tip = TypeInstPtr::make(field_holder->java_mirror());
 137     obj = _gvn.makecon(tip);
 138     if (is_get) {
 139       do_get_xxx(obj, field, is_field);
 140     } else {
 141       do_put_xxx(obj, field, is_field);
 142     }
 143   }
 144 }
 145 
 146 
 147 void Parse::do_get_xxx(Node* obj, ciField* field, bool is_field) {
 148   // Does this field have a constant value?  If so, just push the value.
 149   if (field->is_constant()) {
 150     // final field
 151     if (field->is_static()) {
 152       // final static field
 153       if (C->eliminate_autobox()) {
 154         // The pointers in the autobox arrays are always non-null.
 155         ciSymbol* klass_name = field->holder()->name();
 156         if (field->name() == ciSymbol::cache_field_name() &&
 157             field->holder()->uses_default_loader() &&
 158             (klass_name == ciSymbol::java_lang_Character_CharacterCache() ||
 159              klass_name == ciSymbol::java_lang_Byte_ByteCache() ||
 160              klass_name == ciSymbol::java_lang_Short_ShortCache() ||
 161              klass_name == ciSymbol::java_lang_Integer_IntegerCache() ||
 162              klass_name == ciSymbol::java_lang_Long_LongCache())) {
 163           bool require_const = true;
 164           bool autobox_cache = true;
 165           if (push_constant(field->constant_value(), require_const, autobox_cache)) {
 166             return;
 167           }
 168         }
 169       }
 170       if (push_constant(field->constant_value()))
 171         return;
 172     }
 173     else {
 174       // final non-static field
 175       // Treat final non-static fields of trusted classes (classes in
 176       // java.lang.invoke and sun.invoke packages and subpackages) as
 177       // compile time constants.
 178       if (obj->is_Con()) {
 179         const TypeOopPtr* oop_ptr = obj->bottom_type()->isa_oopptr();
 180         ciObject* constant_oop = oop_ptr->const_oop();
 181         ciConstant constant = field->constant_value_of(constant_oop);
 182         if (push_constant(constant, true))
 183           return;
 184       }
 185     }
 186   }
 187 
 188   ciType* field_klass = field->type();
 189   bool is_vol = field->is_volatile();
 190 
 191   // Compute address and memory type.
 192   int offset = field->offset_in_bytes();
 193   const TypePtr* adr_type = C->alias_type(field)->adr_type();
 194   Node *adr = basic_plus_adr(obj, obj, offset);
 195   BasicType bt = field->layout_type();
 196 
 197   // Build the resultant type of the load
 198   const Type *type;
 199 
 200   bool must_assert_null = false;
 201 
 202   if( bt == T_OBJECT ) {
 203     if (!field->type()->is_loaded()) {
 204       type = TypeInstPtr::BOTTOM;
 205       must_assert_null = true;
 206     } else if (field->is_constant() && field->is_static()) {
 207       // This can happen if the constant oop is non-perm.
 208       ciObject* con = field->constant_value().as_object();
 209       // Do not "join" in the previous type; it doesn't add value,
 210       // and may yield a vacuous result if the field is of interface type.
 211       type = TypeOopPtr::make_from_constant(con)->isa_oopptr();
 212       assert(type != NULL, "field singleton type must be consistent");
 213     } else {
 214       type = TypeOopPtr::make_from_klass(field_klass->as_klass());
 215     }
 216   } else {
 217     type = Type::get_const_basic_type(bt);
 218   }
 219   // Build the load.
 220   Node* ld = make_load(NULL, adr, type, bt, adr_type, is_vol);
 221 
 222   // Adjust Java stack
 223   if (type2size[bt] == 1)
 224     push(ld);
 225   else
 226     push_pair(ld);
 227 
 228   if (must_assert_null) {
 229     // Do not take a trap here.  It's possible that the program
 230     // will never load the field's class, and will happily see
 231     // null values in this field forever.  Don't stumble into a
 232     // trap for such a program, or we might get a long series
 233     // of useless recompilations.  (Or, we might load a class
 234     // which should not be loaded.)  If we ever see a non-null
 235     // value, we will then trap and recompile.  (The trap will
 236     // not need to mention the class index, since the class will
 237     // already have been loaded if we ever see a non-null value.)
 238     // uncommon_trap(iter().get_field_signature_index());
 239 #ifndef PRODUCT
 240     if (PrintOpto && (Verbose || WizardMode)) {
 241       method()->print_name(); tty->print_cr(" asserting nullness of field at bci: %d", bci());
 242     }
 243 #endif
 244     if (C->log() != NULL) {
 245       C->log()->elem("assert_null reason='field' klass='%d'",
 246                      C->log()->identify(field->type()));
 247     }
 248     // If there is going to be a trap, put it at the next bytecode:
 249     set_bci(iter().next_bci());
 250     null_assert(peek());
 251     set_bci(iter().cur_bci()); // put it back
 252   }
 253 
 254   // If reference is volatile, prevent following memory ops from
 255   // floating up past the volatile read.  Also prevents commoning
 256   // another volatile read.
 257   if (field->is_volatile()) {
 258     // Memory barrier includes bogus read of value to force load BEFORE membar
 259     insert_mem_bar(Op_MemBarAcquire, ld);
 260   }
 261 }
 262 
 263 void Parse::do_put_xxx(Node* obj, ciField* field, bool is_field) {
 264   bool is_vol = field->is_volatile();
 265   // If reference is volatile, prevent following memory ops from
 266   // floating down past the volatile write.  Also prevents commoning
 267   // another volatile read.
 268   if (is_vol)  insert_mem_bar(Op_MemBarRelease);
 269 
 270   // Compute address and memory type.
 271   int offset = field->offset_in_bytes();
 272   const TypePtr* adr_type = C->alias_type(field)->adr_type();
 273   Node* adr = basic_plus_adr(obj, obj, offset);
 274   BasicType bt = field->layout_type();
 275   // Value to be stored
 276   Node* val = type2size[bt] == 1 ? pop() : pop_pair();
 277   // Round doubles before storing
 278   if (bt == T_DOUBLE)  val = dstore_rounding(val);
 279 
 280   // Store the value.
 281   Node* store;
 282   if (bt == T_OBJECT) {
 283     const TypeOopPtr* field_type;
 284     if (!field->type()->is_loaded()) {
 285       field_type = TypeInstPtr::BOTTOM;
 286     } else {
 287       field_type = TypeOopPtr::make_from_klass(field->type()->as_klass());
 288     }
 289     store = store_oop_to_object( control(), obj, adr, adr_type, val, field_type, bt);
 290   } else {
 291     store = store_to_memory( control(), adr, val, bt, adr_type, is_vol );
 292   }
 293 
 294   // If reference is volatile, prevent following volatiles ops from
 295   // floating up before the volatile write.
 296   if (is_vol) {
 297     // First place the specific membar for THIS volatile index. This first
 298     // membar is dependent on the store, keeping any other membars generated
 299     // below from floating up past the store.
 300     int adr_idx = C->get_alias_index(adr_type);
 301     insert_mem_bar_volatile(Op_MemBarVolatile, adr_idx, store);
 302 
 303     // Now place a membar for AliasIdxBot for the unknown yet-to-be-parsed
 304     // volatile alias indices. Skip this if the membar is redundant.
 305     if (adr_idx != Compile::AliasIdxBot) {
 306       insert_mem_bar_volatile(Op_MemBarVolatile, Compile::AliasIdxBot, store);
 307     }
 308 
 309     // Finally, place alias-index-specific membars for each volatile index
 310     // that isn't the adr_idx membar. Typically there's only 1 or 2.
 311     for( int i = Compile::AliasIdxRaw; i < C->num_alias_types(); i++ ) {
 312       if (i != adr_idx && C->alias_type(i)->is_volatile()) {
 313         insert_mem_bar_volatile(Op_MemBarVolatile, i, store);
 314       }
 315     }
 316   }
 317 
 318   // If the field is final, the rules of Java say we are in <init> or <clinit>.
 319   // Note the presence of writes to final non-static fields, so that we
 320   // can insert a memory barrier later on to keep the writes from floating
 321   // out of the constructor.
 322   if (is_field && field->is_final()) {
 323     set_wrote_final(true);
 324     // Preserve allocation ptr to create precedent edge to it in membar
 325     // generated on exit from constructor.
 326     if (C->eliminate_autobox() &&
 327         adr_type->isa_oopptr() && adr_type->is_oopptr()->is_ptr_to_boxed_value() &&
 328         AllocateNode::Ideal_allocation(obj, &_gvn) != NULL) {
 329       set_alloc_with_final(obj);
 330     }
 331   }
 332 }
 333 
 334 
 335 bool Parse::push_constant(ciConstant constant, bool require_constant, bool is_autobox_cache) {
 336   switch (constant.basic_type()) {
 337   case T_BOOLEAN:  push( intcon(constant.as_boolean()) ); break;
 338   case T_INT:      push( intcon(constant.as_int())     ); break;
 339   case T_CHAR:     push( intcon(constant.as_char())    ); break;
 340   case T_BYTE:     push( intcon(constant.as_byte())    ); break;
 341   case T_SHORT:    push( intcon(constant.as_short())   ); break;
 342   case T_FLOAT:    push( makecon(TypeF::make(constant.as_float())) );  break;
 343   case T_DOUBLE:   push_pair( makecon(TypeD::make(constant.as_double())) );  break;
 344   case T_LONG:     push_pair( longcon(constant.as_long()) ); break;
 345   case T_ARRAY:
 346   case T_OBJECT: {
 347     // cases:
 348     //   can_be_constant    = (oop not scavengable || ScavengeRootsInCode != 0)
 349     //   should_be_constant = (oop not scavengable || ScavengeRootsInCode >= 2)
 350     // An oop is not scavengable if it is in the perm gen.
 351     ciObject* oop_constant = constant.as_object();
 352     if (oop_constant->is_null_object()) {
 353       push( zerocon(T_OBJECT) );
 354       break;
 355     } else if (require_constant || oop_constant->should_be_constant()) {
 356       push( makecon(TypeOopPtr::make_from_constant(oop_constant, require_constant, is_autobox_cache)) );
 357       break;
 358     } else {
 359       // we cannot inline the oop, but we can use it later to narrow a type
 360       return false;
 361     }
 362   }
 363   case T_ILLEGAL: {
 364     // Invalid ciConstant returned due to OutOfMemoryError in the CI
 365     assert(C->env()->failing(), "otherwise should not see this");
 366     // These always occur because of object types; we are going to
 367     // bail out anyway, so make the stack depths match up
 368     push( zerocon(T_OBJECT) );
 369     return false;
 370   }
 371   default:
 372     ShouldNotReachHere();
 373     return false;
 374   }
 375 
 376   // success
 377   return true;
 378 }
 379 
 380 
 381 
 382 //=============================================================================
 383 void Parse::do_anewarray() {
 384   bool will_link;
 385   ciKlass* klass = iter().get_klass(will_link);
 386 
 387   // Uncommon Trap when class that array contains is not loaded
 388   // we need the loaded class for the rest of graph; do not
 389   // initialize the container class (see Java spec)!!!
 390   assert(will_link, "anewarray: typeflow responsibility");
 391 
 392   ciObjArrayKlass* array_klass = ciObjArrayKlass::make(klass);
 393   // Check that array_klass object is loaded
 394   if (!array_klass->is_loaded()) {
 395     // Generate uncommon_trap for unloaded array_class
 396     uncommon_trap(Deoptimization::Reason_unloaded,
 397                   Deoptimization::Action_reinterpret,
 398                   array_klass);
 399     return;
 400   }
 401 
 402   kill_dead_locals();
 403 
 404   const TypeKlassPtr* array_klass_type = TypeKlassPtr::make(array_klass);
 405   Node* count_val = pop();
 406   Node* obj = new_array(makecon(array_klass_type), count_val, 1);
 407   push(obj);
 408 }
 409 
 410 
 411 void Parse::do_newarray(BasicType elem_type) {
 412   kill_dead_locals();
 413 
 414   Node*   count_val = pop();
 415   const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(elem_type));
 416   Node*   obj = new_array(makecon(array_klass), count_val, 1);
 417   // Push resultant oop onto stack
 418   push(obj);
 419 }
 420 
 421 // Expand simple expressions like new int[3][5] and new Object[2][nonConLen].
 422 // Also handle the degenerate 1-dimensional case of anewarray.
 423 Node* Parse::expand_multianewarray(ciArrayKlass* array_klass, Node* *lengths, int ndimensions, int nargs) {
 424   Node* length = lengths[0];
 425   assert(length != NULL, "");
 426   Node* array = new_array(makecon(TypeKlassPtr::make(array_klass)), length, nargs);
 427   if (ndimensions > 1) {
 428     jint length_con = find_int_con(length, -1);
 429     guarantee(length_con >= 0, "non-constant multianewarray");
 430     ciArrayKlass* array_klass_1 = array_klass->as_obj_array_klass()->element_klass()->as_array_klass();
 431     const TypePtr* adr_type = TypeAryPtr::OOPS;
 432     const TypeOopPtr*    elemtype = _gvn.type(array)->is_aryptr()->elem()->make_oopptr();
 433     const intptr_t header   = arrayOopDesc::base_offset_in_bytes(T_OBJECT);
 434     for (jint i = 0; i < length_con; i++) {
 435       Node*    elem   = expand_multianewarray(array_klass_1, &lengths[1], ndimensions-1, nargs);
 436       intptr_t offset = header + ((intptr_t)i << LogBytesPerHeapOop);
 437       Node*    eaddr  = basic_plus_adr(array, offset);
 438       store_oop_to_array(control(), array, eaddr, adr_type, elem, elemtype, T_OBJECT);
 439     }
 440   }
 441   return array;
 442 }
 443 
 444 void Parse::do_multianewarray() {
 445   int ndimensions = iter().get_dimensions();
 446 
 447   // the m-dimensional array
 448   bool will_link;
 449   ciArrayKlass* array_klass = iter().get_klass(will_link)->as_array_klass();
 450   assert(will_link, "multianewarray: typeflow responsibility");
 451 
 452   // Note:  Array classes are always initialized; no is_initialized check.
 453 
 454   kill_dead_locals();
 455 
 456   // get the lengths from the stack (first dimension is on top)
 457   Node** length = NEW_RESOURCE_ARRAY(Node*, ndimensions + 1);
 458   length[ndimensions] = NULL;  // terminating null for make_runtime_call
 459   int j;
 460   for (j = ndimensions-1; j >= 0 ; j--) length[j] = pop();
 461 
 462   // The original expression was of this form: new T[length0][length1]...
 463   // It is often the case that the lengths are small (except the last).
 464   // If that happens, use the fast 1-d creator a constant number of times.
 465   const jint expand_limit = MIN2((juint)MultiArrayExpandLimit, (juint)100);
 466   jint expand_count = 1;        // count of allocations in the expansion
 467   jint expand_fanout = 1;       // running total fanout
 468   for (j = 0; j < ndimensions-1; j++) {
 469     jint dim_con = find_int_con(length[j], -1);
 470     expand_fanout *= dim_con;
 471     expand_count  += expand_fanout; // count the level-J sub-arrays
 472     if (dim_con <= 0
 473         || dim_con > expand_limit
 474         || expand_count > expand_limit) {
 475       expand_count = 0;
 476       break;
 477     }
 478   }
 479 
 480   // Can use multianewarray instead of [a]newarray if only one dimension,
 481   // or if all non-final dimensions are small constants.
 482   if (ndimensions == 1 || (1 <= expand_count && expand_count <= expand_limit)) {
 483     Node* obj = NULL;
 484     // Set the original stack and the reexecute bit for the interpreter
 485     // to reexecute the multianewarray bytecode if deoptimization happens.
 486     // Do it unconditionally even for one dimension multianewarray.
 487     // Note: the reexecute bit will be set in GraphKit::add_safepoint_edges()
 488     // when AllocateArray node for newarray is created.
 489     { PreserveReexecuteState preexecs(this);
 490       inc_sp(ndimensions);
 491       // Pass 0 as nargs since uncommon trap code does not need to restore stack.
 492       obj = expand_multianewarray(array_klass, &length[0], ndimensions, 0);
 493     } //original reexecute and sp are set back here
 494     push(obj);
 495     return;
 496   }
 497 
 498   address fun = NULL;
 499   switch (ndimensions) {
 500   case 1: ShouldNotReachHere(); break;
 501   case 2: fun = OptoRuntime::multianewarray2_Java(); break;
 502   case 3: fun = OptoRuntime::multianewarray3_Java(); break;
 503   case 4: fun = OptoRuntime::multianewarray4_Java(); break;
 504   case 5: fun = OptoRuntime::multianewarray5_Java(); break;
 505   };
 506   Node* c = NULL;
 507 
 508   if (fun != NULL) {
 509     c = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
 510                           OptoRuntime::multianewarray_Type(ndimensions),
 511                           fun, NULL, TypeRawPtr::BOTTOM,
 512                           makecon(TypeKlassPtr::make(array_klass)),
 513                           length[0], length[1], length[2],
 514                           (ndimensions > 2) ? length[3] : NULL,
 515                           (ndimensions > 3) ? length[4] : NULL);
 516   } else {
 517     // Create a java array for dimension sizes
 518     Node* dims = NULL;
 519     { PreserveReexecuteState preexecs(this);
 520       inc_sp(ndimensions);
 521       Node* dims_array_klass = makecon(TypeKlassPtr::make(ciArrayKlass::make(ciType::make(T_INT))));
 522       dims = new_array(dims_array_klass, intcon(ndimensions), 0);
 523 
 524       // Fill-in it with values
 525       for (j = 0; j < ndimensions; j++) {
 526         Node *dims_elem = array_element_address(dims, intcon(j), T_INT);
 527         store_to_memory(control(), dims_elem, length[j], T_INT, TypeAryPtr::INTS);
 528       }
 529     }
 530 
 531     c = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
 532                           OptoRuntime::multianewarrayN_Type(),
 533                           OptoRuntime::multianewarrayN_Java(), NULL, TypeRawPtr::BOTTOM,
 534                           makecon(TypeKlassPtr::make(array_klass)),
 535                           dims);
 536   }
 537   make_slow_call_ex(c, env()->Throwable_klass(), false);
 538 
 539   Node* res = _gvn.transform(new (C) ProjNode(c, TypeFunc::Parms));
 540 
 541   const Type* type = TypeOopPtr::make_from_klass_raw(array_klass);
 542 
 543   // Improve the type:  We know it's not null, exact, and of a given length.
 544   type = type->is_ptr()->cast_to_ptr_type(TypePtr::NotNull);
 545   type = type->is_aryptr()->cast_to_exactness(true);
 546 
 547   const TypeInt* ltype = _gvn.find_int_type(length[0]);
 548   if (ltype != NULL)
 549     type = type->is_aryptr()->cast_to_size(ltype);
 550 
 551     // We cannot sharpen the nested sub-arrays, since the top level is mutable.
 552 
 553   Node* cast = _gvn.transform( new (C) CheckCastPPNode(control(), res, type) );
 554   push(cast);
 555 
 556   // Possible improvements:
 557   // - Make a fast path for small multi-arrays.  (W/ implicit init. loops.)
 558   // - Issue CastII against length[*] values, to TypeInt::POS.
 559 }