1 /*
2 * Copyright (c) 2009, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.lir.gen;
26
27 import static jdk.vm.ci.code.ValueUtil.asAllocatableValue;
28 import static jdk.vm.ci.code.ValueUtil.isAllocatableValue;
29 import static jdk.vm.ci.code.ValueUtil.isLegal;
30 import static jdk.vm.ci.code.ValueUtil.isStackSlot;
31 import static org.graalvm.compiler.lir.LIRValueUtil.asConstant;
32 import static org.graalvm.compiler.lir.LIRValueUtil.isConstantValue;
33 import static org.graalvm.compiler.lir.LIRValueUtil.isVariable;
34 import static org.graalvm.compiler.lir.LIRValueUtil.isVirtualStackSlot;
35
36 import java.util.ArrayList;
37 import java.util.List;
38 import java.util.Optional;
39
40 import org.graalvm.compiler.asm.Label;
41 import org.graalvm.compiler.core.common.LIRKind;
42 import org.graalvm.compiler.core.common.calc.Condition;
43 import org.graalvm.compiler.core.common.cfg.AbstractBlockBase;
44 import org.graalvm.compiler.core.common.spi.CodeGenProviders;
45 import org.graalvm.compiler.core.common.spi.ForeignCallLinkage;
46 import org.graalvm.compiler.core.common.spi.ForeignCallsProvider;
47 import org.graalvm.compiler.core.common.spi.LIRKindTool;
48 import org.graalvm.compiler.core.common.type.Stamp;
49 import org.graalvm.compiler.debug.DebugCloseable;
50 import org.graalvm.compiler.debug.GraalError;
51 import org.graalvm.compiler.debug.TTY;
52 import org.graalvm.compiler.graph.NodeSourcePosition;
53 import org.graalvm.compiler.lir.ConstantValue;
54 import org.graalvm.compiler.lir.LIR;
55 import org.graalvm.compiler.lir.LIRFrameState;
56 import org.graalvm.compiler.lir.LIRInstruction;
57 import org.graalvm.compiler.lir.LIRVerifier;
58 import org.graalvm.compiler.lir.LabelRef;
59 import org.graalvm.compiler.lir.StandardOp;
60 import org.graalvm.compiler.lir.StandardOp.BlockEndOp;
61 import org.graalvm.compiler.lir.StandardOp.LabelOp;
62 import org.graalvm.compiler.lir.StandardOp.ZapRegistersOp;
63 import org.graalvm.compiler.lir.SwitchStrategy;
64 import org.graalvm.compiler.lir.Variable;
65 import org.graalvm.compiler.lir.hashing.Hasher;
66 import org.graalvm.compiler.options.Option;
67 import org.graalvm.compiler.options.OptionKey;
68 import org.graalvm.compiler.options.OptionType;
69 import org.graalvm.compiler.options.OptionValues;
70
71 import jdk.vm.ci.code.CallingConvention;
72 import jdk.vm.ci.code.CodeCacheProvider;
73 import jdk.vm.ci.code.Register;
74 import jdk.vm.ci.code.RegisterAttributes;
75 import jdk.vm.ci.code.RegisterConfig;
76 import jdk.vm.ci.code.StackSlot;
77 import jdk.vm.ci.code.TargetDescription;
78 import jdk.vm.ci.meta.AllocatableValue;
79 import jdk.vm.ci.meta.Constant;
80 import jdk.vm.ci.meta.JavaConstant;
81 import jdk.vm.ci.meta.JavaKind;
82 import jdk.vm.ci.meta.MetaAccessProvider;
83 import jdk.vm.ci.meta.PlatformKind;
84 import jdk.vm.ci.meta.Value;
85 import jdk.vm.ci.meta.ValueKind;
86
87 /**
88 * This class traverses the HIR instructions and generates LIR instructions from them.
89 */
90 public abstract class LIRGenerator implements LIRGeneratorTool {
91
92 public static class Options {
93 // @formatter:off
94 @Option(help = "Print HIR along side LIR as the latter is generated", type = OptionType.Debug)
95 public static final OptionKey<Boolean> PrintIRWithLIR = new OptionKey<>(false);
96 @Option(help = "The trace level for the LIR generator", type = OptionType.Debug)
97 public static final OptionKey<Integer> TraceLIRGeneratorLevel = new OptionKey<>(0);
98 // @formatter:on
99 }
100
101 private final LIRKindTool lirKindTool;
102
103 private final CodeGenProviders providers;
104
105 private AbstractBlockBase<?> currentBlock;
106
107 private LIRGenerationResult res;
108
109 protected final ArithmeticLIRGenerator arithmeticLIRGen;
110 private final MoveFactory moveFactory;
111
112 private final boolean printIrWithLir;
113 private final int traceLIRGeneratorLevel;
114
115 public LIRGenerator(LIRKindTool lirKindTool, ArithmeticLIRGenerator arithmeticLIRGen, MoveFactory moveFactory, CodeGenProviders providers, LIRGenerationResult res) {
116 this.lirKindTool = lirKindTool;
117 this.arithmeticLIRGen = arithmeticLIRGen;
118 this.res = res;
119 this.providers = providers;
120 OptionValues options = res.getLIR().getOptions();
121 this.printIrWithLir = !TTY.isSuppressed() && Options.PrintIRWithLIR.getValue(options);
122 this.traceLIRGeneratorLevel = TTY.isSuppressed() ? 0 : Options.TraceLIRGeneratorLevel.getValue(options);
123
124 assert arithmeticLIRGen.lirGen == null;
125 arithmeticLIRGen.lirGen = this;
126 this.moveFactory = moveFactory;
127 }
128
129 @Override
130 public ArithmeticLIRGeneratorTool getArithmetic() {
131 return arithmeticLIRGen;
132 }
133
134 @Override
135 public MoveFactory getMoveFactory() {
136 return moveFactory;
137 }
138
139 private MoveFactory spillMoveFactory;
140
141 @Override
142 public MoveFactory getSpillMoveFactory() {
143 if (spillMoveFactory == null) {
144 boolean verify = false;
145 assert (verify = true) == true;
146 if (verify) {
147 spillMoveFactory = new VerifyingMoveFactory(moveFactory);
148 } else {
149 spillMoveFactory = moveFactory;
150 }
151 }
152 return spillMoveFactory;
153 }
154
155 @Override
156 public LIRKind getValueKind(JavaKind javaKind) {
157 return LIRKind.fromJavaKind(target().arch, javaKind);
158 }
159
160 @Override
161 public TargetDescription target() {
162 return getCodeCache().getTarget();
163 }
164
165 @Override
166 public CodeGenProviders getProviders() {
167 return providers;
168 }
169
170 @Override
171 public MetaAccessProvider getMetaAccess() {
172 return providers.getMetaAccess();
173 }
174
175 @Override
176 public CodeCacheProvider getCodeCache() {
177 return providers.getCodeCache();
178 }
179
180 @Override
181 public ForeignCallsProvider getForeignCalls() {
182 return providers.getForeignCalls();
183 }
184
185 public LIRKindTool getLIRKindTool() {
186 return lirKindTool;
187 }
188
189 /**
190 * Hide {@link #nextVariable()} from other users.
191 */
192 public abstract static class VariableProvider {
193 private int numVariables;
194
195 public int numVariables() {
196 return numVariables;
197 }
198
199 private int nextVariable() {
200 return numVariables++;
201 }
202 }
203
204 @Override
205 public Variable newVariable(ValueKind<?> valueKind) {
206 return new Variable(valueKind, ((VariableProvider) res.getLIR()).nextVariable());
207 }
208
209 @Override
210 public RegisterConfig getRegisterConfig() {
211 return res.getRegisterConfig();
212 }
213
214 @Override
215 public RegisterAttributes attributes(Register register) {
216 return getRegisterConfig().getAttributesMap()[register.number];
217 }
218
219 @Override
220 public Variable emitMove(Value input) {
221 assert !(input instanceof Variable) : "Creating a copy of a variable via this method is not supported (and potentially a bug): " + input;
222 Variable result = newVariable(input.getValueKind());
223 emitMove(result, input);
224 return result;
225 }
226
227 @Override
228 public void emitMove(AllocatableValue dst, Value src) {
229 append(moveFactory.createMove(dst, src));
230 }
231
232 @Override
233 public Variable emitReadRegister(Register register, ValueKind<?> kind) {
234 return emitMove(register.asValue(kind));
235 }
236
237 @Override
238 public void emitWriteRegister(Register dst, Value src, ValueKind<?> kind) {
239 emitMove(dst.asValue(kind), src);
240 }
241
242 @Override
243 public void emitMoveConstant(AllocatableValue dst, Constant src) {
244 append(moveFactory.createLoad(dst, src));
245 }
246
247 @Override
248 public boolean canInlineConstant(Constant constant) {
249 return moveFactory.canInlineConstant(constant);
250 }
251
252 @Override
253 public boolean mayEmbedConstantLoad(Constant constant) {
254 return moveFactory.mayEmbedConstantLoad(constant);
255 }
256
257 @Override
258 public Value emitConstant(LIRKind kind, Constant constant) {
259 if (moveFactory.canInlineConstant(constant)) {
260 return new ConstantValue(toRegisterKind(kind), constant);
261 } else {
262 return emitLoadConstant(toRegisterKind(kind), constant);
263 }
264 }
265
266 @Override
267 public Value emitJavaConstant(JavaConstant constant) {
268 return emitConstant(getValueKind(constant.getJavaKind()), constant);
269 }
270
271 @Override
272 public AllocatableValue emitLoadConstant(ValueKind<?> kind, Constant constant) {
273 Variable result = newVariable(kind);
274 emitMoveConstant(result, constant);
275 return result;
276 }
277
278 @Override
279 public AllocatableValue asAllocatable(Value value) {
280 if (isAllocatableValue(value)) {
281 return asAllocatableValue(value);
282 } else if (isConstantValue(value)) {
283 return emitLoadConstant(value.getValueKind(), asConstant(value));
284 } else {
285 return emitMove(value);
286 }
287 }
288
289 @Override
290 public Variable load(Value value) {
291 if (!isVariable(value)) {
292 return emitMove(value);
293 }
294 return (Variable) value;
295 }
296
297 @Override
298 public Value loadNonConst(Value value) {
299 if (isConstantValue(value) && !moveFactory.canInlineConstant(asConstant(value))) {
300 return emitMove(value);
301 }
302 return value;
303 }
304
305 /**
306 * Determines if only oop maps are required for the code generated from the LIR.
307 */
308 @Override
309 public boolean needOnlyOopMaps() {
310 return false;
311 }
312
313 /**
314 * Gets the ABI specific operand used to return a value of a given kind from a method.
315 *
316 * @param javaKind the kind of value being returned
317 * @param valueKind the backend type of the value being returned
318 * @return the operand representing the ABI defined location used return a value of kind
319 * {@code kind}
320 */
321 @Override
322 public AllocatableValue resultOperandFor(JavaKind javaKind, ValueKind<?> valueKind) {
323 Register reg = getRegisterConfig().getReturnRegister(javaKind);
324 assert target().arch.canStoreValue(reg.getRegisterCategory(), valueKind.getPlatformKind()) : reg.getRegisterCategory() + " " + valueKind.getPlatformKind();
325 return reg.asValue(valueKind);
326 }
327
328 NodeSourcePosition currentPosition;
329
330 @Override
331 public void setSourcePosition(NodeSourcePosition position) {
332 currentPosition = position;
333 }
334
335 @Override
336 public <I extends LIRInstruction> I append(I op) {
337 LIR lir = res.getLIR();
338 if (printIrWithLir) {
339 TTY.println(op.toStringWithIdPrefix());
340 TTY.println();
341 }
342 assert LIRVerifier.verify(op);
343 ArrayList<LIRInstruction> lirForBlock = lir.getLIRforBlock(getCurrentBlock());
344 op.setPosition(currentPosition);
345 lirForBlock.add(op);
346 return op;
347 }
348
349 @Override
350 public boolean hasBlockEnd(AbstractBlockBase<?> block) {
351 ArrayList<LIRInstruction> ops = getResult().getLIR().getLIRforBlock(block);
352 if (ops.size() == 0) {
353 return false;
354 }
355 return ops.get(ops.size() - 1) instanceof BlockEndOp;
356 }
357
358 private final class BlockScopeImpl extends BlockScope {
359
360 private BlockScopeImpl(AbstractBlockBase<?> block) {
361 currentBlock = block;
362 }
363
364 private void doBlockStart() {
365 if (printIrWithLir) {
366 TTY.print(currentBlock.toString());
367 }
368
369 // set up the list of LIR instructions
370 assert res.getLIR().getLIRforBlock(currentBlock) == null : "LIR list already computed for this block";
371 res.getLIR().setLIRforBlock(currentBlock, new ArrayList<LIRInstruction>());
372
373 append(new LabelOp(new Label(currentBlock.getId()), currentBlock.isAligned()));
374
375 if (traceLIRGeneratorLevel >= 1) {
376 TTY.println("BEGIN Generating LIR for block B" + currentBlock.getId());
377 }
378 }
379
380 private void doBlockEnd() {
381 if (traceLIRGeneratorLevel >= 1) {
382 TTY.println("END Generating LIR for block B" + currentBlock.getId());
383 }
384
385 if (printIrWithLir) {
386 TTY.println();
387 }
388 currentBlock = null;
389 }
390
391 @Override
392 public AbstractBlockBase<?> getCurrentBlock() {
393 return currentBlock;
394 }
395
396 @Override
397 public void close() {
398 doBlockEnd();
399 }
400
401 }
402
403 @Override
404 public final BlockScope getBlockScope(AbstractBlockBase<?> block) {
405 BlockScopeImpl blockScope = new BlockScopeImpl(block);
406 blockScope.doBlockStart();
407 return blockScope;
408 }
409
410 private final class MatchScope implements DebugCloseable {
411
412 private MatchScope(AbstractBlockBase<?> block) {
413 currentBlock = block;
414 }
415
416 @Override
417 public void close() {
418 currentBlock = null;
419 }
420
421 }
422
423 public final DebugCloseable getMatchScope(AbstractBlockBase<?> block) {
424 MatchScope matchScope = new MatchScope(block);
425 return matchScope;
426 }
427
428 @Override
429 public void emitIncomingValues(Value[] params) {
430 ((LabelOp) res.getLIR().getLIRforBlock(getCurrentBlock()).get(0)).setIncomingValues(params);
431 }
432
433 @Override
434 public abstract void emitJump(LabelRef label);
435
436 @Override
437 public abstract void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueDestination, LabelRef falseDestination,
438 double trueDestinationProbability);
439
440 @Override
441 public abstract void emitOverflowCheckBranch(LabelRef overflow, LabelRef noOverflow, LIRKind cmpKind, double overflowProbability);
442
443 @Override
444 public abstract void emitIntegerTestBranch(Value left, Value right, LabelRef trueDestination, LabelRef falseDestination, double trueSuccessorProbability);
445
446 @Override
447 public abstract Variable emitConditionalMove(PlatformKind cmpKind, Value leftVal, Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue);
448
449 @Override
450 public abstract Variable emitIntegerTestMove(Value leftVal, Value right, Value trueValue, Value falseValue);
451
452 /**
453 * Emits the single call operation at the heart of generating LIR for a
454 * {@linkplain #emitForeignCall(ForeignCallLinkage, LIRFrameState, Value...) foreign call}.
455 */
456 protected abstract void emitForeignCallOp(ForeignCallLinkage linkage, Value result, Value[] arguments, Value[] temps, LIRFrameState info);
457
458 @Override
459 public Variable emitForeignCall(ForeignCallLinkage linkage, LIRFrameState frameState, Value... args) {
460 LIRFrameState state = null;
461 if (linkage.needsDebugInfo()) {
462 if (frameState != null) {
463 state = frameState;
464 } else {
465 assert needOnlyOopMaps();
466 state = new LIRFrameState(null, null, null);
467 }
468 }
469
470 // move the arguments into the correct location
471 CallingConvention linkageCc = linkage.getOutgoingCallingConvention();
472 res.getFrameMapBuilder().callsMethod(linkageCc);
473 assert linkageCc.getArgumentCount() == args.length : "argument count mismatch";
474 Value[] argLocations = new Value[args.length];
475 for (int i = 0; i < args.length; i++) {
476 Value arg = args[i];
477 AllocatableValue loc = linkageCc.getArgument(i);
478 emitMove(loc, arg);
479 argLocations[i] = loc;
480 }
481 res.setForeignCall(true);
482 emitForeignCallOp(linkage, linkageCc.getReturn(), argLocations, linkage.getTemporaries(), state);
483
484 if (isLegal(linkageCc.getReturn())) {
485 return emitMove(linkageCc.getReturn());
486 } else {
487 return null;
488 }
489 }
490
491 @Override
492 public void emitStrategySwitch(JavaConstant[] keyConstants, double[] keyProbabilities, LabelRef[] keyTargets, LabelRef defaultTarget, Variable value) {
493 SwitchStrategy strategy = SwitchStrategy.getBestStrategy(keyProbabilities, keyConstants, keyTargets);
494
495 int keyCount = keyConstants.length;
496 double minDensity = 1 / Math.sqrt(strategy.getAverageEffort());
497 Optional<Hasher> hasher = hasherFor(keyConstants, minDensity);
498 double hashTableSwitchDensity = hasher.map(h -> keyCount / (double) h.cardinality()).orElse(0d);
499 // The value range computation below may overflow, so compute it as a long.
500 long valueRange = (long) keyConstants[keyCount - 1].asInt() - (long) keyConstants[0].asInt() + 1;
501 double tableSwitchDensity = keyCount / (double) valueRange;
502
503 /*
504 * This heuristic tries to find a compromise between the effort for the best switch strategy
505 * and the density of a tableswitch. If the effort for the strategy is at least 4, then a
506 * tableswitch is preferred if better than a certain value that starts at 0.5 and lowers
507 * gradually with additional effort.
508 */
509 if (strategy.getAverageEffort() < 4d || (tableSwitchDensity < minDensity && hashTableSwitchDensity < minDensity)) {
510 emitStrategySwitch(strategy, value, keyTargets, defaultTarget);
511 } else {
512 if (hashTableSwitchDensity > tableSwitchDensity) {
513 Hasher h = hasher.get();
514 int cardinality = h.cardinality();
515 LabelRef[] targets = new LabelRef[cardinality];
516 JavaConstant[] keys = new JavaConstant[cardinality];
517 for (int i = 0; i < cardinality; i++) {
518 keys[i] = JavaConstant.INT_0;
519 targets[i] = defaultTarget;
520 }
521 for (int i = 0; i < keyCount; i++) {
522 int idx = h.hash(keyConstants[i].asInt());
523 keys[idx] = keyConstants[i];
524 targets[idx] = keyTargets[i];
525 }
526 emitHashTableSwitch(h, keys, defaultTarget, targets, value);
527 } else {
528 int minValue = keyConstants[0].asInt();
529 assert valueRange < Integer.MAX_VALUE;
530 LabelRef[] targets = new LabelRef[(int) valueRange];
531 for (int i = 0; i < valueRange; i++) {
532 targets[i] = defaultTarget;
533 }
534 for (int i = 0; i < keyCount; i++) {
535 targets[keyConstants[i].asInt() - minValue] = keyTargets[i];
536 }
537 emitTableSwitch(minValue, defaultTarget, targets, value);
538 }
539 }
540 }
541
542 @Override
543 public abstract void emitStrategySwitch(SwitchStrategy strategy, Variable key, LabelRef[] keyTargets, LabelRef defaultTarget);
544
545 protected abstract void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key);
546
547 @SuppressWarnings("unused")
548 protected Optional<Hasher> hasherFor(JavaConstant[] keyConstants, double minDensity) {
549 return Optional.empty();
550 }
551
552 @SuppressWarnings("unused")
553 protected void emitHashTableSwitch(Hasher hasher, JavaConstant[] keys, LabelRef defaultTarget, LabelRef[] targets, Value value) {
554 throw new UnsupportedOperationException(getClass().getSimpleName() + " doesn't support hash table switches");
555 }
556
557 @Override
558 public void beforeRegisterAllocation() {
559 }
560
561 /**
562 * Gets a garbage value for a given kind.
563 */
564 protected abstract JavaConstant zapValueForKind(PlatformKind kind);
565
566 @Override
567 public LIRKind getLIRKind(Stamp stamp) {
568 return stamp.getLIRKind(lirKindTool);
569 }
570
571 protected LIRKind getAddressKind(Value base, long displacement, Value index) {
572 if (LIRKind.isValue(base) && (index.equals(Value.ILLEGAL) || LIRKind.isValue(index))) {
573 return LIRKind.value(target().arch.getWordKind());
574 } else if (base.getValueKind() instanceof LIRKind && base.getValueKind(LIRKind.class).isReference(0) && displacement == 0L && index.equals(Value.ILLEGAL)) {
575 return LIRKind.reference(target().arch.getWordKind());
576 } else {
577 return LIRKind.unknownReference(target().arch.getWordKind());
578 }
579 }
580
581 @Override
582 public AbstractBlockBase<?> getCurrentBlock() {
583 return currentBlock;
584 }
585
586 @Override
587 public LIRGenerationResult getResult() {
588 return res;
589 }
590
591 @Override
592 public void emitBlackhole(Value operand) {
593 append(new StandardOp.BlackholeOp(operand));
594 }
595
596 @Override
597 public LIRInstruction createBenchmarkCounter(String name, String group, Value increment) {
598 throw GraalError.unimplemented();
599 }
600
601 @Override
602 public LIRInstruction createMultiBenchmarkCounter(String[] names, String[] groups, Value[] increments) {
603 throw GraalError.unimplemented();
604 }
605
606 @Override
607 public abstract ZapRegistersOp createZapRegisters(Register[] zappedRegisters, JavaConstant[] zapValues);
608
609 @Override
610 public ZapRegistersOp createZapRegisters() {
611 Register[] zappedRegisters = getResult().getFrameMap().getRegisterConfig().getAllocatableRegisters().toArray();
612 return createZapRegisters(zappedRegisters);
613 }
614
615 @Override
616 public ZapRegistersOp createZapRegisters(Register[] zappedRegisters) {
617 JavaConstant[] zapValues = new JavaConstant[zappedRegisters.length];
618 for (int i = 0; i < zappedRegisters.length; i++) {
619 PlatformKind kind = target().arch.getLargestStorableKind(zappedRegisters[i].getRegisterCategory());
620 zapValues[i] = zapValueForKind(kind);
621 }
622 return createZapRegisters(zappedRegisters, zapValues);
623 }
624
625 @Override
626 public abstract LIRInstruction createZapArgumentSpace(StackSlot[] zappedStack, JavaConstant[] zapValues);
627
628 @Override
629 public LIRInstruction zapArgumentSpace() {
630 List<StackSlot> slots = null;
631 for (AllocatableValue arg : res.getCallingConvention().getArguments()) {
632 if (isStackSlot(arg)) {
633 if (slots == null) {
634 slots = new ArrayList<>();
635 }
636 slots.add((StackSlot) arg);
637 } else {
638 assert !isVirtualStackSlot(arg);
639 }
640 }
641 if (slots == null) {
642 return null;
643 }
644 StackSlot[] zappedStack = slots.toArray(new StackSlot[slots.size()]);
645 JavaConstant[] zapValues = new JavaConstant[zappedStack.length];
646 for (int i = 0; i < zappedStack.length; i++) {
647 PlatformKind kind = zappedStack[i].getPlatformKind();
648 zapValues[i] = zapValueForKind(kind);
649 }
650 return createZapArgumentSpace(zappedStack, zapValues);
651 }
652 }