1 /*
   2  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   3  *
   4  * This code is free software; you can redistribute it and/or modify it
   5  * under the terms of the GNU General Public License version 2 only, as
   6  * published by the Free Software Foundation.  Oracle designates this
   7  * particular file as subject to the "Classpath" exception as provided
   8  * by Oracle in the LICENSE file that accompanied this code.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  */
  24 
  25 /*
  26  * This file is available under and governed by the GNU General Public
  27  * License version 2 only, as published by the Free Software Foundation.
  28  * However, the following notice accompanied the original version of this
  29  * file:
  30  *
  31  * Written by Doug Lea with assistance from members of JCP JSR-166
  32  * Expert Group and released to the public domain, as explained at
  33  * http://creativecommons.org/publicdomain/zero/1.0/
  34  */
  35 
  36 package java.util;
  37 
  38 import java.util.concurrent.CountedCompleter;
  39 import java.util.concurrent.ForkJoinPool;
  40 import java.util.function.BinaryOperator;
  41 import java.util.function.DoubleBinaryOperator;
  42 import java.util.function.IntBinaryOperator;
  43 import java.util.function.LongBinaryOperator;
  44 
  45 /**
  46  * ForkJoin tasks to perform Arrays.parallelPrefix operations.
  47  *
  48  * @author Doug Lea
  49  * @since 1.8
  50  */
  51 class ArrayPrefixHelpers {
  52     private ArrayPrefixHelpers() {} // non-instantiable
  53 
  54     /*
  55      * Parallel prefix (aka cumulate, scan) task classes
  56      * are based loosely on Guy Blelloch's original
  57      * algorithm (http://www.cs.cmu.edu/~scandal/alg/scan.html):
  58      *  Keep dividing by two to threshold segment size, and then:
  59      *   Pass 1: Create tree of partial sums for each segment
  60      *   Pass 2: For each segment, cumulate with offset of left sibling
  61      *
  62      * This version improves performance within FJ framework mainly by
  63      * allowing the second pass of ready left-hand sides to proceed
  64      * even if some right-hand side first passes are still executing.
  65      * It also combines first and second pass for leftmost segment,
  66      * and skips the first pass for rightmost segment (whose result is
  67      * not needed for second pass).  It similarly manages to avoid
  68      * requiring that users supply an identity basis for accumulations
  69      * by tracking those segments/subtasks for which the first
  70      * existing element is used as base.
  71      *
  72      * Managing this relies on ORing some bits in the pendingCount for
  73      * phases/states: CUMULATE, SUMMED, and FINISHED. CUMULATE is the
  74      * main phase bit. When false, segments compute only their sum.
  75      * When true, they cumulate array elements. CUMULATE is set at
  76      * root at beginning of second pass and then propagated down. But
  77      * it may also be set earlier for subtrees with lo==0 (the left
  78      * spine of tree). SUMMED is a one bit join count. For leafs, it
  79      * is set when summed. For internal nodes, it becomes true when
  80      * one child is summed.  When the second child finishes summing,
  81      * we then moves up tree to trigger the cumulate phase. FINISHED
  82      * is also a one bit join count. For leafs, it is set when
  83      * cumulated. For internal nodes, it becomes true when one child
  84      * is cumulated.  When the second child finishes cumulating, it
  85      * then moves up tree, completing at the root.
  86      *
  87      * To better exploit locality and reduce overhead, the compute
  88      * method loops starting with the current task, moving if possible
  89      * to one of its subtasks rather than forking.
  90      *
  91      * As usual for this sort of utility, there are 4 versions, that
  92      * are simple copy/paste/adapt variants of each other.  (The
  93      * double and int versions differ from long version solely by
  94      * replacing "long" (with case-matching)).
  95      */
  96 
  97     // see above
  98     static final int CUMULATE = 1;
  99     static final int SUMMED   = 2;
 100     static final int FINISHED = 4;
 101 
 102     /** The smallest subtask array partition size to use as threshold */
 103     static final int MIN_PARTITION = 16;
 104 
 105     static final class CumulateTask<T> extends CountedCompleter<Void> {
 106         final T[] array;
 107         final BinaryOperator<T> function;
 108         CumulateTask<T> left, right;
 109         T in, out;
 110         final int lo, hi, origin, fence, threshold;
 111 
 112         /** Root task constructor */
 113         public CumulateTask(CumulateTask<T> parent,
 114                             BinaryOperator<T> function,
 115                             T[] array, int lo, int hi) {
 116             super(parent);
 117             this.function = function; this.array = array;
 118             this.lo = this.origin = lo; this.hi = this.fence = hi;
 119             int p;
 120             this.threshold =
 121                 (p = (hi - lo) / (ForkJoinPool.getCommonPoolParallelism() << 3))
 122                 <= MIN_PARTITION ? MIN_PARTITION : p;
 123         }
 124 
 125         /** Subtask constructor */
 126         CumulateTask(CumulateTask<T> parent, BinaryOperator<T> function,
 127                      T[] array, int origin, int fence, int threshold,
 128                      int lo, int hi) {
 129             super(parent);
 130             this.function = function; this.array = array;
 131             this.origin = origin; this.fence = fence;
 132             this.threshold = threshold;
 133             this.lo = lo; this.hi = hi;
 134         }
 135 
 136         public final void compute() {
 137             final BinaryOperator<T> fn;
 138             final T[] a;
 139             if ((fn = this.function) == null || (a = this.array) == null)
 140                 throw new NullPointerException();    // hoist checks
 141             int th = threshold, org = origin, fnc = fence, l, h;
 142             CumulateTask<T> t = this;
 143             outer: while ((l = t.lo) >= 0 && (h = t.hi) <= a.length) {
 144                 if (h - l > th) {
 145                     CumulateTask<T> lt = t.left, rt = t.right, f;
 146                     if (lt == null) {                // first pass
 147                         int mid = (l + h) >>> 1;
 148                         f = rt = t.right =
 149                             new CumulateTask<T>(t, fn, a, org, fnc, th, mid, h);
 150                         t = lt = t.left =
 151                             new CumulateTask<T>(t, fn, a, org, fnc, th, l, mid);
 152                     }
 153                     else {                           // possibly refork
 154                         T pin = t.in;
 155                         lt.in = pin;
 156                         f = t = null;
 157                         if (rt != null) {
 158                             T lout = lt.out;
 159                             rt.in = (l == org ? lout :
 160                                      fn.apply(pin, lout));
 161                             for (int c;;) {
 162                                 if (((c = rt.getPendingCount()) & CUMULATE) != 0)
 163                                     break;
 164                                 if (rt.compareAndSetPendingCount(c, c|CUMULATE)){
 165                                     t = rt;
 166                                     break;
 167                                 }
 168                             }
 169                         }
 170                         for (int c;;) {
 171                             if (((c = lt.getPendingCount()) & CUMULATE) != 0)
 172                                 break;
 173                             if (lt.compareAndSetPendingCount(c, c|CUMULATE)) {
 174                                 if (t != null)
 175                                     f = t;
 176                                 t = lt;
 177                                 break;
 178                             }
 179                         }
 180                         if (t == null)
 181                             break;
 182                     }
 183                     if (f != null)
 184                         f.fork();
 185                 }
 186                 else {
 187                     int state; // Transition to sum, cumulate, or both
 188                     for (int b;;) {
 189                         if (((b = t.getPendingCount()) & FINISHED) != 0)
 190                             break outer;                      // already done
 191                         state = ((b & CUMULATE) != 0 ? FINISHED :
 192                                  (l > org) ? SUMMED : (SUMMED|FINISHED));
 193                         if (t.compareAndSetPendingCount(b, b|state))
 194                             break;
 195                     }
 196 
 197                     T sum;
 198                     if (state != SUMMED) {
 199                         int first;
 200                         if (l == org) {                       // leftmost; no in
 201                             sum = a[org];
 202                             first = org + 1;
 203                         }
 204                         else {
 205                             sum = t.in;
 206                             first = l;
 207                         }
 208                         for (int i = first; i < h; ++i)       // cumulate
 209                             a[i] = sum = fn.apply(sum, a[i]);
 210                     }
 211                     else if (h < fnc) {                       // skip rightmost
 212                         sum = a[l];
 213                         for (int i = l + 1; i < h; ++i)       // sum only
 214                             sum = fn.apply(sum, a[i]);
 215                     }
 216                     else
 217                         sum = t.in;
 218                     t.out = sum;
 219                     for (CumulateTask<T> par;;) {             // propagate
 220                         @SuppressWarnings("unchecked") CumulateTask<T> partmp
 221                             = (CumulateTask<T>)t.getCompleter();
 222                         if ((par = partmp) == null) {
 223                             if ((state & FINISHED) != 0)      // enable join
 224                                 t.quietlyComplete();
 225                             break outer;
 226                         }
 227                         int b = par.getPendingCount();
 228                         if ((b & state & FINISHED) != 0)
 229                             t = par;                          // both done
 230                         else if ((b & state & SUMMED) != 0) { // both summed
 231                             int nextState; CumulateTask<T> lt, rt;
 232                             if ((lt = par.left) != null &&
 233                                 (rt = par.right) != null) {
 234                                 T lout = lt.out;
 235                                 par.out = (rt.hi == fnc ? lout :
 236                                            fn.apply(lout, rt.out));
 237                             }
 238                             int refork = (((b & CUMULATE) == 0 &&
 239                                            par.lo == org) ? CUMULATE : 0);
 240                             if ((nextState = b|state|refork) == b ||
 241                                 par.compareAndSetPendingCount(b, nextState)) {
 242                                 state = SUMMED;               // drop finished
 243                                 t = par;
 244                                 if (refork != 0)
 245                                     par.fork();
 246                             }
 247                         }
 248                         else if (par.compareAndSetPendingCount(b, b|state))
 249                             break outer;                      // sib not ready
 250                     }
 251                 }
 252             }
 253         }
 254         @java.io.Serial
 255         private static final long serialVersionUID = 5293554502939613543L;
 256     }
 257 
 258     static final class LongCumulateTask extends CountedCompleter<Void> {
 259         final long[] array;
 260         final LongBinaryOperator function;
 261         LongCumulateTask left, right;
 262         long in, out;
 263         final int lo, hi, origin, fence, threshold;
 264 
 265         /** Root task constructor */
 266         public LongCumulateTask(LongCumulateTask parent,
 267                                 LongBinaryOperator function,
 268                                 long[] array, int lo, int hi) {
 269             super(parent);
 270             this.function = function; this.array = array;
 271             this.lo = this.origin = lo; this.hi = this.fence = hi;
 272             int p;
 273             this.threshold =
 274                 (p = (hi - lo) / (ForkJoinPool.getCommonPoolParallelism() << 3))
 275                 <= MIN_PARTITION ? MIN_PARTITION : p;
 276         }
 277 
 278         /** Subtask constructor */
 279         LongCumulateTask(LongCumulateTask parent, LongBinaryOperator function,
 280                          long[] array, int origin, int fence, int threshold,
 281                          int lo, int hi) {
 282             super(parent);
 283             this.function = function; this.array = array;
 284             this.origin = origin; this.fence = fence;
 285             this.threshold = threshold;
 286             this.lo = lo; this.hi = hi;
 287         }
 288 
 289         public final void compute() {
 290             final LongBinaryOperator fn;
 291             final long[] a;
 292             if ((fn = this.function) == null || (a = this.array) == null)
 293                 throw new NullPointerException();    // hoist checks
 294             int th = threshold, org = origin, fnc = fence, l, h;
 295             LongCumulateTask t = this;
 296             outer: while ((l = t.lo) >= 0 && (h = t.hi) <= a.length) {
 297                 if (h - l > th) {
 298                     LongCumulateTask lt = t.left, rt = t.right, f;
 299                     if (lt == null) {                // first pass
 300                         int mid = (l + h) >>> 1;
 301                         f = rt = t.right =
 302                             new LongCumulateTask(t, fn, a, org, fnc, th, mid, h);
 303                         t = lt = t.left =
 304                             new LongCumulateTask(t, fn, a, org, fnc, th, l, mid);
 305                     }
 306                     else {                           // possibly refork
 307                         long pin = t.in;
 308                         lt.in = pin;
 309                         f = t = null;
 310                         if (rt != null) {
 311                             long lout = lt.out;
 312                             rt.in = (l == org ? lout :
 313                                      fn.applyAsLong(pin, lout));
 314                             for (int c;;) {
 315                                 if (((c = rt.getPendingCount()) & CUMULATE) != 0)
 316                                     break;
 317                                 if (rt.compareAndSetPendingCount(c, c|CUMULATE)){
 318                                     t = rt;
 319                                     break;
 320                                 }
 321                             }
 322                         }
 323                         for (int c;;) {
 324                             if (((c = lt.getPendingCount()) & CUMULATE) != 0)
 325                                 break;
 326                             if (lt.compareAndSetPendingCount(c, c|CUMULATE)) {
 327                                 if (t != null)
 328                                     f = t;
 329                                 t = lt;
 330                                 break;
 331                             }
 332                         }
 333                         if (t == null)
 334                             break;
 335                     }
 336                     if (f != null)
 337                         f.fork();
 338                 }
 339                 else {
 340                     int state; // Transition to sum, cumulate, or both
 341                     for (int b;;) {
 342                         if (((b = t.getPendingCount()) & FINISHED) != 0)
 343                             break outer;                      // already done
 344                         state = ((b & CUMULATE) != 0 ? FINISHED :
 345                                  (l > org) ? SUMMED : (SUMMED|FINISHED));
 346                         if (t.compareAndSetPendingCount(b, b|state))
 347                             break;
 348                     }
 349 
 350                     long sum;
 351                     if (state != SUMMED) {
 352                         int first;
 353                         if (l == org) {                       // leftmost; no in
 354                             sum = a[org];
 355                             first = org + 1;
 356                         }
 357                         else {
 358                             sum = t.in;
 359                             first = l;
 360                         }
 361                         for (int i = first; i < h; ++i)       // cumulate
 362                             a[i] = sum = fn.applyAsLong(sum, a[i]);
 363                     }
 364                     else if (h < fnc) {                       // skip rightmost
 365                         sum = a[l];
 366                         for (int i = l + 1; i < h; ++i)       // sum only
 367                             sum = fn.applyAsLong(sum, a[i]);
 368                     }
 369                     else
 370                         sum = t.in;
 371                     t.out = sum;
 372                     for (LongCumulateTask par;;) {            // propagate
 373                         if ((par = (LongCumulateTask)t.getCompleter()) == null) {
 374                             if ((state & FINISHED) != 0)      // enable join
 375                                 t.quietlyComplete();
 376                             break outer;
 377                         }
 378                         int b = par.getPendingCount();
 379                         if ((b & state & FINISHED) != 0)
 380                             t = par;                          // both done
 381                         else if ((b & state & SUMMED) != 0) { // both summed
 382                             int nextState; LongCumulateTask lt, rt;
 383                             if ((lt = par.left) != null &&
 384                                 (rt = par.right) != null) {
 385                                 long lout = lt.out;
 386                                 par.out = (rt.hi == fnc ? lout :
 387                                            fn.applyAsLong(lout, rt.out));
 388                             }
 389                             int refork = (((b & CUMULATE) == 0 &&
 390                                            par.lo == org) ? CUMULATE : 0);
 391                             if ((nextState = b|state|refork) == b ||
 392                                 par.compareAndSetPendingCount(b, nextState)) {
 393                                 state = SUMMED;               // drop finished
 394                                 t = par;
 395                                 if (refork != 0)
 396                                     par.fork();
 397                             }
 398                         }
 399                         else if (par.compareAndSetPendingCount(b, b|state))
 400                             break outer;                      // sib not ready
 401                     }
 402                 }
 403             }
 404         }
 405         @java.io.Serial
 406         private static final long serialVersionUID = -5074099945909284273L;
 407     }
 408 
 409     static final class DoubleCumulateTask extends CountedCompleter<Void> {
 410         final double[] array;
 411         final DoubleBinaryOperator function;
 412         DoubleCumulateTask left, right;
 413         double in, out;
 414         final int lo, hi, origin, fence, threshold;
 415 
 416         /** Root task constructor */
 417         public DoubleCumulateTask(DoubleCumulateTask parent,
 418                                   DoubleBinaryOperator function,
 419                                   double[] array, int lo, int hi) {
 420             super(parent);
 421             this.function = function; this.array = array;
 422             this.lo = this.origin = lo; this.hi = this.fence = hi;
 423             int p;
 424             this.threshold =
 425                 (p = (hi - lo) / (ForkJoinPool.getCommonPoolParallelism() << 3))
 426                 <= MIN_PARTITION ? MIN_PARTITION : p;
 427         }
 428 
 429         /** Subtask constructor */
 430         DoubleCumulateTask(DoubleCumulateTask parent, DoubleBinaryOperator function,
 431                            double[] array, int origin, int fence, int threshold,
 432                            int lo, int hi) {
 433             super(parent);
 434             this.function = function; this.array = array;
 435             this.origin = origin; this.fence = fence;
 436             this.threshold = threshold;
 437             this.lo = lo; this.hi = hi;
 438         }
 439 
 440         public final void compute() {
 441             final DoubleBinaryOperator fn;
 442             final double[] a;
 443             if ((fn = this.function) == null || (a = this.array) == null)
 444                 throw new NullPointerException();    // hoist checks
 445             int th = threshold, org = origin, fnc = fence, l, h;
 446             DoubleCumulateTask t = this;
 447             outer: while ((l = t.lo) >= 0 && (h = t.hi) <= a.length) {
 448                 if (h - l > th) {
 449                     DoubleCumulateTask lt = t.left, rt = t.right, f;
 450                     if (lt == null) {                // first pass
 451                         int mid = (l + h) >>> 1;
 452                         f = rt = t.right =
 453                             new DoubleCumulateTask(t, fn, a, org, fnc, th, mid, h);
 454                         t = lt = t.left =
 455                             new DoubleCumulateTask(t, fn, a, org, fnc, th, l, mid);
 456                     }
 457                     else {                           // possibly refork
 458                         double pin = t.in;
 459                         lt.in = pin;
 460                         f = t = null;
 461                         if (rt != null) {
 462                             double lout = lt.out;
 463                             rt.in = (l == org ? lout :
 464                                      fn.applyAsDouble(pin, lout));
 465                             for (int c;;) {
 466                                 if (((c = rt.getPendingCount()) & CUMULATE) != 0)
 467                                     break;
 468                                 if (rt.compareAndSetPendingCount(c, c|CUMULATE)){
 469                                     t = rt;
 470                                     break;
 471                                 }
 472                             }
 473                         }
 474                         for (int c;;) {
 475                             if (((c = lt.getPendingCount()) & CUMULATE) != 0)
 476                                 break;
 477                             if (lt.compareAndSetPendingCount(c, c|CUMULATE)) {
 478                                 if (t != null)
 479                                     f = t;
 480                                 t = lt;
 481                                 break;
 482                             }
 483                         }
 484                         if (t == null)
 485                             break;
 486                     }
 487                     if (f != null)
 488                         f.fork();
 489                 }
 490                 else {
 491                     int state; // Transition to sum, cumulate, or both
 492                     for (int b;;) {
 493                         if (((b = t.getPendingCount()) & FINISHED) != 0)
 494                             break outer;                      // already done
 495                         state = ((b & CUMULATE) != 0 ? FINISHED :
 496                                  (l > org) ? SUMMED : (SUMMED|FINISHED));
 497                         if (t.compareAndSetPendingCount(b, b|state))
 498                             break;
 499                     }
 500 
 501                     double sum;
 502                     if (state != SUMMED) {
 503                         int first;
 504                         if (l == org) {                       // leftmost; no in
 505                             sum = a[org];
 506                             first = org + 1;
 507                         }
 508                         else {
 509                             sum = t.in;
 510                             first = l;
 511                         }
 512                         for (int i = first; i < h; ++i)       // cumulate
 513                             a[i] = sum = fn.applyAsDouble(sum, a[i]);
 514                     }
 515                     else if (h < fnc) {                       // skip rightmost
 516                         sum = a[l];
 517                         for (int i = l + 1; i < h; ++i)       // sum only
 518                             sum = fn.applyAsDouble(sum, a[i]);
 519                     }
 520                     else
 521                         sum = t.in;
 522                     t.out = sum;
 523                     for (DoubleCumulateTask par;;) {            // propagate
 524                         if ((par = (DoubleCumulateTask)t.getCompleter()) == null) {
 525                             if ((state & FINISHED) != 0)      // enable join
 526                                 t.quietlyComplete();
 527                             break outer;
 528                         }
 529                         int b = par.getPendingCount();
 530                         if ((b & state & FINISHED) != 0)
 531                             t = par;                          // both done
 532                         else if ((b & state & SUMMED) != 0) { // both summed
 533                             int nextState; DoubleCumulateTask lt, rt;
 534                             if ((lt = par.left) != null &&
 535                                 (rt = par.right) != null) {
 536                                 double lout = lt.out;
 537                                 par.out = (rt.hi == fnc ? lout :
 538                                            fn.applyAsDouble(lout, rt.out));
 539                             }
 540                             int refork = (((b & CUMULATE) == 0 &&
 541                                            par.lo == org) ? CUMULATE : 0);
 542                             if ((nextState = b|state|refork) == b ||
 543                                 par.compareAndSetPendingCount(b, nextState)) {
 544                                 state = SUMMED;               // drop finished
 545                                 t = par;
 546                                 if (refork != 0)
 547                                     par.fork();
 548                             }
 549                         }
 550                         else if (par.compareAndSetPendingCount(b, b|state))
 551                             break outer;                      // sib not ready
 552                     }
 553                 }
 554             }
 555         }
 556         @java.io.Serial
 557         private static final long serialVersionUID = -586947823794232033L;
 558     }
 559 
 560     static final class IntCumulateTask extends CountedCompleter<Void> {
 561         final int[] array;
 562         final IntBinaryOperator function;
 563         IntCumulateTask left, right;
 564         int in, out;
 565         final int lo, hi, origin, fence, threshold;
 566 
 567         /** Root task constructor */
 568         public IntCumulateTask(IntCumulateTask parent,
 569                                IntBinaryOperator function,
 570                                int[] array, int lo, int hi) {
 571             super(parent);
 572             this.function = function; this.array = array;
 573             this.lo = this.origin = lo; this.hi = this.fence = hi;
 574             int p;
 575             this.threshold =
 576                 (p = (hi - lo) / (ForkJoinPool.getCommonPoolParallelism() << 3))
 577                 <= MIN_PARTITION ? MIN_PARTITION : p;
 578         }
 579 
 580         /** Subtask constructor */
 581         IntCumulateTask(IntCumulateTask parent, IntBinaryOperator function,
 582                         int[] array, int origin, int fence, int threshold,
 583                         int lo, int hi) {
 584             super(parent);
 585             this.function = function; this.array = array;
 586             this.origin = origin; this.fence = fence;
 587             this.threshold = threshold;
 588             this.lo = lo; this.hi = hi;
 589         }
 590 
 591         public final void compute() {
 592             final IntBinaryOperator fn;
 593             final int[] a;
 594             if ((fn = this.function) == null || (a = this.array) == null)
 595                 throw new NullPointerException();    // hoist checks
 596             int th = threshold, org = origin, fnc = fence, l, h;
 597             IntCumulateTask t = this;
 598             outer: while ((l = t.lo) >= 0 && (h = t.hi) <= a.length) {
 599                 if (h - l > th) {
 600                     IntCumulateTask lt = t.left, rt = t.right, f;
 601                     if (lt == null) {                // first pass
 602                         int mid = (l + h) >>> 1;
 603                         f = rt = t.right =
 604                             new IntCumulateTask(t, fn, a, org, fnc, th, mid, h);
 605                         t = lt = t.left =
 606                             new IntCumulateTask(t, fn, a, org, fnc, th, l, mid);
 607                     }
 608                     else {                           // possibly refork
 609                         int pin = t.in;
 610                         lt.in = pin;
 611                         f = t = null;
 612                         if (rt != null) {
 613                             int lout = lt.out;
 614                             rt.in = (l == org ? lout :
 615                                      fn.applyAsInt(pin, lout));
 616                             for (int c;;) {
 617                                 if (((c = rt.getPendingCount()) & CUMULATE) != 0)
 618                                     break;
 619                                 if (rt.compareAndSetPendingCount(c, c|CUMULATE)){
 620                                     t = rt;
 621                                     break;
 622                                 }
 623                             }
 624                         }
 625                         for (int c;;) {
 626                             if (((c = lt.getPendingCount()) & CUMULATE) != 0)
 627                                 break;
 628                             if (lt.compareAndSetPendingCount(c, c|CUMULATE)) {
 629                                 if (t != null)
 630                                     f = t;
 631                                 t = lt;
 632                                 break;
 633                             }
 634                         }
 635                         if (t == null)
 636                             break;
 637                     }
 638                     if (f != null)
 639                         f.fork();
 640                 }
 641                 else {
 642                     int state; // Transition to sum, cumulate, or both
 643                     for (int b;;) {
 644                         if (((b = t.getPendingCount()) & FINISHED) != 0)
 645                             break outer;                      // already done
 646                         state = ((b & CUMULATE) != 0 ? FINISHED :
 647                                  (l > org) ? SUMMED : (SUMMED|FINISHED));
 648                         if (t.compareAndSetPendingCount(b, b|state))
 649                             break;
 650                     }
 651 
 652                     int sum;
 653                     if (state != SUMMED) {
 654                         int first;
 655                         if (l == org) {                       // leftmost; no in
 656                             sum = a[org];
 657                             first = org + 1;
 658                         }
 659                         else {
 660                             sum = t.in;
 661                             first = l;
 662                         }
 663                         for (int i = first; i < h; ++i)       // cumulate
 664                             a[i] = sum = fn.applyAsInt(sum, a[i]);
 665                     }
 666                     else if (h < fnc) {                       // skip rightmost
 667                         sum = a[l];
 668                         for (int i = l + 1; i < h; ++i)       // sum only
 669                             sum = fn.applyAsInt(sum, a[i]);
 670                     }
 671                     else
 672                         sum = t.in;
 673                     t.out = sum;
 674                     for (IntCumulateTask par;;) {            // propagate
 675                         if ((par = (IntCumulateTask)t.getCompleter()) == null) {
 676                             if ((state & FINISHED) != 0)      // enable join
 677                                 t.quietlyComplete();
 678                             break outer;
 679                         }
 680                         int b = par.getPendingCount();
 681                         if ((b & state & FINISHED) != 0)
 682                             t = par;                          // both done
 683                         else if ((b & state & SUMMED) != 0) { // both summed
 684                             int nextState; IntCumulateTask lt, rt;
 685                             if ((lt = par.left) != null &&
 686                                 (rt = par.right) != null) {
 687                                 int lout = lt.out;
 688                                 par.out = (rt.hi == fnc ? lout :
 689                                            fn.applyAsInt(lout, rt.out));
 690                             }
 691                             int refork = (((b & CUMULATE) == 0 &&
 692                                            par.lo == org) ? CUMULATE : 0);
 693                             if ((nextState = b|state|refork) == b ||
 694                                 par.compareAndSetPendingCount(b, nextState)) {
 695                                 state = SUMMED;               // drop finished
 696                                 t = par;
 697                                 if (refork != 0)
 698                                     par.fork();
 699                             }
 700                         }
 701                         else if (par.compareAndSetPendingCount(b, b|state))
 702                             break outer;                      // sib not ready
 703                     }
 704                 }
 705             }
 706         }
 707         @java.io.Serial
 708         private static final long serialVersionUID = 3731755594596840961L;
 709     }
 710 }