6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 26 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 27 28 ShenandoahWorkerScope::ShenandoahWorkerScope(WorkGang* workers, uint nworkers) : 29 _workers(workers), _n_workers(nworkers) { 30 _workers->update_active_workers(nworkers); 31 } 32 33 ShenandoahWorkerScope::~ShenandoahWorkerScope() { 34 assert(_workers->active_workers() == _n_workers, 35 "Active workers can not be changed within this scope"); 36 } 37 38 ShenandoahPushWorkerScope::ShenandoahPushWorkerScope(WorkGang* workers, uint nworkers) : 39 _workers(workers), _old_workers(workers->active_workers()), _n_workers(nworkers) { 40 _workers->update_active_workers(nworkers); 41 } 42 43 ShenandoahPushWorkerScope::~ShenandoahPushWorkerScope() { 44 assert(_workers->active_workers() == _n_workers, 45 "Active workers can not be changed within this scope"); 46 // Restore old worker value 47 _workers->update_active_workers(_old_workers); 48 } 49 50 AbstractGangWorker* ShenandoahWorkGang::install_worker(uint which) { 51 AbstractGangWorker* worker = WorkGang::install_worker(which); 52 if (_initialize_gclab) { 53 worker->gclab().initialize(true); 54 } 55 56 return worker; 57 } | 6 * published by the Free Software Foundation. 7 * 8 * This code is distributed in the hope that it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 11 * version 2 for more details (a copy is included in the LICENSE file that 12 * accompanied this code). 13 * 14 * You should have received a copy of the GNU General Public License version 15 * 2 along with this work; if not, write to the Free Software Foundation, 16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 17 * 18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 19 * or visit www.oracle.com if you need additional information or have any 20 * questions. 21 * 22 */ 23 24 #include "precompiled.hpp" 25 26 #include "gc/shenandoah/shenandoahPhaser.hpp" 27 #include "gc/shenandoah/shenandoahWorkGroup.hpp" 28 #include "runtime/semaphore.hpp" 29 30 struct ShenandoahWorkGangStats { 31 volatile size_t worker_spins; 32 volatile size_t worker_yields; 33 volatile size_t worker_arrived_during_spin; 34 volatile size_t worker_arrived_during_yield; 35 }; 36 37 static struct ShenandoahWorkGangStats _WG_stats; 38 39 ShenandoahWorkerScope::ShenandoahWorkerScope(ShenandoahWorkGang* workers, uint nworkers) : 40 _workers(workers), _n_workers(nworkers) { 41 _workers->update_active_workers(nworkers); 42 } 43 44 ShenandoahWorkerScope::~ShenandoahWorkerScope() { 45 assert(_workers->active_workers() == _n_workers, 46 "Active workers can not be changed within this scope"); 47 } 48 49 ShenandoahPushWorkerScope::ShenandoahPushWorkerScope(ShenandoahWorkGang* workers, uint nworkers) : 50 _workers(workers), _old_workers(workers->active_workers()), _n_workers(nworkers) { 51 _workers->update_active_workers(nworkers); 52 } 53 54 ShenandoahPushWorkerScope::~ShenandoahPushWorkerScope() { 55 assert(_workers->active_workers() == _n_workers, 56 "Active workers can not be changed within this scope"); 57 // Restore old worker value 58 _workers->update_active_workers(_old_workers); 59 } 60 61 62 ShenandoahWorkerSessionScope::ShenandoahWorkerSessionScope(ShenandoahWorkGang* workers, 63 uint nworkers, SessionType type) 64 : ShenandoahWorkerScope(workers, nworkers) { 65 _with_session = (type == Parallel) ? UseSessionForParallelWorkers : UseSessionForConcWorkers; 66 67 if (_with_session && os::active_processor_count() < (int)nworkers) { 68 log_info(gc, task)("Not enough CPUs available to run worker session."); 69 _with_session = false; 70 } 71 if (_with_session) { 72 workers->start_session(); 73 } 74 } 75 76 ShenandoahWorkerSessionScope::~ShenandoahWorkerSessionScope() { 77 if (_with_session) { 78 _workers->end_session(); 79 } 80 } 81 82 class ShenandoahWorkerPhaser; 83 class ShenandoahSpinPhaser; 84 85 #define INIT_UNPARKED_WORKERS (-3210) 86 87 class ShenandoahWorkerPhaser; 88 89 // Dispatcher that supports session 90 class ShenandoahWorkSessionDispatcher : public GangTaskDispatcher { 91 friend class ShenandoahWorkerPhaser; 92 93 private: 94 enum SessionState { 95 session_none, 96 session_started, 97 session_in_progress, 98 session_ending 99 }; 100 101 SessionState _state; 102 uint _session_workers; 103 104 // The task currently being dispatched to the GangWorkers. 105 AbstractGangTask* _task; 106 107 volatile uint _started; 108 volatile uint _not_finished; 109 110 // Semaphore used to start the GangWorkers. 111 Semaphore* _start_semaphore; 112 // Semaphore used to notify the coordinator that all workers are done. 113 Semaphore* _end_semaphore; 114 115 // Semaphore used to block task scheduler 116 Semaphore* _sched_semaphore; 117 118 ShenandoahWorkerPhaser* _worker_phaser; 119 ShenandoahSpinPhaser* _task_phaser; 120 121 volatile jint _unparked_workers; 122 123 public: 124 ShenandoahWorkSessionDispatcher(); 125 virtual ~ShenandoahWorkSessionDispatcher(); 126 127 // Distributes the task out to num_workers workers. 128 // Returns when the task has been completed by all workers. 129 void coordinator_execute_on_workers(AbstractGangTask* task, uint num_workers); 130 131 WorkData worker_wait_for_task(); 132 133 // Signal to the coordinator that the worker is done with the assigned task. 134 void worker_done_with_task(); 135 136 void session_start(); 137 void session_end(); 138 139 private: 140 SessionState get_state() const { return _state; } 141 void set_state(SessionState state) { _state = state; } 142 143 AbstractGangTask* get_task() const { return _task; } 144 void set_task(AbstractGangTask* task) { _task = task; } 145 146 // Notify by ShenandoahWorkerPhaser that all workers are arrived 147 void notify_workers_arrived(); 148 149 void do_normal_coordination(uint num_workers); 150 }; 151 152 153 ShenandoahWorkGang::ShenandoahWorkGang(const char* name, 154 uint workers, 155 bool are_GC_task_threads, 156 bool are_ConcurrentGC_threads) : 157 WorkGang(name, workers, are_GC_task_threads, are_ConcurrentGC_threads, 158 new ShenandoahWorkSessionDispatcher()), _initialize_gclab(false) { 159 } 160 161 AbstractGangWorker* ShenandoahWorkGang::install_worker(uint which) { 162 AbstractGangWorker* worker = WorkGang::install_worker(which); 163 if (_initialize_gclab) { 164 worker->gclab().initialize(true); 165 } 166 167 return worker; 168 } 169 170 void ShenandoahWorkGang::start_session() { 171 ShenandoahWorkSessionDispatcher* d 172 = (ShenandoahWorkSessionDispatcher*)dispatcher(); 173 d->session_start(); 174 } 175 176 class NoopTask : public AbstractGangTask { 177 public: 178 NoopTask() : AbstractGangTask("Noop Task") { } 179 void work(uint worker_id) { } 180 }; 181 182 void ShenandoahWorkGang::end_session() { 183 ShenandoahWorkSessionDispatcher* d 184 = (ShenandoahWorkSessionDispatcher*)dispatcher(); 185 d->session_end(); 186 // Use NoopTask to bring session state to session_none 187 NoopTask t; 188 run_task(&t); 189 } 190 191 192 // ShenandoahWorkSessionDispatcher Implementation 193 class ShenandoahSpinPhaser : public ShenandoahPhaser { 194 protected: 195 volatile jint _odd_phase; 196 volatile jint _even_phase; 197 198 public: 199 ShenandoahSpinPhaser(int parties) : ShenandoahPhaser(parties), 200 _odd_phase(0), _even_phase(0) { } 201 202 protected: 203 virtual int blockWaiters(int phase); 204 virtual void releaseWaiters(int phase); 205 206 // By default, never terminates 207 virtual bool onAdvance(int phase, int registeredParties) { 208 return false; 209 } 210 211 volatile jint* phase_wait_addr(int phase) { 212 return (phase % 2 == 0) ? &_even_phase : &_odd_phase; 213 } 214 }; 215 216 217 #define SPIN_LOOP_COUNT 4096 218 #define SPIN_COUNT_BEFORE_YIELD 100 219 220 static void spin_loop() { 221 for (int index = 0; index < SPIN_LOOP_COUNT; index ++) { 222 SpinPause(); 223 } 224 } 225 226 int ShenandoahSpinPhaser::blockWaiters(int phase) { 227 volatile jint *addr = phase_wait_addr(phase); 228 int spin_count = 0; 229 230 size_t spin_loops = 0; 231 size_t yields = 0; 232 233 while (OrderAccess::load_acquire(addr) == 0) { 234 spin_loop(); 235 ++ spin_loops; 236 if (++ spin_count >= SPIN_COUNT_BEFORE_YIELD) { 237 os::naked_yield(); 238 ++ yields; 239 spin_count = 0; 240 } 241 } 242 243 if (PrintShenandoahWorkGangStats) { 244 Atomic::add(spin_loops, &_WG_stats.worker_spins); 245 Atomic::add(yields, &_WG_stats.worker_yields); 246 247 if (yields == 0) { 248 Atomic::inc(&_WG_stats.worker_arrived_during_spin); 249 } else { 250 Atomic::inc(&_WG_stats.worker_arrived_during_yield); 251 } 252 } 253 254 return phase; 255 } 256 257 void ShenandoahSpinPhaser::releaseWaiters(int phase) { 258 volatile jint *this_phase = (phase % 2 == 0) ? &_even_phase : &_odd_phase; 259 volatile jint *other_phase = (phase % 2 == 0) ? &_odd_phase : &_even_phase; 260 261 // Reset the other phase 262 *other_phase = 0; 263 264 OrderAccess::release_store_fence(this_phase, 1); 265 } 266 267 268 class ShenandoahWorkerPhaser : public ShenandoahSpinPhaser { 269 private: 270 volatile jint _terminated; 271 ShenandoahWorkSessionDispatcher* _dispatcher; 272 273 public: 274 ShenandoahWorkerPhaser(int parties, ShenandoahWorkSessionDispatcher* dispatcher) 275 : ShenandoahSpinPhaser(parties), _terminated(0), _dispatcher(dispatcher) { 276 } 277 278 void set_terminated() { 279 OrderAccess::release_store_fence(&_terminated, 1); 280 } 281 282 protected: 283 // int blockWaiters(int phase); 284 285 bool onAdvance(int phase, int registeredParties); 286 }; 287 288 bool ShenandoahWorkerPhaser::onAdvance(int phase, int registeredParties) { 289 jint terminated = OrderAccess::load_acquire(&_terminated); 290 // Termination 291 if (terminated == 1 || registeredParties == 0) return true; 292 293 _dispatcher->notify_workers_arrived(); 294 295 return false; 296 } 297 298 void ShenandoahWorkGang::print_stats() { 299 assert(PrintShenandoahWorkGangStats, "Not enabled"); 300 tty->print_cr("ShenandoahWorkGang Session Statistics:"); 301 if (_WG_stats.worker_arrived_during_spin == 0 && 302 _WG_stats.worker_arrived_during_yield == 0) { 303 tty->print_cr("No sessions"); 304 } else { 305 tty->print_cr("\tWorkers: Spins ( " SIZE_FORMAT_W(10) ") Yields (" SIZE_FORMAT_W(10) ")", 306 _WG_stats.worker_spins, _WG_stats.worker_yields); 307 tty->print_cr("\t Arrived during: Spins ( " SIZE_FORMAT_W(10) ") Yields (" SIZE_FORMAT_W(10) ")", 308 _WG_stats.worker_arrived_during_spin, _WG_stats.worker_arrived_during_yield); 309 } 310 } 311 312 ShenandoahWorkSessionDispatcher::ShenandoahWorkSessionDispatcher() : 313 _task(NULL), _started(0), _not_finished(0), 314 _start_semaphore(new Semaphore()), _end_semaphore(new Semaphore()), 315 _sched_semaphore(new Semaphore()), 316 _state(session_none), _session_workers(0), _worker_phaser(NULL), 317 _task_phaser(NULL), _unparked_workers(INIT_UNPARKED_WORKERS) { 318 } 319 320 ShenandoahWorkSessionDispatcher::~ShenandoahWorkSessionDispatcher() { 321 if (_worker_phaser != NULL) { 322 delete _worker_phaser; 323 } 324 if (_task_phaser != NULL) { 325 delete _task_phaser; 326 } 327 } 328 329 void ShenandoahWorkSessionDispatcher::notify_workers_arrived() { 330 _sched_semaphore->signal(); 331 } 332 333 void ShenandoahWorkSessionDispatcher::coordinator_execute_on_workers(AbstractGangTask* task, uint num_workers) { 334 SessionState state = get_state(); 335 336 _not_finished = num_workers; 337 if (state == session_none) { 338 set_task(task); 339 do_normal_coordination(num_workers); 340 } else if (state == session_started) { 341 _session_workers = num_workers; 342 343 // Recreate phasers to avoid overflowing phases 344 if (_worker_phaser != NULL) { 345 assert(_task_phaser != NULL, "Sanity"); 346 delete _worker_phaser; 347 delete _task_phaser; 348 } 349 350 _worker_phaser = new ShenandoahWorkerPhaser(num_workers, this); 351 _task_phaser = new ShenandoahSpinPhaser(num_workers + 1); 352 353 set_task(task); 354 set_state(session_in_progress); 355 356 assert(_unparked_workers == 0 || _unparked_workers == INIT_UNPARKED_WORKERS, 357 "Should have parked all workers before session transition"); 358 _unparked_workers = num_workers; 359 360 assert(_started == 0, "nothing should have started"); 361 OrderAccess::fence(); 362 363 _start_semaphore->signal(num_workers); 364 _sched_semaphore->wait(); 365 _started = 0; 366 set_task(NULL); 367 } else { 368 assert(state == session_in_progress || state == session_ending, "What else?"); 369 assert(num_workers == _session_workers, "must not change workers"); 370 assert(!_worker_phaser->is_terminated(),"must not have been terminated"); 371 372 if (state == session_ending) { 373 // Ending a session. Transition blocking primitive from phaser 374 // to semaphore. 375 // Make sure that all workers have arrived before state tansition. 376 while (_task_phaser->get_arrived_parties() < (int)_session_workers) { 377 spin_loop(); 378 } 379 380 assert(_task_phaser->get_arrived_parties() == (int)_session_workers, 381 "only this thread has yet to arrive"); 382 set_state(session_none); 383 } 384 385 set_task(task); 386 OrderAccess::fence(); 387 388 _task_phaser->arrive(); 389 if (state != session_ending) { 390 _sched_semaphore->wait(); 391 _started = 0; 392 set_task(NULL); 393 } else { 394 // after this, workers will be blocked by semaphore 395 _worker_phaser->set_terminated(); 396 _end_semaphore->wait(); 397 assert(_not_finished == 0, "%d not finished workers?", _not_finished); 398 _started = 0; 399 set_task(NULL); 400 } 401 } 402 } 403 404 WorkData ShenandoahWorkSessionDispatcher::worker_wait_for_task() { 405 SessionState state = get_state(); 406 407 if (state == session_none) { 408 // Wait for the coordinator to dispatch a task. 409 if (_unparked_workers != INIT_UNPARKED_WORKERS && ShenandoahWorkGang::allow_session()) { 410 jint n = Atomic::add(-1, &_unparked_workers); 411 assert(n >= 0, "must be"); 412 } 413 414 _start_semaphore->wait(); 415 } else { 416 // Running inside a hot session, just wait for next task 417 _task_phaser->arriveAndAwaitAdvance(); 418 } 419 420 421 uint num_started = (uint) Atomic::add(1, (volatile jint*)&_started); 422 // Subtract one to get a zero-indexed worker id. 423 uint worker_id = num_started - 1; 424 425 AbstractGangTask* task = get_task(); 426 assert(task != NULL, "No task"); 427 return WorkData(task, worker_id); 428 } 429 430 void ShenandoahWorkSessionDispatcher::worker_done_with_task() { 431 SessionState state = get_state(); 432 if (state == session_none) { 433 uint not_finished = (uint) Atomic::add(-1, (volatile jint*)&_not_finished); 434 435 // The last worker signals to the coordinator that all work is completed. 436 if (not_finished == 0) { 437 _end_semaphore->signal(); 438 } 439 } else { 440 _worker_phaser->arriveAndAwaitAdvance(); 441 } 442 } 443 444 void ShenandoahWorkSessionDispatcher::session_start() { 445 assert(ShenandoahWorkGang::allow_session(), "Session is not allowed"); 446 assert(get_state() == session_none, "Wrong state"); 447 448 // Starting a session involves semaphore to phaser blocking primitive 449 // handoff. Make sure that all workers are blocked on semaphore, before 450 // the handoff. 451 // Setting session_started state too early can result arriving workers to park 452 // on wrong primitive. 453 while (OrderAccess::load_acquire(&_unparked_workers) > 0) { 454 spin_loop(); 455 } 456 457 set_state(session_started); 458 } 459 460 void ShenandoahWorkSessionDispatcher::session_end() { 461 assert(ShenandoahWorkGang::allow_session(), "Session is not allowed"); 462 assert(get_state() == session_in_progress, "Wrong state"); 463 set_state(session_ending); 464 } 465 466 467 void ShenandoahWorkSessionDispatcher::do_normal_coordination(uint num_workers) { 468 _not_finished = num_workers; 469 470 if (_unparked_workers != INIT_UNPARKED_WORKERS && ShenandoahWorkGang::allow_session()) { 471 Atomic::add((jint)num_workers, &_unparked_workers); 472 } else { 473 _unparked_workers = num_workers; 474 } 475 476 // Dispatch 'num_workers' number of tasks. 477 _start_semaphore->signal(num_workers); 478 479 // Wait for the last worker to signal the coordinator. 480 _end_semaphore->wait(); 481 482 // No workers are allowed to read the state variables after the coordinator has been signaled. 483 assert(_not_finished == 0, "%d not finished workers?", _not_finished); 484 _task = NULL; 485 _started = 0; 486 } |