1 /* 2 * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/oopStorage.inline.hpp" 27 #include "gc/shared/oopStorageSet.hpp" 28 #include "logging/log.hpp" 29 #include "memory/iterator.hpp" 30 #include "memory/universe.hpp" 31 #include "oops/access.inline.hpp" 32 #include "oops/oop.inline.hpp" 33 #include "runtime/handles.inline.hpp" 34 #include "runtime/jniHandles.inline.hpp" 35 #include "runtime/mutexLocker.inline.hpp" 36 #include "runtime/thread.inline.hpp" 37 #include "utilities/align.hpp" 38 #include "utilities/debug.hpp" 39 40 static OopStorage* global_handles() { 41 return OopStorageSet::jni_global(); 42 } 43 44 static OopStorage* weak_global_handles() { 45 return OopStorageSet::jni_weak(); 46 } 47 48 // Serviceability agent support. 49 OopStorage* JNIHandles::_global_handles = NULL; 50 OopStorage* JNIHandles::_weak_global_handles = NULL; 51 52 void jni_handles_init() { 53 JNIHandles::_global_handles = global_handles(); 54 JNIHandles::_weak_global_handles = weak_global_handles(); 55 } 56 57 58 jobject JNIHandles::make_local(oop obj) { 59 if (obj == NULL) { 60 return NULL; // ignore null handles 61 } else { 62 Thread* thread = Thread::current(); 63 assert(oopDesc::is_oop(obj), "not an oop"); 64 assert(!current_thread_in_native(), "must not be in native"); 65 return thread->active_handles()->allocate_handle(obj); 66 } 67 } 68 69 70 // optimized versions 71 72 jobject JNIHandles::make_local(Thread* thread, oop obj) { 73 if (obj == NULL) { 74 return NULL; // ignore null handles 75 } else { 76 assert(oopDesc::is_oop(obj), "not an oop"); 77 assert(thread->is_Java_thread(), "not a Java thread"); 78 assert(!current_thread_in_native(), "must not be in native"); 79 return thread->active_handles()->allocate_handle(obj); 80 } 81 } 82 83 84 jobject JNIHandles::make_local(JNIEnv* env, oop obj) { 85 if (obj == NULL) { 86 return NULL; // ignore null handles 87 } else { 88 JavaThread* thread = JavaThread::thread_from_jni_environment(env); 89 assert(oopDesc::is_oop(obj), "not an oop"); 90 assert(!current_thread_in_native(), "must not be in native"); 91 return thread->active_handles()->allocate_handle(obj); 92 } 93 } 94 95 96 static void report_handle_allocation_failure(AllocFailType alloc_failmode, 97 const char* handle_kind) { 98 if (alloc_failmode == AllocFailStrategy::EXIT_OOM) { 99 // Fake size value, since we don't know the min allocation size here. 100 vm_exit_out_of_memory(sizeof(oop), OOM_MALLOC_ERROR, 101 "Cannot create %s JNI handle", handle_kind); 102 } else { 103 assert(alloc_failmode == AllocFailStrategy::RETURN_NULL, "invariant"); 104 } 105 } 106 107 jobject JNIHandles::make_global(Handle obj, AllocFailType alloc_failmode) { 108 assert(!Universe::heap()->is_gc_active(), "can't extend the root set during GC"); 109 assert(!current_thread_in_native(), "must not be in native"); 110 jobject res = NULL; 111 if (!obj.is_null()) { 112 // ignore null handles 113 assert(oopDesc::is_oop(obj()), "not an oop"); 114 oop* ptr = global_handles()->allocate(); 115 // Return NULL on allocation failure. 116 if (ptr != NULL) { 117 assert(*ptr == NULL, "invariant"); 118 NativeAccess<>::oop_store(ptr, obj()); 119 res = reinterpret_cast<jobject>(ptr); 120 } else { 121 report_handle_allocation_failure(alloc_failmode, "global"); 122 } 123 } 124 125 return res; 126 } 127 128 129 jobject JNIHandles::make_weak_global(Handle obj, AllocFailType alloc_failmode) { 130 assert(!Universe::heap()->is_gc_active(), "can't extend the root set during GC"); 131 assert(!current_thread_in_native(), "must not be in native"); 132 jobject res = NULL; 133 if (!obj.is_null()) { 134 // ignore null handles 135 assert(oopDesc::is_oop(obj()), "not an oop"); 136 oop* ptr = weak_global_handles()->allocate(); 137 // Return NULL on allocation failure. 138 if (ptr != NULL) { 139 assert(*ptr == NULL, "invariant"); 140 NativeAccess<ON_PHANTOM_OOP_REF>::oop_store(ptr, obj()); 141 char* tptr = reinterpret_cast<char*>(ptr) + weak_tag_value; 142 res = reinterpret_cast<jobject>(tptr); 143 } else { 144 report_handle_allocation_failure(alloc_failmode, "weak global"); 145 } 146 } 147 return res; 148 } 149 150 // Resolve some erroneous cases to NULL, rather than treating them as 151 // possibly unchecked errors. In particular, deleted handles are 152 // treated as NULL (though a deleted and later reallocated handle 153 // isn't detected). 154 oop JNIHandles::resolve_external_guard(jobject handle) { 155 oop result = NULL; 156 if (handle != NULL) { 157 result = resolve_impl<DECORATORS_NONE, true /* external_guard */>(handle); 158 } 159 return result; 160 } 161 162 bool JNIHandles::is_global_weak_cleared(jweak handle) { 163 assert(handle != NULL, "precondition"); 164 assert(is_jweak(handle), "not a weak handle"); 165 oop* oop_ptr = jweak_ptr(handle); 166 oop value = NativeAccess<ON_PHANTOM_OOP_REF | AS_NO_KEEPALIVE>::oop_load(oop_ptr); 167 return value == NULL; 168 } 169 170 void JNIHandles::destroy_global(jobject handle) { 171 if (handle != NULL) { 172 assert(!is_jweak(handle), "wrong method for detroying jweak"); 173 oop* oop_ptr = jobject_ptr(handle); 174 NativeAccess<>::oop_store(oop_ptr, (oop)NULL); 175 global_handles()->release(oop_ptr); 176 } 177 } 178 179 180 void JNIHandles::destroy_weak_global(jobject handle) { 181 if (handle != NULL) { 182 assert(is_jweak(handle), "JNI handle not jweak"); 183 oop* oop_ptr = jweak_ptr(handle); 184 NativeAccess<ON_PHANTOM_OOP_REF>::oop_store(oop_ptr, (oop)NULL); 185 weak_global_handles()->release(oop_ptr); 186 } 187 } 188 189 190 void JNIHandles::oops_do(OopClosure* f) { 191 global_handles()->oops_do(f); 192 } 193 194 195 void JNIHandles::weak_oops_do(BoolObjectClosure* is_alive, OopClosure* f) { 196 weak_global_handles()->weak_oops_do(is_alive, f); 197 } 198 199 200 void JNIHandles::weak_oops_do(OopClosure* f) { 201 weak_global_handles()->weak_oops_do(f); 202 } 203 204 205 inline bool is_storage_handle(const OopStorage* storage, const oop* ptr) { 206 return storage->allocation_status(ptr) == OopStorage::ALLOCATED_ENTRY; 207 } 208 209 210 jobjectRefType JNIHandles::handle_type(Thread* thread, jobject handle) { 211 assert(handle != NULL, "precondition"); 212 jobjectRefType result = JNIInvalidRefType; 213 if (is_jweak(handle)) { 214 if (is_storage_handle(weak_global_handles(), jweak_ptr(handle))) { 215 result = JNIWeakGlobalRefType; 216 } 217 } else { 218 switch (global_handles()->allocation_status(jobject_ptr(handle))) { 219 case OopStorage::ALLOCATED_ENTRY: 220 result = JNIGlobalRefType; 221 break; 222 223 case OopStorage::UNALLOCATED_ENTRY: 224 break; // Invalid global handle 225 226 case OopStorage::INVALID_ENTRY: 227 // Not in global storage. Might be a local handle. 228 if (is_local_handle(thread, handle) || 229 (thread->is_Java_thread() && 230 is_frame_handle((JavaThread*)thread, handle))) { 231 result = JNILocalRefType; 232 } 233 break; 234 235 default: 236 ShouldNotReachHere(); 237 } 238 } 239 return result; 240 } 241 242 243 bool JNIHandles::is_local_handle(Thread* thread, jobject handle) { 244 assert(handle != NULL, "precondition"); 245 JNIHandleBlock* block = thread->active_handles(); 246 247 // Look back past possible native calls to jni_PushLocalFrame. 248 while (block != NULL) { 249 if (block->chain_contains(handle)) { 250 return true; 251 } 252 block = block->pop_frame_link(); 253 } 254 return false; 255 } 256 257 258 // Determine if the handle is somewhere in the current thread's stack. 259 // We easily can't isolate any particular stack frame the handle might 260 // come from, so we'll check the whole stack. 261 262 bool JNIHandles::is_frame_handle(JavaThread* thr, jobject handle) { 263 assert(handle != NULL, "precondition"); 264 // If there is no java frame, then this must be top level code, such 265 // as the java command executable, in which case, this type of handle 266 // is not permitted. 267 return (thr->has_last_Java_frame() && 268 (void*)handle < (void*)thr->stack_base() && 269 (void*)handle >= (void*)thr->last_Java_sp()); 270 } 271 272 273 bool JNIHandles::is_global_handle(jobject handle) { 274 assert(handle != NULL, "precondition"); 275 return !is_jweak(handle) && is_storage_handle(global_handles(), jobject_ptr(handle)); 276 } 277 278 279 bool JNIHandles::is_weak_global_handle(jobject handle) { 280 assert(handle != NULL, "precondition"); 281 return is_jweak(handle) && is_storage_handle(weak_global_handles(), jweak_ptr(handle)); 282 } 283 284 size_t JNIHandles::global_handle_memory_usage() { 285 return global_handles()->total_memory_usage(); 286 } 287 288 size_t JNIHandles::weak_global_handle_memory_usage() { 289 return weak_global_handles()->total_memory_usage(); 290 } 291 292 293 // We assume this is called at a safepoint: no lock is needed. 294 void JNIHandles::print_on(outputStream* st) { 295 assert(SafepointSynchronize::is_at_safepoint(), "must be at safepoint"); 296 297 st->print_cr("JNI global refs: " SIZE_FORMAT ", weak refs: " SIZE_FORMAT, 298 global_handles()->allocation_count(), 299 weak_global_handles()->allocation_count()); 300 st->cr(); 301 st->flush(); 302 } 303 304 void JNIHandles::print() { print_on(tty); } 305 306 class VerifyJNIHandles: public OopClosure { 307 public: 308 virtual void do_oop(oop* root) { 309 guarantee(oopDesc::is_oop_or_null(RawAccess<>::oop_load(root)), "Invalid oop"); 310 } 311 virtual void do_oop(narrowOop* root) { ShouldNotReachHere(); } 312 }; 313 314 void JNIHandles::verify() { 315 VerifyJNIHandles verify_handle; 316 317 oops_do(&verify_handle); 318 weak_oops_do(&verify_handle); 319 } 320 321 // This method is implemented here to avoid circular includes between 322 // jniHandles.hpp and thread.hpp. 323 bool JNIHandles::current_thread_in_native() { 324 Thread* thread = Thread::current(); 325 return (thread->is_Java_thread() && 326 JavaThread::current()->thread_state() == _thread_in_native); 327 } 328 329 330 int JNIHandleBlock::_blocks_allocated = 0; 331 JNIHandleBlock* JNIHandleBlock::_block_free_list = NULL; 332 #ifndef PRODUCT 333 JNIHandleBlock* JNIHandleBlock::_block_list = NULL; 334 #endif 335 336 static inline bool is_tagged_free_list(uintptr_t value) { 337 return (value & 1u) != 0; 338 } 339 340 static inline uintptr_t tag_free_list(uintptr_t value) { 341 return value | 1u; 342 } 343 344 static inline uintptr_t untag_free_list(uintptr_t value) { 345 return value & ~(uintptr_t)1u; 346 } 347 348 // There is a freelist of handles running through the JNIHandleBlock 349 // with a tagged next pointer, distinguishing these next pointers from 350 // oops. The freelist handling currently relies on the size of oops 351 // being the same as a native pointer. If this ever changes, then 352 // this freelist handling must change too. 353 STATIC_ASSERT(sizeof(oop) == sizeof(uintptr_t)); 354 355 #ifdef ASSERT 356 void JNIHandleBlock::zap() { 357 // Zap block values 358 _top = 0; 359 for (int index = 0; index < block_size_in_oops; index++) { 360 // NOT using Access here; just bare clobbering to NULL, since the 361 // block no longer contains valid oops. 362 _handles[index] = 0; 363 } 364 } 365 #endif // ASSERT 366 367 JNIHandleBlock* JNIHandleBlock::allocate_block(Thread* thread) { 368 assert(thread == NULL || thread == Thread::current(), "sanity check"); 369 JNIHandleBlock* block; 370 // Check the thread-local free list for a block so we don't 371 // have to acquire a mutex. 372 if (thread != NULL && thread->free_handle_block() != NULL) { 373 block = thread->free_handle_block(); 374 thread->set_free_handle_block(block->_next); 375 } 376 else { 377 // locking with safepoint checking introduces a potential deadlock: 378 // - we would hold JNIHandleBlockFreeList_lock and then Threads_lock 379 // - another would hold Threads_lock (jni_AttachCurrentThread) and then 380 // JNIHandleBlockFreeList_lock (JNIHandleBlock::allocate_block) 381 MutexLocker ml(JNIHandleBlockFreeList_lock, 382 Mutex::_no_safepoint_check_flag); 383 if (_block_free_list == NULL) { 384 // Allocate new block 385 block = new JNIHandleBlock(); 386 _blocks_allocated++; 387 block->zap(); 388 #ifndef PRODUCT 389 // Link new block to list of all allocated blocks 390 block->_block_list_link = _block_list; 391 _block_list = block; 392 #endif 393 } else { 394 // Get block from free list 395 block = _block_free_list; 396 _block_free_list = _block_free_list->_next; 397 } 398 } 399 block->_top = 0; 400 block->_next = NULL; 401 block->_pop_frame_link = NULL; 402 block->_planned_capacity = block_size_in_oops; 403 // _last, _free_list & _allocate_before_rebuild initialized in allocate_handle 404 debug_only(block->_last = NULL); 405 debug_only(block->_free_list = NULL); 406 debug_only(block->_allocate_before_rebuild = -1); 407 return block; 408 } 409 410 411 void JNIHandleBlock::release_block(JNIHandleBlock* block, Thread* thread) { 412 assert(thread == NULL || thread == Thread::current(), "sanity check"); 413 JNIHandleBlock* pop_frame_link = block->pop_frame_link(); 414 // Put returned block at the beginning of the thread-local free list. 415 // Note that if thread == NULL, we use it as an implicit argument that 416 // we _don't_ want the block to be kept on the free_handle_block. 417 // See for instance JavaThread::exit(). 418 if (thread != NULL ) { 419 block->zap(); 420 JNIHandleBlock* freelist = thread->free_handle_block(); 421 block->_pop_frame_link = NULL; 422 thread->set_free_handle_block(block); 423 424 // Add original freelist to end of chain 425 if ( freelist != NULL ) { 426 while ( block->_next != NULL ) block = block->_next; 427 block->_next = freelist; 428 } 429 block = NULL; 430 } 431 if (block != NULL) { 432 // Return blocks to free list 433 // locking with safepoint checking introduces a potential deadlock: 434 // - we would hold JNIHandleBlockFreeList_lock and then Threads_lock 435 // - another would hold Threads_lock (jni_AttachCurrentThread) and then 436 // JNIHandleBlockFreeList_lock (JNIHandleBlock::allocate_block) 437 MutexLocker ml(JNIHandleBlockFreeList_lock, 438 Mutex::_no_safepoint_check_flag); 439 while (block != NULL) { 440 block->zap(); 441 JNIHandleBlock* next = block->_next; 442 block->_next = _block_free_list; 443 _block_free_list = block; 444 block = next; 445 } 446 } 447 if (pop_frame_link != NULL) { 448 // As a sanity check we release blocks pointed to by the pop_frame_link. 449 // This should never happen (only if PopLocalFrame is not called the 450 // correct number of times). 451 release_block(pop_frame_link, thread); 452 } 453 } 454 455 456 void JNIHandleBlock::oops_do(OopClosure* f) { 457 JNIHandleBlock* current_chain = this; 458 // Iterate over chain of blocks, followed by chains linked through the 459 // pop frame links. 460 while (current_chain != NULL) { 461 for (JNIHandleBlock* current = current_chain; current != NULL; 462 current = current->_next) { 463 assert(current == current_chain || current->pop_frame_link() == NULL, 464 "only blocks first in chain should have pop frame link set"); 465 for (int index = 0; index < current->_top; index++) { 466 uintptr_t* addr = &(current->_handles)[index]; 467 uintptr_t value = *addr; 468 // traverse heap pointers only, not deleted handles or free list 469 // pointers 470 if (value != 0 && !is_tagged_free_list(value)) { 471 oop* root = (oop*)addr; 472 f->do_oop(root); 473 } 474 } 475 // the next handle block is valid only if current block is full 476 if (current->_top < block_size_in_oops) { 477 break; 478 } 479 } 480 current_chain = current_chain->pop_frame_link(); 481 } 482 } 483 484 485 jobject JNIHandleBlock::allocate_handle(oop obj) { 486 assert(Universe::heap()->is_in(obj), "sanity check"); 487 if (_top == 0) { 488 // This is the first allocation or the initial block got zapped when 489 // entering a native function. If we have any following blocks they are 490 // not valid anymore. 491 for (JNIHandleBlock* current = _next; current != NULL; 492 current = current->_next) { 493 assert(current->_last == NULL, "only first block should have _last set"); 494 assert(current->_free_list == NULL, 495 "only first block should have _free_list set"); 496 if (current->_top == 0) { 497 // All blocks after the first clear trailing block are already cleared. 498 #ifdef ASSERT 499 for (current = current->_next; current != NULL; current = current->_next) { 500 assert(current->_top == 0, "trailing blocks must already be cleared"); 501 } 502 #endif 503 break; 504 } 505 current->_top = 0; 506 current->zap(); 507 } 508 // Clear initial block 509 _free_list = NULL; 510 _allocate_before_rebuild = 0; 511 _last = this; 512 zap(); 513 } 514 515 // Try last block 516 if (_last->_top < block_size_in_oops) { 517 oop* handle = (oop*)&(_last->_handles)[_last->_top++]; 518 NativeAccess<IS_DEST_UNINITIALIZED>::oop_store(handle, obj); 519 return (jobject) handle; 520 } 521 522 // Try free list 523 if (_free_list != NULL) { 524 oop* handle = (oop*)_free_list; 525 _free_list = (uintptr_t*) untag_free_list(*_free_list); 526 NativeAccess<IS_DEST_UNINITIALIZED>::oop_store(handle, obj); 527 return (jobject) handle; 528 } 529 // Check if unused block follow last 530 if (_last->_next != NULL) { 531 // update last and retry 532 _last = _last->_next; 533 return allocate_handle(obj); 534 } 535 536 // No space available, we have to rebuild free list or expand 537 if (_allocate_before_rebuild == 0) { 538 rebuild_free_list(); // updates _allocate_before_rebuild counter 539 } else { 540 // Append new block 541 Thread* thread = Thread::current(); 542 Handle obj_handle(thread, obj); 543 // This can block, so we need to preserve obj across call. 544 _last->_next = JNIHandleBlock::allocate_block(thread); 545 _last = _last->_next; 546 _allocate_before_rebuild--; 547 obj = obj_handle(); 548 } 549 return allocate_handle(obj); // retry 550 } 551 552 void JNIHandleBlock::rebuild_free_list() { 553 assert(_allocate_before_rebuild == 0 && _free_list == NULL, "just checking"); 554 int free = 0; 555 int blocks = 0; 556 for (JNIHandleBlock* current = this; current != NULL; current = current->_next) { 557 for (int index = 0; index < current->_top; index++) { 558 uintptr_t* handle = &(current->_handles)[index]; 559 if (*handle == 0) { 560 // this handle was cleared out by a delete call, reuse it 561 *handle = _free_list == NULL ? 0 : tag_free_list((uintptr_t)_free_list); 562 _free_list = handle; 563 free++; 564 } 565 } 566 // we should not rebuild free list if there are unused handles at the end 567 assert(current->_top == block_size_in_oops, "just checking"); 568 blocks++; 569 } 570 // Heuristic: if more than half of the handles are free we rebuild next time 571 // as well, otherwise we append a corresponding number of new blocks before 572 // attempting a free list rebuild again. 573 int total = blocks * block_size_in_oops; 574 int extra = total - 2*free; 575 if (extra > 0) { 576 // Not as many free handles as we would like - compute number of new blocks to append 577 _allocate_before_rebuild = (extra + block_size_in_oops - 1) / block_size_in_oops; 578 } 579 } 580 581 582 bool JNIHandleBlock::contains(jobject handle) const { 583 return ((jobject)&_handles[0] <= handle && handle<(jobject)&_handles[_top]); 584 } 585 586 587 bool JNIHandleBlock::chain_contains(jobject handle) const { 588 for (JNIHandleBlock* current = (JNIHandleBlock*) this; current != NULL; current = current->_next) { 589 if (current->contains(handle)) { 590 return true; 591 } 592 } 593 return false; 594 } 595 596 597 size_t JNIHandleBlock::length() const { 598 size_t result = 1; 599 for (JNIHandleBlock* current = _next; current != NULL; current = current->_next) { 600 result++; 601 } 602 return result; 603 } 604 605 class CountJNIHandleClosure: public OopClosure { 606 private: 607 int _count; 608 public: 609 CountJNIHandleClosure(): _count(0) {} 610 virtual void do_oop(oop* ooph) { _count++; } 611 virtual void do_oop(narrowOop* unused) { ShouldNotReachHere(); } 612 int count() { return _count; } 613 }; 614 615 const size_t JNIHandleBlock::get_number_of_live_handles() { 616 CountJNIHandleClosure counter; 617 oops_do(&counter); 618 return counter.count(); 619 } 620 621 // This method is not thread-safe, i.e., must be called while holding a lock on the 622 // structure. 623 size_t JNIHandleBlock::memory_usage() const { 624 return length() * sizeof(JNIHandleBlock); 625 } 626 627 628 #ifndef PRODUCT 629 630 bool JNIHandles::is_local_handle(jobject handle) { 631 return JNIHandleBlock::any_contains(handle); 632 } 633 634 bool JNIHandleBlock::any_contains(jobject handle) { 635 assert(handle != NULL, "precondition"); 636 for (JNIHandleBlock* current = _block_list; current != NULL; current = current->_block_list_link) { 637 if (current->contains(handle)) { 638 return true; 639 } 640 } 641 return false; 642 } 643 644 void JNIHandleBlock::print_statistics() { 645 int used_blocks = 0; 646 int free_blocks = 0; 647 int used_handles = 0; 648 int free_handles = 0; 649 JNIHandleBlock* block = _block_list; 650 while (block != NULL) { 651 if (block->_top > 0) { 652 used_blocks++; 653 } else { 654 free_blocks++; 655 } 656 used_handles += block->_top; 657 free_handles += (block_size_in_oops - block->_top); 658 block = block->_block_list_link; 659 } 660 tty->print_cr("JNIHandleBlocks statistics"); 661 tty->print_cr("- blocks allocated: %d", used_blocks + free_blocks); 662 tty->print_cr("- blocks in use: %d", used_blocks); 663 tty->print_cr("- blocks free: %d", free_blocks); 664 tty->print_cr("- handles in use: %d", used_handles); 665 tty->print_cr("- handles free: %d", free_handles); 666 } 667 668 #endif