1 /* 2 * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/g1/g1NUMA.hpp" 27 #include "logging/logStream.hpp" 28 #include "runtime/globals.hpp" 29 #include "runtime/os.hpp" 30 31 G1NUMA* G1NUMA::_inst = NULL; 32 33 size_t G1NUMA::region_size() const { 34 assert(_region_size > 0, "Heap region size is not yet set"); 35 return _region_size; 36 } 37 38 size_t G1NUMA::page_size() const { 39 assert(_page_size > 0, "Page size not is yet set"); 40 return _page_size; 41 } 42 43 bool G1NUMA::is_enabled() const { return num_active_nodes() > 1; } 44 45 G1NUMA* G1NUMA::create() { 46 guarantee(_inst == NULL, "Should be called once."); 47 _inst = new G1NUMA(); 48 49 // NUMA only supported on Linux. 50 #ifdef LINUX 51 _inst->initialize(UseNUMA); 52 #else 53 _inst->initialize(false); 54 #endif /* LINUX */ 55 56 return _inst; 57 } 58 59 // Returns memory node ids 60 const int* G1NUMA::node_ids() const { 61 return _node_ids; 62 } 63 64 uint G1NUMA::index_of_node_id(int node_id) const { 65 assert(node_id >= 0, "invalid node id %d", node_id); 66 assert(node_id < _len_node_id_to_index_map, "invalid node id %d", node_id); 67 uint node_index = _node_id_to_index_map[node_id]; 68 assert(node_index != G1NUMA::UnknownNodeIndex, 69 "invalid node id %d", node_id); 70 return node_index; 71 } 72 73 G1NUMA::G1NUMA() : 74 _node_id_to_index_map(NULL), _len_node_id_to_index_map(0), 75 _node_ids(NULL), _num_active_node_ids(0), 76 _region_size(0), _page_size(0), _stats(NULL) { 77 } 78 79 void G1NUMA::initialize_without_numa() { 80 // If NUMA is not enabled or supported, initialize as having a singel node. 81 _num_active_node_ids = 1; 82 _node_ids = NEW_C_HEAP_ARRAY(int, _num_active_node_ids, mtGC); 83 _node_ids[0] = 0; 84 // Map index 0 to node 0 85 _len_node_id_to_index_map = 1; 86 _node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC); 87 _node_id_to_index_map[0] = 0; 88 } 89 90 void G1NUMA::initialize(bool use_numa) { 91 if (!use_numa) { 92 initialize_without_numa(); 93 return; 94 } 95 96 assert(UseNUMA, "Invariant"); 97 size_t num_node_ids = os::numa_get_groups_num(); 98 99 // Create an array of active node ids. 100 _node_ids = NEW_C_HEAP_ARRAY(int, num_node_ids, mtGC); 101 _num_active_node_ids = (uint)os::numa_get_leaf_groups(_node_ids, num_node_ids); 102 103 int max_node_id = 0; 104 for (uint i = 0; i < _num_active_node_ids; i++) { 105 max_node_id = MAX2(max_node_id, _node_ids[i]); 106 } 107 108 // Create a mapping between node_id and index. 109 _len_node_id_to_index_map = max_node_id + 1; 110 _node_id_to_index_map = NEW_C_HEAP_ARRAY(uint, _len_node_id_to_index_map, mtGC); 111 112 // Set all indices with unknown node id. 113 for (int i = 0; i < _len_node_id_to_index_map; i++) { 114 _node_id_to_index_map[i] = G1NUMA::UnknownNodeIndex; 115 } 116 117 // Set the indices for the actually retrieved node ids. 118 for (uint i = 0; i < _num_active_node_ids; i++) { 119 _node_id_to_index_map[_node_ids[i]] = i; 120 } 121 122 _stats = new G1NUMAStats(_node_ids, _num_active_node_ids); 123 } 124 125 G1NUMA::~G1NUMA() { 126 delete _stats; 127 FREE_C_HEAP_ARRAY(int, _node_id_to_index_map); 128 FREE_C_HEAP_ARRAY(int, _node_ids); 129 } 130 131 void G1NUMA::set_region_info(size_t region_size, size_t page_size) { 132 _region_size = region_size; 133 _page_size = page_size; 134 } 135 136 uint G1NUMA::num_active_nodes() const { 137 assert(_num_active_node_ids > 0, "just checking"); 138 return _num_active_node_ids; 139 } 140 141 uint G1NUMA::index_of_current_thread() const { 142 if (!is_enabled()) { 143 return 0; 144 } 145 return index_of_node_id(os::numa_get_group_id()); 146 } 147 148 uint G1NUMA::preferred_node_index_for_index(uint region_index) const { 149 if (region_size() >= page_size()) { 150 // Simple case, pages are smaller than the region so we 151 // can just alternate over the nodes. 152 return region_index % _num_active_node_ids; 153 } else { 154 // Multiple regions in one page, so we need to make sure the 155 // regions within a page is preferred on the same node. 156 size_t regions_per_page = page_size() / region_size(); 157 return (region_index / regions_per_page) % _num_active_node_ids; 158 } 159 } 160 161 int G1NUMA::numa_id(int index) const { 162 assert(index < _len_node_id_to_index_map, "Index %d out of range: [0,%d)", 163 index, _len_node_id_to_index_map); 164 return _node_ids[index]; 165 } 166 167 uint G1NUMA::index_of_address(HeapWord *address) const { 168 int numa_id = os::numa_get_group_id_for_address((const void*)address); 169 if (numa_id == -1) { 170 return UnknownNodeIndex; 171 } else { 172 return index_of_node_id(numa_id); 173 } 174 } 175 176 uint G1NUMA::index_for_region(HeapRegion* hr) const { 177 if (!is_enabled()) { 178 return 0; 179 } 180 181 if (AlwaysPreTouch) { 182 // If we already pretouched, we can check actual node index here. 183 // However, if node index is still unknown, use preferred node index. 184 uint node_index = index_of_address(hr->bottom()); 185 if (node_index != UnknownNodeIndex) { 186 return node_index; 187 } 188 } 189 190 return preferred_node_index_for_index(hr->hrm_index()); 191 } 192 193 // Request to spread the given memory evenly across the available NUMA 194 // nodes. Which node to request for a given address is given by the 195 // region size and the page size. Below are two examples on 4 NUMA nodes system: 196 // 1. G1HeapRegionSize(_region_size) is larger than or equal to page size. 197 // * Page #: |-0--||-1--||-2--||-3--||-4--||-5--||-6--||-7--||-8--||-9--||-10-||-11-||-12-||-13-||-14-||-15-| 198 // * HeapRegion #: |----#0----||----#1----||----#2----||----#3----||----#4----||----#5----||----#6----||----#7----| 199 // * NUMA node #: |----#0----||----#1----||----#2----||----#3----||----#0----||----#1----||----#2----||----#3----| 200 // 2. G1HeapRegionSize(_region_size) is smaller than page size. 201 // Memory will be touched one page at a time because G1RegionToSpaceMapper commits 202 // pages one by one. 203 // * Page #: |-----0----||-----1----||-----2----||-----3----||-----4----||-----5----||-----6----||-----7----| 204 // * HeapRegion #: |-#0-||-#1-||-#2-||-#3-||-#4-||-#5-||-#6-||-#7-||-#8-||-#9-||#10-||#11-||#12-||#13-||#14-||#15-| 205 // * NUMA node #: |----#0----||----#1----||----#2----||----#3----||----#0----||----#1----||----#2----||----#3----| 206 void G1NUMA::request_memory_on_node(void* aligned_address, size_t size_in_bytes, uint region_index) { 207 if (!is_enabled()) { 208 return; 209 } 210 211 if (size_in_bytes == 0) { 212 return; 213 } 214 215 uint node_index = preferred_node_index_for_index(region_index); 216 217 assert(is_aligned(aligned_address, page_size()), "Given address (" PTR_FORMAT ") should be aligned.", p2i(aligned_address)); 218 assert(is_aligned(size_in_bytes, page_size()), "Given size (" SIZE_FORMAT ") should be aligned.", size_in_bytes); 219 220 log_trace(gc, heap, numa)("Request memory [" PTR_FORMAT ", " PTR_FORMAT ") to be NUMA id (%d)", 221 p2i(aligned_address), p2i((char*)aligned_address + size_in_bytes), _node_ids[node_index]); 222 os::numa_make_local((char*)aligned_address, size_in_bytes, _node_ids[node_index]); 223 } 224 225 uint G1NUMA::max_search_depth() const { 226 // Multiple of 3 is just random number to limit iterations. 227 // There would be some cases that 1 page may be consisted of multiple HeapRegions. 228 return 3 * MAX2((uint)(page_size() / region_size()), (uint)1) * num_active_nodes(); 229 } 230 231 void G1NUMA::update_statistics(G1NUMAStats::NodeDataItems phase, 232 uint requested_node_index, 233 uint allocated_node_index) { 234 if (_stats == NULL) { 235 return; 236 } 237 238 uint converted_req_index; 239 if(requested_node_index < _num_active_node_ids) { 240 converted_req_index = requested_node_index; 241 } else { 242 assert(requested_node_index == AnyNodeIndex, 243 "Requested node index %u should be AnyNodeIndex.", requested_node_index); 244 converted_req_index = _num_active_node_ids; 245 } 246 _stats->update(phase, converted_req_index, allocated_node_index); 247 } 248 249 void G1NUMA::copy_statistics(G1NUMAStats::NodeDataItems phase, 250 uint requested_node_index, 251 size_t* allocated_stat) { 252 if (_stats == NULL) { 253 return; 254 } 255 256 _stats->copy(phase, requested_node_index, allocated_stat); 257 } 258 259 void G1NUMA::print_statistics() const { 260 if (_stats == NULL) { 261 return; 262 } 263 264 _stats->print_statistics(); 265 } 266 267 NodeIndexCheckClosure::NodeIndexCheckClosure(const char* desc, G1NUMA* numa, LogStream* ls) : 268 _desc(desc), _numa(numa), _ls(ls) { 269 270 uint num_nodes = _numa->num_active_nodes(); 271 _matched = NEW_C_HEAP_ARRAY(uint, num_nodes, mtGC); 272 _mismatched = NEW_C_HEAP_ARRAY(uint, num_nodes, mtGC); 273 _total = NEW_C_HEAP_ARRAY(uint, num_nodes, mtGC); 274 memset(_matched, 0, sizeof(uint) * num_nodes); 275 memset(_mismatched, 0, sizeof(uint) * num_nodes); 276 memset(_total, 0, sizeof(uint) * num_nodes); 277 } 278 279 NodeIndexCheckClosure::~NodeIndexCheckClosure() { 280 _ls->print("%s: NUMA region verification (id: matched/mismatched/total): ", _desc); 281 const int* numa_ids = _numa->node_ids(); 282 for (uint i = 0; i < _numa->num_active_nodes(); i++) { 283 _ls->print("%d: %u/%u/%u ", numa_ids[i], _matched[i], _mismatched[i], _total[i]); 284 } 285 286 FREE_C_HEAP_ARRAY(uint, _matched); 287 FREE_C_HEAP_ARRAY(uint, _mismatched); 288 FREE_C_HEAP_ARRAY(uint, _total); 289 } 290 291 bool NodeIndexCheckClosure::do_heap_region(HeapRegion* hr) { 292 // Preferred node index will only have valid node index. 293 uint preferred_node_index = _numa->preferred_node_index_for_index(hr->hrm_index()); 294 // Active node index may have UnknownNodeIndex. 295 uint active_node_index = _numa->index_of_address(hr->bottom()); 296 297 if (preferred_node_index == active_node_index) { 298 _matched[preferred_node_index]++; 299 } else if (preferred_node_index != active_node_index && 300 active_node_index != G1NUMA::UnknownNodeIndex) { 301 _mismatched[preferred_node_index]++; 302 } 303 _total[preferred_node_index]++; 304 305 return false; 306 }