14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zList.inline.hpp"
26 #include "gc/z/zNUMA.hpp"
27 #include "gc/z/zPage.inline.hpp"
28 #include "gc/z/zPageCache.hpp"
29 #include "gc/z/zStat.hpp"
30 #include "logging/log.hpp"
31
32 static const ZStatCounter ZCounterPageCacheHitL1("Memory", "Page Cache Hit L1", ZStatUnitOpsPerSecond);
33 static const ZStatCounter ZCounterPageCacheHitL2("Memory", "Page Cache Hit L2", ZStatUnitOpsPerSecond);
34 static const ZStatCounter ZCounterPageCacheMiss("Memory", "Page Cache Miss", ZStatUnitOpsPerSecond);
35
36 ZPageCache::ZPageCache() :
37 _available(0),
38 _small(),
39 _medium(),
40 _large() {}
41
42 ZPage* ZPageCache::alloc_small_page() {
43 const uint32_t numa_id = ZNUMA::id();
44 const uint32_t numa_count = ZNUMA::count();
45
46 // Try NUMA local page cache
47 ZPage* const l1_page = _small.get(numa_id).remove_first();
48 if (l1_page != NULL) {
49 ZStatInc(ZCounterPageCacheHitL1);
50 return l1_page;
51 }
52
53 // Try NUMA remote page cache(s)
54 uint32_t remote_numa_id = numa_id + 1;
55 const uint32_t remote_numa_count = numa_count - 1;
56 for (uint32_t i = 0; i < remote_numa_count; i++) {
57 if (remote_numa_id == numa_count) {
58 remote_numa_id = 0;
59 }
60
61 ZPage* const l2_page = _small.get(remote_numa_id).remove_first();
62 if (l2_page != NULL) {
63 ZStatInc(ZCounterPageCacheHitL2);
64 return l2_page;
65 }
66
67 remote_numa_id++;
68 }
69
70 ZStatInc(ZCounterPageCacheMiss);
71 return NULL;
72 }
73
74 ZPage* ZPageCache::alloc_medium_page() {
75 ZPage* const l1_page = _medium.remove_first();
76 if (l1_page != NULL) {
77 ZStatInc(ZCounterPageCacheHitL1);
78 return l1_page;
79 }
80
81 ZStatInc(ZCounterPageCacheMiss);
82 return NULL;
83 }
84
85 ZPage* ZPageCache::alloc_large_page(size_t size) {
86 // Find a page with the right size
87 ZListIterator<ZPage> iter(&_large);
88 for (ZPage* l1_page; iter.next(&l1_page);) {
89 if (l1_page->size() == size) {
90 // Page found
91 _large.remove(l1_page);
92 ZStatInc(ZCounterPageCacheHitL1);
93 return l1_page;
94 }
95 }
96
97 ZStatInc(ZCounterPageCacheMiss);
98 return NULL;
99 }
100
101 ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) {
102 ZPage* page;
103
104 if (type == ZPageTypeSmall) {
105 page = alloc_small_page();
106 } else if (type == ZPageTypeMedium) {
107 page = alloc_medium_page();
108 } else {
109 page = alloc_large_page(size);
110 }
111
112 if (page != NULL) {
113 _available -= page->size();
114 }
115
116 return page;
117 }
118
119 void ZPageCache::free_page(ZPage* page) {
120 const uint8_t type = page->type();
121 if (type == ZPageTypeSmall) {
122 _small.get(page->numa_id()).insert_first(page);
123 } else if (type == ZPageTypeMedium) {
124 _medium.insert_first(page);
125 } else {
126 _large.insert_first(page);
127 }
128
129 _available += page->size();
130 }
131
132 bool ZPageCache::flush_list_inner(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
133 ZPage* const page = from->last();
134 if (page == NULL || !cl->do_page(page)) {
135 // Don't flush page
136 return false;
137 }
138
139 // Flush page
140 _available -= page->size();
141 from->remove(page);
142 to->insert_last(page);
143 return true;
144 }
145
146 void ZPageCache::flush_list(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
147 while (flush_list_inner(cl, from, to));
148 }
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zList.inline.hpp"
26 #include "gc/z/zNUMA.hpp"
27 #include "gc/z/zPage.inline.hpp"
28 #include "gc/z/zPageCache.hpp"
29 #include "gc/z/zStat.hpp"
30 #include "logging/log.hpp"
31
32 static const ZStatCounter ZCounterPageCacheHitL1("Memory", "Page Cache Hit L1", ZStatUnitOpsPerSecond);
33 static const ZStatCounter ZCounterPageCacheHitL2("Memory", "Page Cache Hit L2", ZStatUnitOpsPerSecond);
34 static const ZStatCounter ZCounterPageCacheHitL3("Memory", "Page Cache Hit L3", ZStatUnitOpsPerSecond);
35 static const ZStatCounter ZCounterPageCacheMiss("Memory", "Page Cache Miss", ZStatUnitOpsPerSecond);
36
37 ZPageCacheFlushClosure::ZPageCacheFlushClosure(size_t requested) :
38 _requested(requested),
39 _flushed(0) {}
40
41 size_t ZPageCacheFlushClosure::overflushed() const {
42 return _flushed > _requested ? _flushed - _requested : 0;
43 }
44
45 ZPageCache::ZPageCache() :
46 _available(0),
47 _small(),
48 _medium(),
49 _large() {}
50
51 ZPage* ZPageCache::alloc_small_page() {
52 const uint32_t numa_id = ZNUMA::id();
53 const uint32_t numa_count = ZNUMA::count();
54
55 // Try NUMA local page cache
56 ZPage* const l1_page = _small.get(numa_id).remove_first();
57 if (l1_page != NULL) {
58 ZStatInc(ZCounterPageCacheHitL1);
59 return l1_page;
60 }
61
62 // Try NUMA remote page cache(s)
63 uint32_t remote_numa_id = numa_id + 1;
64 const uint32_t remote_numa_count = numa_count - 1;
65 for (uint32_t i = 0; i < remote_numa_count; i++) {
66 if (remote_numa_id == numa_count) {
67 remote_numa_id = 0;
68 }
69
70 ZPage* const l2_page = _small.get(remote_numa_id).remove_first();
71 if (l2_page != NULL) {
72 ZStatInc(ZCounterPageCacheHitL2);
73 return l2_page;
74 }
75
76 remote_numa_id++;
77 }
78
79 return NULL;
80 }
81
82 ZPage* ZPageCache::alloc_medium_page() {
83 ZPage* const page = _medium.remove_first();
84 if (page != NULL) {
85 ZStatInc(ZCounterPageCacheHitL1);
86 return page;
87 }
88
89 return NULL;
90 }
91
92 ZPage* ZPageCache::alloc_large_page(size_t size) {
93 // Find a page with the right size
94 ZListIterator<ZPage> iter(&_large);
95 for (ZPage* page; iter.next(&page);) {
96 if (size == page->size()) {
97 // Page found
98 _large.remove(page);
99 ZStatInc(ZCounterPageCacheHitL1);
100 return page;
101 }
102 }
103
104 return NULL;
105 }
106
107 ZPage* ZPageCache::alloc_oversized_medium_page(size_t size) {
108 if (size <= ZPageSizeMedium) {
109 return _medium.remove_first();
110 }
111
112 return NULL;
113 }
114
115 ZPage* ZPageCache::alloc_oversized_large_page(size_t size) {
116 // Find a page that is large enough
117 ZListIterator<ZPage> iter(&_large);
118 for (ZPage* page; iter.next(&page);) {
119 if (size <= page->size()) {
120 // Page found
121 _large.remove(page);
122 return page;
123 }
124 }
125
126 return NULL;
127 }
128
129 ZPage* ZPageCache::alloc_oversized_page(size_t size) {
130 ZPage* page = alloc_oversized_large_page(size);
131 if (page == NULL) {
132 page = alloc_oversized_medium_page(size);
133 }
134
135 if (page != NULL) {
136 ZStatInc(ZCounterPageCacheHitL3);
137 }
138
139 return page;
140 }
141
142 ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) {
143 ZPage* page;
144
145 // Try allocate exact page
146 if (type == ZPageTypeSmall) {
147 page = alloc_small_page();
148 } else if (type == ZPageTypeMedium) {
149 page = alloc_medium_page();
150 } else {
151 page = alloc_large_page(size);
152 }
153
154 if (page == NULL) {
155 // Try allocate potentially oversized page
156 ZPage* const oversized = alloc_oversized_page(size);
157 if (oversized != NULL) {
158 if (size < oversized->size()) {
159 // Split oversized page
160 page = oversized->split(type, size);
161
162 // Cache remainder
163 free_page_inner(oversized);
164 } else {
165 // Re-type correctly sized page
166 page = oversized->retype(type);
167 }
168 }
169 }
170
171 if (page != NULL) {
172 _available -= page->size();
173 } else {
174 ZStatInc(ZCounterPageCacheMiss);
175 }
176
177 return page;
178 }
179
180 void ZPageCache::free_page_inner(ZPage* page) {
181 const uint8_t type = page->type();
182 if (type == ZPageTypeSmall) {
183 _small.get(page->numa_id()).insert_first(page);
184 } else if (type == ZPageTypeMedium) {
185 _medium.insert_first(page);
186 } else {
187 _large.insert_first(page);
188 }
189 }
190
191 void ZPageCache::free_page(ZPage* page) {
192 free_page_inner(page);
193 _available += page->size();
194 }
195
196 bool ZPageCache::flush_list_inner(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
197 ZPage* const page = from->last();
198 if (page == NULL || !cl->do_page(page)) {
199 // Don't flush page
200 return false;
201 }
202
203 // Flush page
204 _available -= page->size();
205 from->remove(page);
206 to->insert_last(page);
207 return true;
208 }
209
210 void ZPageCache::flush_list(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
211 while (flush_list_inner(cl, from, to));
212 }
|