1 /*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/z/zAddress.inline.hpp"
26 #include "gc/z/zBarrier.inline.hpp"
27 #include "gc/z/zForwarding.inline.hpp"
28 #include "gc/z/zHeap.hpp"
29 #include "gc/z/zOopClosures.inline.hpp"
30 #include "gc/z/zPage.hpp"
31 #include "gc/z/zRelocate.hpp"
32 #include "gc/z/zRelocationSet.inline.hpp"
33 #include "gc/z/zRootsIterator.hpp"
34 #include "gc/z/zStat.hpp"
35 #include "gc/z/zTask.hpp"
36 #include "gc/z/zThread.inline.hpp"
37 #include "gc/z/zThreadLocalAllocBuffer.hpp"
38 #include "gc/z/zWorkers.hpp"
39 #include "logging/log.hpp"
40
41 static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
42
43 ZRelocate::ZRelocate(ZWorkers* workers) :
44 _workers(workers) {}
45
46 class ZRelocateRootsIteratorClosure : public ZRootsIteratorClosure {
47 public:
48 virtual void do_thread(Thread* thread) {
49 // Update thread local address bad mask
50 ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
51
52 // Relocate invisible root
53 ZThreadLocalData::do_invisible_root(thread, ZBarrier::relocate_barrier_on_root_oop_field);
54
55 // Remap TLAB
56 ZThreadLocalAllocBuffer::remap(thread);
57 }
58
59 virtual void do_oop(oop* p) {
60 ZBarrier::relocate_barrier_on_root_oop_field(p);
61 }
62
63 virtual void do_oop(narrowOop* p) {
64 ShouldNotReachHere();
65 }
66 };
67
68 class ZRelocateRootsTask : public ZTask {
69 private:
70 ZRootsIterator _roots;
71 ZRelocateRootsIteratorClosure _cl;
72
73 public:
74 ZRelocateRootsTask() :
75 ZTask("ZRelocateRootsTask"),
76 _roots(true /* visit_jvmti_weak_export */) {}
77
78 virtual void work() {
79 // During relocation we need to visit the JVMTI
80 // export weak roots to rehash the JVMTI tag map
81 _roots.oops_do(&_cl);
82 }
83 };
84
85 void ZRelocate::start() {
86 ZRelocateRootsTask task;
87 _workers->run_parallel(&task);
88 }
89
90 uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t from_index, uintptr_t from_offset) const {
91 ZForwardingCursor cursor;
92
93 // Lookup forwarding entry
94 const ZForwardingEntry entry = forwarding->find(from_index, &cursor);
95 if (entry.populated() && entry.from_index() == from_index) {
96 // Already relocated, return new address
97 return entry.to_offset();
98 }
99
100 assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live");
101
102 if (forwarding->is_pinned()) {
103 // In-place forward
104 return forwarding->insert(from_index, from_offset, &cursor);
105 }
106
107 // Allocate object
108 const uintptr_t from_good = ZAddress::good(from_offset);
109 const size_t size = ZUtils::object_size(from_good);
110 const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
111 if (to_good == 0) {
112 // Failed, in-place forward
113 return forwarding->insert(from_index, from_offset, &cursor);
114 }
115
116 // Copy object
117 ZUtils::object_copy(from_good, to_good, size);
118
119 // Insert forwarding entry
120 const uintptr_t to_offset = ZAddress::offset(to_good);
121 const uintptr_t to_offset_final = forwarding->insert(from_index, to_offset, &cursor);
122 if (to_offset_final == to_offset) {
123 // Relocation succeeded
124 return to_offset;
125 }
126
127 // Relocation contention
128 ZStatInc(ZCounterRelocationContention);
129 log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), forwarding: " PTR_FORMAT
130 ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
131 ZThread::id(), ZThread::name(), p2i(forwarding), cursor, from_good, size);
132
133 // Try undo allocation
134 ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
135
136 return to_offset_final;
137 }
138
139 uintptr_t ZRelocate::relocate_object(ZForwarding* forwarding, uintptr_t from_addr) const {
140 const uintptr_t from_offset = ZAddress::offset(from_addr);
141 const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
142 const uintptr_t to_offset = relocate_object_inner(forwarding, from_index, from_offset);
143
144 if (from_offset == to_offset) {
145 // In-place forwarding, pin page
146 forwarding->set_pinned();
147 }
148
149 return ZAddress::good(to_offset);
150 }
151
152 uintptr_t ZRelocate::forward_object(ZForwarding* forwarding, uintptr_t from_addr) const {
153 const uintptr_t from_offset = ZAddress::offset(from_addr);
154 const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
155 const ZForwardingEntry entry = forwarding->find(from_index);
156
157 assert(entry.populated(), "Should be forwarded");
158 assert(entry.from_index() == from_index, "Should be forwarded");
159
160 return ZAddress::good(entry.to_offset());
161 }
162
163 class ZRelocateObjectClosure : public ObjectClosure {
164 private:
165 ZRelocate* const _relocate;
166 ZForwarding* const _forwarding;
167
168 public:
169 ZRelocateObjectClosure(ZRelocate* relocate, ZForwarding* forwarding) :
170 _relocate(relocate),
171 _forwarding(forwarding) {}
172
173 virtual void do_object(oop o) {
174 _relocate->relocate_object(_forwarding, ZOop::to_address(o));
175 }
176 };
177
178 bool ZRelocate::work(ZRelocationSetParallelIterator* iter) {
179 bool success = true;
180
181 // Relocate pages in the relocation set
182 for (ZForwarding* forwarding; iter->next(&forwarding);) {
183 // Relocate objects in page
184 ZRelocateObjectClosure cl(this, forwarding);
185 forwarding->page()->object_iterate(&cl);
186
187 if (ZVerifyForwarding) {
188 forwarding->verify();
189 }
190
191 if (forwarding->is_pinned()) {
192 // Relocation failed, page is now pinned
193 success = false;
194 } else {
195 // Relocation succeeded, release page
196 forwarding->release_page();
197 }
198 }
199
200 return success;
201 }
202
203 class ZRelocateTask : public ZTask {
204 private:
205 ZRelocate* const _relocate;
206 ZRelocationSetParallelIterator _iter;
207 bool _failed;
208
209 public:
210 ZRelocateTask(ZRelocate* relocate, ZRelocationSet* relocation_set) :
211 ZTask("ZRelocateTask"),
212 _relocate(relocate),
213 _iter(relocation_set),
214 _failed(false) {}
215
216 virtual void work() {
217 if (!_relocate->work(&_iter)) {
218 _failed = true;
219 }
220 }
221
222 bool failed() const {
223 return _failed;
224 }
225 };
226
227 bool ZRelocate::relocate(ZRelocationSet* relocation_set) {
228 ZRelocateTask task(this, relocation_set);
229 _workers->run_concurrent(&task);
230 return !task.failed();
231 }