1 /*
2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26
27 #include "gc/shared/memset_with_concurrent_readers.hpp"
28 #include "runtime/prefetch.inline.hpp"
29 #include "utilities/debug.hpp"
30 #include "utilities/globalDefinitions.hpp"
31 #include "utilities/macros.hpp"
32
33 #if INCLUDE_ALL_GCS
34
35 // An implementation of memset, for use when there may be concurrent
36 // readers of the region being stored into.
37 //
38 // We can't use the standard library memset if it is implemented using
39 // block initializing stores. Doing so can result in concurrent readers
40 // seeing spurious zeros.
41 //
42 // We can't use the obvious C/C++ for-loop, because the compiler may
43 // recognize the idiomatic loop and optimize it into a call to the
44 // standard library memset; we've seen exactly this happen with, for
45 // example, Solaris Studio 12.3. Hence the use of inline assembly
46 // code, hiding loops from the compiler's optimizer.
47 //
48 // We don't attempt to use the standard library memset when it is safe
49 // to do so. We could conservatively do so by detecting the presence
50 // of block initializing stores (VM_Version::has_blk_init()), but the
51 // implementation provided here should be sufficient.
52
53 static void fill_subword(void* start, void* end, int value);
54
55 void memset_with_concurrent_readers(void* to, int value, size_t size) {
56 Prefetch::write(to, 0);
57 void* end = static_cast<char*>(to) + size;
58 if (size >= BytesPerWord) {
59 // Fill any partial word prefix.
60 uintx* aligned_to = static_cast<uintx*>(align_ptr_up(to, BytesPerWord));
61 fill_subword(to, aligned_to, value);
62
63 // Compute fill word.
64 STATIC_ASSERT(BitsPerByte == 8);
65 STATIC_ASSERT(BitsPerWord == 64);
66 uintx xvalue = value & 0xff;
67 xvalue |= (xvalue << 8);
68 xvalue |= (xvalue << 16);
69 xvalue |= (xvalue << 32);
70
71 uintx* aligned_end = static_cast<uintx*>(align_ptr_down(end, BytesPerWord));
72 assert(aligned_to <= aligned_end, "invariant");
73
74 // for ( ; aligned_to < aligned_end; ++aligned_to) {
75 // *aligned_to = xvalue;
76 // }
77 uintptr_t temp;
78 __asm__ volatile(
79 // Unroll loop x8.
80 " sub %[aend], %[ato], %[temp]\n\t"
81 " cmp %[temp], 56\n\t" // cc := (aligned_end - aligned_to) > 7 words
82 " ba %xcc, 2f\n\t"
83 " sub %[aend], 56, %[temp]\n\t" // limit := aligned_end - 7 words
84 "1:\n\t" // unrolled x8 store loop top
85 " cmp %[temp], %[ato]\n\t" // cc := limit > (next) aligned_to
86 " stx %[xvalue], [%[ato]-64]\n\t" // store 8 words, aligned_to pre-incremented
87 " stx %[xvalue], [%[ato]-56]\n\t"
88 " stx %[xvalue], [%[ato]-48]\n\t"
89 " stx %[xvalue], [%[ato]-40]\n\t"
90 " stx %[xvalue], [%[ato]-32]\n\t"
91 " stx %[xvalue], [%[ato]-24]\n\t"
92 " stx %[xvalue], [%[ato]-16]\n\t"
93 " stx %[xvalue], [%[ato]-8]\n\t"
94 "2:\n\t"
95 " bgu,a %xcc, 1b\n\t" // loop if more than 7 words remaining
96 " add %[ato], 64, %[ato]\n\t" // aligned_to += 8, for next iteration
97 // Fill remaining < 8 full words.
98 // Dispatch on (aligned_end - aligned_to).
99 // offset := (7 - (aligned_end - aligned_to)) + 3
100 // 3 instructions from rdpc to dispatch start
101 " sub %[ato], %[aend], %[ato]\n\t" // offset := aligned_to - aligned_end
102 " srax %[ato], 1, %[ato]\n\t" // scale offset for instruction size of 4
103 " add %[ato], 40, %[ato]\n\t" // offset += 10 * instruction size
104 " rd %pc, %[temp]\n\t" // dispatch on scaled offset
105 " jmpl %[temp]+%[ato], %g0\n\t"
106 " nop\n\t"
107 "3:\n\t" // dispatch start
108 " stx %[xvalue], [%[aend]-56]\n\t" // aligned_end[-7] = xvalue
109 " stx %[xvalue], [%[aend]-48]\n\t"
110 " stx %[xvalue], [%[aend]-40]\n\t"
111 " stx %[xvalue], [%[aend]-32]\n\t"
112 " stx %[xvalue], [%[aend]-24]\n\t"
113 " stx %[xvalue], [%[aend]-16]\n\t"
114 " stx %[xvalue], [%[aend]-8]\n\t" // aligned_end[-1] = xvalue
115 : /* no outputs */
116 : [ato] "&+r" (aligned_to),
117 [aend] "r" (aligned_end),
118 [xvalue] "r" (xvalue),
119 [temp] "&=r" (temp)
120 : "cc", "memory");
121 to = aligned_end; // setup for suffix
122 }
123 // Fill any partial word suffix. Also the prefix if size < BytesPerWord.
124 fill_subword(to, end, value);
125 }
126
127 static void fill_subword(void* start, void* end, int value) {
128 STATIC_ASSERT(BytesPerWord == 8);
129 assert(pointer_delta(end, start, 1) < BytesPerWord, "precondition");
130 // Dispatch on (end - start).
131 void* pc;
132 __asm__ volatile(
133 // offset := (7 - (end - start)) + 3
134 // 3 instructions from rdpc to dispatch start
135 " sub %[offset], %[end], %[offset]\n\t" // offset := start - end
136 " sllx %[offset], 2, %[offset]\n\t" // scale offset for instruction size of 4
137 " add %[offset], 40, %[offset]\n\t" // offset += 10 * instruction size
138 " rd %pc, %[pc]\n\t" // dispatch on scaled offset
139 " jmpl %[pc]+%[offset], %g0\n\t"
140 " nop\n\t"
141 "1:\n\t" // dispatch start
142 " stb %[value], [%[end]-7]\n\t" // end[-7] = value
143 " stb %[value], [%[end]-6]\n\t"
144 " stb %[value], [%[end]-5]\n\t"
145 " stb %[value], [%[end]-4]\n\t"
146 " stb %[value], [%[end]-3]\n\t"
147 " stb %[value], [%[end]-2]\n\t"
148 " stb %[value], [%[end]-1]\n\t" // end[-1] = value
149 : /* no outputs */
150 : [offset] "&+r" (start),
151 [end] "r" (end),
152 [value] "r" (value),
153 [pc] "&=r" (pc)
154 : "memory");
155 }
156
157 #endif // INCLUDE_ALL_GCS