61 // Both, memory value and increment, are treated as 32bit signed binary integers.
62 // No overflow exceptions are recognized, and the condition code does not hold
63 // information about the value in memory.
64 //
65 // The value in memory is updated by using a compare-and-swap instruction. The
66 // instruction is retried as often as required.
67 //
68 // The return value of the method is the value that was successfully stored. At the
69 // time the caller receives back control, the value in memory may have changed already.
70
71 // New atomic operations only include specific-operand-serialization, not full
72 // memory barriers. We can use the Fast-BCR-Serialization Facility for them.
73 inline void z196_fast_sync() {
74 __asm__ __volatile__ ("bcr 14, 0" : : : "memory");
75 }
76
77 template<size_t byte_size>
78 struct Atomic::PlatformAdd
79 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
80 {
81 template<typename I, typename D>
82 D add_and_fetch(I add_value, D volatile* dest, atomic_memory_order order) const;
83 };
84
85 template<>
86 template<typename I, typename D>
87 inline D Atomic::PlatformAdd<4>::add_and_fetch(I inc, D volatile* dest,
88 atomic_memory_order order) const {
89 STATIC_ASSERT(4 == sizeof(I));
90 STATIC_ASSERT(4 == sizeof(D));
91
92 D old, upd;
93
94 if (VM_Version::has_LoadAndALUAtomicV1()) {
95 if (order == memory_order_conservative) { z196_fast_sync(); }
96 __asm__ __volatile__ (
97 " LGFR 0,%[inc] \n\t" // save increment
98 " LA 3,%[mem] \n\t" // force data address into ARG2
99 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value
100 // " LAA 2,0,0(3) \n\t" // actually coded instruction
101 " .byte 0xeb \n\t" // LAA main opcode
102 " .byte 0x20 \n\t" // R1,R3
103 " .byte 0x30 \n\t" // R2,disp1
104 " .byte 0x00 \n\t" // disp2,disp3
105 " .byte 0x00 \n\t" // disp4,disp5
106 " .byte 0xf8 \n\t" // LAA minor opcode
107 " AR 2,0 \n\t" // calc new value in register
120 " LLGF %[old],%[mem] \n\t" // get old value
121 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result
122 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem
123 " JNE 0b \n\t" // no success? -> retry
124 //---< outputs >---
125 : [old] "=&a" (old) // write-only, old counter value
126 , [upd] "=&d" (upd) // write-only, updated counter value
127 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically
128 //---< inputs >---
129 : [inc] "a" (inc) // read-only.
130 //---< clobbered >---
131 : "cc", "memory"
132 );
133 }
134
135 return upd;
136 }
137
138
139 template<>
140 template<typename I, typename D>
141 inline D Atomic::PlatformAdd<8>::add_and_fetch(I inc, D volatile* dest,
142 atomic_memory_order order) const {
143 STATIC_ASSERT(8 == sizeof(I));
144 STATIC_ASSERT(8 == sizeof(D));
145
146 D old, upd;
147
148 if (VM_Version::has_LoadAndALUAtomicV1()) {
149 if (order == memory_order_conservative) { z196_fast_sync(); }
150 __asm__ __volatile__ (
151 " LGR 0,%[inc] \n\t" // save increment
152 " LA 3,%[mem] \n\t" // force data address into ARG2
153 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value
154 // " LAAG 2,0,0(3) \n\t" // actually coded instruction
155 " .byte 0xeb \n\t" // LAA main opcode
156 " .byte 0x20 \n\t" // R1,R3
157 " .byte 0x30 \n\t" // R2,disp1
158 " .byte 0x00 \n\t" // disp2,disp3
159 " .byte 0x00 \n\t" // disp4,disp5
160 " .byte 0xe8 \n\t" // LAA minor opcode
161 " AGR 2,0 \n\t" // calc new value in register
|
61 // Both, memory value and increment, are treated as 32bit signed binary integers.
62 // No overflow exceptions are recognized, and the condition code does not hold
63 // information about the value in memory.
64 //
65 // The value in memory is updated by using a compare-and-swap instruction. The
66 // instruction is retried as often as required.
67 //
68 // The return value of the method is the value that was successfully stored. At the
69 // time the caller receives back control, the value in memory may have changed already.
70
71 // New atomic operations only include specific-operand-serialization, not full
72 // memory barriers. We can use the Fast-BCR-Serialization Facility for them.
73 inline void z196_fast_sync() {
74 __asm__ __volatile__ ("bcr 14, 0" : : : "memory");
75 }
76
77 template<size_t byte_size>
78 struct Atomic::PlatformAdd
79 : Atomic::AddAndFetch<Atomic::PlatformAdd<byte_size> >
80 {
81 template<typename D, typename I>
82 D add_and_fetch(D volatile* dest, I add_value, atomic_memory_order order) const;
83 };
84
85 template<>
86 template<typename D, typename I>
87 inline D Atomic::PlatformAdd<4>::add_and_fetch(D volatile* dest, I inc,
88 atomic_memory_order order) const {
89 STATIC_ASSERT(4 == sizeof(I));
90 STATIC_ASSERT(4 == sizeof(D));
91
92 D old, upd;
93
94 if (VM_Version::has_LoadAndALUAtomicV1()) {
95 if (order == memory_order_conservative) { z196_fast_sync(); }
96 __asm__ __volatile__ (
97 " LGFR 0,%[inc] \n\t" // save increment
98 " LA 3,%[mem] \n\t" // force data address into ARG2
99 // " LAA %[upd],%[inc],%[mem] \n\t" // increment and get old value
100 // " LAA 2,0,0(3) \n\t" // actually coded instruction
101 " .byte 0xeb \n\t" // LAA main opcode
102 " .byte 0x20 \n\t" // R1,R3
103 " .byte 0x30 \n\t" // R2,disp1
104 " .byte 0x00 \n\t" // disp2,disp3
105 " .byte 0x00 \n\t" // disp4,disp5
106 " .byte 0xf8 \n\t" // LAA minor opcode
107 " AR 2,0 \n\t" // calc new value in register
120 " LLGF %[old],%[mem] \n\t" // get old value
121 "0: LA %[upd],0(%[inc],%[old]) \n\t" // calc result
122 " CS %[old],%[upd],%[mem] \n\t" // try to xchg res with mem
123 " JNE 0b \n\t" // no success? -> retry
124 //---< outputs >---
125 : [old] "=&a" (old) // write-only, old counter value
126 , [upd] "=&d" (upd) // write-only, updated counter value
127 , [mem] "+Q" (*dest) // read/write, memory to be updated atomically
128 //---< inputs >---
129 : [inc] "a" (inc) // read-only.
130 //---< clobbered >---
131 : "cc", "memory"
132 );
133 }
134
135 return upd;
136 }
137
138
139 template<>
140 template<typename D, typename I>
141 inline D Atomic::PlatformAdd<8>::add_and_fetch(D volatile* dest, I inc,
142 atomic_memory_order order) const {
143 STATIC_ASSERT(8 == sizeof(I));
144 STATIC_ASSERT(8 == sizeof(D));
145
146 D old, upd;
147
148 if (VM_Version::has_LoadAndALUAtomicV1()) {
149 if (order == memory_order_conservative) { z196_fast_sync(); }
150 __asm__ __volatile__ (
151 " LGR 0,%[inc] \n\t" // save increment
152 " LA 3,%[mem] \n\t" // force data address into ARG2
153 // " LAAG %[upd],%[inc],%[mem] \n\t" // increment and get old value
154 // " LAAG 2,0,0(3) \n\t" // actually coded instruction
155 " .byte 0xeb \n\t" // LAA main opcode
156 " .byte 0x20 \n\t" // R1,R3
157 " .byte 0x30 \n\t" // R2,disp1
158 " .byte 0x00 \n\t" // disp2,disp3
159 " .byte 0x00 \n\t" // disp4,disp5
160 " .byte 0xe8 \n\t" // LAA minor opcode
161 " AGR 2,0 \n\t" // calc new value in register
|