3168 br(Assembler::GE, L_by16_loop);
3169 adds(len, len, 16-4);
3170 br(Assembler::GE, L_by4_loop);
3171 adds(len, len, 4);
3172 br(Assembler::GT, L_by1_loop);
3173 BIND(L_exit);
3174 ornw(crc, zr, crc);
3175 }
3176
3177 /**
3178 * @param crc register containing existing CRC (32-bit)
3179 * @param buf register pointing to input byte buffer (byte*)
3180 * @param len register containing number of bytes
3181 * @param table register that will contain address of CRC table
3182 * @param tmp scratch register
3183 */
3184 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len,
3185 Register table0, Register table1, Register table2, Register table3,
3186 Register tmp, Register tmp2, Register tmp3) {
3187 Label L_exit;
3188 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop;
3189
3190 subs(len, len, 64);
3191 br(Assembler::GE, CRC_by64_loop);
3192 adds(len, len, 64-4);
3193 br(Assembler::GE, CRC_by4_loop);
3194 adds(len, len, 4);
3195 br(Assembler::GT, CRC_by1_loop);
3196 b(L_exit);
3197
3198 BIND(CRC_by4_loop);
3199 ldrw(tmp, Address(post(buf, 4)));
3200 subs(len, len, 4);
3201 crc32cw(crc, crc, tmp);
3202 br(Assembler::GE, CRC_by4_loop);
3203 adds(len, len, 4);
3204 br(Assembler::LE, L_exit);
3205 BIND(CRC_by1_loop);
3206 ldrb(tmp, Address(post(buf, 1)));
3207 subs(len, len, 1);
3208 crc32cb(crc, crc, tmp);
3209 br(Assembler::GT, CRC_by1_loop);
3210 b(L_exit);
3211
3212 align(CodeEntryAlignment);
3213 BIND(CRC_by64_loop);
3214 subs(len, len, 64);
3215 ldp(tmp, tmp3, Address(post(buf, 16)));
3216 crc32cx(crc, crc, tmp);
3217 crc32cx(crc, crc, tmp3);
3218 ldp(tmp, tmp3, Address(post(buf, 16)));
3219 crc32cx(crc, crc, tmp);
3220 crc32cx(crc, crc, tmp3);
3221 ldp(tmp, tmp3, Address(post(buf, 16)));
3222 crc32cx(crc, crc, tmp);
3223 crc32cx(crc, crc, tmp3);
3224 ldp(tmp, tmp3, Address(post(buf, 16)));
3225 crc32cx(crc, crc, tmp);
3226 crc32cx(crc, crc, tmp3);
3227 br(Assembler::GE, CRC_by64_loop);
3228 adds(len, len, 64-4);
3229 br(Assembler::GE, CRC_by4_loop);
3230 adds(len, len, 4);
3231 br(Assembler::GT, CRC_by1_loop);
3232 BIND(L_exit);
3233 return;
3234 }
3235
3236 SkipIfEqual::SkipIfEqual(
3237 MacroAssembler* masm, const bool* flag_addr, bool value) {
3238 _masm = masm;
3239 unsigned long offset;
3240 _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset);
3241 _masm->ldrb(rscratch1, Address(rscratch1, offset));
3242 _masm->cbzw(rscratch1, _label);
3243 }
3244
3245 SkipIfEqual::~SkipIfEqual() {
3246 _masm->bind(_label);
3247 }
3248
3249 void MacroAssembler::addptr(const Address &dst, int32_t src) {
3250 Address adr;
3251 switch(dst.getMode()) {
3252 case Address::base_plus_offset:
3253 // This is the expected mode, although we allow all the other
3254 // forms below.
|
3168 br(Assembler::GE, L_by16_loop);
3169 adds(len, len, 16-4);
3170 br(Assembler::GE, L_by4_loop);
3171 adds(len, len, 4);
3172 br(Assembler::GT, L_by1_loop);
3173 BIND(L_exit);
3174 ornw(crc, zr, crc);
3175 }
3176
3177 /**
3178 * @param crc register containing existing CRC (32-bit)
3179 * @param buf register pointing to input byte buffer (byte*)
3180 * @param len register containing number of bytes
3181 * @param table register that will contain address of CRC table
3182 * @param tmp scratch register
3183 */
3184 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len,
3185 Register table0, Register table1, Register table2, Register table3,
3186 Register tmp, Register tmp2, Register tmp3) {
3187 Label L_exit;
3188 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32;
3189
3190 subs(len, len, 128);
3191 br(Assembler::GE, CRC_by64_pre);
3192 BIND(CRC_less64);
3193 adds(len, len, 128-32);
3194 br(Assembler::GE, CRC_by32_loop);
3195 BIND(CRC_less32)
3196 adds(len, len, 32-4);
3197 br(Assembler::GE, CRC_by4_loop);
3198 adds(len, len, 4);
3199 br(Assembler::GT, CRC_by1_loop);
3200 b(L_exit);
3201
3202 BIND(CRC_by32_loop);
3203 ldp(table0, table1, Address(post(buf, 16)));
3204 subs(len, len, 32);
3205 crc32cx(crc, crc, table0);
3206 ldr(table2, Address(post(buf, 8)));
3207 crc32cx(crc, crc, table1);
3208 ldr(table3, Address(post(buf, 8)));
3209 crc32cx(crc, crc, table2);
3210 crc32cx(crc, crc, table3);
3211 br(Assembler::GE, CRC_by32_loop);
3212 cmn(len, 32);
3213 br(Assembler::NE, CRC_less32);
3214 b(L_exit);
3215
3216 BIND(CRC_by4_loop);
3217 ldrw(tmp, Address(post(buf, 4)));
3218 subs(len, len, 4);
3219 crc32cw(crc, crc, tmp);
3220 br(Assembler::GE, CRC_by4_loop);
3221 adds(len, len, 4);
3222 br(Assembler::LE, L_exit);
3223 BIND(CRC_by1_loop);
3224 ldrb(tmp, Address(post(buf, 1)));
3225 subs(len, len, 1);
3226 crc32cb(crc, crc, tmp);
3227 br(Assembler::GT, CRC_by1_loop);
3228 b(L_exit);
3229
3230 BIND(CRC_by64_pre);
3231 sub(buf, buf, 8);
3232 ldp(table0, table1, Address(buf, 8));
3233 crc32cx(crc, crc, table0);
3234 ldr(table2, Address(buf, 24));
3235 crc32cx(crc, crc, table1);
3236 ldr(table3, Address(buf, 32));
3237 crc32cx(crc, crc, table2);
3238 ldr(table0, Address(buf, 40));
3239 crc32cx(crc, crc, table3);
3240 ldr(table1, Address(buf, 48));
3241 crc32cx(crc, crc, table0);
3242 ldr(table2, Address(buf, 56));
3243 crc32cx(crc, crc, table1);
3244 ldr(table3, Address(pre(buf, 64)));
3245
3246 b(CRC_by64_loop);
3247
3248 align(CodeEntryAlignment);
3249 BIND(CRC_by64_loop);
3250 subs(len, len, 64);
3251 crc32cx(crc, crc, table2);
3252 ldr(table0, Address(buf, 8));
3253 crc32cx(crc, crc, table3);
3254 ldr(table1, Address(buf, 16));
3255 crc32cx(crc, crc, table0);
3256 ldr(table2, Address(buf, 24));
3257 crc32cx(crc, crc, table1);
3258 ldr(table3, Address(buf, 32));
3259 crc32cx(crc, crc, table2);
3260 ldr(table0, Address(buf, 40));
3261 crc32cx(crc, crc, table3);
3262 ldr(table1, Address(buf, 48));
3263 crc32cx(crc, crc, table0);
3264 ldr(table2, Address(buf, 56));
3265 crc32cx(crc, crc, table1);
3266 ldr(table3, Address(pre(buf, 64)));
3267 br(Assembler::GE, CRC_by64_loop);
3268
3269 // post-loop
3270 crc32cx(crc, crc, table2);
3271 crc32cx(crc, crc, table3);
3272
3273 sub(len, len, 64);
3274 add(buf, buf, 8);
3275 cmn(len, 128);
3276 br(Assembler::NE, CRC_less64);
3277 BIND(L_exit);
3278 return;
3279 }
3280
3281
3282 SkipIfEqual::SkipIfEqual(
3283 MacroAssembler* masm, const bool* flag_addr, bool value) {
3284 _masm = masm;
3285 unsigned long offset;
3286 _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset);
3287 _masm->ldrb(rscratch1, Address(rscratch1, offset));
3288 _masm->cbzw(rscratch1, _label);
3289 }
3290
3291 SkipIfEqual::~SkipIfEqual() {
3292 _masm->bind(_label);
3293 }
3294
3295 void MacroAssembler::addptr(const Address &dst, int32_t src) {
3296 Address adr;
3297 switch(dst.getMode()) {
3298 case Address::base_plus_offset:
3299 // This is the expected mode, although we allow all the other
3300 // forms below.
|