3211 br(Assembler::GT, L_by1_loop); 3212 b(L_exit); 3213 3214 align(CodeEntryAlignment); 3215 BIND(L_by16_loop); 3216 subs(len, len, 16); 3217 ldp(tmp, tmp3, Address(post(buf, 16))); 3218 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 3219 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 3220 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, false); 3221 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, true); 3222 br(Assembler::GE, L_by16_loop); 3223 adds(len, len, 16-4); 3224 br(Assembler::GE, L_by4_loop); 3225 adds(len, len, 4); 3226 br(Assembler::GT, L_by1_loop); 3227 BIND(L_exit); 3228 mvnw(crc, crc); 3229 } 3230 3231 /** 3232 * @param crc register containing existing CRC (32-bit) 3233 * @param buf register pointing to input byte buffer (byte*) 3234 * @param len register containing number of bytes 3235 * @param table register that will contain address of CRC table 3236 * @param tmp scratch register 3237 */ 3238 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len, 3239 Register table0, Register table1, Register table2, Register table3, 3240 Register tmp, Register tmp2, Register tmp3) { 3241 Label L_exit; 3242 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop; 3243 3244 subs(len, len, 64); 3245 br(Assembler::GE, CRC_by64_loop); 3246 adds(len, len, 64-4); 3247 br(Assembler::GE, CRC_by4_loop); 3248 adds(len, len, 4); 3249 br(Assembler::GT, CRC_by1_loop); 3250 b(L_exit); 3251 3252 BIND(CRC_by4_loop); 3253 ldrw(tmp, Address(post(buf, 4))); 3254 subs(len, len, 4); 3255 crc32cw(crc, crc, tmp); 3256 br(Assembler::GE, CRC_by4_loop); 3257 adds(len, len, 4); 3258 br(Assembler::LE, L_exit); 3259 BIND(CRC_by1_loop); 3260 ldrb(tmp, Address(post(buf, 1))); 3261 subs(len, len, 1); 3262 crc32cb(crc, crc, tmp); 3263 br(Assembler::GT, CRC_by1_loop); 3264 b(L_exit); 3265 3266 align(CodeEntryAlignment); 3267 BIND(CRC_by64_loop); 3268 subs(len, len, 64); 3269 ldp(tmp, tmp3, Address(post(buf, 16))); 3270 crc32cx(crc, crc, tmp); 3271 crc32cx(crc, crc, tmp3); 3272 ldp(tmp, tmp3, Address(post(buf, 16))); 3273 crc32cx(crc, crc, tmp); 3274 crc32cx(crc, crc, tmp3); 3275 ldp(tmp, tmp3, Address(post(buf, 16))); 3276 crc32cx(crc, crc, tmp); 3277 crc32cx(crc, crc, tmp3); 3278 ldp(tmp, tmp3, Address(post(buf, 16))); 3279 crc32cx(crc, crc, tmp); 3280 crc32cx(crc, crc, tmp3); 3281 br(Assembler::GE, CRC_by64_loop); 3282 adds(len, len, 64-4); 3283 br(Assembler::GE, CRC_by4_loop); 3284 adds(len, len, 4); 3285 br(Assembler::GT, CRC_by1_loop); 3286 BIND(L_exit); 3287 return; 3288 } 3289 3290 SkipIfEqual::SkipIfEqual( 3291 MacroAssembler* masm, const bool* flag_addr, bool value) { 3292 _masm = masm; 3293 unsigned long offset; 3294 _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset); 3295 _masm->ldrb(rscratch1, Address(rscratch1, offset)); 3296 _masm->cbzw(rscratch1, _label); 3297 } 3298 3299 SkipIfEqual::~SkipIfEqual() { 3300 _masm->bind(_label); 3301 } 3302 3303 void MacroAssembler::addptr(const Address &dst, int32_t src) { 3304 Address adr; 3305 switch(dst.getMode()) { 3306 case Address::base_plus_offset: 3307 // This is the expected mode, although we allow all the other 3308 // forms below. | 3211 br(Assembler::GT, L_by1_loop); 3212 b(L_exit); 3213 3214 align(CodeEntryAlignment); 3215 BIND(L_by16_loop); 3216 subs(len, len, 16); 3217 ldp(tmp, tmp3, Address(post(buf, 16))); 3218 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, false); 3219 update_word_crc32(crc, tmp, tmp2, table0, table1, table2, table3, true); 3220 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, false); 3221 update_word_crc32(crc, tmp3, tmp2, table0, table1, table2, table3, true); 3222 br(Assembler::GE, L_by16_loop); 3223 adds(len, len, 16-4); 3224 br(Assembler::GE, L_by4_loop); 3225 adds(len, len, 4); 3226 br(Assembler::GT, L_by1_loop); 3227 BIND(L_exit); 3228 mvnw(crc, crc); 3229 } 3230 3231 void MacroAssembler::kernel_crc32c_using_crc32c(Register crc, Register buf, 3232 Register len, Register tmp0, Register tmp1, Register tmp2, 3233 Register tmp3) { 3234 Label CRC_by64_loop, CRC_by4_loop, CRC_by1_loop, CRC_less64, CRC_by64_pre, CRC_by32_loop, CRC_less32, L_exit; 3235 assert_different_registers(crc, buf, len, tmp0, tmp1, tmp2, tmp3); 3236 3237 subs(len, len, 128); 3238 br(Assembler::GE, CRC_by64_pre); 3239 BIND(CRC_less64); 3240 adds(len, len, 128-32); 3241 br(Assembler::GE, CRC_by32_loop); 3242 BIND(CRC_less32); 3243 adds(len, len, 32-4); 3244 br(Assembler::GE, CRC_by4_loop); 3245 adds(len, len, 4); 3246 br(Assembler::GT, CRC_by1_loop); 3247 b(L_exit); 3248 3249 BIND(CRC_by32_loop); 3250 ldp(tmp0, tmp1, Address(post(buf, 16))); 3251 subs(len, len, 32); 3252 crc32cx(crc, crc, tmp0); 3253 ldr(tmp2, Address(post(buf, 8))); 3254 crc32cx(crc, crc, tmp1); 3255 ldr(tmp3, Address(post(buf, 8))); 3256 crc32cx(crc, crc, tmp2); 3257 crc32cx(crc, crc, tmp3); 3258 br(Assembler::GE, CRC_by32_loop); 3259 cmn(len, 32); 3260 br(Assembler::NE, CRC_less32); 3261 b(L_exit); 3262 3263 BIND(CRC_by4_loop); 3264 ldrw(tmp0, Address(post(buf, 4))); 3265 subs(len, len, 4); 3266 crc32cw(crc, crc, tmp0); 3267 br(Assembler::GE, CRC_by4_loop); 3268 adds(len, len, 4); 3269 br(Assembler::LE, L_exit); 3270 BIND(CRC_by1_loop); 3271 ldrb(tmp0, Address(post(buf, 1))); 3272 subs(len, len, 1); 3273 crc32cb(crc, crc, tmp0); 3274 br(Assembler::GT, CRC_by1_loop); 3275 b(L_exit); 3276 3277 BIND(CRC_by64_pre); 3278 sub(buf, buf, 8); 3279 ldp(tmp0, tmp1, Address(buf, 8)); 3280 crc32cx(crc, crc, tmp0); 3281 ldr(tmp2, Address(buf, 24)); 3282 crc32cx(crc, crc, tmp1); 3283 ldr(tmp3, Address(buf, 32)); 3284 crc32cx(crc, crc, tmp2); 3285 ldr(tmp0, Address(buf, 40)); 3286 crc32cx(crc, crc, tmp3); 3287 ldr(tmp1, Address(buf, 48)); 3288 crc32cx(crc, crc, tmp0); 3289 ldr(tmp2, Address(buf, 56)); 3290 crc32cx(crc, crc, tmp1); 3291 ldr(tmp3, Address(pre(buf, 64))); 3292 3293 b(CRC_by64_loop); 3294 3295 align(CodeEntryAlignment); 3296 BIND(CRC_by64_loop); 3297 subs(len, len, 64); 3298 crc32cx(crc, crc, tmp2); 3299 ldr(tmp0, Address(buf, 8)); 3300 crc32cx(crc, crc, tmp3); 3301 ldr(tmp1, Address(buf, 16)); 3302 crc32cx(crc, crc, tmp0); 3303 ldr(tmp2, Address(buf, 24)); 3304 crc32cx(crc, crc, tmp1); 3305 ldr(tmp3, Address(buf, 32)); 3306 crc32cx(crc, crc, tmp2); 3307 ldr(tmp0, Address(buf, 40)); 3308 crc32cx(crc, crc, tmp3); 3309 ldr(tmp1, Address(buf, 48)); 3310 crc32cx(crc, crc, tmp0); 3311 ldr(tmp2, Address(buf, 56)); 3312 crc32cx(crc, crc, tmp1); 3313 ldr(tmp3, Address(pre(buf, 64))); 3314 br(Assembler::GE, CRC_by64_loop); 3315 3316 // post-loop 3317 crc32cx(crc, crc, tmp2); 3318 crc32cx(crc, crc, tmp3); 3319 3320 sub(len, len, 64); 3321 add(buf, buf, 8); 3322 cmn(len, 128); 3323 br(Assembler::NE, CRC_less64); 3324 BIND(L_exit); 3325 } 3326 3327 /** 3328 * @param crc register containing existing CRC (32-bit) 3329 * @param buf register pointing to input byte buffer (byte*) 3330 * @param len register containing number of bytes 3331 * @param table register that will contain address of CRC table 3332 * @param tmp scratch register 3333 */ 3334 void MacroAssembler::kernel_crc32c(Register crc, Register buf, Register len, 3335 Register table0, Register table1, Register table2, Register table3, 3336 Register tmp, Register tmp2, Register tmp3) { 3337 kernel_crc32c_using_crc32c(crc, buf, len, table0, table1, table2, table3); 3338 } 3339 3340 3341 SkipIfEqual::SkipIfEqual( 3342 MacroAssembler* masm, const bool* flag_addr, bool value) { 3343 _masm = masm; 3344 unsigned long offset; 3345 _masm->adrp(rscratch1, ExternalAddress((address)flag_addr), offset); 3346 _masm->ldrb(rscratch1, Address(rscratch1, offset)); 3347 _masm->cbzw(rscratch1, _label); 3348 } 3349 3350 SkipIfEqual::~SkipIfEqual() { 3351 _masm->bind(_label); 3352 } 3353 3354 void MacroAssembler::addptr(const Address &dst, int32_t src) { 3355 Address adr; 3356 switch(dst.getMode()) { 3357 case Address::base_plus_offset: 3358 // This is the expected mode, although we allow all the other 3359 // forms below. |