4371 __ jcc(Assembler::belowEqual, Lcbc_exit);
4372 __ evpxorq(RK11, RK11, RK11, Assembler::AVX_512bit);
4373 __ evpxorq(RK12, RK12, RK12, Assembler::AVX_512bit);
4374 __ cmpl(rounds, 52);
4375 __ jcc(Assembler::belowEqual, Lcbc_exit);
4376 __ evpxorq(RK13, RK13, RK13, Assembler::AVX_512bit);
4377 __ evpxorq(RK14, RK14, RK14, Assembler::AVX_512bit);
4378
4379 __ BIND(Lcbc_exit);
4380 __ pop(rbx);
4381 #ifdef _WIN64
4382 __ movl(rax, len_mem);
4383 #else
4384 __ pop(rax); // return length
4385 #endif
4386 __ leave(); // required for proper stackwalking of RuntimeStub frame
4387 __ ret(0);
4388 return start;
4389 }
4390
4391 // byte swap x86 long
4392 address generate_ghash_long_swap_mask() {
4393 __ align(CodeEntryAlignment);
4394 StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
4395 address start = __ pc();
4396 __ emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
4397 __ emit_data64(0x0706050403020100, relocInfo::none );
4398 return start;
4399 }
4400
4401 // byte swap x86 byte array
4402 address generate_ghash_byte_swap_mask() {
4403 __ align(CodeEntryAlignment);
4404 StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
4405 address start = __ pc();
4406 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
4407 __ emit_data64(0x0001020304050607, relocInfo::none );
4408 return start;
4409 }
4410
5871 for (int ii = 0; ii < 16; ++ii) {
5872 memcpy(dst + 32 * ii, src + 16 * ii, 16);
5873 memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
5874 }
5875 StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
5876 StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
5877 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
5878 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
5879 }
5880 if (UseSHA512Intrinsics) {
5881 StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
5882 StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
5883 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
5884 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
5885 }
5886
5887 // Generate GHASH intrinsics code
5888 if (UseGHASHIntrinsics) {
5889 StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
5890 StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
5891 StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
5892 }
5893
5894 if (UseBASE64Intrinsics) {
5895 StubRoutines::x86::_and_mask = base64_and_mask_addr();
5896 StubRoutines::x86::_bswap_mask = base64_bswap_mask_addr();
5897 StubRoutines::x86::_base64_charset = base64_charset_addr();
5898 StubRoutines::x86::_url_charset = base64url_charset_addr();
5899 StubRoutines::x86::_gather_mask = base64_gather_mask_addr();
5900 StubRoutines::x86::_left_shift_mask = base64_left_shift_mask_addr();
5901 StubRoutines::x86::_right_shift_mask = base64_right_shift_mask_addr();
5902 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
5903 }
5904
5905 // Safefetch stubs.
5906 generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
5907 &StubRoutines::_safefetch32_fault_pc,
5908 &StubRoutines::_safefetch32_continuation_pc);
5909 generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
5910 &StubRoutines::_safefetchN_fault_pc,
5911 &StubRoutines::_safefetchN_continuation_pc);
|
4371 __ jcc(Assembler::belowEqual, Lcbc_exit);
4372 __ evpxorq(RK11, RK11, RK11, Assembler::AVX_512bit);
4373 __ evpxorq(RK12, RK12, RK12, Assembler::AVX_512bit);
4374 __ cmpl(rounds, 52);
4375 __ jcc(Assembler::belowEqual, Lcbc_exit);
4376 __ evpxorq(RK13, RK13, RK13, Assembler::AVX_512bit);
4377 __ evpxorq(RK14, RK14, RK14, Assembler::AVX_512bit);
4378
4379 __ BIND(Lcbc_exit);
4380 __ pop(rbx);
4381 #ifdef _WIN64
4382 __ movl(rax, len_mem);
4383 #else
4384 __ pop(rax); // return length
4385 #endif
4386 __ leave(); // required for proper stackwalking of RuntimeStub frame
4387 __ ret(0);
4388 return start;
4389 }
4390
4391 // Polynomial x^128+x^127+x^126+x^121+1
4392 address ghash_polynomial_addr() {
4393 __ align(CodeEntryAlignment);
4394 StubCodeMark mark(this, "StubRoutines", "_ghash_poly_addr");
4395 address start = __ pc();
4396 __ emit_data64(0x0000000000000001, relocInfo::none);
4397 __ emit_data64(0xc200000000000000, relocInfo::none);
4398 return start;
4399 }
4400
4401 address ghash_shufflemask_addr() {
4402 __ align(CodeEntryAlignment);
4403 StubCodeMark mark(this, "StubRoutines", "_ghash_shuffmask_addr");
4404 address start = __ pc();
4405 __ emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none);
4406 __ emit_data64(0x0f0f0f0f0f0f0f0f, relocInfo::none);
4407 return start;
4408 }
4409
4410 // Ghash single and multi block operations using AVX instructions
4411 address generate_avx_ghash_processBlocks() {
4412 __ align(CodeEntryAlignment);
4413
4414 StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
4415 address start = __ pc();
4416
4417 // arguments
4418 const Register state = c_rarg0;
4419 const Register htbl = c_rarg1;
4420 const Register data = c_rarg2;
4421 const Register blocks = c_rarg3;
4422 __ enter();
4423 // Save state before entering routine
4424 __ push(r12);
4425 __ push(r13);
4426 __ push(r14);
4427 __ push(r15);
4428 __ push(rbx);
4429 __ push(rbp);
4430
4431 __ avx_ghash(state, htbl, data, blocks);
4432
4433 __ pop(rbp);
4434 __ pop(rbx);
4435 __ pop(r15);
4436 __ pop(r14);
4437 __ pop(r13);
4438 __ pop(r12);
4439 __ leave(); // required for proper stackwalking of RuntimeStub frame
4440 __ ret(0);
4441 return start;
4442 }
4443
4444 // byte swap x86 long
4445 address generate_ghash_long_swap_mask() {
4446 __ align(CodeEntryAlignment);
4447 StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
4448 address start = __ pc();
4449 __ emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
4450 __ emit_data64(0x0706050403020100, relocInfo::none );
4451 return start;
4452 }
4453
4454 // byte swap x86 byte array
4455 address generate_ghash_byte_swap_mask() {
4456 __ align(CodeEntryAlignment);
4457 StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
4458 address start = __ pc();
4459 __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
4460 __ emit_data64(0x0001020304050607, relocInfo::none );
4461 return start;
4462 }
4463
5924 for (int ii = 0; ii < 16; ++ii) {
5925 memcpy(dst + 32 * ii, src + 16 * ii, 16);
5926 memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
5927 }
5928 StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
5929 StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
5930 StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
5931 StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
5932 }
5933 if (UseSHA512Intrinsics) {
5934 StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
5935 StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
5936 StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
5937 StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
5938 }
5939
5940 // Generate GHASH intrinsics code
5941 if (UseGHASHIntrinsics) {
5942 StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
5943 StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
5944 if (VM_Version::supports_avx()) {
5945 StubRoutines::x86::_ghash_shuffmask_addr = ghash_shufflemask_addr();
5946 StubRoutines::x86::_ghash_poly_addr = ghash_polynomial_addr();
5947 StubRoutines::_ghash_processBlocks = generate_avx_ghash_processBlocks();
5948 } else {
5949 StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
5950 }
5951 }
5952
5953 if (UseBASE64Intrinsics) {
5954 StubRoutines::x86::_and_mask = base64_and_mask_addr();
5955 StubRoutines::x86::_bswap_mask = base64_bswap_mask_addr();
5956 StubRoutines::x86::_base64_charset = base64_charset_addr();
5957 StubRoutines::x86::_url_charset = base64url_charset_addr();
5958 StubRoutines::x86::_gather_mask = base64_gather_mask_addr();
5959 StubRoutines::x86::_left_shift_mask = base64_left_shift_mask_addr();
5960 StubRoutines::x86::_right_shift_mask = base64_right_shift_mask_addr();
5961 StubRoutines::_base64_encodeBlock = generate_base64_encodeBlock();
5962 }
5963
5964 // Safefetch stubs.
5965 generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
5966 &StubRoutines::_safefetch32_fault_pc,
5967 &StubRoutines::_safefetch32_continuation_pc);
5968 generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
5969 &StubRoutines::_safefetchN_fault_pc,
5970 &StubRoutines::_safefetchN_continuation_pc);
|