src/share/native/sun/security/ec/impl/ec.c

rev 13649 : 8147502: Digest is incorrectly truncated for ECDSA signatures when the bit length of n is less than the field size
Summary: Truncate the digest according to the group order, not the field size
Reviewed-by: jnimeh

*** 1,7 ****
  /*
!  * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved.
   * Use is subject to license terms.
   *
   * This library is free software; you can redistribute it and/or
   * modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either
--- 1,7 ----
  /*
!  * Copyright (c) 2007, 2019, Oracle and/or its affiliates. All rights reserved.
   * Use is subject to license terms.
   *
   * This library is free software; you can redistribute it and/or
   * modify it under the terms of the GNU Lesser General Public
   * License as published by the Free Software Foundation; either
*** 657,666 ****
--- 657,667 ----
      mp_err err = MP_OKAY;
      ECParams *ecParams = NULL;
      SECItem kGpoint = { siBuffer, NULL, 0};
      int flen = 0;    /* length in bytes of the field size */
      unsigned olen;   /* length in bytes of the base point order */
+     unsigned int orderBitSize;
  
  #if EC_DEBUG
      char mpstr[256];
  #endif
  
*** 759,772 ****
      ** s = (k**-1 * (HASH(M) + d*r)) mod n
      */
      SECITEM_TO_MPINT(*digest, &s);        /* s = HASH(M)     */
  
      /* In the definition of EC signing, digests are truncated
!      * to the length of n in bits.
       * (see SEC 1 "Elliptic Curve Digit Signature Algorithm" section 4.1.*/
!     if (digest->len*8 > (unsigned int)ecParams->fieldID.size) {
!         mpl_rsh(&s,&s,digest->len*8 - ecParams->fieldID.size);
      }
  
  #if EC_DEBUG
      mp_todecimal(&n, mpstr);
      printf("n : %s (dec)\n", mpstr);
--- 760,774 ----
      ** s = (k**-1 * (HASH(M) + d*r)) mod n
      */
      SECITEM_TO_MPINT(*digest, &s);        /* s = HASH(M)     */
  
      /* In the definition of EC signing, digests are truncated
!      * to the order length
       * (see SEC 1 "Elliptic Curve Digit Signature Algorithm" section 4.1.*/
!     orderBitSize = mpl_significant_bits(&n);
!     if (digest->len*8 > orderBitSize) {
!         mpl_rsh(&s,&s,digest->len*8 - orderBitSize);
      }
  
  #if EC_DEBUG
      mp_todecimal(&n, mpstr);
      printf("n : %s (dec)\n", mpstr);
*** 895,904 ****
--- 897,907 ----
      ECParams *ecParams = NULL;
      SECItem pointC = { siBuffer, NULL, 0 };
      int slen;       /* length in bytes of a half signature (r or s) */
      int flen;       /* length in bytes of the field size */
      unsigned olen;  /* length in bytes of the base point order */
+     unsigned int orderBitSize;
  
  #if EC_DEBUG
      char mpstr[256];
      printf("ECDSA verification called\n");
  #endif
*** 974,988 ****
      ** u1 = ((HASH(M')) * c) mod n
      */
      SECITEM_TO_MPINT(*digest, &u1);                  /* u1 = HASH(M)     */
  
      /* In the definition of EC signing, digests are truncated
!      * to the length of n in bits.
       * (see SEC 1 "Elliptic Curve Digit Signature Algorithm" section 4.1.*/
      /* u1 = HASH(M')     */
!     if (digest->len*8 > (unsigned int)ecParams->fieldID.size) {
!         mpl_rsh(&u1,&u1,digest->len*8- ecParams->fieldID.size);
      }
  
  #if EC_DEBUG
      mp_todecimal(&r_, mpstr);
      printf("r_: %s (dec)\n", mpstr);
--- 977,992 ----
      ** u1 = ((HASH(M')) * c) mod n
      */
      SECITEM_TO_MPINT(*digest, &u1);                  /* u1 = HASH(M)     */
  
      /* In the definition of EC signing, digests are truncated
!      * to the order length, in bits.
       * (see SEC 1 "Elliptic Curve Digit Signature Algorithm" section 4.1.*/
      /* u1 = HASH(M')     */
!     orderBitSize = mpl_significant_bits(&n);
!     if (digest->len*8 > orderBitSize) {
!         mpl_rsh(&u1,&u1,digest->len*8- orderBitSize);
      }
  
  #if EC_DEBUG
      mp_todecimal(&r_, mpstr);
      printf("r_: %s (dec)\n", mpstr);