Print this page
4896 Performance improvements for KCF AES modes

@@ -18,10 +18,11 @@
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
  */
 
 /*
  * AES provider for the Kernel Cryptographic Framework (KCF)
  */

@@ -384,21 +385,21 @@
 
         return (CRYPTO_SUCCESS);
 }
 
 static void
-aes_copy_block64(uint8_t *in, uint64_t *out)
+aes_copy_block64(const uint8_t *in, uint64_t *out)
 {
         if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
                 /* LINTED: pointer alignment */
                 out[0] = *(uint64_t *)&in[0];
                 /* LINTED: pointer alignment */
                 out[1] = *(uint64_t *)&in[8];
         } else {
                 uint8_t *iv8 = (uint8_t *)&out[0];
 
-                AES_COPY_BLOCK(in, iv8);
+                AES_COPY_BLOCK_UNALIGNED(in, iv8);
         }
 }
 
 
 static int

@@ -473,11 +474,11 @@
 
                 /* order of following 2 lines MUST not be reversed */
                 ciphertext->cd_offset = ciphertext->cd_length;
                 ciphertext->cd_length = saved_length - ciphertext->cd_length;
                 ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
                 if (ret != CRYPTO_SUCCESS) {
                         return (ret);
                 }
 
                 if (plaintext != ciphertext) {

@@ -494,12 +495,12 @@
 
                 /* order of following 2 lines MUST not be reversed */
                 ciphertext->cd_offset = ciphertext->cd_length;
                 ciphertext->cd_length = saved_length - ciphertext->cd_length;
                 ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-                    aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+                    AES_XOR_BLOCK);
                 if (ret != CRYPTO_SUCCESS) {
                         return (ret);
                 }
 
                 if (plaintext != ciphertext) {

@@ -588,12 +589,12 @@
                 /* order of following 2 lines MUST not be reversed */
                 plaintext->cd_offset = plaintext->cd_length;
                 plaintext->cd_length = saved_length - plaintext->cd_length;
 
                 ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-                    aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+                    AES_XOR_BLOCK);
                 if (ret == CRYPTO_SUCCESS) {
                         if (plaintext != ciphertext) {
                                 plaintext->cd_length =
                                     plaintext->cd_offset - saved_offset;
                         }

@@ -606,11 +607,12 @@
                 /* order of following 2 lines MUST not be reversed */
                 plaintext->cd_offset = plaintext->cd_length;
                 plaintext->cd_length = saved_length - plaintext->cd_length;
 
                 ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK,
+                    AES_COPY_BLOCK, aes_ctr_mode);
                 if (ret == CRYPTO_SUCCESS) {
                         if (plaintext != ciphertext) {
                                 plaintext->cd_length =
                                     plaintext->cd_offset - saved_offset;
                         }

@@ -720,17 +722,19 @@
 
         AES_ARG_INPLACE(ciphertext, plaintext);
 
         /*
          * Compute number of bytes that will hold the plaintext.
-         * This is not necessary for CCM, GCM, and GMAC since these
+         * This is not necessary for CCM and GMAC since these
          * mechanisms never return plaintext for update operations.
          */
-        if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
+        if ((aes_ctx->ac_flags & (CCM_MODE|GMAC_MODE)) == 0) {
                 out_len = aes_ctx->ac_remainder_len;
                 out_len += ciphertext->cd_length;
                 out_len &= ~(AES_BLOCK_LEN - 1);
+                if (aes_ctx->ac_flags & GCM_MODE)
+                        out_len -= ((gcm_ctx_t *)aes_ctx)->gcm_tag_len;
 
                 /* return length needed to store the output */
                 if (plaintext->cd_length < out_len) {
                         plaintext->cd_length = out_len;
                         return (CRYPTO_BUFFER_TOO_SMALL);

@@ -816,20 +820,20 @@
                         if (ret != CRYPTO_SUCCESS)
                                 return (ret);
                 }
         } else if (aes_ctx->ac_flags & CCM_MODE) {
                 ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
                 if (ret != CRYPTO_SUCCESS) {
                         return (ret);
                 }
         } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
                 size_t saved_offset = data->cd_offset;
 
                 ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-                    aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+                    AES_XOR_BLOCK);
                 if (ret != CRYPTO_SUCCESS) {
                         return (ret);
                 }
                 data->cd_length = data->cd_offset - saved_offset;
                 data->cd_offset = saved_offset;

@@ -901,12 +905,12 @@
                 ASSERT(aes_ctx->ac_processed_data_len == pt_len);
                 ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
                 saved_offset = data->cd_offset;
                 saved_length = data->cd_length;
                 ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-                    aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+                    AES_XOR_BLOCK);
                 if (ret == CRYPTO_SUCCESS) {
                         data->cd_length = data->cd_offset - saved_offset;
                 } else {
                         data->cd_length = saved_length;
                 }

@@ -915,25 +919,25 @@
                 if (ret != CRYPTO_SUCCESS) {
                         return (ret);
                 }
         } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
                 /*
-                 * This is where all the plaintext is returned, make sure
-                 * the plaintext buffer is big enough
+                 * Check to make sure there is enough space for remaining
+                 * plaintext.
                  */
                 gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
-                size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
+                size_t pt_len = ctx->gcm_last_input_fill - ctx->gcm_tag_len;
 
                 if (data->cd_length < pt_len) {
                         data->cd_length = pt_len;
                         return (CRYPTO_BUFFER_TOO_SMALL);
                 }
-
                 saved_offset = data->cd_offset;
                 saved_length = data->cd_length;
                 ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+                    AES_XOR_BLOCK, aes_ctr_mode);
                 if (ret == CRYPTO_SUCCESS) {
                         data->cd_length = data->cd_offset - saved_offset;
                 } else {
                         data->cd_length = saved_length;
                 }

@@ -942,11 +946,10 @@
                 if (ret != CRYPTO_SUCCESS) {
                         return (ret);
                 }
         }
 
-
         if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
                 data->cd_length = 0;
         }
 
         (void) aes_free_context(ctx);

@@ -1041,19 +1044,19 @@
 
         if (ret == CRYPTO_SUCCESS) {
                 if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
                         ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
                             ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
-                            aes_xor_block);
+                            AES_XOR_BLOCK);
                         if (ret != CRYPTO_SUCCESS)
                                 goto out;
                         ASSERT(aes_ctx.ac_remainder_len == 0);
                 } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
                     mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
                         ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
                             ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
-                            aes_copy_block, aes_xor_block);
+                            AES_COPY_BLOCK, AES_XOR_BLOCK);
                         if (ret != CRYPTO_SUCCESS)
                                 goto out;
                         ASSERT(aes_ctx.ac_remainder_len == 0);
                 } else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) {
                         if (aes_ctx.ac_remainder_len > 0) {

@@ -1180,11 +1183,11 @@
                             == aes_ctx.ac_data_len);
                         ASSERT(aes_ctx.ac_processed_mac_len
                             == aes_ctx.ac_mac_len);
                         ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
                             plaintext, AES_BLOCK_LEN, aes_encrypt_block,
-                            aes_copy_block, aes_xor_block);
+                            AES_COPY_BLOCK, AES_XOR_BLOCK);
                         ASSERT(aes_ctx.ac_remainder_len == 0);
                         if ((ret == CRYPTO_SUCCESS) &&
                             (ciphertext != plaintext)) {
                                 plaintext->cd_length =
                                     plaintext->cd_offset - saved_offset;

@@ -1193,11 +1196,11 @@
                         }
                 } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
                     mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
                         ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
                             plaintext, AES_BLOCK_LEN, aes_encrypt_block,
-                            aes_xor_block);
+                            AES_COPY_BLOCK, AES_XOR_BLOCK, aes_ctr_mode);
                         ASSERT(aes_ctx.ac_remainder_len == 0);
                         if ((ret == CRYPTO_SUCCESS) &&
                             (ciphertext != plaintext)) {
                                 plaintext->cd_length =
                                     plaintext->cd_offset - saved_offset;

@@ -1235,16 +1238,11 @@
 
         if (aes_ctx.ac_flags & CCM_MODE) {
                 if (aes_ctx.ac_pt_buf != NULL) {
                         kmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len);
                 }
-        } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) {
-                if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) {
-                        kmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
-                            ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
                 }
-        }
 
         return (ret);
 }
 
 /*

@@ -1350,39 +1348,39 @@
                     mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) {
                         return (CRYPTO_MECHANISM_PARAM_INVALID);
                 }
                 pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
                 rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
-                    pp->cb, aes_copy_block);
+                    pp->cb, AES_COPY_BLOCK);
                 break;
         }
         case AES_CCM_MECH_INFO_TYPE:
                 if (mechanism->cm_param == NULL ||
                     mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) {
                         return (CRYPTO_MECHANISM_PARAM_INVALID);
                 }
                 rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
                     kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
-                    aes_xor_block);
+                    AES_XOR_BLOCK);
                 break;
         case AES_GCM_MECH_INFO_TYPE:
                 if (mechanism->cm_param == NULL ||
                     mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) {
                         return (CRYPTO_MECHANISM_PARAM_INVALID);
                 }
                 rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-                    aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+                    AES_XOR_BLOCK);
                 break;
         case AES_GMAC_MECH_INFO_TYPE:
                 if (mechanism->cm_param == NULL ||
                     mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) {
                         return (CRYPTO_MECHANISM_PARAM_INVALID);
                 }
                 rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
-                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-                    aes_xor_block);
+                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+                    AES_XOR_BLOCK);
                 break;
         case AES_ECB_MECH_INFO_TYPE:
                 aes_ctx->ac_flags |= ECB_MODE;
         }