Print this page
4896 Performance improvements for KCF AES modes
@@ -18,10 +18,11 @@
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
*/
/*
* AES provider for the Kernel Cryptographic Framework (KCF)
*/
@@ -384,21 +385,21 @@
return (CRYPTO_SUCCESS);
}
static void
-aes_copy_block64(uint8_t *in, uint64_t *out)
+aes_copy_block64(const uint8_t *in, uint64_t *out)
{
if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
/* LINTED: pointer alignment */
out[0] = *(uint64_t *)&in[0];
/* LINTED: pointer alignment */
out[1] = *(uint64_t *)&in[8];
} else {
uint8_t *iv8 = (uint8_t *)&out[0];
- AES_COPY_BLOCK(in, iv8);
+ AES_COPY_BLOCK_UNALIGNED(in, iv8);
}
}
static int
@@ -473,11 +474,11 @@
/* order of following 2 lines MUST not be reversed */
ciphertext->cd_offset = ciphertext->cd_length;
ciphertext->cd_length = saved_length - ciphertext->cd_length;
ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
- AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
if (plaintext != ciphertext) {
@@ -494,12 +495,12 @@
/* order of following 2 lines MUST not be reversed */
ciphertext->cd_offset = ciphertext->cd_length;
ciphertext->cd_length = saved_length - ciphertext->cd_length;
ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
- AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
- aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+ AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
if (plaintext != ciphertext) {
@@ -588,12 +589,12 @@
/* order of following 2 lines MUST not be reversed */
plaintext->cd_offset = plaintext->cd_length;
plaintext->cd_length = saved_length - plaintext->cd_length;
ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
- AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
- aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+ AES_XOR_BLOCK);
if (ret == CRYPTO_SUCCESS) {
if (plaintext != ciphertext) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
}
@@ -606,11 +607,12 @@
/* order of following 2 lines MUST not be reversed */
plaintext->cd_offset = plaintext->cd_length;
plaintext->cd_length = saved_length - plaintext->cd_length;
ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
- AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK,
+ AES_COPY_BLOCK, aes_ctr_mode);
if (ret == CRYPTO_SUCCESS) {
if (plaintext != ciphertext) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
}
@@ -720,17 +722,19 @@
AES_ARG_INPLACE(ciphertext, plaintext);
/*
* Compute number of bytes that will hold the plaintext.
- * This is not necessary for CCM, GCM, and GMAC since these
+ * This is not necessary for CCM and GMAC since these
* mechanisms never return plaintext for update operations.
*/
- if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
+ if ((aes_ctx->ac_flags & (CCM_MODE|GMAC_MODE)) == 0) {
out_len = aes_ctx->ac_remainder_len;
out_len += ciphertext->cd_length;
out_len &= ~(AES_BLOCK_LEN - 1);
+ if (aes_ctx->ac_flags & GCM_MODE)
+ out_len -= ((gcm_ctx_t *)aes_ctx)->gcm_tag_len;
/* return length needed to store the output */
if (plaintext->cd_length < out_len) {
plaintext->cd_length = out_len;
return (CRYPTO_BUFFER_TOO_SMALL);
@@ -816,20 +820,20 @@
if (ret != CRYPTO_SUCCESS)
return (ret);
}
} else if (aes_ctx->ac_flags & CCM_MODE) {
ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
- AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
size_t saved_offset = data->cd_offset;
ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
- AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
- aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+ AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
data->cd_length = data->cd_offset - saved_offset;
data->cd_offset = saved_offset;
@@ -901,12 +905,12 @@
ASSERT(aes_ctx->ac_processed_data_len == pt_len);
ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
saved_offset = data->cd_offset;
saved_length = data->cd_length;
ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
- AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
- aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+ AES_XOR_BLOCK);
if (ret == CRYPTO_SUCCESS) {
data->cd_length = data->cd_offset - saved_offset;
} else {
data->cd_length = saved_length;
}
@@ -915,25 +919,25 @@
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
/*
- * This is where all the plaintext is returned, make sure
- * the plaintext buffer is big enough
+ * Check to make sure there is enough space for remaining
+ * plaintext.
*/
gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
- size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
+ size_t pt_len = ctx->gcm_last_input_fill - ctx->gcm_tag_len;
if (data->cd_length < pt_len) {
data->cd_length = pt_len;
return (CRYPTO_BUFFER_TOO_SMALL);
}
-
saved_offset = data->cd_offset;
saved_length = data->cd_length;
ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
- AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+ AES_XOR_BLOCK, aes_ctr_mode);
if (ret == CRYPTO_SUCCESS) {
data->cd_length = data->cd_offset - saved_offset;
} else {
data->cd_length = saved_length;
}
@@ -942,11 +946,10 @@
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
}
-
if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
data->cd_length = 0;
}
(void) aes_free_context(ctx);
@@ -1041,19 +1044,19 @@
if (ret == CRYPTO_SUCCESS) {
if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
- aes_xor_block);
+ AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS)
goto out;
ASSERT(aes_ctx.ac_remainder_len == 0);
} else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
- aes_copy_block, aes_xor_block);
+ AES_COPY_BLOCK, AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS)
goto out;
ASSERT(aes_ctx.ac_remainder_len == 0);
} else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) {
if (aes_ctx.ac_remainder_len > 0) {
@@ -1180,11 +1183,11 @@
== aes_ctx.ac_data_len);
ASSERT(aes_ctx.ac_processed_mac_len
== aes_ctx.ac_mac_len);
ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
plaintext, AES_BLOCK_LEN, aes_encrypt_block,
- aes_copy_block, aes_xor_block);
+ AES_COPY_BLOCK, AES_XOR_BLOCK);
ASSERT(aes_ctx.ac_remainder_len == 0);
if ((ret == CRYPTO_SUCCESS) &&
(ciphertext != plaintext)) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
@@ -1193,11 +1196,11 @@
}
} else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
plaintext, AES_BLOCK_LEN, aes_encrypt_block,
- aes_xor_block);
+ AES_COPY_BLOCK, AES_XOR_BLOCK, aes_ctr_mode);
ASSERT(aes_ctx.ac_remainder_len == 0);
if ((ret == CRYPTO_SUCCESS) &&
(ciphertext != plaintext)) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
@@ -1235,16 +1238,11 @@
if (aes_ctx.ac_flags & CCM_MODE) {
if (aes_ctx.ac_pt_buf != NULL) {
kmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len);
}
- } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) {
- if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) {
- kmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
- ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
}
- }
return (ret);
}
/*
@@ -1350,39 +1348,39 @@
mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
- pp->cb, aes_copy_block);
+ pp->cb, AES_COPY_BLOCK);
break;
}
case AES_CCM_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
- aes_xor_block);
+ AES_XOR_BLOCK);
break;
case AES_GCM_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
- AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
- aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+ AES_XOR_BLOCK);
break;
case AES_GMAC_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
- AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
- aes_xor_block);
+ AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+ AES_XOR_BLOCK);
break;
case AES_ECB_MECH_INFO_TYPE:
aes_ctx->ac_flags |= ECB_MODE;
}