Print this page
4896 Performance improvements for KCF AES modes
*** 18,27 ****
--- 18,28 ----
*
* CDDL HEADER END
*/
/*
* Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
*/
/*
* AES provider for the Kernel Cryptographic Framework (KCF)
*/
*** 384,404 ****
return (CRYPTO_SUCCESS);
}
static void
! aes_copy_block64(uint8_t *in, uint64_t *out)
{
if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
/* LINTED: pointer alignment */
out[0] = *(uint64_t *)&in[0];
/* LINTED: pointer alignment */
out[1] = *(uint64_t *)&in[8];
} else {
uint8_t *iv8 = (uint8_t *)&out[0];
! AES_COPY_BLOCK(in, iv8);
}
}
static int
--- 385,405 ----
return (CRYPTO_SUCCESS);
}
static void
! aes_copy_block64(const uint8_t *in, uint64_t *out)
{
if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
/* LINTED: pointer alignment */
out[0] = *(uint64_t *)&in[0];
/* LINTED: pointer alignment */
out[1] = *(uint64_t *)&in[8];
} else {
uint8_t *iv8 = (uint8_t *)&out[0];
! AES_COPY_BLOCK_UNALIGNED(in, iv8);
}
}
static int
*** 473,483 ****
/* order of following 2 lines MUST not be reversed */
ciphertext->cd_offset = ciphertext->cd_length;
ciphertext->cd_length = saved_length - ciphertext->cd_length;
ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
! AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
if (plaintext != ciphertext) {
--- 474,484 ----
/* order of following 2 lines MUST not be reversed */
ciphertext->cd_offset = ciphertext->cd_length;
ciphertext->cd_length = saved_length - ciphertext->cd_length;
ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
! AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
if (plaintext != ciphertext) {
*** 494,505 ****
/* order of following 2 lines MUST not be reversed */
ciphertext->cd_offset = ciphertext->cd_length;
ciphertext->cd_length = saved_length - ciphertext->cd_length;
ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
! AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
! aes_xor_block);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
if (plaintext != ciphertext) {
--- 495,506 ----
/* order of following 2 lines MUST not be reversed */
ciphertext->cd_offset = ciphertext->cd_length;
ciphertext->cd_length = saved_length - ciphertext->cd_length;
ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
! AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
! AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
if (plaintext != ciphertext) {
*** 588,599 ****
/* order of following 2 lines MUST not be reversed */
plaintext->cd_offset = plaintext->cd_length;
plaintext->cd_length = saved_length - plaintext->cd_length;
ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
! AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
! aes_xor_block);
if (ret == CRYPTO_SUCCESS) {
if (plaintext != ciphertext) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
}
--- 589,600 ----
/* order of following 2 lines MUST not be reversed */
plaintext->cd_offset = plaintext->cd_length;
plaintext->cd_length = saved_length - plaintext->cd_length;
ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
! AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
! AES_XOR_BLOCK);
if (ret == CRYPTO_SUCCESS) {
if (plaintext != ciphertext) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
}
*** 606,616 ****
/* order of following 2 lines MUST not be reversed */
plaintext->cd_offset = plaintext->cd_length;
plaintext->cd_length = saved_length - plaintext->cd_length;
ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
! AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
if (ret == CRYPTO_SUCCESS) {
if (plaintext != ciphertext) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
}
--- 607,618 ----
/* order of following 2 lines MUST not be reversed */
plaintext->cd_offset = plaintext->cd_length;
plaintext->cd_length = saved_length - plaintext->cd_length;
ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
! AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK,
! AES_COPY_BLOCK, aes_ctr_mode);
if (ret == CRYPTO_SUCCESS) {
if (plaintext != ciphertext) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
}
*** 720,736 ****
AES_ARG_INPLACE(ciphertext, plaintext);
/*
* Compute number of bytes that will hold the plaintext.
! * This is not necessary for CCM, GCM, and GMAC since these
* mechanisms never return plaintext for update operations.
*/
! if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
out_len = aes_ctx->ac_remainder_len;
out_len += ciphertext->cd_length;
out_len &= ~(AES_BLOCK_LEN - 1);
/* return length needed to store the output */
if (plaintext->cd_length < out_len) {
plaintext->cd_length = out_len;
return (CRYPTO_BUFFER_TOO_SMALL);
--- 722,740 ----
AES_ARG_INPLACE(ciphertext, plaintext);
/*
* Compute number of bytes that will hold the plaintext.
! * This is not necessary for CCM and GMAC since these
* mechanisms never return plaintext for update operations.
*/
! if ((aes_ctx->ac_flags & (CCM_MODE|GMAC_MODE)) == 0) {
out_len = aes_ctx->ac_remainder_len;
out_len += ciphertext->cd_length;
out_len &= ~(AES_BLOCK_LEN - 1);
+ if (aes_ctx->ac_flags & GCM_MODE)
+ out_len -= ((gcm_ctx_t *)aes_ctx)->gcm_tag_len;
/* return length needed to store the output */
if (plaintext->cd_length < out_len) {
plaintext->cd_length = out_len;
return (CRYPTO_BUFFER_TOO_SMALL);
*** 816,835 ****
if (ret != CRYPTO_SUCCESS)
return (ret);
}
} else if (aes_ctx->ac_flags & CCM_MODE) {
ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
size_t saved_offset = data->cd_offset;
ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
! aes_xor_block);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
data->cd_length = data->cd_offset - saved_offset;
data->cd_offset = saved_offset;
--- 820,839 ----
if (ret != CRYPTO_SUCCESS)
return (ret);
}
} else if (aes_ctx->ac_flags & CCM_MODE) {
ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
size_t saved_offset = data->cd_offset;
ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
! AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
data->cd_length = data->cd_offset - saved_offset;
data->cd_offset = saved_offset;
*** 901,912 ****
ASSERT(aes_ctx->ac_processed_data_len == pt_len);
ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
saved_offset = data->cd_offset;
saved_length = data->cd_length;
ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
! aes_xor_block);
if (ret == CRYPTO_SUCCESS) {
data->cd_length = data->cd_offset - saved_offset;
} else {
data->cd_length = saved_length;
}
--- 905,916 ----
ASSERT(aes_ctx->ac_processed_data_len == pt_len);
ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
saved_offset = data->cd_offset;
saved_length = data->cd_length;
ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
! AES_XOR_BLOCK);
if (ret == CRYPTO_SUCCESS) {
data->cd_length = data->cd_offset - saved_offset;
} else {
data->cd_length = saved_length;
}
*** 915,939 ****
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
/*
! * This is where all the plaintext is returned, make sure
! * the plaintext buffer is big enough
*/
gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
! size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
if (data->cd_length < pt_len) {
data->cd_length = pt_len;
return (CRYPTO_BUFFER_TOO_SMALL);
}
-
saved_offset = data->cd_offset;
saved_length = data->cd_length;
ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
if (ret == CRYPTO_SUCCESS) {
data->cd_length = data->cd_offset - saved_offset;
} else {
data->cd_length = saved_length;
}
--- 919,943 ----
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
/*
! * Check to make sure there is enough space for remaining
! * plaintext.
*/
gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
! size_t pt_len = ctx->gcm_last_input_fill - ctx->gcm_tag_len;
if (data->cd_length < pt_len) {
data->cd_length = pt_len;
return (CRYPTO_BUFFER_TOO_SMALL);
}
saved_offset = data->cd_offset;
saved_length = data->cd_length;
ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
! AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
! AES_XOR_BLOCK, aes_ctr_mode);
if (ret == CRYPTO_SUCCESS) {
data->cd_length = data->cd_offset - saved_offset;
} else {
data->cd_length = saved_length;
}
*** 942,952 ****
if (ret != CRYPTO_SUCCESS) {
return (ret);
}
}
-
if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
data->cd_length = 0;
}
(void) aes_free_context(ctx);
--- 946,955 ----
*** 1041,1059 ****
if (ret == CRYPTO_SUCCESS) {
if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
! aes_xor_block);
if (ret != CRYPTO_SUCCESS)
goto out;
ASSERT(aes_ctx.ac_remainder_len == 0);
} else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
! aes_copy_block, aes_xor_block);
if (ret != CRYPTO_SUCCESS)
goto out;
ASSERT(aes_ctx.ac_remainder_len == 0);
} else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) {
if (aes_ctx.ac_remainder_len > 0) {
--- 1044,1062 ----
if (ret == CRYPTO_SUCCESS) {
if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
! AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS)
goto out;
ASSERT(aes_ctx.ac_remainder_len == 0);
} else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
! AES_COPY_BLOCK, AES_XOR_BLOCK);
if (ret != CRYPTO_SUCCESS)
goto out;
ASSERT(aes_ctx.ac_remainder_len == 0);
} else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) {
if (aes_ctx.ac_remainder_len > 0) {
*** 1180,1190 ****
== aes_ctx.ac_data_len);
ASSERT(aes_ctx.ac_processed_mac_len
== aes_ctx.ac_mac_len);
ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
plaintext, AES_BLOCK_LEN, aes_encrypt_block,
! aes_copy_block, aes_xor_block);
ASSERT(aes_ctx.ac_remainder_len == 0);
if ((ret == CRYPTO_SUCCESS) &&
(ciphertext != plaintext)) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
--- 1183,1193 ----
== aes_ctx.ac_data_len);
ASSERT(aes_ctx.ac_processed_mac_len
== aes_ctx.ac_mac_len);
ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
plaintext, AES_BLOCK_LEN, aes_encrypt_block,
! AES_COPY_BLOCK, AES_XOR_BLOCK);
ASSERT(aes_ctx.ac_remainder_len == 0);
if ((ret == CRYPTO_SUCCESS) &&
(ciphertext != plaintext)) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
*** 1193,1203 ****
}
} else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
plaintext, AES_BLOCK_LEN, aes_encrypt_block,
! aes_xor_block);
ASSERT(aes_ctx.ac_remainder_len == 0);
if ((ret == CRYPTO_SUCCESS) &&
(ciphertext != plaintext)) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
--- 1196,1206 ----
}
} else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
plaintext, AES_BLOCK_LEN, aes_encrypt_block,
! AES_COPY_BLOCK, AES_XOR_BLOCK, aes_ctr_mode);
ASSERT(aes_ctx.ac_remainder_len == 0);
if ((ret == CRYPTO_SUCCESS) &&
(ciphertext != plaintext)) {
plaintext->cd_length =
plaintext->cd_offset - saved_offset;
*** 1235,1250 ****
if (aes_ctx.ac_flags & CCM_MODE) {
if (aes_ctx.ac_pt_buf != NULL) {
kmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len);
}
- } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) {
- if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) {
- kmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
- ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
}
- }
return (ret);
}
/*
--- 1238,1248 ----
*** 1350,1388 ****
mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
! pp->cb, aes_copy_block);
break;
}
case AES_CCM_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
! aes_xor_block);
break;
case AES_GCM_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
! AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
! aes_xor_block);
break;
case AES_GMAC_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
! AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
! aes_xor_block);
break;
case AES_ECB_MECH_INFO_TYPE:
aes_ctx->ac_flags |= ECB_MODE;
}
--- 1348,1386 ----
mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
! pp->cb, AES_COPY_BLOCK);
break;
}
case AES_CCM_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
! AES_XOR_BLOCK);
break;
case AES_GCM_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
! AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
! AES_XOR_BLOCK);
break;
case AES_GMAC_MECH_INFO_TYPE:
if (mechanism->cm_param == NULL ||
mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) {
return (CRYPTO_MECHANISM_PARAM_INVALID);
}
rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
! AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
! AES_XOR_BLOCK);
break;
case AES_ECB_MECH_INFO_TYPE:
aes_ctx->ac_flags |= ECB_MODE;
}