Print this page
4896 Performance improvements for KCF AES modes


   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.

  23  */
  24 
  25 #ifndef _KERNEL
  26 #include <strings.h>
  27 #include <limits.h>
  28 #include <assert.h>
  29 #include <security/cryptoki.h>
  30 #endif
  31 
  32 #include <sys/types.h>
  33 #include <sys/kmem.h>

  34 #include <modes/modes.h>
  35 #include <sys/crypto/common.h>
  36 #include <sys/crypto/impl.h>
  37 #include <sys/byteorder.h>
  38 
  39 #if defined(__i386) || defined(__amd64)
  40 #define UNALIGNED_POINTERS_PERMITTED
  41 #endif
  42 
  43 /*
  44  * Encrypt multiple blocks of data in CCM mode.  Decrypt for CCM mode
  45  * is done in another function.
  46  */
  47 int
  48 ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
  49     crypto_data_t *out, size_t block_size,
  50     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
  51     void (*copy_block)(uint8_t *, uint8_t *),
  52     void (*xor_block)(uint8_t *, uint8_t *))
  53 {
  54         size_t remainder = length;
  55         size_t need;
  56         uint8_t *datap = (uint8_t *)data;
  57         uint8_t *blockp;
  58         uint8_t *lastp;
  59         void *iov_or_mp;
  60         offset_t offset;
  61         uint8_t *out_data_1;
  62         uint8_t *out_data_2;
  63         size_t out_data_1_len;
  64         uint64_t counter;
  65         uint8_t *mac_buf;
  66 
  67         if (length + ctx->ccm_remainder_len < block_size) {
  68                 /* accumulate bytes here and return */
  69                 bcopy(datap,
  70                     (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
  71                     length);
  72                 ctx->ccm_remainder_len += length;


 189 
 190         mac_buf = (uint8_t *)ctx->ccm_mac_buf;
 191 
 192         /* first counter block start with index 0 */
 193         counter = 0;
 194         ctx->ccm_cb[1] = (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
 195 
 196         counterp = (uint8_t *)ctx->ccm_tmp;
 197         encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
 198 
 199         /* calculate XOR of MAC with first counter block */
 200         for (i = 0; i < ctx->ccm_mac_len; i++) {
 201                 ccm_mac[i] = mac_buf[i] ^ counterp[i];
 202         }
 203 }
 204 
 205 /* ARGSUSED */
 206 int
 207 ccm_encrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
 208     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 209     void (*xor_block)(uint8_t *, uint8_t *))
 210 {
 211         uint8_t *lastp, *mac_buf, *ccm_mac_p, *macp;
 212         void *iov_or_mp;
 213         offset_t offset;
 214         uint8_t *out_data_1;
 215         uint8_t *out_data_2;
 216         size_t out_data_1_len;
 217         int i;
 218 
 219         if (out->cd_length < (ctx->ccm_remainder_len + ctx->ccm_mac_len)) {
 220                 return (CRYPTO_DATA_LEN_RANGE);
 221         }
 222 
 223         /*
 224          * When we get here, the number of bytes of payload processed
 225          * plus whatever data remains, if any,
 226          * should be the same as the number of bytes that's being
 227          * passed in the argument during init time.
 228          */
 229         if ((ctx->ccm_processed_data_len + ctx->ccm_remainder_len)


 342 
 343         counterp = (uint8_t *)ctx->ccm_tmp;
 344         encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
 345 
 346         /* XOR with counter block */
 347         for (i = 0; i < ctx->ccm_remainder_len; i++) {
 348                 outp[i] = datap[i] ^ counterp[i];
 349         }
 350 }
 351 
 352 /*
 353  * This will decrypt the cipher text.  However, the plaintext won't be
 354  * returned to the caller.  It will be returned when decrypt_final() is
 355  * called if the MAC matches
 356  */
 357 /* ARGSUSED */
 358 int
 359 ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
 360     crypto_data_t *out, size_t block_size,
 361     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 362     void (*copy_block)(uint8_t *, uint8_t *),
 363     void (*xor_block)(uint8_t *, uint8_t *))
 364 {
 365         size_t remainder = length;
 366         size_t need;
 367         uint8_t *datap = (uint8_t *)data;
 368         uint8_t *blockp;
 369         uint8_t *cbp;
 370         uint64_t counter;
 371         size_t pt_len, total_decrypted_len, mac_len, pm_len, pd_len;
 372         uint8_t *resultp;
 373 
 374 
 375         pm_len = ctx->ccm_processed_mac_len;
 376 
 377         if (pm_len > 0) {
 378                 uint8_t *tmp;
 379                 /*
 380                  * all ciphertext has been processed, just waiting for
 381                  * part of the value of the mac
 382                  */
 383                 if ((pm_len + length) > ctx->ccm_mac_len) {


 509                                  * compute plaintext for the remaining input
 510                                  */
 511                                 ccm_decrypt_incomplete_block(ctx,
 512                                     encrypt_block);
 513                                 ctx->ccm_processed_data_len += remainder;
 514                                 ctx->ccm_remainder_len = 0;
 515                         }
 516                         goto out;
 517                 }
 518                 ctx->ccm_copy_to = NULL;
 519 
 520         } while (remainder > 0);
 521 
 522 out:
 523         return (CRYPTO_SUCCESS);
 524 }
 525 
 526 int
 527 ccm_decrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
 528     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 529     void (*copy_block)(uint8_t *, uint8_t *),
 530     void (*xor_block)(uint8_t *, uint8_t *))
 531 {
 532         size_t mac_remain, pt_len;
 533         uint8_t *pt, *mac_buf, *macp, *ccm_mac_p;
 534         int rv;
 535 
 536         pt_len = ctx->ccm_data_len;
 537 
 538         /* Make sure output buffer can fit all of the plaintext */
 539         if (out->cd_length < pt_len) {
 540                 return (CRYPTO_DATA_LEN_RANGE);
 541         }
 542 
 543         pt = ctx->ccm_pt_buf;
 544         mac_remain = ctx->ccm_processed_data_len;
 545         mac_buf = (uint8_t *)ctx->ccm_mac_buf;
 546 
 547         macp = (uint8_t *)ctx->ccm_tmp;
 548 
 549         while (mac_remain > 0) {
 550 


 753                 encoded[3] = (auth_data_len & 0xff000000000000) >> 48;
 754                 encoded[4] = (auth_data_len & 0xff0000000000) >> 40;
 755                 encoded[5] = (auth_data_len & 0xff00000000) >> 32;
 756                 encoded[6] = (auth_data_len & 0xff000000) >> 24;
 757                 encoded[7] = (auth_data_len & 0xff0000) >> 16;
 758                 encoded[8] = (auth_data_len & 0xff00) >> 8;
 759                 encoded[9] = auth_data_len & 0xff;
 760 #endif  /* UNALIGNED_POINTERS_PERMITTED */
 761 #endif  /* _LP64 */
 762         }
 763 }
 764 
 765 /*
 766  * The following function should be call at encrypt or decrypt init time
 767  * for AES CCM mode.
 768  */
 769 int
 770 ccm_init(ccm_ctx_t *ctx, unsigned char *nonce, size_t nonce_len,
 771     unsigned char *auth_data, size_t auth_data_len, size_t block_size,
 772     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 773     void (*xor_block)(uint8_t *, uint8_t *))
 774 {
 775         uint8_t *mac_buf, *datap, *ivp, *authp;
 776         size_t remainder, processed;
 777         uint8_t encoded_a[10]; /* max encoded auth data length is 10 octets */
 778         size_t encoded_a_len = 0;
 779 
 780         mac_buf = (uint8_t *)&(ctx->ccm_mac_buf);
 781 
 782         /*
 783          * Format the 1st block for CBC-MAC and construct the
 784          * 1st counter block.
 785          *
 786          * aes_ctx->ccm_iv is used for storing the counter block
 787          * mac_buf will store b0 at this time.
 788          */
 789         ccm_format_initial_blocks(nonce, nonce_len,
 790             auth_data_len, mac_buf, ctx);
 791 
 792         /* The IV for CBC MAC for AES CCM mode is always zero */
 793         ivp = (uint8_t *)ctx->ccm_tmp;


 837                         datap = (uint8_t *)authp;
 838                         remainder = 0;
 839                 } else {
 840                         datap = (uint8_t *)(&(auth_data[processed]));
 841                         processed += block_size;
 842                         remainder -= block_size;
 843                 }
 844 
 845                 xor_block(datap, mac_buf);
 846                 encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
 847 
 848         } while (remainder > 0);
 849 
 850         return (CRYPTO_SUCCESS);
 851 }
 852 
 853 int
 854 ccm_init_ctx(ccm_ctx_t *ccm_ctx, char *param, int kmflag,
 855     boolean_t is_encrypt_init, size_t block_size,
 856     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 857     void (*xor_block)(uint8_t *, uint8_t *))
 858 {
 859         int rv;
 860         CK_AES_CCM_PARAMS *ccm_param;
 861 
 862         if (param != NULL) {
 863                 ccm_param = (CK_AES_CCM_PARAMS *)(void *)param;
 864 
 865                 if ((rv = ccm_validate_args(ccm_param,
 866                     is_encrypt_init)) != 0) {
 867                         return (rv);
 868                 }
 869 
 870                 ccm_ctx->ccm_mac_len = ccm_param->ulMACSize;
 871                 if (is_encrypt_init) {
 872                         ccm_ctx->ccm_data_len = ccm_param->ulDataSize;
 873                 } else {
 874                         ccm_ctx->ccm_data_len =
 875                             ccm_param->ulDataSize - ccm_ctx->ccm_mac_len;
 876                         ccm_ctx->ccm_processed_mac_len = 0;
 877                 }




   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2015 by Saso Kiselkov. All rights reserved.
  24  */
  25 
  26 #ifndef _KERNEL
  27 #include <strings.h>
  28 #include <limits.h>
  29 #include <assert.h>
  30 #include <security/cryptoki.h>
  31 #endif
  32 
  33 #include <sys/types.h>
  34 #include <sys/kmem.h>
  35 #define INLINE_CRYPTO_GET_PTRS
  36 #include <modes/modes.h>
  37 #include <sys/crypto/common.h>
  38 #include <sys/crypto/impl.h>
  39 #include <sys/byteorder.h>
  40 
  41 #if defined(__i386) || defined(__amd64)
  42 #define UNALIGNED_POINTERS_PERMITTED
  43 #endif
  44 
  45 /*
  46  * Encrypt multiple blocks of data in CCM mode.  Decrypt for CCM mode
  47  * is done in another function.
  48  */
  49 int
  50 ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
  51     crypto_data_t *out, size_t block_size,
  52     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
  53     void (*copy_block)(const uint8_t *, uint8_t *),
  54     void (*xor_block)(const uint8_t *, uint8_t *))
  55 {
  56         size_t remainder = length;
  57         size_t need;
  58         uint8_t *datap = (uint8_t *)data;
  59         uint8_t *blockp;
  60         uint8_t *lastp;
  61         void *iov_or_mp;
  62         offset_t offset;
  63         uint8_t *out_data_1;
  64         uint8_t *out_data_2;
  65         size_t out_data_1_len;
  66         uint64_t counter;
  67         uint8_t *mac_buf;
  68 
  69         if (length + ctx->ccm_remainder_len < block_size) {
  70                 /* accumulate bytes here and return */
  71                 bcopy(datap,
  72                     (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len,
  73                     length);
  74                 ctx->ccm_remainder_len += length;


 191 
 192         mac_buf = (uint8_t *)ctx->ccm_mac_buf;
 193 
 194         /* first counter block start with index 0 */
 195         counter = 0;
 196         ctx->ccm_cb[1] = (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter;
 197 
 198         counterp = (uint8_t *)ctx->ccm_tmp;
 199         encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
 200 
 201         /* calculate XOR of MAC with first counter block */
 202         for (i = 0; i < ctx->ccm_mac_len; i++) {
 203                 ccm_mac[i] = mac_buf[i] ^ counterp[i];
 204         }
 205 }
 206 
 207 /* ARGSUSED */
 208 int
 209 ccm_encrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
 210     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 211     void (*xor_block)(const uint8_t *, uint8_t *))
 212 {
 213         uint8_t *lastp, *mac_buf, *ccm_mac_p, *macp;
 214         void *iov_or_mp;
 215         offset_t offset;
 216         uint8_t *out_data_1;
 217         uint8_t *out_data_2;
 218         size_t out_data_1_len;
 219         int i;
 220 
 221         if (out->cd_length < (ctx->ccm_remainder_len + ctx->ccm_mac_len)) {
 222                 return (CRYPTO_DATA_LEN_RANGE);
 223         }
 224 
 225         /*
 226          * When we get here, the number of bytes of payload processed
 227          * plus whatever data remains, if any,
 228          * should be the same as the number of bytes that's being
 229          * passed in the argument during init time.
 230          */
 231         if ((ctx->ccm_processed_data_len + ctx->ccm_remainder_len)


 344 
 345         counterp = (uint8_t *)ctx->ccm_tmp;
 346         encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp);
 347 
 348         /* XOR with counter block */
 349         for (i = 0; i < ctx->ccm_remainder_len; i++) {
 350                 outp[i] = datap[i] ^ counterp[i];
 351         }
 352 }
 353 
 354 /*
 355  * This will decrypt the cipher text.  However, the plaintext won't be
 356  * returned to the caller.  It will be returned when decrypt_final() is
 357  * called if the MAC matches
 358  */
 359 /* ARGSUSED */
 360 int
 361 ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
 362     crypto_data_t *out, size_t block_size,
 363     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 364     void (*copy_block)(const uint8_t *, uint8_t *),
 365     void (*xor_block)(const uint8_t *, uint8_t *))
 366 {
 367         size_t remainder = length;
 368         size_t need;
 369         uint8_t *datap = (uint8_t *)data;
 370         uint8_t *blockp;
 371         uint8_t *cbp;
 372         uint64_t counter;
 373         size_t pt_len, total_decrypted_len, mac_len, pm_len, pd_len;
 374         uint8_t *resultp;
 375 
 376 
 377         pm_len = ctx->ccm_processed_mac_len;
 378 
 379         if (pm_len > 0) {
 380                 uint8_t *tmp;
 381                 /*
 382                  * all ciphertext has been processed, just waiting for
 383                  * part of the value of the mac
 384                  */
 385                 if ((pm_len + length) > ctx->ccm_mac_len) {


 511                                  * compute plaintext for the remaining input
 512                                  */
 513                                 ccm_decrypt_incomplete_block(ctx,
 514                                     encrypt_block);
 515                                 ctx->ccm_processed_data_len += remainder;
 516                                 ctx->ccm_remainder_len = 0;
 517                         }
 518                         goto out;
 519                 }
 520                 ctx->ccm_copy_to = NULL;
 521 
 522         } while (remainder > 0);
 523 
 524 out:
 525         return (CRYPTO_SUCCESS);
 526 }
 527 
 528 int
 529 ccm_decrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
 530     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 531     void (*copy_block)(const uint8_t *, uint8_t *),
 532     void (*xor_block)(const uint8_t *, uint8_t *))
 533 {
 534         size_t mac_remain, pt_len;
 535         uint8_t *pt, *mac_buf, *macp, *ccm_mac_p;
 536         int rv;
 537 
 538         pt_len = ctx->ccm_data_len;
 539 
 540         /* Make sure output buffer can fit all of the plaintext */
 541         if (out->cd_length < pt_len) {
 542                 return (CRYPTO_DATA_LEN_RANGE);
 543         }
 544 
 545         pt = ctx->ccm_pt_buf;
 546         mac_remain = ctx->ccm_processed_data_len;
 547         mac_buf = (uint8_t *)ctx->ccm_mac_buf;
 548 
 549         macp = (uint8_t *)ctx->ccm_tmp;
 550 
 551         while (mac_remain > 0) {
 552 


 755                 encoded[3] = (auth_data_len & 0xff000000000000) >> 48;
 756                 encoded[4] = (auth_data_len & 0xff0000000000) >> 40;
 757                 encoded[5] = (auth_data_len & 0xff00000000) >> 32;
 758                 encoded[6] = (auth_data_len & 0xff000000) >> 24;
 759                 encoded[7] = (auth_data_len & 0xff0000) >> 16;
 760                 encoded[8] = (auth_data_len & 0xff00) >> 8;
 761                 encoded[9] = auth_data_len & 0xff;
 762 #endif  /* UNALIGNED_POINTERS_PERMITTED */
 763 #endif  /* _LP64 */
 764         }
 765 }
 766 
 767 /*
 768  * The following function should be call at encrypt or decrypt init time
 769  * for AES CCM mode.
 770  */
 771 int
 772 ccm_init(ccm_ctx_t *ctx, unsigned char *nonce, size_t nonce_len,
 773     unsigned char *auth_data, size_t auth_data_len, size_t block_size,
 774     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 775     void (*xor_block)(const uint8_t *, uint8_t *))
 776 {
 777         uint8_t *mac_buf, *datap, *ivp, *authp;
 778         size_t remainder, processed;
 779         uint8_t encoded_a[10]; /* max encoded auth data length is 10 octets */
 780         size_t encoded_a_len = 0;
 781 
 782         mac_buf = (uint8_t *)&(ctx->ccm_mac_buf);
 783 
 784         /*
 785          * Format the 1st block for CBC-MAC and construct the
 786          * 1st counter block.
 787          *
 788          * aes_ctx->ccm_iv is used for storing the counter block
 789          * mac_buf will store b0 at this time.
 790          */
 791         ccm_format_initial_blocks(nonce, nonce_len,
 792             auth_data_len, mac_buf, ctx);
 793 
 794         /* The IV for CBC MAC for AES CCM mode is always zero */
 795         ivp = (uint8_t *)ctx->ccm_tmp;


 839                         datap = (uint8_t *)authp;
 840                         remainder = 0;
 841                 } else {
 842                         datap = (uint8_t *)(&(auth_data[processed]));
 843                         processed += block_size;
 844                         remainder -= block_size;
 845                 }
 846 
 847                 xor_block(datap, mac_buf);
 848                 encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf);
 849 
 850         } while (remainder > 0);
 851 
 852         return (CRYPTO_SUCCESS);
 853 }
 854 
 855 int
 856 ccm_init_ctx(ccm_ctx_t *ccm_ctx, char *param, int kmflag,
 857     boolean_t is_encrypt_init, size_t block_size,
 858     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
 859     void (*xor_block)(const uint8_t *, uint8_t *))
 860 {
 861         int rv;
 862         CK_AES_CCM_PARAMS *ccm_param;
 863 
 864         if (param != NULL) {
 865                 ccm_param = (CK_AES_CCM_PARAMS *)(void *)param;
 866 
 867                 if ((rv = ccm_validate_args(ccm_param,
 868                     is_encrypt_init)) != 0) {
 869                         return (rv);
 870                 }
 871 
 872                 ccm_ctx->ccm_mac_len = ccm_param->ulMACSize;
 873                 if (is_encrypt_init) {
 874                         ccm_ctx->ccm_data_len = ccm_param->ulDataSize;
 875                 } else {
 876                         ccm_ctx->ccm_data_len =
 877                             ccm_param->ulDataSize - ccm_ctx->ccm_mac_len;
 878                         ccm_ctx->ccm_processed_mac_len = 0;
 879                 }