Print this page
4896 Performance improvements for KCF AES modes


   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */



  25 
  26 #ifndef _KERNEL
  27 #include <strings.h>
  28 #include <limits.h>
  29 #include <assert.h>
  30 #include <security/cryptoki.h>
  31 #endif
  32 
  33 #include <sys/types.h>

  34 #include <modes/modes.h>
  35 #include <sys/crypto/common.h>
  36 #include <sys/crypto/impl.h>
  37 #include <sys/byteorder.h>

  38 


  39 /*
  40  * Encrypt and decrypt multiple blocks of data in counter mode.
  41  */
  42 int
  43 ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
  44     crypto_data_t *out, size_t block_size,
  45     int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
  46     void (*xor_block)(uint8_t *, uint8_t *))


  47 {
  48         size_t remainder = length;
  49         size_t need;
  50         uint8_t *datap = (uint8_t *)data;
  51         uint8_t *blockp;
  52         uint8_t *lastp;
  53         void *iov_or_mp;
  54         offset_t offset;
  55         uint8_t *out_data_1;
  56         uint8_t *out_data_2;
  57         size_t out_data_1_len;
  58         uint64_t lower_counter, upper_counter;
  59 


















  60         if (length + ctx->ctr_remainder_len < block_size) {
  61                 /* accumulate bytes here and return */
  62                 bcopy(datap,
  63                     (uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len,
  64                     length);
  65                 ctx->ctr_remainder_len += length;
  66                 ctx->ctr_copy_to = datap;
  67                 return (CRYPTO_SUCCESS);
  68         }
  69 
  70         lastp = (uint8_t *)ctx->ctr_cb;
  71         if (out != NULL)
  72                 crypto_init_ptrs(out, &iov_or_mp, &offset);
  73 
  74         do {
  75                 /* Unprocessed data from last call. */
  76                 if (ctx->ctr_remainder_len > 0) {
  77                         need = block_size - ctx->ctr_remainder_len;
  78 
  79                         if (need > remainder)


 188         for (i = 0; i < ctx->ctr_remainder_len; i++) {
 189                 p[i] ^= lastp[i];
 190         }
 191 
 192         crypto_init_ptrs(out, &iov_or_mp, &offset);
 193         crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
 194             &out_data_1_len, &out_data_2, ctx->ctr_remainder_len);
 195 
 196         bcopy(p, out_data_1, out_data_1_len);
 197         if (out_data_2 != NULL) {
 198                 bcopy((uint8_t *)p + out_data_1_len,
 199                     out_data_2, ctx->ctr_remainder_len - out_data_1_len);
 200         }
 201         out->cd_offset += ctx->ctr_remainder_len;
 202         ctx->ctr_remainder_len = 0;
 203         return (CRYPTO_SUCCESS);
 204 }
 205 
 206 int
 207 ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
 208 void (*copy_block)(uint8_t *, uint8_t *))
 209 {
 210         uint64_t upper_mask = 0;
 211         uint64_t lower_mask = 0;
 212 
 213         if (count == 0 || count > 128) {
 214                 return (CRYPTO_MECHANISM_PARAM_INVALID);
 215         }
 216         /* upper 64 bits of the mask */
 217         if (count >= 64) {
 218                 count -= 64;
 219                 upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1;
 220                 lower_mask = UINT64_MAX;
 221         } else {
 222                 /* now the lower 63 bits */
 223                 lower_mask = (1ULL << count) - 1;
 224         }
 225         ctr_ctx->ctr_lower_mask = htonll(lower_mask);
 226         ctr_ctx->ctr_upper_mask = htonll(upper_mask);
 227 
 228         copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb);




   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 /*
  26  * Copyright 2015 by Saso Kiselkov. All rights reserved.
  27  */
  28 
  29 #ifndef _KERNEL
  30 #include <strings.h>
  31 #include <limits.h>
  32 #include <assert.h>
  33 #include <security/cryptoki.h>
  34 #endif
  35 
  36 #include <sys/types.h>
  37 #define INLINE_CRYPTO_GET_PTRS
  38 #include <modes/modes.h>
  39 #include <sys/crypto/common.h>
  40 #include <sys/crypto/impl.h>
  41 #include <sys/byteorder.h>
  42 #include <sys/cmn_err.h>
  43 
  44 boolean_t ctr_fastpath_enabled = B_TRUE;
  45 
  46 /*
  47  * Encrypt and decrypt multiple blocks of data in counter mode.
  48  */
  49 int
  50 ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
  51     crypto_data_t *out, size_t block_size,
  52     int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
  53     void (*xor_block)(const uint8_t *, uint8_t *),
  54     int (*cipher_ctr)(const void *ks, const uint8_t *pt, uint8_t *ct,
  55     uint64_t len, uint64_t counter[2]))
  56 {
  57         size_t remainder = length;
  58         size_t need;
  59         uint8_t *datap = (uint8_t *)data;
  60         uint8_t *blockp;
  61         uint8_t *lastp;
  62         void *iov_or_mp;
  63         offset_t offset;
  64         uint8_t *out_data_1;
  65         uint8_t *out_data_2;
  66         size_t out_data_1_len;
  67         uint64_t lower_counter, upper_counter;
  68 
  69         /*
  70          * CTR encryption/decryption fastpath requirements:
  71          * - fastpath is enabled
  72          * - algorithm-specific acceleration function is available
  73          * - input is block-aligned
  74          * - the counter value won't overflow the lower counter mask
  75          * - output is a single contiguous region and doesn't alias input
  76          */
  77         if (ctr_fastpath_enabled && cipher_ctr != NULL &&
  78             ctx->ctr_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
  79             ntohll(ctx->ctr_cb[1]) <= ctx->ctr_lower_mask -
  80             length / block_size && CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
  81                 cipher_ctr(ctx->ctr_keysched, (uint8_t *)data,
  82                     CRYPTO_DATA_FIRST_BLOCK(out), length, ctx->ctr_cb);
  83                 out->cd_offset += length;
  84                 return (CRYPTO_SUCCESS);
  85         }
  86 
  87         if (length + ctx->ctr_remainder_len < block_size) {
  88                 /* accumulate bytes here and return */
  89                 bcopy(datap,
  90                     (uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len,
  91                     length);
  92                 ctx->ctr_remainder_len += length;
  93                 ctx->ctr_copy_to = datap;
  94                 return (CRYPTO_SUCCESS);
  95         }
  96 
  97         lastp = (uint8_t *)ctx->ctr_cb;
  98         if (out != NULL)
  99                 crypto_init_ptrs(out, &iov_or_mp, &offset);
 100 
 101         do {
 102                 /* Unprocessed data from last call. */
 103                 if (ctx->ctr_remainder_len > 0) {
 104                         need = block_size - ctx->ctr_remainder_len;
 105 
 106                         if (need > remainder)


 215         for (i = 0; i < ctx->ctr_remainder_len; i++) {
 216                 p[i] ^= lastp[i];
 217         }
 218 
 219         crypto_init_ptrs(out, &iov_or_mp, &offset);
 220         crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
 221             &out_data_1_len, &out_data_2, ctx->ctr_remainder_len);
 222 
 223         bcopy(p, out_data_1, out_data_1_len);
 224         if (out_data_2 != NULL) {
 225                 bcopy((uint8_t *)p + out_data_1_len,
 226                     out_data_2, ctx->ctr_remainder_len - out_data_1_len);
 227         }
 228         out->cd_offset += ctx->ctr_remainder_len;
 229         ctx->ctr_remainder_len = 0;
 230         return (CRYPTO_SUCCESS);
 231 }
 232 
 233 int
 234 ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
 235     void (*copy_block)(const uint8_t *, uint8_t *))
 236 {
 237         uint64_t upper_mask = 0;
 238         uint64_t lower_mask = 0;
 239 
 240         if (count == 0 || count > 128) {
 241                 return (CRYPTO_MECHANISM_PARAM_INVALID);
 242         }
 243         /* upper 64 bits of the mask */
 244         if (count >= 64) {
 245                 count -= 64;
 246                 upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1;
 247                 lower_mask = UINT64_MAX;
 248         } else {
 249                 /* now the lower 63 bits */
 250                 lower_mask = (1ULL << count) - 1;
 251         }
 252         ctr_ctx->ctr_lower_mask = htonll(lower_mask);
 253         ctr_ctx->ctr_upper_mask = htonll(upper_mask);
 254 
 255         copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb);