4896 Performance improvements for KCF AES modes

*** 20,51 ****
   */
  /*
   * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
   */
  
  #ifndef _KERNEL
  #include <strings.h>
  #include <limits.h>
  #include <assert.h>
  #include <security/cryptoki.h>
  #endif
  
  #include <sys/types.h>
  #include <modes/modes.h>
  #include <sys/crypto/common.h>
  #include <sys/crypto/impl.h>
  #include <sys/byteorder.h>
  
  /*
   * Encrypt and decrypt multiple blocks of data in counter mode.
   */
  int
  ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
      crypto_data_t *out, size_t block_size,
      int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
!     void (*xor_block)(uint8_t *, uint8_t *))
  {
          size_t remainder = length;
          size_t need;
          uint8_t *datap = (uint8_t *)data;
          uint8_t *blockp;
--- 20,60 ----
   */
  /*
   * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
   * Use is subject to license terms.
   */
+ /*
+  * Copyright 2015 by Saso Kiselkov. All rights reserved.
+  */
  
  #ifndef _KERNEL
  #include <strings.h>
  #include <limits.h>
  #include <assert.h>
  #include <security/cryptoki.h>
  #endif
  
  #include <sys/types.h>
+ #define INLINE_CRYPTO_GET_PTRS
  #include <modes/modes.h>
  #include <sys/crypto/common.h>
  #include <sys/crypto/impl.h>
  #include <sys/byteorder.h>
+ #include <sys/cmn_err.h>
  
+ boolean_t ctr_fastpath_enabled = B_TRUE;
+ 
  /*
   * Encrypt and decrypt multiple blocks of data in counter mode.
   */
  int
  ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
      crypto_data_t *out, size_t block_size,
      int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
!     void (*xor_block)(const uint8_t *, uint8_t *),
!     int (*cipher_ctr)(const void *ks, const uint8_t *pt, uint8_t *ct,
!     uint64_t len, uint64_t counter[2]))
  {
          size_t remainder = length;
          size_t need;
          uint8_t *datap = (uint8_t *)data;
          uint8_t *blockp;
*** 55,64 ****
--- 64,91 ----
          uint8_t *out_data_1;
          uint8_t *out_data_2;
          size_t out_data_1_len;
          uint64_t lower_counter, upper_counter;
  
+         /*
+          * CTR encryption/decryption fastpath requirements:
+          * - fastpath is enabled
+          * - algorithm-specific acceleration function is available
+          * - input is block-aligned
+          * - the counter value won't overflow the lower counter mask
+          * - output is a single contiguous region and doesn't alias input
+          */
+         if (ctr_fastpath_enabled && cipher_ctr != NULL &&
+             ctx->ctr_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
+             ntohll(ctx->ctr_cb[1]) <= ctx->ctr_lower_mask -
+             length / block_size && CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
+                 cipher_ctr(ctx->ctr_keysched, (uint8_t *)data,
+                     CRYPTO_DATA_FIRST_BLOCK(out), length, ctx->ctr_cb);
+                 out->cd_offset += length;
+                 return (CRYPTO_SUCCESS);
+         }
+ 
          if (length + ctx->ctr_remainder_len < block_size) {
                  /* accumulate bytes here and return */
                  bcopy(datap,
                      (uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len,
                      length);
*** 203,213 ****
          return (CRYPTO_SUCCESS);
  }
  
  int
  ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
! void (*copy_block)(uint8_t *, uint8_t *))
  {
          uint64_t upper_mask = 0;
          uint64_t lower_mask = 0;
  
          if (count == 0 || count > 128) {
--- 230,240 ----
          return (CRYPTO_SUCCESS);
  }
  
  int
  ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
!     void (*copy_block)(const uint8_t *, uint8_t *))
  {
          uint64_t upper_mask = 0;
          uint64_t lower_mask = 0;
  
          if (count == 0 || count > 128) {