Print this page
4896 Performance improvements for KCF AES modes

Split Close
Expand all
Collapse all
          --- old/usr/src/uts/common/crypto/io/aes.c
          +++ new/usr/src/uts/common/crypto/io/aes.c
↓ open down ↓ 12 lines elided ↑ open up ↑
  13   13   * When distributing Covered Code, include this CDDL HEADER in each
  14   14   * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15   15   * If applicable, add the following below this CDDL HEADER, with the
  16   16   * fields enclosed by brackets "[]" replaced with your own identifying
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
  21   21  /*
  22   22   * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
       23 + * Copyright 2015 by Saso Kiselkov. All rights reserved.
  23   24   */
  24   25  
  25   26  /*
  26   27   * AES provider for the Kernel Cryptographic Framework (KCF)
  27   28   */
  28   29  
  29   30  #include <sys/types.h>
  30   31  #include <sys/systm.h>
  31   32  #include <sys/modctl.h>
  32   33  #include <sys/cmn_err.h>
↓ open down ↓ 346 lines elided ↑ open up ↑
 379  380                  crypto_free_mode_ctx(aes_ctx);
 380  381                  return (rv);
 381  382          }
 382  383  
 383  384          ctx->cc_provider_private = aes_ctx;
 384  385  
 385  386          return (CRYPTO_SUCCESS);
 386  387  }
 387  388  
 388  389  static void
 389      -aes_copy_block64(uint8_t *in, uint64_t *out)
      390 +aes_copy_block64(const uint8_t *in, uint64_t *out)
 390  391  {
 391  392          if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
 392  393                  /* LINTED: pointer alignment */
 393  394                  out[0] = *(uint64_t *)&in[0];
 394  395                  /* LINTED: pointer alignment */
 395  396                  out[1] = *(uint64_t *)&in[8];
 396  397          } else {
 397  398                  uint8_t *iv8 = (uint8_t *)&out[0];
 398  399  
 399      -                AES_COPY_BLOCK(in, iv8);
      400 +                AES_COPY_BLOCK_UNALIGNED(in, iv8);
 400  401          }
 401  402  }
 402  403  
 403  404  
 404  405  static int
 405  406  aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
 406  407      crypto_data_t *ciphertext, crypto_req_handle_t req)
 407  408  {
 408  409          int ret = CRYPTO_FAILED;
 409  410  
↓ open down ↓ 58 lines elided ↑ open up ↑
 468  469                  /*
 469  470                   * ccm_encrypt_final() will compute the MAC and append
 470  471                   * it to existing ciphertext. So, need to adjust the left over
 471  472                   * length value accordingly
 472  473                   */
 473  474  
 474  475                  /* order of following 2 lines MUST not be reversed */
 475  476                  ciphertext->cd_offset = ciphertext->cd_length;
 476  477                  ciphertext->cd_length = saved_length - ciphertext->cd_length;
 477  478                  ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
 478      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
      479 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
 479  480                  if (ret != CRYPTO_SUCCESS) {
 480  481                          return (ret);
 481  482                  }
 482  483  
 483  484                  if (plaintext != ciphertext) {
 484  485                          ciphertext->cd_length =
 485  486                              ciphertext->cd_offset - saved_offset;
 486  487                  }
 487  488                  ciphertext->cd_offset = saved_offset;
 488  489          } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
 489  490                  /*
 490  491                   * gcm_encrypt_final() will compute the MAC and append
 491  492                   * it to existing ciphertext. So, need to adjust the left over
 492  493                   * length value accordingly
 493  494                   */
 494  495  
 495  496                  /* order of following 2 lines MUST not be reversed */
 496  497                  ciphertext->cd_offset = ciphertext->cd_length;
 497  498                  ciphertext->cd_length = saved_length - ciphertext->cd_length;
 498  499                  ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
 499      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
 500      -                    aes_xor_block);
      500 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
      501 +                    AES_XOR_BLOCK);
 501  502                  if (ret != CRYPTO_SUCCESS) {
 502  503                          return (ret);
 503  504                  }
 504  505  
 505  506                  if (plaintext != ciphertext) {
 506  507                          ciphertext->cd_length =
 507  508                              ciphertext->cd_offset - saved_offset;
 508  509                  }
 509  510                  ciphertext->cd_offset = saved_offset;
 510  511          }
↓ open down ↓ 72 lines elided ↑ open up ↑
 583  584  
 584  585          if (aes_ctx->ac_flags & CCM_MODE) {
 585  586                  ASSERT(aes_ctx->ac_processed_data_len == aes_ctx->ac_data_len);
 586  587                  ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
 587  588  
 588  589                  /* order of following 2 lines MUST not be reversed */
 589  590                  plaintext->cd_offset = plaintext->cd_length;
 590  591                  plaintext->cd_length = saved_length - plaintext->cd_length;
 591  592  
 592  593                  ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
 593      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
 594      -                    aes_xor_block);
      594 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
      595 +                    AES_XOR_BLOCK);
 595  596                  if (ret == CRYPTO_SUCCESS) {
 596  597                          if (plaintext != ciphertext) {
 597  598                                  plaintext->cd_length =
 598  599                                      plaintext->cd_offset - saved_offset;
 599  600                          }
 600  601                  } else {
 601  602                          plaintext->cd_length = saved_length;
 602  603                  }
 603  604  
 604  605                  plaintext->cd_offset = saved_offset;
 605  606          } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
 606  607                  /* order of following 2 lines MUST not be reversed */
 607  608                  plaintext->cd_offset = plaintext->cd_length;
 608  609                  plaintext->cd_length = saved_length - plaintext->cd_length;
 609  610  
 610  611                  ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
 611      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
      612 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK,
      613 +                    AES_COPY_BLOCK, aes_ctr_mode);
 612  614                  if (ret == CRYPTO_SUCCESS) {
 613  615                          if (plaintext != ciphertext) {
 614  616                                  plaintext->cd_length =
 615  617                                      plaintext->cd_offset - saved_offset;
 616  618                          }
 617  619                  } else {
 618  620                          plaintext->cd_length = saved_length;
 619  621                  }
 620  622  
 621  623                  plaintext->cd_offset = saved_offset;
↓ open down ↓ 93 lines elided ↑ open up ↑
 715  717          int ret = CRYPTO_SUCCESS;
 716  718          aes_ctx_t *aes_ctx;
 717  719  
 718  720          ASSERT(ctx->cc_provider_private != NULL);
 719  721          aes_ctx = ctx->cc_provider_private;
 720  722  
 721  723          AES_ARG_INPLACE(ciphertext, plaintext);
 722  724  
 723  725          /*
 724  726           * Compute number of bytes that will hold the plaintext.
 725      -         * This is not necessary for CCM, GCM, and GMAC since these
      727 +         * This is not necessary for CCM and GMAC since these
 726  728           * mechanisms never return plaintext for update operations.
 727  729           */
 728      -        if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
      730 +        if ((aes_ctx->ac_flags & (CCM_MODE|GMAC_MODE)) == 0) {
 729  731                  out_len = aes_ctx->ac_remainder_len;
 730  732                  out_len += ciphertext->cd_length;
 731  733                  out_len &= ~(AES_BLOCK_LEN - 1);
      734 +                if (aes_ctx->ac_flags & GCM_MODE)
      735 +                        out_len -= ((gcm_ctx_t *)aes_ctx)->gcm_tag_len;
 732  736  
 733  737                  /* return length needed to store the output */
 734  738                  if (plaintext->cd_length < out_len) {
 735  739                          plaintext->cd_length = out_len;
 736  740                          return (CRYPTO_BUFFER_TOO_SMALL);
 737  741                  }
 738  742          }
 739  743  
 740  744          saved_offset = plaintext->cd_offset;
 741  745          saved_length = plaintext->cd_length;
↓ open down ↓ 69 lines elided ↑ open up ↑
 811  815  
 812  816          if (aes_ctx->ac_flags & CTR_MODE) {
 813  817                  if (aes_ctx->ac_remainder_len > 0) {
 814  818                          ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data,
 815  819                              aes_encrypt_block);
 816  820                          if (ret != CRYPTO_SUCCESS)
 817  821                                  return (ret);
 818  822                  }
 819  823          } else if (aes_ctx->ac_flags & CCM_MODE) {
 820  824                  ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
 821      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
      825 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
 822  826                  if (ret != CRYPTO_SUCCESS) {
 823  827                          return (ret);
 824  828                  }
 825  829          } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
 826  830                  size_t saved_offset = data->cd_offset;
 827  831  
 828  832                  ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
 829      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
 830      -                    aes_xor_block);
      833 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
      834 +                    AES_XOR_BLOCK);
 831  835                  if (ret != CRYPTO_SUCCESS) {
 832  836                          return (ret);
 833  837                  }
 834  838                  data->cd_length = data->cd_offset - saved_offset;
 835  839                  data->cd_offset = saved_offset;
 836  840          } else {
 837  841                  /*
 838  842                   * There must be no unprocessed plaintext.
 839  843                   * This happens if the length of the last data is
 840  844                   * not a multiple of the AES block length.
↓ open down ↓ 55 lines elided ↑ open up ↑
 896  900                  if (data->cd_length < pt_len) {
 897  901                          data->cd_length = pt_len;
 898  902                          return (CRYPTO_BUFFER_TOO_SMALL);
 899  903                  }
 900  904  
 901  905                  ASSERT(aes_ctx->ac_processed_data_len == pt_len);
 902  906                  ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
 903  907                  saved_offset = data->cd_offset;
 904  908                  saved_length = data->cd_length;
 905  909                  ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
 906      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
 907      -                    aes_xor_block);
      910 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
      911 +                    AES_XOR_BLOCK);
 908  912                  if (ret == CRYPTO_SUCCESS) {
 909  913                          data->cd_length = data->cd_offset - saved_offset;
 910  914                  } else {
 911  915                          data->cd_length = saved_length;
 912  916                  }
 913  917  
 914  918                  data->cd_offset = saved_offset;
 915  919                  if (ret != CRYPTO_SUCCESS) {
 916  920                          return (ret);
 917  921                  }
 918  922          } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
 919  923                  /*
 920      -                 * This is where all the plaintext is returned, make sure
 921      -                 * the plaintext buffer is big enough
      924 +                 * Check to make sure there is enough space for remaining
      925 +                 * plaintext.
 922  926                   */
 923  927                  gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
 924      -                size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
      928 +                size_t pt_len = ctx->gcm_last_input_fill - ctx->gcm_tag_len;
 925  929  
 926  930                  if (data->cd_length < pt_len) {
 927  931                          data->cd_length = pt_len;
 928  932                          return (CRYPTO_BUFFER_TOO_SMALL);
 929  933                  }
 930      -
 931  934                  saved_offset = data->cd_offset;
 932  935                  saved_length = data->cd_length;
 933  936                  ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
 934      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
      937 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
      938 +                    AES_XOR_BLOCK, aes_ctr_mode);
 935  939                  if (ret == CRYPTO_SUCCESS) {
 936  940                          data->cd_length = data->cd_offset - saved_offset;
 937  941                  } else {
 938  942                          data->cd_length = saved_length;
 939  943                  }
 940  944  
 941  945                  data->cd_offset = saved_offset;
 942  946                  if (ret != CRYPTO_SUCCESS) {
 943  947                          return (ret);
 944  948                  }
 945  949          }
 946  950  
 947      -
 948  951          if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
 949  952                  data->cd_length = 0;
 950  953          }
 951  954  
 952  955          (void) aes_free_context(ctx);
 953  956  
 954  957          return (CRYPTO_SUCCESS);
 955  958  }
 956  959  
 957  960  /* ARGSUSED */
↓ open down ↓ 78 lines elided ↑ open up ↑
1036 1039                      aes_encrypt_contiguous_blocks, aes_copy_block64);
1037 1040                  break;
1038 1041          default:
1039 1042                  ret = CRYPTO_ARGUMENTS_BAD;
1040 1043          }
1041 1044  
1042 1045          if (ret == CRYPTO_SUCCESS) {
1043 1046                  if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
1044 1047                          ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
1045 1048                              ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
1046      -                            aes_xor_block);
     1049 +                            AES_XOR_BLOCK);
1047 1050                          if (ret != CRYPTO_SUCCESS)
1048 1051                                  goto out;
1049 1052                          ASSERT(aes_ctx.ac_remainder_len == 0);
1050 1053                  } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
1051 1054                      mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
1052 1055                          ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
1053 1056                              ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
1054      -                            aes_copy_block, aes_xor_block);
     1057 +                            AES_COPY_BLOCK, AES_XOR_BLOCK);
1055 1058                          if (ret != CRYPTO_SUCCESS)
1056 1059                                  goto out;
1057 1060                          ASSERT(aes_ctx.ac_remainder_len == 0);
1058 1061                  } else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) {
1059 1062                          if (aes_ctx.ac_remainder_len > 0) {
1060 1063                                  ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx,
1061 1064                                      ciphertext, aes_encrypt_block);
1062 1065                                  if (ret != CRYPTO_SUCCESS)
1063 1066                                          goto out;
1064 1067                          }
↓ open down ↓ 110 lines elided ↑ open up ↑
1175 1178          }
1176 1179  
1177 1180          if (ret == CRYPTO_SUCCESS) {
1178 1181                  if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
1179 1182                          ASSERT(aes_ctx.ac_processed_data_len
1180 1183                              == aes_ctx.ac_data_len);
1181 1184                          ASSERT(aes_ctx.ac_processed_mac_len
1182 1185                              == aes_ctx.ac_mac_len);
1183 1186                          ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
1184 1187                              plaintext, AES_BLOCK_LEN, aes_encrypt_block,
1185      -                            aes_copy_block, aes_xor_block);
     1188 +                            AES_COPY_BLOCK, AES_XOR_BLOCK);
1186 1189                          ASSERT(aes_ctx.ac_remainder_len == 0);
1187 1190                          if ((ret == CRYPTO_SUCCESS) &&
1188 1191                              (ciphertext != plaintext)) {
1189 1192                                  plaintext->cd_length =
1190 1193                                      plaintext->cd_offset - saved_offset;
1191 1194                          } else {
1192 1195                                  plaintext->cd_length = saved_length;
1193 1196                          }
1194 1197                  } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
1195 1198                      mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
1196 1199                          ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
1197 1200                              plaintext, AES_BLOCK_LEN, aes_encrypt_block,
1198      -                            aes_xor_block);
     1201 +                            AES_COPY_BLOCK, AES_XOR_BLOCK, aes_ctr_mode);
1199 1202                          ASSERT(aes_ctx.ac_remainder_len == 0);
1200 1203                          if ((ret == CRYPTO_SUCCESS) &&
1201 1204                              (ciphertext != plaintext)) {
1202 1205                                  plaintext->cd_length =
1203 1206                                      plaintext->cd_offset - saved_offset;
1204 1207                          } else {
1205 1208                                  plaintext->cd_length = saved_length;
1206 1209                          }
1207 1210                  } else if (mechanism->cm_type != AES_CTR_MECH_INFO_TYPE) {
1208 1211                          ASSERT(aes_ctx.ac_remainder_len == 0);
↓ open down ↓ 21 lines elided ↑ open up ↑
1230 1233  out:
1231 1234          if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
1232 1235                  bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
1233 1236                  kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
1234 1237          }
1235 1238  
1236 1239          if (aes_ctx.ac_flags & CCM_MODE) {
1237 1240                  if (aes_ctx.ac_pt_buf != NULL) {
1238 1241                          kmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len);
1239 1242                  }
1240      -        } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) {
1241      -                if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) {
1242      -                        kmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
1243      -                            ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
1244      -                }
1245 1243          }
1246 1244  
1247 1245          return (ret);
1248 1246  }
1249 1247  
1250 1248  /*
1251 1249   * KCF software provider context template entry points.
1252 1250   */
1253 1251  /* ARGSUSED */
1254 1252  static int
↓ open down ↓ 90 lines elided ↑ open up ↑
1345 1343                  break;
1346 1344          case AES_CTR_MECH_INFO_TYPE: {
1347 1345                  CK_AES_CTR_PARAMS *pp;
1348 1346  
1349 1347                  if (mechanism->cm_param == NULL ||
1350 1348                      mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) {
1351 1349                          return (CRYPTO_MECHANISM_PARAM_INVALID);
1352 1350                  }
1353 1351                  pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
1354 1352                  rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
1355      -                    pp->cb, aes_copy_block);
     1353 +                    pp->cb, AES_COPY_BLOCK);
1356 1354                  break;
1357 1355          }
1358 1356          case AES_CCM_MECH_INFO_TYPE:
1359 1357                  if (mechanism->cm_param == NULL ||
1360 1358                      mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) {
1361 1359                          return (CRYPTO_MECHANISM_PARAM_INVALID);
1362 1360                  }
1363 1361                  rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
1364 1362                      kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
1365      -                    aes_xor_block);
     1363 +                    AES_XOR_BLOCK);
1366 1364                  break;
1367 1365          case AES_GCM_MECH_INFO_TYPE:
1368 1366                  if (mechanism->cm_param == NULL ||
1369 1367                      mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) {
1370 1368                          return (CRYPTO_MECHANISM_PARAM_INVALID);
1371 1369                  }
1372 1370                  rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
1373      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
1374      -                    aes_xor_block);
     1371 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
     1372 +                    AES_XOR_BLOCK);
1375 1373                  break;
1376 1374          case AES_GMAC_MECH_INFO_TYPE:
1377 1375                  if (mechanism->cm_param == NULL ||
1378 1376                      mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) {
1379 1377                          return (CRYPTO_MECHANISM_PARAM_INVALID);
1380 1378                  }
1381 1379                  rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
1382      -                    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
1383      -                    aes_xor_block);
     1380 +                    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
     1381 +                    AES_XOR_BLOCK);
1384 1382                  break;
1385 1383          case AES_ECB_MECH_INFO_TYPE:
1386 1384                  aes_ctx->ac_flags |= ECB_MODE;
1387 1385          }
1388 1386  
1389 1387          if (rv != CRYPTO_SUCCESS) {
1390 1388                  if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
1391 1389                          bzero(keysched, size);
1392 1390                          kmem_free(keysched, size);
1393 1391                  }
↓ open down ↓ 79 lines elided ↑ open up ↑
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX