1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 /*
  26  * Copyright 2015 by Saso Kiselkov. All rights reserved.
  27  */
  28 
  29 #ifndef _KERNEL
  30 #include <strings.h>
  31 #include <limits.h>
  32 #include <assert.h>
  33 #include <security/cryptoki.h>
  34 #endif
  35 
  36 #include <sys/types.h>
  37 #define INLINE_CRYPTO_GET_PTRS
  38 #include <modes/modes.h>
  39 #include <sys/crypto/common.h>
  40 #include <sys/crypto/impl.h>
  41 
  42 boolean_t cbc_fastpath_enabled = B_TRUE;
  43 
  44 static void
  45 cbc_decrypt_fastpath(cbc_ctx_t *ctx, const uint8_t *data, size_t length,
  46     uint8_t *out, size_t block_size,
  47     int (*decrypt)(const void *, const uint8_t *, uint8_t *),
  48     int (*decrypt_ecb)(const void *, const uint8_t *, uint8_t *, uint64_t),
  49     void (*xor_block)(const uint8_t *, uint8_t *),
  50     void (*xor_block_range)(const uint8_t *, uint8_t *, uint64_t))
  51 {
  52         const uint8_t *iv = (uint8_t *)ctx->cbc_iv;
  53 
  54         /* Use bulk decryption when available. */
  55         if (decrypt_ecb != NULL) {
  56                 decrypt_ecb(ctx->cbc_keysched, data, out, length);
  57         } else {
  58                 for (size_t i = 0; i < length; i += block_size)
  59                         decrypt(ctx->cbc_keysched, &data[i], &out[i]);
  60         }
  61 
  62         /* Use bulk XOR when available. */
  63         if (xor_block_range != NULL && length >= 2 * block_size) {
  64                 xor_block(iv, out);
  65                 xor_block_range(data, &out[block_size], length - block_size);
  66         } else {
  67                 for (size_t i = 0; i < length; i += block_size) {
  68                         xor_block(iv, &out[i]);
  69                         iv = &data[i];
  70                 }
  71         }
  72 }
  73 
  74 /*
  75  * Algorithm independent CBC functions.
  76  */
  77 int
  78 cbc_encrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
  79     crypto_data_t *out, size_t block_size,
  80     int (*encrypt)(const void *, const uint8_t *, uint8_t *),
  81     void (*copy_block)(const uint8_t *, uint8_t *),
  82     void (*xor_block)(const uint8_t *, uint8_t *),
  83     int (*encrypt_cbc)(const void *, const uint8_t *, uint8_t *,
  84     const uint8_t *, uint64_t))
  85 {
  86         size_t remainder = length;
  87         size_t need;
  88         uint8_t *datap = (uint8_t *)data;
  89         uint8_t *blockp;
  90         uint8_t *lastp;
  91         void *iov_or_mp;
  92         offset_t offset;
  93         uint8_t *out_data_1;
  94         uint8_t *out_data_2;
  95         size_t out_data_1_len;
  96 
  97         /*
  98          * CBC encryption fastpath requirements:
  99          * - fastpath is enabled
 100          * - algorithm-specific acceleration function is available
 101          * - input is block-aligned
 102          * - output is a single contiguous region or the user requested that
 103          *   we overwrite their input buffer (input/output aliasing allowed)
 104          */
 105         if (cbc_fastpath_enabled && encrypt_cbc != NULL && length != 0 &&
 106             ctx->cbc_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
 107             (out == NULL || CRYPTO_DATA_IS_SINGLE_BLOCK(out))) {
 108                 if (out == NULL) {
 109                         encrypt_cbc(ctx->cbc_keysched, (uint8_t *)data,
 110                             (uint8_t *)data, (uint8_t *)ctx->cbc_iv, length);
 111                         ctx->cbc_lastp = (uint8_t *)&data[length - block_size];
 112                 } else {
 113                         uint8_t *outp = CRYPTO_DATA_FIRST_BLOCK(out);
 114                         encrypt_cbc(ctx->cbc_keysched, (uint8_t *)data, outp,
 115                             (uint8_t *)ctx->cbc_iv, length);
 116                         out->cd_offset += length;
 117                         ctx->cbc_lastp = &outp[length - block_size];
 118                 }
 119                 goto out;
 120         }
 121 
 122         if (length + ctx->cbc_remainder_len < block_size) {
 123                 /* accumulate bytes here and return */
 124                 bcopy(datap,
 125                     (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
 126                     length);
 127                 ctx->cbc_remainder_len += length;
 128                 ctx->cbc_copy_to = datap;
 129                 return (CRYPTO_SUCCESS);
 130         }
 131 
 132         lastp = (uint8_t *)ctx->cbc_iv;
 133         if (out != NULL)
 134                 crypto_init_ptrs(out, &iov_or_mp, &offset);
 135 
 136         do {
 137                 /* Unprocessed data from last call. */
 138                 if (ctx->cbc_remainder_len > 0) {
 139                         need = block_size - ctx->cbc_remainder_len;
 140 
 141                         if (need > remainder)
 142                                 return (CRYPTO_DATA_LEN_RANGE);
 143 
 144                         bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
 145                             [ctx->cbc_remainder_len], need);
 146 
 147                         blockp = (uint8_t *)ctx->cbc_remainder;
 148                 } else {
 149                         blockp = datap;
 150                 }
 151 
 152                 if (out == NULL) {
 153                         /*
 154                          * XOR the previous cipher block or IV with the
 155                          * current clear block.
 156                          */
 157                         xor_block(lastp, blockp);
 158                         encrypt(ctx->cbc_keysched, blockp, blockp);
 159 
 160                         ctx->cbc_lastp = blockp;
 161                         lastp = blockp;
 162 
 163                         if (ctx->cbc_remainder_len > 0) {
 164                                 bcopy(blockp, ctx->cbc_copy_to,
 165                                     ctx->cbc_remainder_len);
 166                                 bcopy(blockp + ctx->cbc_remainder_len, datap,
 167                                     need);
 168                         }
 169                 } else {
 170                         /*
 171                          * XOR the previous cipher block or IV with the
 172                          * current clear block.
 173                          */
 174                         xor_block(blockp, lastp);
 175                         encrypt(ctx->cbc_keysched, lastp, lastp);
 176                         crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
 177                             &out_data_1_len, &out_data_2, block_size);
 178 
 179                         /* copy block to where it belongs */
 180                         if (out_data_1_len == block_size) {
 181                                 copy_block(lastp, out_data_1);
 182                         } else {
 183                                 bcopy(lastp, out_data_1, out_data_1_len);
 184                                 if (out_data_2 != NULL) {
 185                                         bcopy(lastp + out_data_1_len,
 186                                             out_data_2,
 187                                             block_size - out_data_1_len);
 188                                 }
 189                         }
 190                         /* update offset */
 191                         out->cd_offset += block_size;
 192                 }
 193 
 194                 /* Update pointer to next block of data to be processed. */
 195                 if (ctx->cbc_remainder_len != 0) {
 196                         datap += need;
 197                         ctx->cbc_remainder_len = 0;
 198                 } else {
 199                         datap += block_size;
 200                 }
 201 
 202                 remainder = (size_t)&data[length] - (size_t)datap;
 203 
 204                 /* Incomplete last block. */
 205                 if (remainder > 0 && remainder < block_size) {
 206                         bcopy(datap, ctx->cbc_remainder, remainder);
 207                         ctx->cbc_remainder_len = remainder;
 208                         ctx->cbc_copy_to = datap;
 209                         goto out;
 210                 }
 211                 ctx->cbc_copy_to = NULL;
 212 
 213         } while (remainder > 0);
 214 
 215 out:
 216         /*
 217          * Save the last encrypted block in the context.
 218          */
 219         if (ctx->cbc_lastp != NULL) {
 220                 copy_block((uint8_t *)ctx->cbc_lastp, (uint8_t *)ctx->cbc_iv);
 221                 ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv;
 222         }
 223 
 224         return (CRYPTO_SUCCESS);
 225 }
 226 
 227 #define OTHER(a, ctx) \
 228         (((a) == (ctx)->cbc_lastblock) ? (ctx)->cbc_iv : (ctx)->cbc_lastblock)
 229 
 230 /* ARGSUSED */
 231 int
 232 cbc_decrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
 233     crypto_data_t *out, size_t block_size,
 234     int (*decrypt)(const void *, const uint8_t *, uint8_t *),
 235     void (*copy_block)(const uint8_t *, uint8_t *),
 236     void (*xor_block)(const uint8_t *, uint8_t *),
 237     int (*decrypt_ecb)(const void *, const uint8_t *, uint8_t *, uint64_t),
 238     void (*xor_block_range)(const uint8_t *, uint8_t *, uint64_t))
 239 {
 240         size_t remainder = length;
 241         size_t need;
 242         uint8_t *datap = (uint8_t *)data;
 243         uint8_t *blockp;
 244         uint8_t *lastp;
 245         void *iov_or_mp;
 246         offset_t offset;
 247         uint8_t *out_data_1;
 248         uint8_t *out_data_2;
 249         size_t out_data_1_len;
 250 
 251         /*
 252          * CBC decryption fastpath requirements:
 253          * - fastpath is enabled
 254          * - input is block-aligned
 255          * - output is a single contiguous region and doesn't alias input
 256          */
 257         if (cbc_fastpath_enabled && ctx->cbc_remainder_len == 0 &&
 258             length != 0 && (length & (block_size - 1)) == 0 &&
 259             CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
 260                 uint8_t *outp = CRYPTO_DATA_FIRST_BLOCK(out);
 261 
 262                 cbc_decrypt_fastpath(ctx, (uint8_t *)data, length, outp,
 263                     block_size, decrypt, decrypt_ecb, xor_block,
 264                     xor_block_range);
 265                 out->cd_offset += length;
 266                 bcopy(&data[length - block_size], ctx->cbc_iv, block_size);
 267                 ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv;
 268                 return (CRYPTO_SUCCESS);
 269         }
 270 
 271         if (length + ctx->cbc_remainder_len < block_size) {
 272                 /* accumulate bytes here and return */
 273                 bcopy(datap,
 274                     (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len,
 275                     length);
 276                 ctx->cbc_remainder_len += length;
 277                 ctx->cbc_copy_to = datap;
 278                 return (CRYPTO_SUCCESS);
 279         }
 280 
 281         lastp = ctx->cbc_lastp;
 282         if (out != NULL)
 283                 crypto_init_ptrs(out, &iov_or_mp, &offset);
 284 
 285         do {
 286                 /* Unprocessed data from last call. */
 287                 if (ctx->cbc_remainder_len > 0) {
 288                         need = block_size - ctx->cbc_remainder_len;
 289 
 290                         if (need > remainder)
 291                                 return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
 292 
 293                         bcopy(datap, &((uint8_t *)ctx->cbc_remainder)
 294                             [ctx->cbc_remainder_len], need);
 295 
 296                         blockp = (uint8_t *)ctx->cbc_remainder;
 297                 } else {
 298                         blockp = datap;
 299                 }
 300 
 301                 /* LINTED: pointer alignment */
 302                 copy_block(blockp, (uint8_t *)OTHER((uint64_t *)lastp, ctx));
 303 
 304                 if (out != NULL) {
 305                         decrypt(ctx->cbc_keysched, blockp,
 306                             (uint8_t *)ctx->cbc_remainder);
 307                         blockp = (uint8_t *)ctx->cbc_remainder;
 308                 } else {
 309                         decrypt(ctx->cbc_keysched, blockp, blockp);
 310                 }
 311 
 312                 /*
 313                  * XOR the previous cipher block or IV with the
 314                  * currently decrypted block.
 315                  */
 316                 xor_block(lastp, blockp);
 317 
 318                 /* LINTED: pointer alignment */
 319                 lastp = (uint8_t *)OTHER((uint64_t *)lastp, ctx);
 320 
 321                 if (out != NULL) {
 322                         crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1,
 323                             &out_data_1_len, &out_data_2, block_size);
 324 
 325                         bcopy(blockp, out_data_1, out_data_1_len);
 326                         if (out_data_2 != NULL) {
 327                                 bcopy(blockp + out_data_1_len, out_data_2,
 328                                     block_size - out_data_1_len);
 329                         }
 330 
 331                         /* update offset */
 332                         out->cd_offset += block_size;
 333 
 334                 } else if (ctx->cbc_remainder_len > 0) {
 335                         /* copy temporary block to where it belongs */
 336                         bcopy(blockp, ctx->cbc_copy_to, ctx->cbc_remainder_len);
 337                         bcopy(blockp + ctx->cbc_remainder_len, datap, need);
 338                 }
 339 
 340                 /* Update pointer to next block of data to be processed. */
 341                 if (ctx->cbc_remainder_len != 0) {
 342                         datap += need;
 343                         ctx->cbc_remainder_len = 0;
 344                 } else {
 345                         datap += block_size;
 346                 }
 347 
 348                 remainder = (size_t)&data[length] - (size_t)datap;
 349 
 350                 /* Incomplete last block. */
 351                 if (remainder > 0 && remainder < block_size) {
 352                         bcopy(datap, ctx->cbc_remainder, remainder);
 353                         ctx->cbc_remainder_len = remainder;
 354                         ctx->cbc_lastp = lastp;
 355                         ctx->cbc_copy_to = datap;
 356                         return (CRYPTO_SUCCESS);
 357                 }
 358                 ctx->cbc_copy_to = NULL;
 359 
 360         } while (remainder > 0);
 361 
 362         ctx->cbc_lastp = lastp;
 363         return (CRYPTO_SUCCESS);
 364 }
 365 
 366 int
 367 cbc_init_ctx(cbc_ctx_t *cbc_ctx, char *param, size_t param_len,
 368     size_t block_size, void (*copy_block)(const uint8_t *, uint64_t *))
 369 {
 370         /*
 371          * Copy IV into context.
 372          *
 373          * If cm_param == NULL then the IV comes from the
 374          * cd_miscdata field in the crypto_data structure.
 375          */
 376         if (param != NULL) {
 377 #ifdef _KERNEL
 378                 ASSERT(param_len == block_size);
 379 #else
 380                 assert(param_len == block_size);
 381 #endif
 382                 copy_block((uchar_t *)param, cbc_ctx->cbc_iv);
 383         }
 384 
 385         cbc_ctx->cbc_lastp = (uint8_t *)&cbc_ctx->cbc_iv[0];
 386         cbc_ctx->cbc_flags |= CBC_MODE;
 387         return (CRYPTO_SUCCESS);
 388 }
 389 
 390 /* ARGSUSED */
 391 void *
 392 cbc_alloc_ctx(int kmflag)
 393 {
 394         cbc_ctx_t *cbc_ctx;
 395 
 396 #ifdef _KERNEL
 397         if ((cbc_ctx = kmem_zalloc(sizeof (cbc_ctx_t), kmflag)) == NULL)
 398 #else
 399         if ((cbc_ctx = calloc(1, sizeof (cbc_ctx_t))) == NULL)
 400 #endif
 401                 return (NULL);
 402 
 403         cbc_ctx->cbc_flags = CBC_MODE;
 404         return (cbc_ctx);
 405 }