--- old/usr/src/common/crypto/aes/aes_impl.c	Thu Apr 30 20:52:29 2015
+++ new/usr/src/common/crypto/aes/aes_impl.c	Thu Apr 30 20:52:29 2015
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
  */
 
 #include <sys/types.h>
@@ -30,6 +31,7 @@
 #ifndef	_KERNEL
 #include <strings.h>
 #include <stdlib.h>
+#include <sys/note.h>
 #endif	/* !_KERNEL */
 
 #ifdef __amd64
@@ -94,8 +96,8 @@
 extern void aes_decrypt_impl(const uint32_t rk[], int Nr, const uint32_t ct[4],
 	uint32_t pt[4]);
 
-#define	AES_ENCRYPT_IMPL(a, b, c, d, e)	aes_encrypt_impl(a, b, c, d)
-#define	AES_DECRYPT_IMPL(a, b, c, d, e)	aes_decrypt_impl(a, b, c, d)
+#define	AES_ENCRYPT_IMPL(a, b, c, d)	aes_encrypt_impl(a, b, c, d)
+#define	AES_DECRYPT_IMPL(a, b, c, d)	aes_decrypt_impl(a, b, c, d)
 
 #elif defined(__amd64)
 
@@ -118,17 +120,37 @@
 	const uint32_t pt[4], uint32_t ct[4]);
 extern void aes_decrypt_intel(const uint32_t rk[], int Nr,
 	const uint32_t ct[4], uint32_t pt[4]);
+extern void aes_encrypt_intel8(const uint32_t rk[], int Nr,
+	const void *pt, void *ct);
+extern void aes_decrypt_intel8(const uint32_t rk[], int Nr,
+	const void *ct, void *pt);
+extern void aes_encrypt_cbc_intel8(const uint32_t rk[], int Nr,
+	const void *pt, void *ct, const void *iv);
+extern void aes_ctr_intel8(const uint32_t rk[], int Nr,
+	const void *input, void *output, uint64_t counter_upper_BE,
+	uint64_t counter_lower_LE);
+extern void aes_xor_intel(const uint8_t *, uint8_t *);
 
-static int intel_aes_instructions_present(void);
+static inline int intel_aes_instructions_present(void);
 
-#define	AES_ENCRYPT_IMPL(a, b, c, d, e) rijndael_encrypt(a, b, c, d, e)
-#define	AES_DECRYPT_IMPL(a, b, c, d, e) rijndael_decrypt(a, b, c, d, e)
+#ifdef	_KERNEL
+/*
+ * Some form of floating-point acceleration is available, so declare these.
+ * The implementations will be in a platform-specific assembly file (e.g.
+ * amd64/aes_intel.s for SSE2/AES-NI).
+ */
+extern void aes_accel_save(void *savestate);
+extern void aes_accel_restore(void *savestate);
+#endif	/* _KERNEL */
 
 #else /* Generic C implementation */
-
-#define	AES_ENCRYPT_IMPL(a, b, c, d, e)	rijndael_encrypt(a, b, c, d)
-#define	AES_DECRYPT_IMPL(a, b, c, d, e)	rijndael_decrypt(a, b, c, d)
+static void rijndael_encrypt(const uint32_t rk[], int Nr, const uint32_t pt[4],
+    uint32_t ct[4]);
+static void rijndael_decrypt(const uint32_t rk[], int Nr, const uint32_t pt[4],
+    uint32_t ct[4]);
 #define	rijndael_key_setup_enc_raw	rijndael_key_setup_enc
+#define	AES_ENCRYPT_IMPL(a, b, c, d)	rijndael_encrypt(a, b, c, d)
+#define	AES_DECRYPT_IMPL(a, b, c, d)	rijndael_decrypt(a, b, c, d)
 #endif	/* sun4u || __amd64 */
 
 #if defined(_LITTLE_ENDIAN) && !defined(__amd64)
@@ -1140,16 +1162,15 @@
 static void
 aes_setupkeys(aes_key_t *key, const uint32_t *keyarr32, int keybits)
 {
+	AES_ACCEL_SAVESTATE(savestate);
+	aes_accel_enter(savestate);
+
 	if (intel_aes_instructions_present()) {
-		key->flags = INTEL_AES_NI_CAPABLE;
-		KPREEMPT_DISABLE;
 		key->nr = rijndael_key_setup_enc_intel(&(key->encr_ks.ks32[0]),
 		    keyarr32, keybits);
 		key->nr = rijndael_key_setup_dec_intel(&(key->decr_ks.ks32[0]),
 		    keyarr32, keybits);
-		KPREEMPT_ENABLE;
 	} else {
-		key->flags = 0;
 		key->nr = rijndael_key_setup_enc_amd64(&(key->encr_ks.ks32[0]),
 		    keyarr32, keybits);
 		key->nr = rijndael_key_setup_dec_amd64(&(key->decr_ks.ks32[0]),
@@ -1156,64 +1177,10 @@
 		    keyarr32, keybits);
 	}
 
+	aes_accel_exit(savestate);
 	key->type = AES_32BIT_KS;
 }
 
-/*
- * Encrypt one block of data. The block is assumed to be an array
- * of four uint32_t values, so copy for alignment (and byte-order
- * reversal for little endian systems might be necessary on the
- * input and output byte streams.
- * The size of the key schedule depends on the number of rounds
- * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
- *
- * Parameters:
- * rk		Key schedule, of aes_ks_t (60 32-bit integers)
- * Nr		Number of rounds
- * pt		Input block (plain text)
- * ct		Output block (crypto text).  Can overlap with pt
- * flags	Indicates whether we're on Intel AES-NI-capable hardware
- */
-static void
-rijndael_encrypt(const uint32_t rk[], int Nr, const uint32_t pt[4],
-    uint32_t ct[4], int flags) {
-	if (flags & INTEL_AES_NI_CAPABLE) {
-		KPREEMPT_DISABLE;
-		aes_encrypt_intel(rk, Nr, pt, ct);
-		KPREEMPT_ENABLE;
-	} else {
-		aes_encrypt_amd64(rk, Nr, pt, ct);
-	}
-}
-
-/*
- * Decrypt one block of data. The block is assumed to be an array
- * of four uint32_t values, so copy for alignment (and byte-order
- * reversal for little endian systems might be necessary on the
- * input and output byte streams.
- * The size of the key schedule depends on the number of rounds
- * (which can be computed from the size of the key), i.e. 4*(Nr + 1).
- *
- * Parameters:
- * rk		Key schedule, of aes_ks_t (60 32-bit integers)
- * Nr		Number of rounds
- * ct		Input block (crypto text)
- * pt		Output block (plain text). Can overlap with pt
- * flags	Indicates whether we're on Intel AES-NI-capable hardware
- */
-static void
-rijndael_decrypt(const uint32_t rk[], int Nr, const uint32_t ct[4],
-    uint32_t pt[4], int flags) {
-	if (flags & INTEL_AES_NI_CAPABLE) {
-		KPREEMPT_DISABLE;
-		aes_decrypt_intel(rk, Nr, ct, pt);
-		KPREEMPT_ENABLE;
-	} else {
-		aes_decrypt_amd64(rk, Nr, ct, pt);
-	}
-}
-
-
 #else /* generic C implementation */
 
 /*
@@ -1622,7 +1589,22 @@
 	aes_setupkeys(newbie, keyarr.ka32, keyBits);
 }
 
+#if	defined(__amd64) && defined(_KERNEL)
+void
+aes_accel_enter(void *savestate)
+{
+	KPREEMPT_DISABLE;
+	aes_accel_save(savestate);
+}
 
+void
+aes_accel_exit(void *savestate)
+{
+	aes_accel_restore(savestate);
+	KPREEMPT_ENABLE;
+}
+#endif	/* defined(__amd64) && defined(_KERNEL) */
+
 /*
  * Encrypt one block using AES.
  * Align if needed and (for x86 32-bit only) byte-swap.
@@ -1637,12 +1619,21 @@
 {
 	aes_key_t	*ksch = (aes_key_t *)ks;
 
+#ifdef	__amd64
+	if (intel_aes_instructions_present())
+		aes_encrypt_intel(&ksch->encr_ks.ks32[0], ksch->nr,
+		    /* LINTED:  pointer alignment */
+		    (uint32_t *)pt, (uint32_t *)ct);
+	else
+		aes_encrypt_amd64(&ksch->encr_ks.ks32[0], ksch->nr,
+		    /* LINTED:  pointer alignment */
+		    (uint32_t *)pt, (uint32_t *)ct);
+#else	/* !__amd64 */
 #ifndef	AES_BYTE_SWAP
 	if (IS_P2ALIGNED2(pt, ct, sizeof (uint32_t))) {
-		/* LINTED:  pointer alignment */
 		AES_ENCRYPT_IMPL(&ksch->encr_ks.ks32[0], ksch->nr,
 		    /* LINTED:  pointer alignment */
-		    (uint32_t *)pt, (uint32_t *)ct, ksch->flags);
+		    (uint32_t *)pt, (uint32_t *)ct);
 	} else {
 #endif
 		uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
@@ -1656,10 +1647,10 @@
 		buffer[1] = htonl(*(uint32_t *)(void *)&pt[4]);
 		buffer[2] = htonl(*(uint32_t *)(void *)&pt[8]);
 		buffer[3] = htonl(*(uint32_t *)(void *)&pt[12]);
-#endif
+#endif	/* byte swap */
 
 		AES_ENCRYPT_IMPL(&ksch->encr_ks.ks32[0], ksch->nr,
-		    buffer, buffer, ksch->flags);
+		    buffer, buffer);
 
 		/* Copy result from buffer to output block */
 #ifndef	AES_BYTE_SWAP
@@ -1671,7 +1662,9 @@
 		*(uint32_t *)(void *)&ct[4] = htonl(buffer[1]);
 		*(uint32_t *)(void *)&ct[8] = htonl(buffer[2]);
 		*(uint32_t *)(void *)&ct[12] = htonl(buffer[3]);
-#endif
+#endif	/* byte swap */
+#endif	/* !__amd64 */
+
 	return (CRYPTO_SUCCESS);
 }
 
@@ -1690,12 +1683,21 @@
 {
 	aes_key_t	*ksch = (aes_key_t *)ks;
 
+#ifdef	__amd64
+	if (intel_aes_instructions_present())
+		aes_decrypt_intel(&ksch->decr_ks.ks32[0], ksch->nr,
+		    /* LINTED:  pointer alignment */
+		    (uint32_t *)ct, (uint32_t *)pt);
+	else
+		aes_decrypt_amd64(&ksch->decr_ks.ks32[0], ksch->nr,
+		    /* LINTED:  pointer alignment */
+		    (uint32_t *)ct, (uint32_t *)pt);
+#else	/* !__amd64 */
 #ifndef	AES_BYTE_SWAP
 	if (IS_P2ALIGNED2(ct, pt, sizeof (uint32_t))) {
-		/* LINTED:  pointer alignment */
 		AES_DECRYPT_IMPL(&ksch->decr_ks.ks32[0], ksch->nr,
 		    /* LINTED:  pointer alignment */
-		    (uint32_t *)ct, (uint32_t *)pt, ksch->flags);
+		    (uint32_t *)ct, (uint32_t *)pt);
 	} else {
 #endif
 		uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)];
@@ -1709,10 +1711,10 @@
 		buffer[1] = htonl(*(uint32_t *)(void *)&ct[4]);
 		buffer[2] = htonl(*(uint32_t *)(void *)&ct[8]);
 		buffer[3] = htonl(*(uint32_t *)(void *)&ct[12]);
-#endif
+#endif	/* byte swap */
 
 		AES_DECRYPT_IMPL(&ksch->decr_ks.ks32[0], ksch->nr,
-		    buffer, buffer, ksch->flags);
+		    buffer, buffer);
 
 		/* Copy result from buffer to output block */
 #ifndef	AES_BYTE_SWAP
@@ -1724,13 +1726,227 @@
 	*(uint32_t *)(void *)&pt[4] = htonl(buffer[1]);
 	*(uint32_t *)(void *)&pt[8] = htonl(buffer[2]);
 	*(uint32_t *)(void *)&pt[12] = htonl(buffer[3]);
-#endif
+#endif	/* byte swap */
+#endif	/* !__amd64 */
 
 	return (CRYPTO_SUCCESS);
 }
 
+#define	ECB_LOOP(ciph_func)						\
+	do {								\
+		for (; i < length; i += AES_BLOCK_LEN)			\
+			ciph_func;					\
+		_NOTE(CONSTCOND)					\
+	} while (0)
+#define	ECB_LOOP_4P(ciph_func, enc_or_dec, in, out)			\
+	ECB_LOOP(ciph_func(&ksch->enc_or_dec ## r_ks.ks32[0],		\
+	    ksch->nr, (void *)&in[i], (void *)&out[i]))
+#define	ECB_LOOP_3P(ciph_func, in, out)					\
+	ECB_LOOP(ciph_func(ksch, (void *)&in[i], (void *)&out[i]))
 
+#ifdef	__amd64
+#define	ECB_INTEL_IMPL(enc_or_dec, in, out)				\
+	do {								\
+		if (intel_aes_instructions_present()) {			\
+			/* first use the accelerated function */	\
+			for (; i + 8 * AES_BLOCK_LEN <= length;		\
+			    i += 8 * AES_BLOCK_LEN)			\
+				aes_ ## enc_or_dec ## rypt_intel8(	\
+				    &ksch->enc_or_dec ## r_ks.ks32[0],	\
+				    ksch->nr, &in[i], &out[i]);		\
+			/* finish off the remainder per-block */	\
+			ECB_LOOP_4P(aes_ ## enc_or_dec ## rypt_intel,	\
+			    enc_or_dec,	in, out);			\
+		} else {						\
+			ECB_LOOP_4P(aes_ ## enc_or_dec ## rypt_amd64,	\
+			    enc_or_dec, in, out);			\
+		}							\
+		_NOTE(CONSTCOND)					\
+	} while (0)
+#endif	/* __amd64 */
+
 /*
+ * Perform AES ECB encryption on a sequence of blocks. On x86-64 CPUs with
+ * the AES-NI extension, this performs the encryption in increments of 8
+ * blocks at a time, exploiting instruction parallelism more efficiently.
+ * On other platforms, this simply encrypts the blocks in sequence.
+ */
+int
+aes_encrypt_ecb(const void *ks, const uint8_t *pt, uint8_t *ct, uint64_t length)
+{
+	aes_key_t *ksch = (aes_key_t *)ks;
+	uint64_t i = 0;
+
+#ifdef	__amd64
+	ECB_INTEL_IMPL(enc, pt, ct);
+#elif	defined(sun4u)
+	ECB_LOOP_4P(aes_encrypt_impl, enc, pt, ct);
+#else	/* Generic C implementation */
+	ECB_LOOP_3P((void) aes_encrypt_block, pt, ct);
+#endif	/* Generic C implementation */
+
+	return (CRYPTO_SUCCESS);
+}
+
+/*
+ * Same as aes_encrypt_ecb, but performs decryption.
+ */
+int
+aes_decrypt_ecb(const void *ks, const uint8_t *ct, uint8_t *pt, uint64_t length)
+{
+	aes_key_t *ksch = (aes_key_t *)ks;
+	uint64_t i = 0;
+
+#ifdef	__amd64
+	ECB_INTEL_IMPL(dec, ct, pt);
+#elif	defined(sun4u)
+	ECB_LOOP_4P(aes_decrypt_impl, dec, ct, pt);
+#else	/* Generic C implementation */
+	ECB_LOOP_3P((void) aes_decrypt_block, ct, pt);
+#endif	/* Generic C implementation */
+
+	return (CRYPTO_SUCCESS);
+}
+#ifdef	__amd64
+#undef	ECB_INTEL_IMPL
+#endif	/* __amd64 */
+
+#undef	ECB_LOOP
+#undef	ECB_LOOP_4P
+#undef	ECB_LOOP_3P
+
+#define	CBC_LOOP(enc_func, xor_func)					\
+	do {								\
+		for (; i < length; i += AES_BLOCK_LEN) {		\
+			/* copy IV to ciphertext */			\
+			bcopy(iv, &ct[i], AES_BLOCK_LEN);		\
+			/* XOR IV with plaintext with input */		\
+			xor_func(&pt[i], &ct[i]);			\
+			/* encrypt counter in output region */		\
+			enc_func;					\
+			iv = &ct[i];					\
+		}							\
+		_NOTE(CONSTCOND)					\
+	} while (0)
+#define	CBC_LOOP_4P(enc_func, xor_func)					\
+	CBC_LOOP(enc_func(&ksch->encr_ks.ks32[0],			\
+	    ksch->nr, (void *)&ct[i], (void *)&ct[i]), xor_func)
+#define	CBC_LOOP_3P(enc_func, xor_func)					\
+	CBC_LOOP(enc_func(ksch, (void *)&ct[i], (void *)&ct[i]), xor_func)
+
+/*
+ * Encrypts a sequence of consecutive AES blocks in CBC mode. On x86-64
+ * with the AES-NI extension, the encryption is performed on 8 blocks at
+ * a time using an optimized assembly implementation, giving a speed boost
+ * of around 75%. On other platforms, this simply performs CBC encryption
+ * in sequence on the blocks.
+ *
+ * Decryption acceleration is implemented in the kernel kcf block cipher
+ * modes code (cbc.c), because that doesn't require a complete hand-tuned
+ * CBC implementation in assembly.
+ */
+int
+aes_encrypt_cbc(const void *ks, const uint8_t *pt, uint8_t *ct,
+    const uint8_t *iv, uint64_t length)
+{
+	aes_key_t *ksch = (aes_key_t *)ks;
+	size_t i = 0;
+
+#ifdef	__amd64
+	if (intel_aes_instructions_present()) {
+		for (; i + 8 * AES_BLOCK_LEN <= length;
+		    i += 8 * AES_BLOCK_LEN) {
+			aes_encrypt_cbc_intel8(&ksch->encr_ks.ks32[0],
+			    ksch->nr, &ct[i], &ct[i], iv);
+			iv = &ct[7 * AES_BLOCK_LEN];
+		}
+		CBC_LOOP_4P(aes_encrypt_intel, aes_xor_intel);
+	} else {
+		CBC_LOOP_4P(aes_encrypt_amd64, aes_xor_intel);
+	}
+#elif	defined(sun4u)
+	CBC_LOOP_4P(aes_encrypt_impl, aes_xor_block);
+#else	/* Generic C implementation */
+	CBC_LOOP_3P((void) aes_encrypt_block, aes_xor_block);
+#endif	/* Generic C implementation */
+
+	return (CRYPTO_SUCCESS);
+}
+#undef	CBC_LOOP
+#undef	CBC_LOOP_4P
+#undef	CBC_LOOP_3P
+
+#define	CTR_LOOP(enc_func, xor_func)					\
+	do {								\
+		for (; i < length; i += AES_BLOCK_LEN) {		\
+			/* set up counter in output region */		\
+			*(uint64_t *)(void *)&output[i] = counter[0];	\
+			*(uint64_t *)(void *)&output[i + 8] =		\
+			    htonll(counter[1]++);			\
+			/* encrypt counter in output region */		\
+			enc_func;					\
+			/* XOR encrypted counter with input */		\
+			xor_func(&input[i], &output[i]);		\
+		}							\
+		_NOTE(CONSTCOND)					\
+	} while (0)
+#define	CTR_LOOP_4P(enc_func, xor_func)					\
+	CTR_LOOP(enc_func(&ksch->encr_ks.ks32[0], ksch->nr,		\
+	    (void *)&output[i], (void *)&output[i]), xor_func)
+#define	CTR_LOOP_3P(enc_func, xor_func)					\
+	CTR_LOOP(enc_func(ksch, (void *)&output[i], (void *)&output[i]),\
+	    xor_func)
+/*
+ * Performs high-performance counter mode encryption and decryption on
+ * a sequence of blocks. In CTR mode, encryption and decryption are the
+ * same operation, just with the plaintext and ciphertext reversed:
+ * plaintext = CTR(CTR(plaintext, K), K)
+ * Blocks also do not interdepend on each other, so it is an excellent
+ * mode when high performance is required and data authentication/integrity
+ * checking is provided via some other means, or isn't necessary.
+ *
+ * On x86-64 with the AES-NI extension, this code performs CTR mode
+ * encryption in parallel on 8 blocks at a time and can provide in
+ * excess of 3GB/s/core of encryption/decryption performance (<1 CPB).
+ */
+int
+aes_ctr_mode(const void *ks, const uint8_t *input, uint8_t *output,
+    uint64_t length, uint64_t counter[2])
+{
+	aes_key_t *ksch = (aes_key_t *)ks;
+	uint64_t i = 0;
+
+	// swap lower part to host order for computations
+	counter[1] = ntohll(counter[1]);
+
+#ifdef	__amd64
+	if (intel_aes_instructions_present()) {
+		/* first use the wide-register accelerated function */
+		for (; i + 8 * AES_BLOCK_LEN <= length;
+		    i += 8 * AES_BLOCK_LEN) {
+			aes_ctr_intel8(&ksch->encr_ks.ks32[0], ksch->nr,
+			    &input[i], &output[i], counter[0], counter[1]);
+			counter[1] += 8;
+		}
+		/* finish off the remainder using the slow per-block method */
+		CTR_LOOP_4P(aes_encrypt_intel, aes_xor_intel);
+	} else {
+		CTR_LOOP_4P(aes_encrypt_amd64, aes_xor_intel);
+	}
+#elif	defined(sun4u)
+	CTR_LOOP_4P(aes_encrypt_impl, aes_xor_block);
+#else	/* Generic C implementation */
+	CTR_LOOP_3P((void) aes_encrypt_block, aes_xor_block);
+#endif	/* Generic C implementation */
+
+	// swap lower part back to big endian
+	counter[1] = htonll(counter[1]);
+
+	return (CRYPTO_SUCCESS);
+}
+#undef	CTR_LOOP
+
+/*
  * Allocate key schedule for AES.
  *
  * Return the pointer and set size to the number of bytes allocated.
@@ -1762,14 +1978,13 @@
 
 #ifdef __amd64
 /*
- * Return 1 if executing on Intel with AES-NI instructions,
- * otherwise 0 (i.e., Intel without AES-NI or AMD64).
+ * Return 1 if executing on x86-64 with AES-NI instructions, otherwise 0.
  * Cache the result, as the CPU can't change.
  *
  * Note: the userland version uses getisax().  The kernel version uses
  * global variable x86_featureset.
  */
-static int
+static inline int
 intel_aes_instructions_present(void)
 {
 	static int	cached_result = -1;
--- old/usr/src/common/crypto/aes/aes_impl.h	Thu Apr 30 20:52:29 2015
+++ new/usr/src/common/crypto/aes/aes_impl.h	Thu Apr 30 20:52:29 2015
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef	_AES_IMPL_H
 #define	_AES_IMPL_H
@@ -45,7 +48,7 @@
 /* Round constant length, in number of 32-bit elements: */
 #define	RC_LENGTH	(5 * ((AES_BLOCK_LEN) / 4 - 2))
 
-#define	AES_COPY_BLOCK(src, dst) \
+#define	AES_COPY_BLOCK_UNALIGNED(src, dst) \
 	(dst)[0] = (src)[0]; \
 	(dst)[1] = (src)[1]; \
 	(dst)[2] = (src)[2]; \
@@ -63,7 +66,7 @@
 	(dst)[14] = (src)[14]; \
 	(dst)[15] = (src)[15]
 
-#define	AES_XOR_BLOCK(src, dst) \
+#define	AES_XOR_BLOCK_UNALIGNED(src, dst) \
 	(dst)[0] ^= (src)[0]; \
 	(dst)[1] ^= (src)[1]; \
 	(dst)[2] ^= (src)[2]; \
@@ -81,6 +84,14 @@
 	(dst)[14] ^= (src)[14]; \
 	(dst)[15] ^= (src)[15]
 
+#define	AES_COPY_BLOCK_ALIGNED(src, dst) \
+	((uint64_t *)(void *)(dst))[0] = ((uint64_t *)(void *)(src))[0]; \
+	((uint64_t *)(void *)(dst))[1] = ((uint64_t *)(void *)(src))[1]
+
+#define	AES_XOR_BLOCK_ALIGNED(src, dst) \
+	((uint64_t *)(void *)(dst))[0] ^= ((uint64_t *)(void *)(src))[0]; \
+	((uint64_t *)(void *)(dst))[1] ^= ((uint64_t *)(void *)(src))[1]
+
 /* AES key size definitions */
 #define	AES_MINBITS		128
 #define	AES_MINBYTES		((AES_MINBITS) >> 3)
@@ -99,6 +110,43 @@
 #define	MAX_AES_NR		14 /* Maximum number of rounds */
 #define	MAX_AES_NB		4  /* Number of columns comprising a state */
 
+/*
+ * Architecture-specific acceleration support autodetection.
+ * Some architectures provide hardware-assisted acceleration using floating
+ * point registers, which need special handling inside of the kernel, so the
+ * macros below define the auxiliary functions needed to utilize them.
+ */
+#if	defined(__amd64) && defined(_KERNEL)
+/*
+ * Using floating point registers requires temporarily disabling kernel
+ * thread preemption, so we need to operate on small-enough chunks to
+ * prevent scheduling latency bubbles.
+ * A typical 64-bit CPU can sustain around 300-400MB/s/core even in the
+ * slowest encryption modes (CBC), which with 32k per run works out to ~100us
+ * per run. CPUs with AES-NI in fast modes (ECB, CTR, CBC decryption) can
+ * easily sustain 3GB/s/core, so the latency potential essentially vanishes.
+ */
+#define	AES_OPSZ	32768
+
+#if	defined(lint) || defined(__lint)
+#define	AES_ACCEL_SAVESTATE(name)	uint8_t name[16 * 16 + 8]
+#else	/* lint || __lint */
+#define	AES_ACCEL_SAVESTATE(name) \
+	/* stack space for xmm0--xmm15 and cr0 (16 x 128 bits + 64 bits) */ \
+	uint8_t name[16 * 16 + 8] __attribute__((aligned(16)))
+#endif	/* lint || __lint */
+
+#else	/* !defined(__amd64) || !defined(_KERNEL) */
+/*
+ * All other accel support
+ */
+#define	AES_OPSZ	((size_t)-1)
+/* On other architectures or outside of the kernel these get stubbed out */
+#define	AES_ACCEL_SAVESTATE(name)
+#define	aes_accel_enter(savestate)
+#define	aes_accel_exit(savestate)
+#endif	/* !defined(__amd64) || !defined(_KERNEL) */
+
 typedef union {
 #ifdef	sun4u
 	uint64_t	ks64[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
@@ -106,9 +154,6 @@
 	uint32_t	ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)];
 } aes_ks_t;
 
-/* aes_key.flags value: */
-#define	INTEL_AES_NI_CAPABLE	0x1	/* AES-NI instructions present */
-
 typedef struct aes_key aes_key_t;
 struct aes_key {
 	aes_ks_t	encr_ks;  /* encryption key schedule */
@@ -115,7 +160,6 @@
 	aes_ks_t	decr_ks;  /* decryption key schedule */
 #ifdef __amd64
 	long double	align128; /* Align fields above for Intel AES-NI */
-	int		flags;	  /* implementation-dependent flags */
 #endif	/* __amd64 */
 	int		nr;	  /* number of rounds (10, 12, or 14) */
 	int		type;	  /* key schedule size (32 or 64 bits) */
@@ -132,14 +176,33 @@
 	void *keysched);
 extern int aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct);
 extern int aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt);
+extern int aes_encrypt_ecb(const void *ks, const uint8_t *pt, uint8_t *ct,
+    uint64_t length);
+extern int aes_decrypt_ecb(const void *ks, const uint8_t *pt, uint8_t *ct,
+    uint64_t length);
+extern int aes_encrypt_cbc(const void *ks, const uint8_t *pt, uint8_t *ct,
+    const uint8_t *iv, uint64_t length);
+extern int aes_ctr_mode(const void *ks, const uint8_t *pt, uint8_t *ct,
+    uint64_t length, uint64_t counter[2]);
 
 /*
  * AES mode functions.
  * The first 2 functions operate on 16-byte AES blocks.
  */
-extern void aes_copy_block(uint8_t *in, uint8_t *out);
-extern void aes_xor_block(uint8_t *data, uint8_t *dst);
+#ifdef	__amd64
+#define	AES_COPY_BLOCK	aes_copy_intel
+#define	AES_XOR_BLOCK	aes_xor_intel
+extern void aes_copy_intel(const uint8_t *src, uint8_t *dst);
+extern void aes_xor_intel(const uint8_t *src, uint8_t *dst);
+extern void aes_xor_intel8(const uint8_t *src, uint8_t *dst);
+#else	/* !__amd64 */
+#define	AES_COPY_BLOCK	aes_copy_block
+#define	AES_XOR_BLOCK	aes_xor_block
+#endif	/* !__amd64 */
 
+extern void aes_copy_block(const uint8_t *src, uint8_t *dst);
+extern void aes_xor_block(const uint8_t *src, uint8_t *dst);
+
 /* Note: ctx is a pointer to aes_ctx_t defined in modes.h */
 extern int aes_encrypt_contiguous_blocks(void *ctx, char *data, size_t length,
     crypto_data_t *out);
@@ -146,7 +209,17 @@
 extern int aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length,
     crypto_data_t *out);
 
+#if	defined(__amd64) && defined(_KERNEL)
 /*
+ * When AES floating-point acceleration is available, these will be called
+ * by the worker functions to clear and restore floating point state and
+ * control kernel thread preemption.
+ */
+extern void aes_accel_enter(void *savestate);
+extern void aes_accel_exit(void *savestate);
+#endif	/* __amd64 && _KERNEL */
+
+/*
  * The following definitions and declarations are only used by AES FIPS POST
  */
 #ifdef _AES_IMPL
--- old/usr/src/common/crypto/aes/aes_modes.c	Thu Apr 30 20:52:29 2015
+++ new/usr/src/common/crypto/aes/aes_modes.c	Thu Apr 30 20:52:29 2015
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #include <sys/types.h>
 #include <sys/sysmacros.h>
@@ -31,45 +34,79 @@
 #include <stdlib.h>
 #endif	/* !_KERNEL */
 
+#if defined(__amd64)
 
+/*
+ * XORs a range of contiguous AES blocks in `data' with blocks in 'dst'
+ * and places the result in `dst'. On x86-64 this exploits the 128-bit
+ * floating point registers (xmm) to maximize performance.
+ */
+static void
+aes_xor_range(const uint8_t *data, uint8_t *dst, uint64_t length)
+{
+	uint64_t i = 0;
+
+	/* First use the unrolled version. */
+	for (; i + 8 * AES_BLOCK_LEN <= length; i += 8 * AES_BLOCK_LEN)
+		aes_xor_intel8(&data[i], &dst[i]);
+	/* Finish the rest in single blocks. */
+	for (; i < length; i += AES_BLOCK_LEN)
+		aes_xor_intel(&data[i], &dst[i]);
+}
+
+#else	/* !__amd64 */
+
+/*
+ * XORs a range of contiguous AES blocks in `data' with blocks in 'dst'
+ * and places the result in `dst'.
+ */
+static void
+aes_xor_range(const uint8_t *data, uint8_t *dst, uint64_t length)
+{
+	uint64_t i = 0;
+
+	if (IS_P2ALIGNED2(dst, data, sizeof (uint64_t))) {
+		/* Unroll the loop to enable efficiency. */
+		for (; i + 8 * AES_BLOCK_LEN < length; i += 8 * AES_BLOCK_LEN) {
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x00], &dst[i + 0x00]);
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x10], &dst[i + 0x10]);
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x20], &dst[i + 0x20]);
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x30], &dst[i + 0x30]);
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x40], &dst[i + 0x40]);
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x50], &dst[i + 0x50]);
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x60], &dst[i + 0x60]);
+			AES_XOR_BLOCK_ALIGNED(&data[i + 0x70], &dst[i + 0x70]);
+		}
+	}
+	/* Finish the rest in single blocks. */
+	for (; i < length; i += AES_BLOCK_LEN)
+		AES_XOR_BLOCK(&data[i], &dst[i]);
+}
+
+#endif	/* !__amd64 */
+
 /* Copy a 16-byte AES block from "in" to "out" */
 void
-aes_copy_block(uint8_t *in, uint8_t *out)
+aes_copy_block(const uint8_t *in, uint8_t *out)
 {
 	if (IS_P2ALIGNED2(in, out, sizeof (uint32_t))) {
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&out[0] = *(uint32_t *)&in[0];
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&out[4] = *(uint32_t *)&in[4];
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&out[8] = *(uint32_t *)&in[8];
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&out[12] = *(uint32_t *)&in[12];
+		AES_COPY_BLOCK_ALIGNED(in, out);
 	} else {
-		AES_COPY_BLOCK(in, out);
+		AES_COPY_BLOCK_UNALIGNED(in, out);
 	}
 }
 
-
 /* XOR a 16-byte AES block of data into dst */
 void
-aes_xor_block(uint8_t *data, uint8_t *dst)
+aes_xor_block(const uint8_t *data, uint8_t *dst)
 {
 	if (IS_P2ALIGNED2(dst, data, sizeof (uint32_t))) {
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&dst[0] ^= *(uint32_t *)&data[0];
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&dst[4] ^= *(uint32_t *)&data[4];
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&dst[8] ^= *(uint32_t *)&data[8];
-		/* LINTED: pointer alignment */
-		*(uint32_t *)&dst[12] ^= *(uint32_t *)&data[12];
+		AES_XOR_BLOCK_ALIGNED(data, dst);
 	} else {
-		AES_XOR_BLOCK(data, dst);
+		AES_XOR_BLOCK_UNALIGNED(data, dst);
 	}
 }
 
-
 /*
  * Encrypt multiple blocks of data according to mode.
  */
@@ -78,33 +115,46 @@
     crypto_data_t *out)
 {
 	aes_ctx_t *aes_ctx = ctx;
-	int rv;
+	int rv = CRYPTO_SUCCESS;
 
-	if (aes_ctx->ac_flags & CTR_MODE) {
-		rv = ctr_mode_contiguous_blocks(ctx, data, length, out,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+	for (size_t i = 0; i < length; i += AES_OPSZ) {
+		size_t opsz = MIN(length - i, AES_OPSZ);
+		AES_ACCEL_SAVESTATE(savestate);
+		aes_accel_enter(savestate);
+
+		if (aes_ctx->ac_flags & CTR_MODE) {
+			rv = ctr_mode_contiguous_blocks(ctx, &data[i], opsz,
+			    out, AES_BLOCK_LEN, aes_encrypt_block,
+			    AES_XOR_BLOCK, aes_ctr_mode);
 #ifdef _KERNEL
-	} else if (aes_ctx->ac_flags & CCM_MODE) {
-		rv = ccm_mode_encrypt_contiguous_blocks(ctx, data, length,
-		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
-	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
-		rv = gcm_mode_encrypt_contiguous_blocks(ctx, data, length,
-		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		} else if (aes_ctx->ac_flags & CCM_MODE) {
+			rv = ccm_mode_encrypt_contiguous_blocks(ctx, &data[i],
+			    opsz, out, AES_BLOCK_LEN, aes_encrypt_block,
+			    AES_COPY_BLOCK, AES_XOR_BLOCK);
+		} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+			rv = gcm_mode_encrypt_contiguous_blocks(ctx, &data[i],
+			    opsz, out, AES_BLOCK_LEN, aes_encrypt_block,
+			    AES_COPY_BLOCK, AES_XOR_BLOCK, aes_ctr_mode);
 #endif
-	} else if (aes_ctx->ac_flags & CBC_MODE) {
-		rv = cbc_encrypt_contiguous_blocks(ctx,
-		    data, length, out, AES_BLOCK_LEN, aes_encrypt_block,
-		    aes_copy_block, aes_xor_block);
-	} else {
-		rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
-		    AES_BLOCK_LEN, aes_encrypt_block);
+		} else if (aes_ctx->ac_flags & CBC_MODE) {
+			rv = cbc_encrypt_contiguous_blocks(ctx, &data[i], opsz,
+			    out, AES_BLOCK_LEN, aes_encrypt_block,
+			    AES_COPY_BLOCK, AES_XOR_BLOCK, aes_encrypt_cbc);
+		} else {
+			rv = ecb_cipher_contiguous_blocks(ctx, &data[i], opsz,
+			    out, AES_BLOCK_LEN, aes_encrypt_block,
+			    aes_encrypt_ecb);
+		}
+
+		aes_accel_exit(savestate);
+
+		if (rv != CRYPTO_SUCCESS)
+				break;
 	}
+
 	return (rv);
 }
 
-
 /*
  * Decrypt multiple blocks of data according to mode.
  */
@@ -113,32 +163,48 @@
     crypto_data_t *out)
 {
 	aes_ctx_t *aes_ctx = ctx;
-	int rv;
+	int rv = CRYPTO_SUCCESS;
 
-	if (aes_ctx->ac_flags & CTR_MODE) {
-		rv = ctr_mode_contiguous_blocks(ctx, data, length, out,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
-		if (rv == CRYPTO_DATA_LEN_RANGE)
-			rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+
+	for (size_t i = 0; i < length; i += AES_OPSZ) {
+		size_t opsz = MIN(length - i, AES_OPSZ);
+		AES_ACCEL_SAVESTATE(savestate);
+		aes_accel_enter(savestate);
+
+		if (aes_ctx->ac_flags & CTR_MODE) {
+			rv = ctr_mode_contiguous_blocks(ctx, &data[i], opsz,
+			    out, AES_BLOCK_LEN, aes_encrypt_block,
+			    AES_XOR_BLOCK, aes_ctr_mode);
+			if (rv == CRYPTO_DATA_LEN_RANGE)
+				rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
 #ifdef _KERNEL
-	} else if (aes_ctx->ac_flags & CCM_MODE) {
-		rv = ccm_mode_decrypt_contiguous_blocks(ctx, data, length,
-		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
-	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
-		rv = gcm_mode_decrypt_contiguous_blocks(ctx, data, length,
-		    out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		} else if (aes_ctx->ac_flags & CCM_MODE) {
+			rv = ccm_mode_decrypt_contiguous_blocks(ctx, &data[i],
+			    opsz, out, AES_BLOCK_LEN, aes_encrypt_block,
+			    AES_COPY_BLOCK, AES_XOR_BLOCK);
+		} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
+			rv = gcm_mode_decrypt_contiguous_blocks(ctx, &data[i],
+			    opsz, out, AES_BLOCK_LEN, aes_encrypt_block,
+			    AES_COPY_BLOCK, AES_XOR_BLOCK, aes_ctr_mode);
 #endif
-	} else if (aes_ctx->ac_flags & CBC_MODE) {
-		rv = cbc_decrypt_contiguous_blocks(ctx, data, length, out,
-		    AES_BLOCK_LEN, aes_decrypt_block, aes_copy_block,
-		    aes_xor_block);
-	} else {
-		rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
-		    AES_BLOCK_LEN, aes_decrypt_block);
-		if (rv == CRYPTO_DATA_LEN_RANGE)
-			rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+		} else if (aes_ctx->ac_flags & CBC_MODE) {
+			rv = cbc_decrypt_contiguous_blocks(ctx, &data[i],
+			    opsz, out, AES_BLOCK_LEN, aes_decrypt_block,
+			    AES_COPY_BLOCK, AES_XOR_BLOCK, aes_decrypt_ecb,
+			    aes_xor_range);
+		} else {
+			rv = ecb_cipher_contiguous_blocks(ctx, &data[i],
+			    opsz, out, AES_BLOCK_LEN, aes_decrypt_block,
+			    aes_decrypt_ecb);
+			if (rv == CRYPTO_DATA_LEN_RANGE)
+				rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
+		}
+
+		aes_accel_exit(savestate);
+
+		if (rv != CRYPTO_SUCCESS)
+				break;
 	}
+
 	return (rv);
 }
--- old/usr/src/common/crypto/aes/amd64/aes_intel.s	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/aes/amd64/aes_intel.s	Thu Apr 30 20:52:30 2015
@@ -148,6 +148,9 @@
  *
  * ====================================================================
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #if defined(lint) || defined(__lint)
 
@@ -281,7 +284,74 @@
 	mov	%rbp, %rsp; \
 	pop	%rbp
 
+/*
+ * void aes_accel_save(void *savestate);
+ *
+ * Saves all 16 XMM registers and CR0 to a temporary location pointed to
+ * in the first argument and clears TS in CR0. This must be invoked before
+ * executing any floating point operations inside the kernel (and kernel
+ * thread preemption must be disabled as well). The memory region to which
+ * all state is saved must be at least 16x 128-bit + 64-bit long and must
+ * be 128-bit aligned.
+ */
+ENTRY_NP(aes_accel_save)
+	movq	%cr0, %rax
+	movq	%rax, 0x100(%rdi)
+	testq	$CR0_TS, %rax
+	jnz	1f
+	movaps	%xmm0, 0x00(%rdi)
+	movaps	%xmm1, 0x10(%rdi)
+	movaps	%xmm2, 0x20(%rdi)
+	movaps	%xmm3, 0x30(%rdi)
+	movaps	%xmm4, 0x40(%rdi)
+	movaps	%xmm5, 0x50(%rdi)
+	movaps	%xmm6, 0x60(%rdi)
+	movaps	%xmm7, 0x70(%rdi)
+	movaps	%xmm8, 0x80(%rdi)
+	movaps	%xmm9, 0x90(%rdi)
+	movaps	%xmm10, 0xa0(%rdi)
+	movaps	%xmm11, 0xb0(%rdi)
+	movaps	%xmm12, 0xc0(%rdi)
+	movaps	%xmm13, 0xd0(%rdi)
+	movaps	%xmm14, 0xe0(%rdi)
+	movaps	%xmm15, 0xf0(%rdi)
+	ret
+1:
+	PROTECTED_CLTS
+	ret
+	SET_SIZE(aes_accel_save)
 
+/*
+ * void aes_accel_restore(void *savestate);
+ *
+ * Restores the saved XMM and CR0.TS state from aes_accel_save.
+ */
+ENTRY_NP(aes_accel_restore)
+	mov	0x100(%rdi), %rax
+	testq	$CR0_TS, %rax
+	jnz	1f
+	movaps	0x00(%rdi), %xmm0
+	movaps	0x10(%rdi), %xmm1
+	movaps	0x20(%rdi), %xmm2
+	movaps	0x30(%rdi), %xmm3
+	movaps	0x40(%rdi), %xmm4
+	movaps	0x50(%rdi), %xmm5
+	movaps	0x60(%rdi), %xmm6
+	movaps	0x70(%rdi), %xmm7
+	movaps	0x80(%rdi), %xmm8
+	movaps	0x90(%rdi), %xmm9
+	movaps	0xa0(%rdi), %xmm10
+	movaps	0xb0(%rdi), %xmm11
+	movaps	0xc0(%rdi), %xmm12
+	movaps	0xd0(%rdi), %xmm13
+	movaps	0xe0(%rdi), %xmm14
+	movaps	0xf0(%rdi), %xmm15
+	ret
+1:
+	STTS(%rax)
+	ret
+	SET_SIZE(aes_accel_restore)
+
 #else
 #define	PROTECTED_CLTS
 #define	CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg)
@@ -378,8 +448,78 @@
 	ret
 	SET_SIZE(_key_expansion_256b)
 
+/*
+ * void aes_copy_intel(const uint8_t *src, uint8_t *dst);
+ *
+ * Copies one unaligned 128-bit block from `src' to `dst'. The copy is
+ * performed using FPU registers, so make sure FPU state is saved when
+ * running this in the kernel.
+ */
+ENTRY_NP(aes_copy_intel)
+	movdqu	(%rdi), %xmm0
+	movdqu	%xmm0, (%rsi)
+	ret
+	SET_SIZE(aes_copy_intel)
 
 /*
+ * void aes_xor_intel(const uint8_t *src, uint8_t *dst);
+ *
+ * XORs one pair of unaligned 128-bit blocks from `src' and `dst' and
+ * stores the result at `dst'. The XOR is performed using FPU registers,
+ * so make sure FPU state is saved when running this in the kernel.
+ */
+ENTRY_NP(aes_xor_intel)
+	movdqu	(%rdi), %xmm0
+	movdqu	(%rsi), %xmm1
+	pxor	%xmm1, %xmm0
+	movdqu	%xmm0, (%rsi)
+	ret
+	SET_SIZE(aes_xor_intel)
+
+/*
+ * void aes_xor_intel8(const uint8_t *src, uint8_t *dst);
+ *
+ * XORs eight pairs of consecutive unaligned 128-bit blocks from `src' and
+ * 'dst' and stores the results at `dst'. The XOR is performed using FPU
+ * registers, so make sure FPU state is saved when running this in the kernel.
+ */
+ENTRY_NP(aes_xor_intel8)
+	movdqu	0x00(%rdi), %xmm0
+	movdqu	0x00(%rsi), %xmm1
+	movdqu	0x10(%rdi), %xmm2
+	movdqu	0x10(%rsi), %xmm3
+	movdqu	0x20(%rdi), %xmm4
+	movdqu	0x20(%rsi), %xmm5
+	movdqu	0x30(%rdi), %xmm6
+	movdqu	0x30(%rsi), %xmm7
+	movdqu	0x40(%rdi), %xmm8
+	movdqu	0x40(%rsi), %xmm9
+	movdqu	0x50(%rdi), %xmm10
+	movdqu	0x50(%rsi), %xmm11
+	movdqu	0x60(%rdi), %xmm12
+	movdqu	0x60(%rsi), %xmm13
+	movdqu	0x70(%rdi), %xmm14
+	movdqu	0x70(%rsi), %xmm15
+	pxor	%xmm1, %xmm0
+	pxor	%xmm3, %xmm2
+	pxor	%xmm5, %xmm4
+	pxor	%xmm7, %xmm6
+	pxor	%xmm9, %xmm8
+	pxor	%xmm11, %xmm10
+	pxor	%xmm13, %xmm12
+	pxor	%xmm15, %xmm14
+	movdqu	%xmm0, 0x00(%rsi)
+	movdqu	%xmm2, 0x10(%rsi)
+	movdqu	%xmm4, 0x20(%rsi)
+	movdqu	%xmm6, 0x30(%rsi)
+	movdqu	%xmm8, 0x40(%rsi)
+	movdqu	%xmm10, 0x50(%rsi)
+	movdqu	%xmm12, 0x60(%rsi)
+	movdqu	%xmm14, 0x70(%rsi)
+	ret
+	SET_SIZE(aes_xor_intel8)
+
+/*
  * rijndael_key_setup_enc_intel()
  * Expand the cipher key into the encryption key schedule.
  *
@@ -659,29 +799,6 @@
 	SET_SIZE(rijndael_key_setup_dec_intel)
 
 
-/*
- * aes_encrypt_intel()
- * Encrypt a single block (in and out can overlap).
- *
- * For kernel code, caller is responsible for ensuring kpreempt_disable()
- * has been called.  This is because %xmm registers are not saved/restored.
- * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
- * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
- * on the stack.
- *
- * Temporary register usage:
- * %xmm0	State
- * %xmm1	Key
- *
- * Original OpenSolaris Interface:
- * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
- *	const uint32_t pt[4], uint32_t ct[4])
- *
- * Original Intel OpenSSL Interface:
- * void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
- *	const AES_KEY *key)
- */
-
 #ifdef	OPENSSL_INTERFACE
 #define	aes_encrypt_intel	intel_AES_encrypt
 #define	aes_decrypt_intel	intel_AES_decrypt
@@ -699,16 +816,112 @@
 #define	NROUNDS		esi	/* P2, 32 bits */
 #define	INP		rdx	/* P3, 64 bits */
 #define	OUTP		rcx	/* P4, 64 bits */
+#define	LENGTH		r8	/* P5, 64 bits */
 #endif	/* OPENSSL_INTERFACE */
 
-#define	STATE		xmm0	/* temporary, 128 bits */
-#define	KEY		xmm1	/* temporary, 128 bits */
+#define	KEY		xmm0	/* temporary, 128 bits */
+#define	STATE0		xmm8	/* temporary, 128 bits */
+#define	STATE1		xmm9	/* temporary, 128 bits */
+#define	STATE2		xmm10	/* temporary, 128 bits */
+#define	STATE3		xmm11	/* temporary, 128 bits */
+#define	STATE4		xmm12	/* temporary, 128 bits */
+#define	STATE5		xmm13	/* temporary, 128 bits */
+#define	STATE6		xmm14	/* temporary, 128 bits */
+#define	STATE7		xmm15	/* temporary, 128 bits */
 
-ENTRY_NP(aes_encrypt_intel)
-	CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
+/*
+ * Runs the first two rounds of AES256 on a state register. `op' should be
+ * aesenc or aesdec.
+ */
+#define	AES256_ROUNDS(op, statereg)	\
+	movaps	-0x60(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	-0x50(%KEYP), %KEY;	\
+	op	%KEY, %statereg
 
-	movups	(%INP), %STATE			/ input
+/*
+ * Runs the first two rounds of AES192, or the 3rd & 4th round of AES256 on
+ * a state register. `op' should be aesenc or aesdec.
+ */
+#define	AES192_ROUNDS(op, statereg)	\
+	movaps	-0x40(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	-0x30(%KEYP), %KEY;	\
+	op	%KEY, %statereg
+
+/*
+ * Runs the full 10 rounds of AES128, or the last 10 rounds of AES192/AES256
+ * on a state register. `op' should be aesenc or aesdec and `lastop' should
+ * be aesenclast or aesdeclast.
+ */
+#define	AES128_ROUNDS(op, lastop, statereg) \
+	movaps	-0x20(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	-0x10(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	(%KEYP), %KEY;		\
+	op	%KEY, %statereg;	\
+	movaps	0x10(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	0x20(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	0x30(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	0x40(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	0x50(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	0x60(%KEYP), %KEY;	\
+	op	%KEY, %statereg;	\
+	movaps	0x70(%KEYP), %KEY;	\
+	lastop	%KEY, %statereg
+
+/*
+ * Macros to run AES encryption rounds. Input must be prefilled in state
+ * register - output will be left there as well.
+ * To run AES256, invoke all of these macros in sequence. To run AES192,
+ * invoke only the -192 and -128 variants. To run AES128, invoke only the
+ * -128 variant.
+ */
+#define	AES256_ENC_ROUNDS(statereg) \
+	AES256_ROUNDS(aesenc, statereg)
+#define	AES192_ENC_ROUNDS(statereg) \
+	AES192_ROUNDS(aesenc, statereg)
+#define	AES128_ENC_ROUNDS(statereg) \
+	AES128_ROUNDS(aesenc, aesenclast, statereg)
+
+/* Same as the AES*_ENC_ROUNDS macros, but for decryption. */
+#define	AES256_DEC_ROUNDS(statereg) \
+	AES256_ROUNDS(aesdec, statereg)
+#define	AES192_DEC_ROUNDS(statereg) \
+	AES192_ROUNDS(aesdec, statereg)
+#define	AES128_DEC_ROUNDS(statereg) \
+	AES128_ROUNDS(aesdec, aesdeclast, statereg)
+
+
+/*
+ * aes_encrypt_intel()
+ * Encrypt a single block (in and out can overlap).
+ *
+ * For kernel code, caller is responsible for bracketing this call with
+ * disabling kernel thread preemption and calling aes_accel_save/restore().
+ *
+ * Temporary register usage:
+ * %xmm0	Key
+ * %xmm8	State
+ *
+ * Original OpenSolaris Interface:
+ * void aes_encrypt_intel(const aes_ks_t *ks, int Nr,
+ *	const uint32_t pt[4], uint32_t ct[4])
+ *
+ * Original Intel OpenSSL Interface:
+ * void intel_AES_encrypt(const unsigned char *in, unsigned char *out,
+ *	const AES_KEY *key)
+ */
+ENTRY_NP(aes_encrypt_intel)
+	movups	(%INP), %STATE0			/ input
 	movaps	(%KEYP), %KEY			/ key
+
 #ifdef	OPENSSL_INTERFACE
 	mov	240(%KEYP), %NROUNDS32		/ round count
 #else	/* OpenSolaris Interface */
@@ -715,7 +928,7 @@
 	/* Round count is already present as P2 in %rsi/%esi */
 #endif	/* OPENSSL_INTERFACE */
 
-	pxor	%KEY, %STATE			/ round 0
+	pxor	%KEY, %STATE0			/ round 0
 	lea	0x30(%KEYP), %KEYP
 	cmp	$12, %NROUNDS
 	jb	.Lenc128
@@ -724,58 +937,28 @@
 
 	/ AES 256
 	lea	0x20(%KEYP), %KEYP
-	movaps	-0x60(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	-0x50(%KEYP), %KEY
-	aesenc	%KEY, %STATE
+	AES256_ENC_ROUNDS(STATE0)
 
 .align 4
 .Lenc192:
 	/ AES 192 and 256
-	movaps	-0x40(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	-0x30(%KEYP), %KEY
-	aesenc	%KEY, %STATE
+	AES192_ENC_ROUNDS(STATE0)
 
 .align 4
 .Lenc128:
 	/ AES 128, 192, and 256
-	movaps	-0x20(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	-0x10(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	0x10(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	0x20(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	0x30(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	0x40(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	0x50(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	0x60(%KEYP), %KEY
-	aesenc	%KEY, %STATE
-	movaps	0x70(%KEYP), %KEY
-	aesenclast	 %KEY, %STATE		/ last round
-	movups	%STATE, (%OUTP)			/ output
+	AES128_ENC_ROUNDS(STATE0)
+	movups	%STATE0, (%OUTP)		/ output
 
-	SET_TS_OR_POP_XMM0_XMM1(%r10)
 	ret
 	SET_SIZE(aes_encrypt_intel)
 
-
 /*
  * aes_decrypt_intel()
  * Decrypt a single block (in and out can overlap).
  *
- * For kernel code, caller is responsible for ensuring kpreempt_disable()
- * has been called.  This is because %xmm registers are not saved/restored.
- * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
- * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
- * on the stack.
+ * For kernel code, caller is responsible for bracketing this call with
+ * disabling kernel thread preemption and calling aes_accel_save/restore().
  *
  * Temporary register usage:
  * %xmm0	State
@@ -783,7 +966,7 @@
  *
  * Original OpenSolaris Interface:
  * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
- *	const uint32_t pt[4], uint32_t ct[4])/
+ *	const uint32_t pt[4], uint32_t ct[4])
  *
  * Original Intel OpenSSL Interface:
  * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
@@ -790,10 +973,9 @@
  *	const AES_KEY *key);
  */
 ENTRY_NP(aes_decrypt_intel)
-	CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
-
-	movups	(%INP), %STATE			/ input
+	movups	(%INP), %STATE0			/ input
 	movaps	(%KEYP), %KEY			/ key
+
 #ifdef	OPENSSL_INTERFACE
 	mov	240(%KEYP), %NROUNDS32		/ round count
 #else	/* OpenSolaris Interface */
@@ -800,7 +982,7 @@
 	/* Round count is already present as P2 in %rsi/%esi */
 #endif	/* OPENSSL_INTERFACE */
 
-	pxor	%KEY, %STATE			/ round 0
+	pxor	%KEY, %STATE0			/ round 0
 	lea	0x30(%KEYP), %KEYP
 	cmp	$12, %NROUNDS
 	jb	.Ldec128
@@ -809,46 +991,417 @@
 
 	/ AES 256
 	lea	0x20(%KEYP), %KEYP
-	movaps	-0x60(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	-0x50(%KEYP), %KEY
-	aesdec	%KEY, %STATE
+	AES256_DEC_ROUNDS(STATE0)
 
 .align 4
 .Ldec192:
 	/ AES 192 and 256
-	movaps	-0x40(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	-0x30(%KEYP), %KEY
-	aesdec	%KEY, %STATE
+	AES192_DEC_ROUNDS(STATE0)
 
 .align 4
 .Ldec128:
 	/ AES 128, 192, and 256
-	movaps	-0x20(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	-0x10(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	0x10(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	0x20(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	0x30(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	0x40(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	0x50(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	0x60(%KEYP), %KEY
-	aesdec	%KEY, %STATE
-	movaps	0x70(%KEYP), %KEY
-	aesdeclast	%KEY, %STATE		/ last round
-	movups	%STATE, (%OUTP)			/ output
+	AES128_DEC_ROUNDS(STATE0)
+	movups	%STATE0, (%OUTP)		/ output
 
-	SET_TS_OR_POP_XMM0_XMM1(%r10)
 	ret
 	SET_SIZE(aes_decrypt_intel)
 
+/* Does a pipelined load of eight input blocks into our AES state registers. */
+#define	AES_LOAD_INPUT_8BLOCKS		\
+	movups	0x00(%INP), %STATE0;	\
+	movups	0x10(%INP), %STATE1;	\
+	movups	0x20(%INP), %STATE2;	\
+	movups	0x30(%INP), %STATE3;	\
+	movups	0x40(%INP), %STATE4;	\
+	movups	0x50(%INP), %STATE5;	\
+	movups	0x60(%INP), %STATE6;	\
+	movups	0x70(%INP), %STATE7;
+
+/* Does a pipelined store of eight AES state registers to the output. */
+#define	AES_STORE_OUTPUT_8BLOCKS	\
+	movups	%STATE0, 0x00(%OUTP);	\
+	movups	%STATE1, 0x10(%OUTP);	\
+	movups	%STATE2, 0x20(%OUTP);	\
+	movups	%STATE3, 0x30(%OUTP);	\
+	movups	%STATE4, 0x40(%OUTP);	\
+	movups	%STATE5, 0x50(%OUTP);	\
+	movups	%STATE6, 0x60(%OUTP);	\
+	movups	%STATE7, 0x70(%OUTP);
+
+/* Performs a pipelined AES instruction with the key on all state registers. */
+#define	AES_KEY_STATE_OP_8BLOCKS(op)	\
+	op	%KEY, %STATE0;		\
+	op	%KEY, %STATE1;		\
+	op	%KEY, %STATE2;		\
+	op	%KEY, %STATE3;		\
+	op	%KEY, %STATE4;		\
+	op	%KEY, %STATE5;		\
+	op	%KEY, %STATE6;		\
+	op	%KEY, %STATE7
+
+/* XOR all AES state regs with key to initiate encryption/decryption. */
+#define	AES_XOR_STATE_8BLOCKS		\
+	AES_KEY_STATE_OP_8BLOCKS(pxor)
+
+/*
+ * Loads a round key from the key schedule offset `off' into the KEY
+ * register and performs `op' using the KEY on all 8 STATE registers.
+ */
+#define	AES_RND_8BLOCKS(op, off)	\
+	movaps	off(%KEYP), %KEY;	\
+	AES_KEY_STATE_OP_8BLOCKS(op)
+
+/*
+ * void aes_encrypt_intel8(const uint32_t roundkeys[], int numrounds,
+ *	const void *plaintext, void *ciphertext)
+ *
+ * Same as aes_encrypt_intel, but performs the encryption operation on
+ * 8 independent blocks in sequence, exploiting instruction pipelining.
+ * This function doesn't support the OpenSSL interface, it's only meant
+ * for kernel use.
+ */
+ENTRY_NP(aes_encrypt_intel8)
+	AES_LOAD_INPUT_8BLOCKS		/ load input
+	movaps	(%KEYP), %KEY		/ key
+	AES_XOR_STATE_8BLOCKS		/ round 0
+
+	lea	0x30(%KEYP), %KEYP	/ point to key schedule
+	cmp	$12, %NROUNDS		/ determine AES variant
+	jb	.Lenc8_128
+	lea	0x20(%KEYP), %KEYP	/ AES192 has larger key schedule
+	je	.Lenc8_192
+
+	lea	0x20(%KEYP), %KEYP	/ AES256 has even larger key schedule
+	AES_RND_8BLOCKS(aesenc, -0x60)	/ AES256 R.1
+	AES_RND_8BLOCKS(aesenc, -0x50)	/ AES256 R.2
+
+.align 4
+.Lenc8_192:
+	AES_RND_8BLOCKS(aesenc, -0x40)	/ AES192 R.1; AES256 R.3
+	AES_RND_8BLOCKS(aesenc, -0x30)	/ AES192 R.2; AES256 R.4
+
+.align 4
+.Lenc8_128:
+	AES_RND_8BLOCKS(aesenc, -0x20)	/ AES128 R.1; AES192 R.3; AES256 R.5
+	AES_RND_8BLOCKS(aesenc, -0x10)	/ AES128 R.2; AES192 R.4; AES256 R.6
+	AES_RND_8BLOCKS(aesenc, 0x00)	/ AES128 R.3; AES192 R.5; AES256 R.7
+	AES_RND_8BLOCKS(aesenc, 0x10)	/ AES128 R.4; AES192 R.6; AES256 R.8
+	AES_RND_8BLOCKS(aesenc, 0x20)	/ AES128 R.5; AES192 R.7; AES256 R.9
+	AES_RND_8BLOCKS(aesenc, 0x30)	/ AES128 R.6; AES192 R.8; AES256 R.10
+	AES_RND_8BLOCKS(aesenc, 0x40)	/ AES128 R.7; AES192 R.9; AES256 R.11
+	AES_RND_8BLOCKS(aesenc, 0x50)	/ AES128 R.8; AES192 R.10; AES256 R.12
+	AES_RND_8BLOCKS(aesenc, 0x60)	/ AES128 R.9; AES192 R.11; AES256 R.13
+	AES_RND_8BLOCKS(aesenclast, 0x70)/ AES128 R.10; AES192 R.12; AES256 R.14
+
+	AES_STORE_OUTPUT_8BLOCKS	/ store output
+	ret
+	SET_SIZE(aes_encrypt_intel8)
+
+
+/*
+ * void aes_decrypt_intel8(const uint32_t roundkeys[], int numrounds,
+ *	const void *ciphertext, void *plaintext)
+ *
+ * Same as aes_decrypt_intel, but performs the decryption operation on
+ * 8 independent blocks in sequence, exploiting instruction pipelining.
+ * This function doesn't support the OpenSSL interface, it's only meant
+ * for kernel use.
+ */
+ENTRY_NP(aes_decrypt_intel8)
+	AES_LOAD_INPUT_8BLOCKS		/ load input
+	movaps	(%KEYP), %KEY		/ key
+	AES_XOR_STATE_8BLOCKS		/ round 0
+
+	lea	0x30(%KEYP), %KEYP	/ point to key schedule
+	cmp	$12, %NROUNDS		/ determine AES variant
+	jb	.Ldec8_128
+	lea	0x20(%KEYP), %KEYP	/ AES192 has larger key schedule
+	je	.Ldec8_192
+
+	lea	0x20(%KEYP), %KEYP	/ AES256 has even larger key schedule
+	AES_RND_8BLOCKS(aesdec, -0x60)	/ AES256 R.1
+	AES_RND_8BLOCKS(aesdec, -0x50)	/ AES256 R.2
+
+.align 4
+.Ldec8_192:
+	AES_RND_8BLOCKS(aesdec, -0x40)	/ AES192 R.1; AES256 R.3
+	AES_RND_8BLOCKS(aesdec, -0x30)	/ AES192 R.2; AES256 R.4
+
+.align 4
+.Ldec8_128:
+	AES_RND_8BLOCKS(aesdec, -0x20)	/ AES128 R.1; AES192 R.3; AES256 R.5
+	AES_RND_8BLOCKS(aesdec, -0x10)	/ AES128 R.2; AES192 R.4; AES256 R.6
+	AES_RND_8BLOCKS(aesdec, 0x00)	/ AES128 R.3; AES192 R.5; AES256 R.7
+	AES_RND_8BLOCKS(aesdec, 0x10)	/ AES128 R.4; AES192 R.6; AES256 R.8
+	AES_RND_8BLOCKS(aesdec, 0x20)	/ AES128 R.5; AES192 R.7; AES256 R.9
+	AES_RND_8BLOCKS(aesdec, 0x30)	/ AES128 R.6; AES192 R.8; AES256 R.10
+	AES_RND_8BLOCKS(aesdec, 0x40)	/ AES128 R.7; AES192 R.9; AES256 R.11
+	AES_RND_8BLOCKS(aesdec, 0x50)	/ AES128 R.8; AES192 R.10; AES256 R.12
+	AES_RND_8BLOCKS(aesdec, 0x60)	/ AES128 R.9; AES192 R.11; AES256 R.13
+	AES_RND_8BLOCKS(aesdeclast, 0x70)/ AES128 R.10; AES192 R.12; AES256 R.14
+
+	AES_STORE_OUTPUT_8BLOCKS	/ store output
+	ret
+	SET_SIZE(aes_decrypt_intel8)
+
+
+/*
+ * This macro encapsulates the entire AES encryption algo for a single
+ * block, which is prefilled in statereg and which will be replaced by
+ * the encrypted output. The KEYP register must already point to the
+ * AES128 key schedule ("lea 0x30(%KEYP), %KEYP" from encryption
+ * function call) so that consecutive invocations of this macro are
+ * supported (KEYP is restored after each invocation).
+ */
+#define	AES_ENC(statereg, label_128, label_192, label_out)	\
+	cmp	$12, %NROUNDS;					\
+	jb	label_128;					\
+	je	label_192;					\
+	/* AES 256 only */					\
+	lea	0x40(%KEYP), %KEYP;				\
+	AES256_ENC_ROUNDS(statereg);				\
+	AES192_ENC_ROUNDS(statereg);				\
+	AES128_ENC_ROUNDS(statereg);				\
+	lea	-0x40(%KEYP), %KEYP;				\
+	jmp	label_out;					\
+.align 4;							\
+label_192:							\
+	lea	0x20(%KEYP), %KEYP;				\
+	/* AES 192 only */					\
+	AES192_ENC_ROUNDS(statereg);				\
+	AES128_ENC_ROUNDS(statereg);				\
+	lea	-0x20(%KEYP), %KEYP;				\
+	jmp	label_out;					\
+.align 4;							\
+label_128:							\
+	/* AES 128 only */					\
+	AES128_ENC_ROUNDS(statereg);				\
+.align 4;							\
+label_out:
+
+
+/*
+ * void aes_encrypt_cbc_intel8(const uint32_t roundkeys[], int numrounds,
+ *	const void *plaintext, void *ciphertext, const void *IV)
+ *
+ * Encrypts 8 consecutive AES blocks in the CBC mode. Input and output
+ * may overlap. This provides a modest performance boost over invoking
+ * the encryption and XOR in separate functions because we can avoid
+ * copying the ciphertext block to and from memory between encryption
+ * and XOR calls.
+ */
+#define	CBC_IV			r8	/* input - IV blk pointer */
+#define	CBC_IV_XMM		xmm1	/* tmp IV location for alignment */
+
+ENTRY_NP(aes_encrypt_cbc_intel8)
+	AES_LOAD_INPUT_8BLOCKS		/ load input
+	movaps	(%KEYP), %KEY		/ key
+	AES_XOR_STATE_8BLOCKS		/ round 0
+
+	lea	0x30(%KEYP), %KEYP	/ point to key schedule
+	movdqu	(%CBC_IV), %CBC_IV_XMM	/ load IV from unaligned memory
+	pxor	%CBC_IV_XMM, %STATE0	/ XOR IV with input block and encrypt
+	AES_ENC(STATE0, .Lenc_cbc_0_128, .Lenc_cbc_0_192, .Lenc_cbc_0_out)
+	pxor	%STATE0, %STATE1
+	AES_ENC(STATE1, .Lenc_cbc_1_128, .Lenc_cbc_1_192, .Lenc_cbc_1_out)
+	pxor	%STATE1, %STATE2
+	AES_ENC(STATE2, .Lenc_cbc_2_128, .Lenc_cbc_2_192, .Lenc_cbc_2_out)
+	pxor	%STATE2, %STATE3
+	AES_ENC(STATE3, .Lenc_cbc_3_128, .Lenc_cbc_3_192, .Lenc_cbc_3_out)
+	pxor	%STATE3, %STATE4
+	AES_ENC(STATE4, .Lenc_cbc_4_128, .Lenc_cbc_4_192, .Lenc_cbc_4_out)
+	pxor	%STATE4, %STATE5
+	AES_ENC(STATE5, .Lenc_cbc_5_128, .Lenc_cbc_5_192, .Lenc_cbc_5_out)
+	pxor	%STATE5, %STATE6
+	AES_ENC(STATE6, .Lenc_cbc_6_128, .Lenc_cbc_6_192, .Lenc_cbc_6_out)
+	pxor	%STATE6, %STATE7
+	AES_ENC(STATE7, .Lenc_cbc_7_128, .Lenc_cbc_7_192, .Lenc_cbc_7_out)
+
+	AES_STORE_OUTPUT_8BLOCKS	/ store output
+	ret
+	SET_SIZE(aes_encrypt_cbc_intel8)
+
+/*
+ * Prefills register state with counters suitable for the CTR encryption
+ * mode. The counter is assumed to consist of two portions:
+ * - A lower monotonically increasing 64-bit counter. If the caller wants
+ *   a smaller counter, they are responsible for checking that it doesn't
+ *   overflow between encryption calls.
+ * - An upper static "nonce" portion, in big endian, preloaded into the
+ *   lower portion of an XMM register.
+ * This macro adds `ctridx' to the lower_LE counter, swaps it to big
+ * endian and by way of a temporary general-purpose register loads the
+ * lower and upper counter portions into a target XMM result register,
+ * which can then be handed off to the encryption process.
+ */
+#define	PREP_CTR_BLOCKS(lower_LE, upper_BE_xmm, ctridx, tmpreg, resreg)	\
+	lea	ctridx(%lower_LE), %tmpreg;				\
+	bswap	%tmpreg;						\
+	movq	%tmpreg, %resreg;					\
+	movlhps	%upper_BE_xmm, %resreg;					\
+	pshufd	$0b01001110, %resreg, %resreg
+
+#define	CTR_UPPER_BE		r8	/* input - counter upper 64 bits (BE) */
+#define	CTR_UPPER_BE_XMM	xmm1	/* tmp for upper counter bits */
+#define	CTR_LOWER_LE		r9	/* input - counter lower 64 bits (LE) */
+#define	CTR_TMP0		rax	/* tmp for lower 64 bit add & bswap */
+#define	CTR_TMP1		rbx	/* tmp for lower 64 bit add & bswap */
+#define	CTR_TMP2		r10	/* tmp for lower 64 bit add & bswap */
+#define	CTR_TMP3		r11	/* tmp for lower 64 bit add & bswap */
+#define	CTR_TMP4		r12	/* tmp for lower 64 bit add & bswap */
+#define	CTR_TMP5		r13	/* tmp for lower 64 bit add & bswap */
+#define	CTR_TMP6		r14	/* tmp for lower 64 bit add & bswap */
+#define	CTR_TMP7		r15	/* tmp for lower 64 bit add & bswap */
+
+/*
+ * These are used in case CTR encryption input is unaligned before XORing.
+ * Must not overlap with any STATE[0-7] register.
+ */
+#define	TMP_INPUT0	xmm0
+#define	TMP_INPUT1	xmm1
+#define	TMP_INPUT2	xmm2
+#define	TMP_INPUT3	xmm3
+#define	TMP_INPUT4	xmm4
+#define	TMP_INPUT5	xmm5
+#define	TMP_INPUT6	xmm6
+#define	TMP_INPUT7	xmm7
+
+/*
+ * void aes_ctr_intel8(const uint32_t roundkeys[], int numrounds,
+ *	const void *input, void *output, uint64_t counter_upper_BE,
+ *	uint64_t counter_lower_LE)
+ *
+ * Runs AES on 8 consecutive blocks in counter mode (encryption and
+ * decryption in counter mode are the same).
+ */
+ENTRY_NP(aes_ctr_intel8)
+	/* save caller's regs */
+	pushq	%rbp
+	movq	%rsp, %rbp
+	subq	$0x38, %rsp
+	/ CTR_TMP0 is rax, no need to save
+	movq	%CTR_TMP1, -0x38(%rbp)
+	movq	%CTR_TMP2, -0x30(%rbp)
+	movq	%CTR_TMP3, -0x28(%rbp)
+	movq	%CTR_TMP4, -0x20(%rbp)
+	movq	%CTR_TMP5, -0x18(%rbp)
+	movq	%CTR_TMP6, -0x10(%rbp)
+	movq	%CTR_TMP7, -0x08(%rbp)
+
+	/*
+	 * CTR step 1: prepare big-endian formatted 128-bit counter values,
+	 * placing the result in the AES-NI input state registers.
+	 */
+	movq	%CTR_UPPER_BE, %CTR_UPPER_BE_XMM
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 0, CTR_TMP0, STATE0)
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 1, CTR_TMP1, STATE1)
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 2, CTR_TMP2, STATE2)
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 3, CTR_TMP3, STATE3)
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 4, CTR_TMP4, STATE4)
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 5, CTR_TMP5, STATE5)
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 6, CTR_TMP6, STATE6)
+	PREP_CTR_BLOCKS(CTR_LOWER_LE, CTR_UPPER_BE_XMM, 7, CTR_TMP7, STATE7)
+
+	/*
+	 * CTR step 2: Encrypt the counters.
+	 */
+	movaps	(%KEYP), %KEY		/ key
+	AES_XOR_STATE_8BLOCKS		/ round 0
+
+	/* Determine the AES variant we're going to compute */
+	lea	0x30(%KEYP), %KEYP	/ point to key schedule
+	cmp	$12, %NROUNDS		/ determine AES variant
+	jb	.Lctr8_128
+	lea	0x20(%KEYP), %KEYP	/ AES192 has larger key schedule
+	je	.Lctr8_192
+
+	/* AES 256 */
+	lea	0x20(%KEYP), %KEYP	/ AES256 has even larger key schedule
+	AES_RND_8BLOCKS(aesenc, -0x60)	/ AES256 R.1
+	AES_RND_8BLOCKS(aesenc, -0x50)	/ AES256 R.2
+
+.align 4
+.Lctr8_192:
+	/* AES 192 and 256 */
+	AES_RND_8BLOCKS(aesenc, -0x40)	/ AES192 R.1; AES256 R.3
+	AES_RND_8BLOCKS(aesenc, -0x30)	/ AES192 R.2; AES256 R.4
+
+.align 4
+.Lctr8_128:
+	/* AES 128, 192, and 256 */
+	AES_RND_8BLOCKS(aesenc, -0x20)	/ AES128 R.1; AES192 R.3; AES256 R.5
+	AES_RND_8BLOCKS(aesenc, -0x10)	/ AES128 R.2; AES192 R.4; AES256 R.6
+	AES_RND_8BLOCKS(aesenc, 0x00)	/ AES128 R.3; AES192 R.5; AES256 R.7
+	AES_RND_8BLOCKS(aesenc, 0x10)	/ AES128 R.4; AES192 R.6; AES256 R.8
+	AES_RND_8BLOCKS(aesenc, 0x20)	/ AES128 R.5; AES192 R.7; AES256 R.9
+	AES_RND_8BLOCKS(aesenc, 0x30)	/ AES128 R.6; AES192 R.8; AES256 R.10
+	AES_RND_8BLOCKS(aesenc, 0x40)	/ AES128 R.7; AES192 R.9; AES256 R.11
+	AES_RND_8BLOCKS(aesenc, 0x50)	/ AES128 R.8; AES192 R.10; AES256 R.12
+	AES_RND_8BLOCKS(aesenc, 0x60)	/ AES128 R.9; AES192 R.11; AES256 R.13
+	AES_RND_8BLOCKS(aesenclast, 0x70)/ AES128 R.10; AES192 R.12; AES256 R.14
+
+	/*
+	 * CTR step 3: XOR input data blocks with encrypted counters to
+	 * produce result.
+	 */
+	mov	%INP, %rax		/ pxor requires alignment, so check
+	andq	$0xf, %rax
+	jnz	.Lctr_input_unaligned
+	pxor	0x00(%INP), %STATE0
+	pxor	0x10(%INP), %STATE1
+	pxor	0x20(%INP), %STATE2
+	pxor	0x30(%INP), %STATE3
+	pxor	0x40(%INP), %STATE4
+	pxor	0x50(%INP), %STATE5
+	pxor	0x60(%INP), %STATE6
+	pxor	0x70(%INP), %STATE7
+	jmp	.Lctr_out
+
+.align 4
+.Lctr_input_unaligned:
+	movdqu	0x00(%INP), %TMP_INPUT0
+	movdqu	0x10(%INP), %TMP_INPUT1
+	movdqu	0x20(%INP), %TMP_INPUT2
+	movdqu	0x30(%INP), %TMP_INPUT3
+	movdqu	0x40(%INP), %TMP_INPUT4
+	movdqu	0x50(%INP), %TMP_INPUT5
+	movdqu	0x60(%INP), %TMP_INPUT6
+	movdqu	0x70(%INP), %TMP_INPUT7
+	pxor	%TMP_INPUT0, %STATE0
+	pxor	%TMP_INPUT1, %STATE1
+	pxor	%TMP_INPUT2, %STATE2
+	pxor	%TMP_INPUT3, %STATE3
+	pxor	%TMP_INPUT4, %STATE4
+	pxor	%TMP_INPUT5, %STATE5
+	pxor	%TMP_INPUT6, %STATE6
+	pxor	%TMP_INPUT7, %STATE7
+
+.align 4
+.Lctr_out:
+	/*
+	 * Step 4: Write out processed blocks to memory.
+	 */
+	movdqu	%STATE0, 0x00(%OUTP)
+	movdqu	%STATE1, 0x10(%OUTP)
+	movdqu	%STATE2, 0x20(%OUTP)
+	movdqu	%STATE3, 0x30(%OUTP)
+	movdqu	%STATE4, 0x40(%OUTP)
+	movdqu	%STATE5, 0x50(%OUTP)
+	movdqu	%STATE6, 0x60(%OUTP)
+	movdqu	%STATE7, 0x70(%OUTP)
+
+	/* restore caller's regs */
+	/ CTR_TMP0 is rax, no need to restore
+	movq	-0x38(%rbp), %CTR_TMP1
+	movq	-0x30(%rbp), %CTR_TMP2
+	movq	-0x28(%rbp), %CTR_TMP3
+	movq	-0x20(%rbp), %CTR_TMP4
+	movq	-0x18(%rbp), %CTR_TMP5
+	movq	-0x10(%rbp), %CTR_TMP6
+	movq	-0x08(%rbp), %CTR_TMP7
+	leave
+	ret
+	SET_SIZE(aes_ctr_intel8)
+
 #endif	/* lint || __lint */
--- old/usr/src/common/crypto/blowfish/blowfish_impl.c	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/blowfish/blowfish_impl.c	Thu Apr 30 20:52:30 2015
@@ -22,6 +22,9 @@
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 /*
  * Blowfish encryption/decryption and keyschedule code.
@@ -699,7 +702,7 @@
 }
 
 void
-blowfish_copy_block(uint8_t *in, uint8_t *out)
+blowfish_copy_block(const uint8_t *in, uint8_t *out)
 {
 	if (IS_P2ALIGNED(in, sizeof (uint32_t)) &&
 	    IS_P2ALIGNED(out, sizeof (uint32_t))) {
@@ -714,7 +717,7 @@
 
 /* XOR block of data into dest */
 void
-blowfish_xor_block(uint8_t *data, uint8_t *dst)
+blowfish_xor_block(const uint8_t *data, uint8_t *dst)
 {
 	if (IS_P2ALIGNED(dst, sizeof (uint32_t)) &&
 	    IS_P2ALIGNED(data, sizeof (uint32_t))) {
@@ -740,10 +743,10 @@
 	if (blowfish_ctx->bc_flags & CBC_MODE) {
 		rv = cbc_encrypt_contiguous_blocks(ctx, data, length, out,
 		    BLOWFISH_BLOCK_LEN, blowfish_encrypt_block,
-		    blowfish_copy_block, blowfish_xor_block);
+		    blowfish_copy_block, blowfish_xor_block, NULL);
 	} else {
 		rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
-		    BLOWFISH_BLOCK_LEN, blowfish_encrypt_block);
+		    BLOWFISH_BLOCK_LEN, blowfish_encrypt_block, NULL);
 	}
 	return (rv);
 }
@@ -761,10 +764,10 @@
 	if (blowfish_ctx->bc_flags & CBC_MODE) {
 		rv = cbc_decrypt_contiguous_blocks(ctx, data, length, out,
 		    BLOWFISH_BLOCK_LEN, blowfish_decrypt_block,
-		    blowfish_copy_block, blowfish_xor_block);
+		    blowfish_copy_block, blowfish_xor_block, NULL, NULL);
 	} else {
 		rv = ecb_cipher_contiguous_blocks(ctx, data, length, out,
-		    BLOWFISH_BLOCK_LEN, blowfish_decrypt_block);
+		    BLOWFISH_BLOCK_LEN, blowfish_decrypt_block, NULL);
 		if (rv == CRYPTO_DATA_LEN_RANGE)
 			rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
 	}
--- old/usr/src/common/crypto/blowfish/blowfish_impl.h	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/blowfish/blowfish_impl.h	Thu Apr 30 20:52:30 2015
@@ -22,12 +22,13 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef	_BLOWFISH_IMPL_H
 #define	_BLOWFISH_IMPL_H
 
-#pragma ident	"%Z%%M%	%I%	%E% SMI"
-
 /*
  * Common definitions used by Blowfish.
  */
@@ -74,8 +75,8 @@
 extern int blowfish_decrypt_block(const void *, const uint8_t *, uint8_t *);
 extern void blowfish_init_keysched(uint8_t *, uint_t, void *);
 extern void *blowfish_alloc_keysched(size_t *, int);
-extern void blowfish_copy_block(uint8_t *, uint8_t *);
-extern void blowfish_xor_block(uint8_t *, uint8_t *);
+extern void blowfish_copy_block(const uint8_t *, uint8_t *);
+extern void blowfish_xor_block(const uint8_t *, uint8_t *);
 #ifdef	__cplusplus
 }
 #endif
--- old/usr/src/common/crypto/des/des_impl.c	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/des/des_impl.c	Thu Apr 30 20:52:30 2015
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #include <sys/types.h>
 #include <sys/systm.h>
@@ -986,7 +989,7 @@
 }
 
 void
-des_copy_block(uint8_t *in, uint8_t *out)
+des_copy_block(const uint8_t *in, uint8_t *out)
 {
 	if (IS_P2ALIGNED(in, sizeof (uint32_t)) &&
 	    IS_P2ALIGNED(out, sizeof (uint32_t))) {
@@ -1001,7 +1004,7 @@
 
 /* XOR block of data into dest */
 void
-des_xor_block(uint8_t *data, uint8_t *dst)
+des_xor_block(const uint8_t *data, uint8_t *dst)
 {
 	if (IS_P2ALIGNED(dst, sizeof (uint32_t)) &&
 	    IS_P2ALIGNED(data, sizeof (uint32_t))) {
@@ -1056,19 +1059,19 @@
 		if (des_ctx->dc_flags & CBC_MODE) {
 			rv = cbc_encrypt_contiguous_blocks(ctx, data,
 			    length, out, DES_BLOCK_LEN, des3_encrypt_block,
-			    des_copy_block, des_xor_block);
+			    des_copy_block, des_xor_block, NULL);
 		} else {
 			rv = ecb_cipher_contiguous_blocks(ctx, data, length,
-			    out, DES_BLOCK_LEN, des3_encrypt_block);
+			    out, DES_BLOCK_LEN, des3_encrypt_block, NULL);
 		}
 	} else {
 		if (des_ctx->dc_flags & CBC_MODE) {
 			rv = cbc_encrypt_contiguous_blocks(ctx, data,
 			    length, out, DES_BLOCK_LEN, des_encrypt_block,
-			    des_copy_block, des_xor_block);
+			    des_copy_block, des_xor_block, NULL);
 		} else {
 			rv = ecb_cipher_contiguous_blocks(ctx, data, length,
-			    out, DES_BLOCK_LEN, des_encrypt_block);
+			    out, DES_BLOCK_LEN, des_encrypt_block, NULL);
 		}
 	}
 	return (rv);
@@ -1088,10 +1091,10 @@
 		if (des_ctx->dc_flags & CBC_MODE) {
 			rv = cbc_decrypt_contiguous_blocks(ctx, data,
 			    length, out, DES_BLOCK_LEN, des3_decrypt_block,
-			    des_copy_block, des_xor_block);
+			    des_copy_block, des_xor_block, NULL, NULL);
 		} else {
 			rv = ecb_cipher_contiguous_blocks(ctx, data, length,
-			    out, DES_BLOCK_LEN, des3_decrypt_block);
+			    out, DES_BLOCK_LEN, des3_decrypt_block, NULL);
 			if (rv == CRYPTO_DATA_LEN_RANGE)
 				rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
 		}
@@ -1099,10 +1102,10 @@
 		if (des_ctx->dc_flags & CBC_MODE) {
 			rv = cbc_decrypt_contiguous_blocks(ctx, data,
 			    length, out, DES_BLOCK_LEN, des_decrypt_block,
-			    des_copy_block, des_xor_block);
+			    des_copy_block, des_xor_block, NULL, NULL);
 		} else {
 			rv = ecb_cipher_contiguous_blocks(ctx, data, length,
-			    out, DES_BLOCK_LEN, des_decrypt_block);
+			    out, DES_BLOCK_LEN, des_decrypt_block, NULL);
 			if (rv == CRYPTO_DATA_LEN_RANGE)
 				rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE;
 		}
--- old/usr/src/common/crypto/des/des_impl.h	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/des/des_impl.h	Thu Apr 30 20:52:30 2015
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef	_DES_IMPL_H
 #define	_DES_IMPL_H
@@ -97,8 +100,8 @@
 extern void *des_alloc_keysched(size_t *, des_strength_t, int);
 extern boolean_t des_keycheck(uint8_t *, des_strength_t, uint8_t *);
 extern void des_parity_fix(uint8_t *, des_strength_t, uint8_t *);
-extern void des_copy_block(uint8_t *, uint8_t *);
-extern void des_xor_block(uint8_t *, uint8_t *);
+extern void des_copy_block(const uint8_t *, uint8_t *);
+extern void des_xor_block(const uint8_t *, uint8_t *);
 extern int des_encrypt_block(const void *, const uint8_t *, uint8_t *);
 extern int des3_encrypt_block(const void *, const uint8_t *, uint8_t *);
 extern int des_decrypt_block(const void *, const uint8_t *, uint8_t *);
--- old/usr/src/common/crypto/modes/amd64/gcm_intel.s	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/modes/amd64/gcm_intel.s	Thu Apr 30 20:52:30 2015
@@ -27,6 +27,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 /*
  * Accelerated GHASH implementation with Intel PCLMULQDQ-NI
@@ -90,6 +93,20 @@
 gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res) {
 }
 
+#ifdef	_KERNEL
+/*ARGSUSED*/
+void
+gcm_intel_save(void *savestate)
+{
+}
+
+/*ARGSUSED*/
+void
+gcm_accel_restore(void *savestate)
+{
+}
+#endif	/* _KERNEL */
+
 #else	/* lint */
 
 #include <sys/asm_linkage.h>
@@ -116,81 +133,90 @@
 #define	PROTECTED_CLTS \
 	CLTS
 #endif	/* __xpv */
-
-	/*
-	 * If CR0_TS is not set, align stack (with push %rbp) and push
-	 * %xmm0 - %xmm10 on stack, otherwise clear CR0_TS
-	 */
-#define	CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg) \
-	push	%rbp; \
-	mov	%rsp, %rbp; \
-	movq	%cr0, tmpreg; \
-	testq	$CR0_TS, tmpreg; \
-	jnz	1f; \
-	and	$-XMM_ALIGN, %rsp; \
-	sub	$[XMM_SIZE * 11], %rsp; \
-	movaps	%xmm0, 160(%rsp); \
-	movaps	%xmm1, 144(%rsp); \
-	movaps	%xmm2, 128(%rsp); \
-	movaps	%xmm3, 112(%rsp); \
-	movaps	%xmm4, 96(%rsp); \
-	movaps	%xmm5, 80(%rsp); \
-	movaps	%xmm6, 64(%rsp); \
-	movaps	%xmm7, 48(%rsp); \
-	movaps	%xmm8, 32(%rsp); \
-	movaps	%xmm9, 16(%rsp); \
-	movaps	%xmm10, (%rsp); \
-	jmp	2f; \
-1: \
-	PROTECTED_CLTS; \
-2:
-
-
-	/*
-	 * If CR0_TS was not set above, pop %xmm0 - %xmm10 off stack,
-	 * otherwise set CR0_TS.
-	 */
-#define	SET_TS_OR_POP_XMM_REGISTERS(tmpreg) \
-	testq	$CR0_TS, tmpreg; \
-	jnz	1f; \
-	movaps	(%rsp), %xmm10; \
-	movaps	16(%rsp), %xmm9; \
-	movaps	32(%rsp), %xmm8; \
-	movaps	48(%rsp), %xmm7; \
-	movaps	64(%rsp), %xmm6; \
-	movaps	80(%rsp), %xmm5; \
-	movaps	96(%rsp), %xmm4; \
-	movaps	112(%rsp), %xmm3; \
-	movaps	128(%rsp), %xmm2; \
-	movaps	144(%rsp), %xmm1; \
-	movaps	160(%rsp), %xmm0; \
-	jmp	2f; \
-1: \
-	STTS(tmpreg); \
-2: \
-	mov	%rbp, %rsp; \
-	pop	%rbp
-
-
-#else
-#define	PROTECTED_CLTS
-#define	CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg)
-#define	SET_TS_OR_POP_XMM_REGISTERS(tmpreg)
 #endif	/* _KERNEL */
 
-/*
- * Use this mask to byte-swap a 16-byte integer with the pshufb instruction
- */
-
-// static uint8_t byte_swap16_mask[] = {
-//	 15, 14, 13, 12, 11, 10, 9, 8, 7, 6 ,5, 4, 3, 2, 1, 0 };
 .text
 .align XMM_ALIGN
+/*
+ * Use this mask to byte-swap a 16-byte integer with the pshufb instruction:
+ * static uint8_t byte_swap16_mask[] = {
+ *	15, 14, 13, 12, 11, 10, 9, 8, 7, 6 ,5, 4, 3, 2, 1, 0 };
+ */
 .Lbyte_swap16_mask:
 	.byte	15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
 
+#ifdef	_KERNEL
+/*
+ * void gcm_intel_save(void *savestate)
+ *
+ * Saves the XMM0--XMM14 registers and CR0 to a temporary location pointed
+ * to in the first argument and clears TS in CR0. This must be invoked before
+ * executing accelerated GCM computations inside the kernel (and kernel
+ * thread preemption must be disabled as well). The memory region to which
+ * all state is saved must be at least 16x 128-bit + 64-bit long and must
+ * be 128-bit aligned.
+ */
+ENTRY_NP(gcm_accel_save)
+	movq	%cr0, %rax
+	movq	%rax, 0x100(%rdi)
+	testq	$CR0_TS, %rax
+	jnz	1f
+	/* FPU is in use, save registers */
+	movaps	%xmm0, 0x00(%rdi)
+	movaps	%xmm1, 0x10(%rdi)
+	movaps	%xmm2, 0x20(%rdi)
+	movaps	%xmm3, 0x30(%rdi)
+	movaps	%xmm4, 0x40(%rdi)
+	movaps	%xmm5, 0x50(%rdi)
+	movaps	%xmm6, 0x60(%rdi)
+	movaps	%xmm7, 0x70(%rdi)
+	movaps	%xmm8, 0x80(%rdi)
+	movaps	%xmm9, 0x90(%rdi)
+	movaps	%xmm10, 0xa0(%rdi)
+	movaps	%xmm11, 0xb0(%rdi)
+	movaps	%xmm12, 0xc0(%rdi)
+	movaps	%xmm13, 0xd0(%rdi)
+	movaps	%xmm14, 0xe0(%rdi)
+	movaps	%xmm15, 0xf0(%rdi)
+	ret
+1:
+	PROTECTED_CLTS
+	ret
+	SET_SIZE(gcm_accel_save)
 
+/*
+ * void gcm_accel_restore(void *savestate)
+ *
+ * Restores the saved XMM and CR0.TS state from aes_accel_save.
+ */
+ENTRY_NP(gcm_accel_restore)
+	movq	0x100(%rdi), %rax
+	testq	$CR0_TS, %rax
+	jnz	1f
+	movaps	0x00(%rdi), %xmm0
+	movaps	0x10(%rdi), %xmm1
+	movaps	0x20(%rdi), %xmm2
+	movaps	0x30(%rdi), %xmm3
+	movaps	0x40(%rdi), %xmm4
+	movaps	0x50(%rdi), %xmm5
+	movaps	0x60(%rdi), %xmm6
+	movaps	0x70(%rdi), %xmm7
+	movaps	0x80(%rdi), %xmm8
+	movaps	0x90(%rdi), %xmm9
+	movaps	0xa0(%rdi), %xmm10
+	movaps	0xb0(%rdi), %xmm11
+	movaps	0xc0(%rdi), %xmm12
+	movaps	0xd0(%rdi), %xmm13
+	movaps	0xe0(%rdi), %xmm14
+	movaps	0xf0(%rdi), %xmm15
+	ret
+1:
+	STTS(%rax)
+	ret
+	SET_SIZE(gcm_accel_restore)
 
+#endif	/* _KERNEL */
+
 /*
  * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
  *
@@ -199,14 +225,11 @@
  *
  * Byte swap the input and the output.
  *
- * Note: x_in, y, and res all point to a block of 20-byte numbers
+ * Note: x_in, y, and res all point to a block of 16-byte numbers
  * (an array of two 64-bit integers).
  *
- * Note2: For kernel code, caller is responsible for ensuring
- * kpreempt_disable() has been called.  This is because %xmm registers are
- * not saved/restored.  Clear and set the CR0.TS bit on entry and exit,
- * respectively, if TS is set on entry.  Otherwise, if TS is not set,
- * save and restore %xmm registers on the stack.
+ * Note2: For kernel code, caller is responsible for bracketing this call with
+ * disabling kernel thread preemption and calling gcm_accel_save/restore().
  *
  * Note3: Original Intel definition:
  * void galois_hash_asm(unsigned char *hk, unsigned char *s,
@@ -224,8 +247,6 @@
  */
 
 ENTRY_NP(gcm_mul_pclmulqdq)
-	CLEAR_TS_OR_PUSH_XMM_REGISTERS(%r10)
-
 	//
 	// Copy Parameters
 	//
@@ -330,7 +351,6 @@
 	//
 	// Cleanup and Return
 	//
-	SET_TS_OR_POP_XMM_REGISTERS(%r10)
 	ret
 	SET_SIZE(gcm_mul_pclmulqdq)
 
--- /dev/null	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/modes/amd64/gcm_intel_cryptogams.s	Thu Apr 30 20:52:30 2015
@@ -0,0 +1,512 @@
+/*
+ * Copyright (c) 2013, CRYPTOGAMS by <appro@openssl.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * *	Redistributions of source code must retain copyright notices,
+ *	this list of conditions and the following disclaimer.
+ *
+ * *	Redistributions in binary form must reproduce the above
+ *	copyright notice, this list of conditions and the following
+ *	disclaimer in the documentation and/or other materials
+ *	provided with the distribution.
+ *
+ * *	Neither the name of the CRYPTOGAMS nor the names of its
+ *	copyright holder and contributors may be used to endorse or
+ *	promote products derived from this software without specific
+ *	prior written permission.
+ *
+ * ALTERNATIVELY, provided that this notice is retained in full, this
+ * product may be distributed under the terms of the GNU General Public
+ * License (GPL), in which case the provisions of the GPL apply INSTEAD OF
+ * those given above.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * Copyright 2015 by Saso Kiselkov on Illumos port sections.
+ */
+
+#if defined(lint) || defined(__lint)
+
+#include <sys/types.h>
+
+/*ARGSUSED*/
+void
+gcm_ghash_clmul(uint64_t ghash[2], const uint8_t Htable[256],
+    const uint8_t *inp, size_t len)
+{
+}
+
+/*ARGSUSED*/
+void
+gcm_init_clmul(const uint64_t hash_init[2], uint8_t Htable[256])
+{
+}
+
+#else	/* lint */
+
+#include <sys/asm_linkage.h>
+#include <sys/controlregs.h>
+#ifdef _KERNEL
+#include <sys/machprivregs.h>
+#endif
+
+.text
+.align XMM_ALIGN
+.Lbyte_swap16_mask:
+	.byte	15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
+.L0x1c2_polynomial:
+	.byte	1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xc2
+.L7_mask:
+	.long	7, 0, 7, 0
+
+#define	Xi	xmm0	/* hash value */
+#define	Xhi	xmm1	/* hash value high order 64 bits */
+#define	Hkey	xmm2	/* hash key */
+#define	T1	xmm3	/* temp1 */
+#define	T2	xmm4	/* temp2 */
+#define	T3	xmm5	/* temp3 */
+#define	Xb0	xmm6	/* cipher block #0 */
+#define	Xb1	xmm7	/* cipher block #1 */
+#define	Xb2	xmm8	/* cipher block #2 */
+#define	Xb3	xmm9	/* cipher block #3 */
+#define	Xb4	xmm10	/* cipher block #4 */
+#define	Xb5	xmm11	/* cipher block #5 */
+#define	Xb6	xmm12	/* cipher block #6 */
+#define	Xb7	xmm13	/* cipher block #7 */
+
+#define	clmul64x64_T2(tmpreg)				\
+	movdqa		%Xi, %Xhi;			\
+	pshufd		$0b01001110, %Xi, %T1;		\
+	pxor		%Xi, %T1;			\
+							\
+	pclmulqdq	$0x00, %Hkey, %Xi;		\
+	pclmulqdq	$0x11, %Hkey, %Xhi;		\
+	pclmulqdq	$0x00, %tmpreg, %T1;		\
+	pxor		%Xi, %T1;			\
+	pxor		%Xhi, %T1;			\
+							\
+	movdqa		%T1, %T2;			\
+	psrldq		$8, %T1;			\
+	pslldq		$8, %T2;			\
+	pxor		%T1, %Xhi;			\
+	pxor		%T2, %Xi
+
+#define	reduction_alg9					\
+	/* 1st phase */					\
+	movdqa		%Xi, %T2;			\
+	movdqa		%Xi, %T1;			\
+	psllq		$5, %Xi;			\
+	pxor		%Xi, %T1;			\
+	psllq		$1, %Xi;			\
+	pxor		%T1, %Xi;			\
+	psllq		$57, %Xi;			\
+	movdqa		%Xi, %T1;			\
+	pslldq		$8, %Xi;			\
+	psrldq		$8, %T1;			\
+	pxor		%T2, %Xi;			\
+	pxor		%T1, %Xhi;			\
+	/* 2nd phase */					\
+	movdqa		%Xi, %T2;			\
+	psrlq		$1, %Xi;			\
+	pxor		%T2, %Xhi;			\
+	pxor		%Xi, %T2;			\
+	psrlq		$5, %Xi;			\
+	pxor		%T2, %Xi;			\
+	psrlq		$1, %Xi;			\
+	pxor		%Xhi, %Xi
+
+#define	Xip	rdi
+#define	Htbl	rsi
+#define	inp	rdx
+#define	len	rcx
+
+#define	Xln	xmm6
+#define	Xmn	xmm7
+#define	Xhn	xmm8
+#define	Hkey2	xmm9
+#define	HK	xmm10
+#define	Xl	xmm11
+#define	Xm	xmm12
+#define	Xh	xmm13
+#define	Hkey3	xmm14
+#define	Hkey4	xmm15
+
+/*
+ * void gcm_ghash_clmul(uint64_t Xi[2], const uint64_t Htable[32],
+ *	const uint8_t *inp, size_t len)
+ */
+ENTRY_NP(gcm_ghash_clmul)
+	movdqa		.Lbyte_swap16_mask(%rip), %T3
+	mov		$0xA040608020C0E000, %rax	/ ((7..0) ? 0xE0) & 0xff
+
+	movdqu		(%Xip), %Xi
+	movdqu		(%Htbl), %Hkey
+	movdqu		0x20(%Htbl), %HK
+	pshufb		%T3, %Xi
+
+	sub		$0x10, %len
+	jz		.Lodd_tail
+
+	movdqu		0x10(%Htbl), %Hkey2
+
+	cmp		$0x30, %len
+	jb		.Lskip4x
+
+	sub		$0x30, %len
+	movdqu		0x30(%Htbl),%Hkey3
+	movdqu		0x40(%Htbl),%Hkey4
+
+	/* Xi+4 =[(H*Ii+3) + (H^2*Ii+2) + (H^3*Ii+1) + H^4*(Ii+Xi)] mod P */
+	movdqu		0x30(%inp), %Xln
+	movdqu		0x20(%inp), %Xl
+	pshufb		%T3, %Xln
+	pshufb		%T3, %Xl
+	movdqa		%Xln, %Xhn
+	pshufd		$0b01001110, %Xln, %Xmn
+	pxor		%Xln, %Xmn
+	pclmulqdq	$0x00, %Hkey, %Xln
+	pclmulqdq	$0x11, %Hkey, %Xhn
+	pclmulqdq	$0x00, %HK, %Xmn
+
+	movdqa		%Xl, %Xh
+	pshufd		$0b01001110, %Xl, %Xm
+	pxor		%Xl, %Xm
+	pclmulqdq	$0x00, %Hkey2, %Xl
+	pclmulqdq	$0x11, %Hkey2, %Xh
+	xorps		%Xl, %Xln
+	pclmulqdq	$0x10, %HK, %Xm
+	xorps		%Xh, %Xhn
+	movups		0x50(%Htbl), %HK
+	xorps		%Xm, %Xmn
+
+	movdqu		0x10(%inp), %Xl
+	movdqu		0x00(%inp), %T1
+	pshufb		%T3, %Xl
+	pshufb		%T3, %T1
+	movdqa		%Xl, %Xh
+	pshufd		$0b01001110, %Xl, %Xm
+	pxor		%T1, %Xi
+	pxor		%Xl, %Xm
+	pclmulqdq	$0x00, %Hkey3, %Xl
+	movdqa		%Xi, %Xhi
+	pshufd		$0b01001110, %Xi, %T1
+	pxor		%Xi, %T1
+	pclmulqdq	$0x11, %Hkey3, %Xh
+	xorps		%Xl, %Xln
+	pclmulqdq	$0x00, %HK, %Xm
+	xorps		%Xh, %Xhn
+
+	lea		0x40(%inp), %inp
+	sub		$0x40, %len
+	jc		.Ltail4x
+
+	jmp		.Lmod4_loop
+
+.align	32
+.Lmod4_loop:
+	pclmulqdq	$0x00, %Hkey4, %Xi
+	xorps		%Xm, %Xmn
+	movdqu		0x30(%inp), %Xl
+	pshufb		%T3, %Xl
+	pclmulqdq	$0x11, %Hkey4, %Xhi
+	xorps		%Xln, %Xi
+	movdqu		0x20(%inp), %Xln
+	movdqa		%Xl, %Xh
+	pshufd		$0b01001110, %Xl, %Xm
+	pclmulqdq	$0x10, %HK, %T1
+	xorps		%Xhn, %Xhi
+	pxor		%Xl, %Xm
+	pshufb		%T3, %Xln
+	movups		0x20(%Htbl), %HK
+	pclmulqdq	$0x00, %Hkey, %Xl
+	xorps		%Xmn, %T1
+	movdqa		%Xln, %Xhn
+	pshufd		$0b01001110, %Xln, %Xmn
+
+	pxor		%Xi, %T1	/ aggregated Karatsuba post-processing
+	pxor		%Xln, %Xmn
+	pxor		%Xhi, %T1
+	movdqa		%T1, %T2
+	pslldq		$8, %T1
+	pclmulqdq	$0x11, %Hkey, %Xh
+	psrldq		$8, %T2
+	pxor		%T1, %Xi
+	movdqa		.L7_mask(%rip), %T1
+	pxor		%T2, %Xhi
+	movq		%rax, %T2
+
+	pand		%Xi, %T1	/ 1st phase
+	pshufb		%T1, %T2
+	pclmulqdq	$0x00, %HK, %Xm
+	pxor		%Xi, %T2
+	psllq		$57, %T2
+	movdqa		%T2, %T1
+	pslldq		$8, %T2
+	pclmulqdq	$0x00, %Hkey2, %Xln
+	psrldq		$8, %T1
+	pxor		%T2, %Xi
+	pxor		%T1, %Xhi
+	movdqu		0(%inp), %T1
+
+	movdqa		%Xi, %T2	/ 2nd phase
+	psrlq		$1, %Xi
+	pclmulqdq	$0x11, %Hkey2, %Xhn
+	xorps		%Xl, %Xln
+	movdqu		0x10(%inp), %Xl
+	pshufb		%T3, %Xl
+	pclmulqdq	$0x10, %HK, %Xmn
+	xorps		%Xh, %Xhn
+	movups		0x50(%Htbl), %HK
+	pshufb		%T3, %T1
+	pxor		%T2, %Xhi
+	pxor		%Xi, %T2
+	psrlq		$5, %Xi
+
+	movdqa		%Xl, %Xh
+	pxor		%Xm, %Xmn
+	pshufd		$0b01001110, %Xl, %Xm
+	pxor		%Xl, %Xm
+	pclmulqdq	$0x00, %Hkey3, %Xl
+	pxor		%T2, %Xi
+	pxor		%T1, %Xhi
+	psrlq		$1, %Xi
+	pclmulqdq	$0x11, %Hkey3, %Xh
+	xorps		%Xl, %Xln
+	pxor		%Xhi, %Xi
+
+	pclmulqdq	$0x00, %HK, %Xm
+	xorps		%Xh, %Xhn
+
+	movdqa		%Xi, %Xhi
+	pshufd		$0b01001110, %Xi, %T1
+	pxor		%Xi, %T1
+
+	lea		0x40(%inp), %inp
+	sub		$0x40, %len
+	jnc		.Lmod4_loop
+
+.Ltail4x:
+	pclmulqdq	$0x00, %Hkey4, %Xi
+	xorps		%Xm, %Xmn
+	pclmulqdq	$0x11, %Hkey4, %Xhi
+	xorps		%Xln, %Xi
+	pclmulqdq	$0x10, %HK, %T1
+	xorps		%Xhn, %Xhi
+	pxor		%Xi, %Xhi	/ aggregated Karatsuba post-processing
+	pxor		%Xmn, %T1
+
+	pxor		%Xhi, %T1
+	pxor		%Xi, %Xhi
+
+	movdqa		%T1, %T2
+	psrldq		$8, %T1
+	pslldq		$8, %T2
+	pxor		%T1, %Xhi
+	pxor		%T2, %Xi
+
+	reduction_alg9
+
+	add	$0x40, %len
+	jz	.Ldone
+	movdqu	0x20(%Htbl), %HK
+	sub	$0x10, %len
+	jz	.Lodd_tail
+.Lskip4x:
+
+	/*
+	 * Xi+2 =[H*(Ii+1 + Xi+1)] mod P =
+	 *	[(H*Ii+1) + (H*Xi+1)] mod P =
+	 *	[(H*Ii+1) + H^2*(Ii+Xi)] mod P
+	 */
+	movdqu		(%inp), %T1		/ Ii
+	movdqu		16(%inp), %Xln		/ Ii+1
+	pshufb		%T3, %T1
+	pshufb		%T3, %Xln
+	pxor		%T1, %Xi		/ Ii+Xi
+
+	movdqa		%Xln, %Xhn
+	pshufd		$0b01001110, %Xln, %T1
+	pxor		%Xln, %T1
+	pclmulqdq	$0x00, %Hkey, %Xln
+	pclmulqdq	$0x11, %Hkey, %Xhn
+	pclmulqdq	$0x00, %HK, %T1
+
+	lea		32(%inp), %inp		/ i+=2
+	sub		$0x20, %len
+	jbe		.Leven_tail
+	jmp		.Lmod_loop
+
+.align	32
+.Lmod_loop:
+	movdqa		%Xi, %Xhi
+	pshufd		$0b01001110, %Xi, %T2
+	pxor		%Xi, %T2
+
+	pclmulqdq	$0x00, %Hkey2, %Xi
+	pclmulqdq	$0x11, %Hkey2, %Xhi
+	pclmulqdq	$0x10, %HK, %T2
+
+	pxor		%Xln, %Xi		/ (H*Ii+1) + H^2*(Ii+Xi)
+	pxor		%Xhn, %Xhi
+	movdqu		(%inp), %Xhn		/ Ii
+	pshufb		%T3, %Xhn
+	movdqu		16(%inp), %Xln		/ Ii+1
+
+	pxor		%Xi, %T1		/ aggregated Karatsuba post-proc
+	pxor		%Xhi, %T1
+	pxor		%Xhn, %Xhi		/ "Ii+Xi",  consume early
+	pxor		%T1, %T2
+	pshufb		%T3, %Xln
+	movdqa		%T2, %T1
+	psrldq		$8, %T1
+	pslldq		$8, %T2
+	pxor		%T1, %Xhi
+	pxor		%T2, %Xi
+
+	movdqa		%Xln, %Xhn
+
+	movdqa		%Xi, %T2		/ 1st phase
+	movdqa		%Xi, %T1
+	psllq		$5, %Xi
+	pclmulqdq	$0x00, %Hkey, %Xln
+	pxor		%Xi, %T1
+	psllq		$1, %Xi
+	pxor		%T1, %Xi
+	psllq		$57, %Xi
+	movdqa		%Xi, %T1
+	pslldq		$8, %Xi
+	psrldq		$8, %T1
+	pxor		%T2, %Xi
+	pxor		%T1, %Xhi
+	pshufd		$0b01001110, %Xhn, %T1
+	pxor		%Xhn, %T1
+
+	pclmulqdq	$0x11, %Hkey, %Xhn
+	movdqa		%Xi, %T2		/ 2nd phase
+	psrlq		$1, %Xi
+	pxor		%T2, %Xhi
+	pxor		%Xi, %T2
+	psrlq		$5, %Xi
+	pxor		%T2, %Xi
+	psrlq		$1, %Xi
+	pclmulqdq	$0x00, %HK, %T1
+	pxor		%Xhi, %Xi
+
+	lea		32(%inp), %inp
+	sub		$0x20, %len
+	ja		.Lmod_loop
+
+.Leven_tail:
+	movdqa		%Xi, %Xhi
+	pshufd		$0b01001110, %Xi, %T2
+	pxor		%Xi, %T2
+
+	pclmulqdq	$0x00, %Hkey2, %Xi
+	pclmulqdq	$0x11, %Hkey2, %Xhi
+	pclmulqdq	$0x10, %HK, %T2
+
+	pxor		%Xln, %Xi		/* (H*Ii+1) + H^2*(Ii+Xi) */
+	pxor		%Xhn, %Xhi
+	pxor		%Xi, %T1
+	pxor		%Xhi, %T1
+	pxor		%T1, %T2
+	movdqa		%T2, %T1
+	psrldq		$8, %T1
+	pslldq		$8, %T2
+	pxor		%T1, %Xhi
+	pxor		%T2, %Xi
+
+	reduction_alg9
+
+	test		%len, %len
+	jnz		.Ldone
+
+.Lodd_tail:
+	movdqu		(%inp), %T1			/ Ii
+	pshufb		%T3, %T1
+	pxor		%T1, %Xi			/ Ii+Xi
+
+	clmul64x64_T2(HK)				/ H*(Ii+Xi)
+	reduction_alg9
+
+.Ldone:
+	pshufb		%T3, %Xi
+	movdqu		%Xi, (%Xip)
+
+	ret
+	SET_SIZE(gcm_ghash_clmul)
+
+/*
+ * void gcm_init_clmul(const void *Xi, void *Htable)
+ */
+ENTRY_NP(gcm_init_clmul)
+	movdqu		(%Xip), %Hkey
+	pshufd		$0b01001110, %Hkey, %Hkey	/ dword swap
+
+	/ <<1 twist
+	pshufd		$0b11111111, %Hkey, %T2	/ broadcast uppermost dword
+	movdqa		%Hkey, %T1
+	psllq		$1, %Hkey
+	pxor		%T3, %T3
+	psrlq		$63, %T1
+	pcmpgtd		%T2, %T3		/ broadcast carry bit
+	pslldq		$8, %T1
+	por		%T1, %Hkey		/ H<<=1
+
+	/ magic reduction
+	pand		.L0x1c2_polynomial(%rip), %T3
+	pxor		%T3, %Hkey		/ if(carry) H^=0x1c2_polynomial
+
+	/ calculate H^2
+	pshufd		$0b01001110, %Hkey, %HK
+	movdqa		%Hkey, %Xi
+	pxor		%Hkey, %HK
+
+	clmul64x64_T2(HK)
+	reduction_alg9
+
+	pshufd		$0b01001110, %Hkey, %T1
+	pshufd		$0b01001110, %Xi, %T2
+	pxor		%Hkey, %T1		/ Karatsuba pre-processing
+	movdqu		%Hkey, 0x00(%Htbl)	/ save H
+	pxor		%Xi, %T2		/ Karatsuba pre-processing
+	movdqu		%Xi, 0x10(%Htbl)	/ save H^2
+	palignr		$8, %T1, %T2		/ low part is H.lo^H.hi...
+	movdqu		%T2, 0x20(%Htbl)	/ save Karatsuba "salt"
+
+	clmul64x64_T2(HK)			/ H^3
+	reduction_alg9
+
+	movdqa		%Xi, %T3
+
+	clmul64x64_T2(HK)			/ H^4
+	reduction_alg9
+
+	pshufd		$0b01001110, %T3, %T1
+	pshufd		$0b01001110, %Xi, %T2
+	pxor		%T3, %T1		/ Karatsuba pre-processing
+	movdqu		%T3, 0x30(%Htbl)	/ save H^3
+	pxor		%Xi, %T2		/ Karatsuba pre-processing
+	movdqu		%Xi, 0x40(%Htbl)	/ save H^4
+	palignr		$8, %T1, %T2		/ low part is H^3.lo^H^3.hi...
+	movdqu		%T2, 0x50(%Htbl)	/ save Karatsuba "salt"
+
+	ret
+	SET_SIZE(gcm_init_clmul)
+
+#endif	/* lint || __lint */
--- old/usr/src/common/crypto/modes/cbc.c	Thu Apr 30 20:52:30 2015
+++ new/usr/src/common/crypto/modes/cbc.c	Thu Apr 30 20:52:30 2015
@@ -22,6 +22,9 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef _KERNEL
 #include <strings.h>
@@ -31,10 +34,43 @@
 #endif
 
 #include <sys/types.h>
+#define	INLINE_CRYPTO_GET_PTRS
 #include <modes/modes.h>
 #include <sys/crypto/common.h>
 #include <sys/crypto/impl.h>
 
+boolean_t cbc_fastpath_enabled = B_TRUE;
+
+static void
+cbc_decrypt_fastpath(cbc_ctx_t *ctx, const uint8_t *data, size_t length,
+    uint8_t *out, size_t block_size,
+    int (*decrypt)(const void *, const uint8_t *, uint8_t *),
+    int (*decrypt_ecb)(const void *, const uint8_t *, uint8_t *, uint64_t),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    void (*xor_block_range)(const uint8_t *, uint8_t *, uint64_t))
+{
+	const uint8_t *iv = (uint8_t *)ctx->cbc_iv;
+
+	/* Use bulk decryption when available. */
+	if (decrypt_ecb != NULL) {
+		decrypt_ecb(ctx->cbc_keysched, data, out, length);
+	} else {
+		for (size_t i = 0; i < length; i += block_size)
+			decrypt(ctx->cbc_keysched, &data[i], &out[i]);
+	}
+
+	/* Use bulk XOR when available. */
+	if (xor_block_range != NULL && length >= 2 * block_size) {
+		xor_block(iv, out);
+		xor_block_range(data, &out[block_size], length - block_size);
+	} else {
+		for (size_t i = 0; i < length; i += block_size) {
+			xor_block(iv, &out[i]);
+			iv = &data[i];
+		}
+	}
+}
+
 /*
  * Algorithm independent CBC functions.
  */
@@ -42,8 +78,10 @@
 cbc_encrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
     int (*encrypt)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*encrypt_cbc)(const void *, const uint8_t *, uint8_t *,
+    const uint8_t *, uint64_t))
 {
 	size_t remainder = length;
 	size_t need;
@@ -56,6 +94,31 @@
 	uint8_t *out_data_2;
 	size_t out_data_1_len;
 
+	/*
+	 * CBC encryption fastpath requirements:
+	 * - fastpath is enabled
+	 * - algorithm-specific acceleration function is available
+	 * - input is block-aligned
+	 * - output is a single contiguous region or the user requested that
+	 *   we overwrite their input buffer (input/output aliasing allowed)
+	 */
+	if (cbc_fastpath_enabled && encrypt_cbc != NULL && length != 0 &&
+	    ctx->cbc_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
+	    (out == NULL || CRYPTO_DATA_IS_SINGLE_BLOCK(out))) {
+		if (out == NULL) {
+			encrypt_cbc(ctx->cbc_keysched, (uint8_t *)data,
+			    (uint8_t *)data, (uint8_t *)ctx->cbc_iv, length);
+			ctx->cbc_lastp = (uint8_t *)&data[length - block_size];
+		} else {
+			uint8_t *outp = CRYPTO_DATA_FIRST_BLOCK(out);
+			encrypt_cbc(ctx->cbc_keysched, (uint8_t *)data, outp,
+			    (uint8_t *)ctx->cbc_iv, length);
+			out->cd_offset += length;
+			ctx->cbc_lastp = &outp[length - block_size];
+		}
+		goto out;
+	}
+
 	if (length + ctx->cbc_remainder_len < block_size) {
 		/* accumulate bytes here and return */
 		bcopy(datap,
@@ -169,8 +232,10 @@
 cbc_decrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
     int (*decrypt)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*decrypt_ecb)(const void *, const uint8_t *, uint8_t *, uint64_t),
+    void (*xor_block_range)(const uint8_t *, uint8_t *, uint64_t))
 {
 	size_t remainder = length;
 	size_t need;
@@ -183,6 +248,26 @@
 	uint8_t *out_data_2;
 	size_t out_data_1_len;
 
+	/*
+	 * CBC decryption fastpath requirements:
+	 * - fastpath is enabled
+	 * - input is block-aligned
+	 * - output is a single contiguous region and doesn't alias input
+	 */
+	if (cbc_fastpath_enabled && ctx->cbc_remainder_len == 0 &&
+	    length != 0 && (length & (block_size - 1)) == 0 &&
+	    CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
+		uint8_t *outp = CRYPTO_DATA_FIRST_BLOCK(out);
+
+		cbc_decrypt_fastpath(ctx, (uint8_t *)data, length, outp,
+		    block_size, decrypt, decrypt_ecb, xor_block,
+		    xor_block_range);
+		out->cd_offset += length;
+		bcopy(&data[length - block_size], ctx->cbc_iv, block_size);
+		ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv;
+		return (CRYPTO_SUCCESS);
+	}
+
 	if (length + ctx->cbc_remainder_len < block_size) {
 		/* accumulate bytes here and return */
 		bcopy(datap,
@@ -280,7 +365,7 @@
 
 int
 cbc_init_ctx(cbc_ctx_t *cbc_ctx, char *param, size_t param_len,
-    size_t block_size, void (*copy_block)(uint8_t *, uint64_t *))
+    size_t block_size, void (*copy_block)(const uint8_t *, uint64_t *))
 {
 	/*
 	 * Copy IV into context.
--- old/usr/src/common/crypto/modes/ccm.c	Thu Apr 30 20:52:31 2015
+++ new/usr/src/common/crypto/modes/ccm.c	Thu Apr 30 20:52:31 2015
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
  */
 
 #ifndef _KERNEL
@@ -31,6 +32,7 @@
 
 #include <sys/types.h>
 #include <sys/kmem.h>
+#define	INLINE_CRYPTO_GET_PTRS
 #include <modes/modes.h>
 #include <sys/crypto/common.h>
 #include <sys/crypto/impl.h>
@@ -48,8 +50,8 @@
 ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	size_t remainder = length;
 	size_t need;
@@ -206,7 +208,7 @@
 int
 ccm_encrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint8_t *lastp, *mac_buf, *ccm_mac_p, *macp;
 	void *iov_or_mp;
@@ -359,8 +361,8 @@
 ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	size_t remainder = length;
 	size_t need;
@@ -526,8 +528,8 @@
 int
 ccm_decrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	size_t mac_remain, pt_len;
 	uint8_t *pt, *mac_buf, *macp, *ccm_mac_p;
@@ -770,7 +772,7 @@
 ccm_init(ccm_ctx_t *ctx, unsigned char *nonce, size_t nonce_len,
     unsigned char *auth_data, size_t auth_data_len, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint8_t *mac_buf, *datap, *ivp, *authp;
 	size_t remainder, processed;
@@ -854,7 +856,7 @@
 ccm_init_ctx(ccm_ctx_t *ccm_ctx, char *param, int kmflag,
     boolean_t is_encrypt_init, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	int rv;
 	CK_AES_CCM_PARAMS *ccm_param;
--- old/usr/src/common/crypto/modes/ctr.c	Thu Apr 30 20:52:31 2015
+++ new/usr/src/common/crypto/modes/ctr.c	Thu Apr 30 20:52:31 2015
@@ -22,6 +22,9 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef _KERNEL
 #include <strings.h>
@@ -31,11 +34,15 @@
 #endif
 
 #include <sys/types.h>
+#define	INLINE_CRYPTO_GET_PTRS
 #include <modes/modes.h>
 #include <sys/crypto/common.h>
 #include <sys/crypto/impl.h>
 #include <sys/byteorder.h>
+#include <sys/cmn_err.h>
 
+boolean_t ctr_fastpath_enabled = B_TRUE;
+
 /*
  * Encrypt and decrypt multiple blocks of data in counter mode.
  */
@@ -43,7 +50,9 @@
 ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
     int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *ks, const uint8_t *pt, uint8_t *ct,
+    uint64_t len, uint64_t counter[2]))
 {
 	size_t remainder = length;
 	size_t need;
@@ -57,6 +66,24 @@
 	size_t out_data_1_len;
 	uint64_t lower_counter, upper_counter;
 
+	/*
+	 * CTR encryption/decryption fastpath requirements:
+	 * - fastpath is enabled
+	 * - algorithm-specific acceleration function is available
+	 * - input is block-aligned
+	 * - the counter value won't overflow the lower counter mask
+	 * - output is a single contiguous region and doesn't alias input
+	 */
+	if (ctr_fastpath_enabled && cipher_ctr != NULL &&
+	    ctx->ctr_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
+	    ntohll(ctx->ctr_cb[1]) <= ctx->ctr_lower_mask -
+	    length / block_size && CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
+		cipher_ctr(ctx->ctr_keysched, (uint8_t *)data,
+		    CRYPTO_DATA_FIRST_BLOCK(out), length, ctx->ctr_cb);
+		out->cd_offset += length;
+		return (CRYPTO_SUCCESS);
+	}
+
 	if (length + ctx->ctr_remainder_len < block_size) {
 		/* accumulate bytes here and return */
 		bcopy(datap,
@@ -205,7 +232,7 @@
 
 int
 ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb,
-void (*copy_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *))
 {
 	uint64_t upper_mask = 0;
 	uint64_t lower_mask = 0;
--- old/usr/src/common/crypto/modes/ecb.c	Thu Apr 30 20:52:31 2015
+++ new/usr/src/common/crypto/modes/ecb.c	Thu Apr 30 20:52:31 2015
@@ -22,6 +22,9 @@
  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef _KERNEL
 #include <strings.h>
@@ -31,17 +34,26 @@
 #endif
 
 #include <sys/types.h>
+#define	INLINE_CRYPTO_GET_PTRS
 #include <modes/modes.h>
 #include <sys/crypto/common.h>
 #include <sys/crypto/impl.h>
 
+boolean_t ecb_fastpath_enabled = B_TRUE;
+
 /*
  * Algorithm independent ECB functions.
+ * `cipher' is a single-block version of the cipher function to be performed
+ * on each input block. `cipher_ecb' is an optional parameter, which if
+ * passed and the input/output conditions allow it, will be invoked for the
+ * entire input buffer once to accelerate the operation.
  */
 int
 ecb_cipher_contiguous_blocks(ecb_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
-    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct))
+    int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct),
+    int (*cipher_ecb)(const void *ks, const uint8_t *pt, uint8_t *ct,
+    uint64_t len))
 {
 	size_t remainder = length;
 	size_t need;
@@ -54,6 +66,28 @@
 	uint8_t *out_data_2;
 	size_t out_data_1_len;
 
+	/*
+	 * ECB encryption/decryption fastpath requirements:
+	 * - fastpath is enabled
+	 * - caller passed an accelerated ECB version of the cipher function
+	 * - input is block-aligned
+	 * - output is a single contiguous region or the user requested that
+	 *   we overwrite their input buffer (input/output aliasing allowed)
+	 */
+	if (ecb_fastpath_enabled && cipher_ecb != NULL &&
+	    ctx->ecb_remainder_len == 0 && length % block_size == 0 &&
+	    (out == NULL || CRYPTO_DATA_IS_SINGLE_BLOCK(out))) {
+		if (out == NULL) {
+			cipher_ecb(ctx->ecb_keysched, (uint8_t *)data,
+			    (uint8_t *)data, length);
+		} else {
+			cipher_ecb(ctx->ecb_keysched, (uint8_t *)data,
+			    CRYPTO_DATA_FIRST_BLOCK(out), length);
+			out->cd_offset += length;
+		}
+		return (CRYPTO_SUCCESS);
+	}
+
 	if (length + ctx->ecb_remainder_len < block_size) {
 		/* accumulate bytes here and return */
 		bcopy(datap,
--- old/usr/src/common/crypto/modes/gcm.c	Thu Apr 30 20:52:31 2015
+++ new/usr/src/common/crypto/modes/gcm.c	Thu Apr 30 20:52:31 2015
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
  */
 
 
@@ -30,14 +31,21 @@
 #include <security/cryptoki.h>
 #endif	/* _KERNEL */
 
-
+#include <sys/cmn_err.h>
 #include <sys/types.h>
 #include <sys/kmem.h>
+#define	INLINE_CRYPTO_GET_PTRS
 #include <modes/modes.h>
 #include <sys/crypto/common.h>
 #include <sys/crypto/impl.h>
 #include <sys/byteorder.h>
 
+#define	COUNTER_MASK	0x00000000ffffffffULL
+
+#ifdef	_KERNEL
+#include <sys/sdt.h>		/* SET_ERROR */
+#endif	/* _KERNEL */
+
 #ifdef __amd64
 
 #ifdef _KERNEL
@@ -45,19 +53,57 @@
 #include <sys/x86_archext.h>	/* x86_featureset, X86FSET_*, CPUID_* */
 #include <sys/disp.h>		/* kpreempt_disable(), kpreempt_enable */
 /* Workaround for no XMM kernel thread save/restore */
-#define	KPREEMPT_DISABLE	kpreempt_disable()
-#define	KPREEMPT_ENABLE		kpreempt_enable()
+extern void gcm_accel_save(void *savestate);
+extern void gcm_accel_restore(void *savestate);
 
+#if	defined(lint) || defined(__lint)
+#define	GCM_ACCEL_SAVESTATE(name)	uint8_t name[16 * 16 + 8]
 #else
+#define	GCM_ACCEL_SAVESTATE(name) \
+	/* stack space for xmm0--xmm15 and cr0 (16 x 128 bits + 64 bits) */ \
+	uint8_t name[16 * 16 + 8] __attribute__((aligned(16)))
+#endif
+
+/*
+ * Disables kernel thread preemption and conditionally gcm_accel_save() iff
+ * Intel PCLMULQDQ support is present. Must be balanced by GCM_ACCEL_EXIT.
+ * This must be present in all externally callable GCM functions which
+ * invoke GHASH operations using FPU-accelerated implementations, or call
+ * static functions which do (such as gcm_encrypt_fastpath128()).
+ */
+#define	GCM_ACCEL_ENTER \
+	GCM_ACCEL_SAVESTATE(savestate); \
+	do { \
+		if (intel_pclmulqdq_instruction_present()) { \
+			kpreempt_disable(); \
+			gcm_accel_save(savestate); \
+		} \
+		_NOTE(CONSTCOND) \
+	} while (0)
+#define	GCM_ACCEL_EXIT \
+	do { \
+		if (intel_pclmulqdq_instruction_present()) { \
+			gcm_accel_restore(savestate); \
+			kpreempt_enable(); \
+		} \
+		_NOTE(CONSTCOND) \
+	} while (0)
+
+#else	/* _KERNEL */
 #include <sys/auxv.h>		/* getisax() */
 #include <sys/auxv_386.h>	/* AV_386_PCLMULQDQ bit */
-#define	KPREEMPT_DISABLE
-#define	KPREEMPT_ENABLE
+#define	SET_ERROR(x)	(x)
 #endif	/* _KERNEL */
 
 extern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
-static int intel_pclmulqdq_instruction_present(void);
-#endif	/* __amd64 */
+extern void gcm_init_clmul(const uint64_t hash_init[2], uint8_t Htable[256]);
+extern void gcm_ghash_clmul(uint64_t ghash[2], const uint8_t Htable[256],
+    const uint8_t *inp, size_t length);
+static inline int intel_pclmulqdq_instruction_present(void);
+#else	/* !__amd64 */
+#define	GCM_ACCEL_ENTER
+#define	GCM_ACCEL_EXIT
+#endif	/* !__amd64 */
 
 struct aes_block {
 	uint64_t a;
@@ -75,14 +121,16 @@
  * Note: x_in, y, and res all point to 16-byte numbers (an array of two
  * 64-bit integers).
  */
-void
+static inline void
 gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res)
 {
 #ifdef __amd64
 	if (intel_pclmulqdq_instruction_present()) {
-		KPREEMPT_DISABLE;
+		/*
+		 * FPU context will have been saved and kernel thread
+		 * preemption disabled already.
+		 */
 		gcm_mul_pclmulqdq(x_in, y, res);
-		KPREEMPT_ENABLE;
 	} else
 #endif	/* __amd64 */
 	{
@@ -116,24 +164,93 @@
 	}
 }
 
-
 #define	GHASH(c, d, t) \
-	xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
-	gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
-	(uint64_t *)(void *)(t));
+	do { \
+		xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
+		gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
+		    (uint64_t *)(void *)(t)); \
+		_NOTE(CONSTCOND) \
+	} while (0)
 
+boolean_t gcm_fastpath_enabled = B_TRUE;
 
-/*
- * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
- * is done in another function.
- */
-int
-gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
-    crypto_data_t *out, size_t block_size,
+static void
+gcm_fastpath128(gcm_ctx_t *ctx, const uint8_t *data, size_t length,
+    uint8_t *out, boolean_t encrypt,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
 {
+	/* When decrypting, `data' holds the ciphertext we need to GHASH. */
+	if (!encrypt) {
+#ifdef	__amd64
+		if (intel_pclmulqdq_instruction_present())
+			gcm_ghash_clmul(ctx->gcm_ghash, ctx->gcm_H_table,
+			    data, length);
+		else
+#endif	/* __amd64 */
+			for (size_t i = 0; i < length; i += 16)
+				GHASH(ctx, &data[i], ctx->gcm_ghash);
+	}
+
+	if (cipher_ctr != NULL) {
+		/*
+		 * GCM is almost but not quite like CTR. GCM increments the
+		 * counter value *before* processing the first input block,
+		 * whereas CTR does so afterwards. So we need to increment
+		 * the counter before calling CTR and decrement it afterwards.
+		 */
+		uint64_t counter = ntohll(ctx->gcm_cb[1]);
+
+		ctx->gcm_cb[1] = htonll((counter & ~COUNTER_MASK) |
+		    ((counter & COUNTER_MASK) + 1));
+		cipher_ctr(ctx->gcm_keysched, data, out, length, ctx->gcm_cb);
+		counter = ntohll(ctx->gcm_cb[1]);
+		ctx->gcm_cb[1] = htonll((counter & ~COUNTER_MASK) |
+		    ((counter & COUNTER_MASK) - 1));
+	} else {
+		uint64_t counter = ntohll(ctx->gcm_cb[1]);
+
+		for (size_t i = 0; i < length; i += 16) {
+			/*LINTED(E_BAD_PTR_CAST_ALIGN)*/
+			*(uint64_t *)&out[i] = ctx->gcm_cb[0];
+			/*LINTED(E_BAD_PTR_CAST_ALIGN)*/
+			*(uint64_t *)&out[i + 8] = htonll(counter++);
+			encrypt_block(ctx->gcm_keysched, &out[i], &out[i]);
+			xor_block(&data[i], &out[i]);
+		}
+
+		ctx->gcm_cb[1] = htonll(counter);
+	}
+
+	/* When encrypting, `out' holds the ciphertext we need to GHASH. */
+	if (encrypt) {
+#ifdef	__amd64
+		if (intel_pclmulqdq_instruction_present())
+			gcm_ghash_clmul(ctx->gcm_ghash, ctx->gcm_H_table,
+			    out, length);
+		else
+#endif	/* __amd64 */
+			for (size_t i = 0; i < length; i += 16)
+				GHASH(ctx, &out[i], ctx->gcm_ghash);
+
+		/* If no more data comes in, the last block is the auth tag. */
+		bcopy(&out[length - 16], ctx->gcm_tmp, 16);
+	}
+
+	ctx->gcm_processed_data_len += length;
+}
+
+static int
+gcm_process_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size, boolean_t encrypt,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
+{
 	size_t remainder = length;
 	size_t need;
 	uint8_t *datap = (uint8_t *)data;
@@ -146,7 +263,29 @@
 	size_t out_data_1_len;
 	uint64_t counter;
 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+	int rv = CRYPTO_SUCCESS;
 
+	GCM_ACCEL_ENTER;
+
+	/*
+	 * GCM mode fastpath requirements:
+	 * - fastpath is enabled
+	 * - block size is 128 bits
+	 * - input is block-aligned
+	 * - the counter value won't overflow
+	 * - output is a single contiguous region and doesn't alias input
+	 */
+	if (gcm_fastpath_enabled && block_size == 16 &&
+	    ctx->gcm_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
+	    ntohll(ctx->gcm_cb[1] & counter_mask) <= ntohll(counter_mask) -
+	    length / block_size && CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
+		gcm_fastpath128(ctx, (uint8_t *)data, length,
+		    CRYPTO_DATA_FIRST_BLOCK(out), encrypt, encrypt_block,
+		    xor_block, cipher_ctr);
+		out->cd_offset += length;
+		goto out;
+	}
+
 	if (length + ctx->gcm_remainder_len < block_size) {
 		/* accumulate bytes here and return */
 		bcopy(datap,
@@ -154,7 +293,7 @@
 		    length);
 		ctx->gcm_remainder_len += length;
 		ctx->gcm_copy_to = datap;
-		return (CRYPTO_SUCCESS);
+		goto out;
 	}
 
 	lastp = (uint8_t *)ctx->gcm_cb;
@@ -166,8 +305,10 @@
 		if (ctx->gcm_remainder_len > 0) {
 			need = block_size - ctx->gcm_remainder_len;
 
-			if (need > remainder)
-				return (CRYPTO_DATA_LEN_RANGE);
+			if (need > remainder) {
+				rv = SET_ERROR(CRYPTO_DATA_LEN_RANGE);
+				goto out;
+			}
 
 			bcopy(datap, &((uint8_t *)ctx->gcm_remainder)
 			    [ctx->gcm_remainder_len], need);
@@ -177,6 +318,10 @@
 			blockp = datap;
 		}
 
+		/* add ciphertext to the hash */
+		if (!encrypt)
+			GHASH(ctx, blockp, ctx->gcm_ghash);
+
 		/*
 		 * Increment counter. Counter bits are confined
 		 * to the bottom 32 bits of the counter block.
@@ -221,7 +366,8 @@
 		}
 
 		/* add ciphertext to the hash */
-		GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+		if (encrypt)
+			GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
 
 		/* Update pointer to next block of data to be processed. */
 		if (ctx->gcm_remainder_len != 0) {
@@ -244,23 +390,47 @@
 
 	} while (remainder > 0);
 out:
-	return (CRYPTO_SUCCESS);
+	GCM_ACCEL_EXIT;
+
+	return (rv);
 }
 
+
+/*
+ * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
+ * is done in another function.
+ */
+/*ARGSUSED*/
+int
+gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
+{
+	return (gcm_process_contiguous_blocks(ctx, data, length, out,
+	    block_size, B_TRUE, encrypt_block, copy_block, xor_block,
+	    cipher_ctr));
+}
+
 /* ARGSUSED */
 int
 gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
 	uint8_t *ghash, *macp;
 	int i, rv;
 
-	if (out->cd_length <
-	    (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
-		return (CRYPTO_DATA_LEN_RANGE);
+	GCM_ACCEL_ENTER;
+
+	if (out->cd_length < (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
+		rv = CRYPTO_DATA_LEN_RANGE;
+		goto out;
 	}
 
 	ghash = (uint8_t *)ctx->gcm_ghash;
@@ -310,16 +480,17 @@
 	if (ctx->gcm_remainder_len > 0) {
 		rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len);
 		if (rv != CRYPTO_SUCCESS)
-			return (rv);
+			goto out;
 	}
 	out->cd_offset += ctx->gcm_remainder_len;
 	ctx->gcm_remainder_len = 0;
 	rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
 	if (rv != CRYPTO_SUCCESS)
-		return (rv);
+		goto out;
 	out->cd_offset += ctx->gcm_tag_len;
-
-	return (CRYPTO_SUCCESS);
+out:
+	GCM_ACCEL_EXIT;
+	return (rv);
 }
 
 /*
@@ -326,19 +497,25 @@
  * This will only deal with decrypting the last block of the input that
  * might not be a multiple of block length.
  */
+/*ARGSUSED*/
 static void
-gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
+gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, uint8_t *data, size_t length,
+    size_t block_size, crypto_data_t *out,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
-	uint8_t *datap, *outp, *counterp;
 	uint64_t counter;
 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
-	int i;
 
+	/* padd last block and add to GHASH */
+	bcopy(data, ctx->gcm_tmp, length);
+	bzero(((uint8_t *)ctx->gcm_tmp) + length,
+	    sizeof (ctx->gcm_tmp) - length);
+	GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+
 	/*
 	 * Increment counter.
-	 * Counter bits are confined to the bottom 32 bits
+	 * Counter bits are confined to the bottom 32 bits.
 	 */
 	counter = ntohll(ctx->gcm_cb[1] & counter_mask);
 	counter = htonll(counter + 1);
@@ -345,23 +522,19 @@
 	counter &= counter_mask;
 	ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
 
-	datap = (uint8_t *)ctx->gcm_remainder;
-	outp = &((ctx->gcm_pt_buf)[index]);
-	counterp = (uint8_t *)ctx->gcm_tmp;
+	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
+	    (uint8_t *)ctx->gcm_tmp);
 
-	/* authentication tag */
-	bzero((uint8_t *)ctx->gcm_tmp, block_size);
-	bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len);
-
-	/* add ciphertext to the hash */
-	GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
-
-	/* decrypt remaining ciphertext */
-	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp);
-
 	/* XOR with counter block */
-	for (i = 0; i < ctx->gcm_remainder_len; i++) {
-		outp[i] = datap[i] ^ counterp[i];
+	for (size_t i = 0; i < length; i++)
+		((uint8_t *)ctx->gcm_tmp)[i] ^= data[i];
+
+	if (out != NULL) {
+		(void) crypto_put_output_data((uchar_t *)ctx->gcm_tmp, out,
+		    length);
+		out->cd_offset += length;
+	} else {
+		bcopy(ctx->gcm_tmp, data, length);
 	}
 }
 
@@ -370,115 +543,173 @@
 gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
 {
-	size_t new_len;
-	uint8_t *new;
+	int rv = CRYPTO_SUCCESS;
 
+	GCM_ACCEL_ENTER;
+
 	/*
-	 * Copy contiguous ciphertext input blocks to plaintext buffer.
-	 * Ciphertext will be decrypted in the final.
+	 * Previous calls accumulate data in the input buffer to make sure
+	 * we have the auth tag (the last part of the ciphertext) when we
+	 * receive a final() call.
 	 */
-	if (length > 0) {
-		new_len = ctx->gcm_pt_buf_len + length;
-#ifdef _KERNEL
-		new = kmem_alloc(new_len, ctx->gcm_kmflag);
-		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
-		kmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
-#else
-		new = malloc(new_len);
-		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
-		free(ctx->gcm_pt_buf);
-#endif
-		if (new == NULL)
-			return (CRYPTO_HOST_MEMORY);
+	if (ctx->gcm_last_input_fill > 0) {
+		/* Try to complete the input buffer */
+		size_t to_copy = MIN(length,
+		    sizeof (ctx->gcm_last_input) - ctx->gcm_last_input_fill);
 
-		ctx->gcm_pt_buf = new;
-		ctx->gcm_pt_buf_len = new_len;
-		bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
-		    length);
-		ctx->gcm_processed_data_len += length;
+		bcopy(data, ctx->gcm_last_input + ctx->gcm_last_input_fill,
+		    to_copy);
+		data += to_copy;
+		ctx->gcm_last_input_fill += to_copy;
+		length -= to_copy;
+
+		if (ctx->gcm_last_input_fill < sizeof (ctx->gcm_last_input))
+			/* Not enough input data to continue */
+			goto out;
+
+		if (length < ctx->gcm_tag_len) {
+			/*
+			 * There isn't enough data ahead to constitute a full
+			 * auth tag, so only crunch one input block and copy
+			 * the remainder of the input into our buffer.
+			 */
+			rv = gcm_process_contiguous_blocks(ctx,
+			    (char *)ctx->gcm_last_input, block_size, out,
+			    block_size, B_FALSE, encrypt_block, copy_block,
+			    xor_block, cipher_ctr);
+			if (rv != CRYPTO_SUCCESS)
+				goto out;
+			ctx->gcm_last_input_fill -= block_size;
+			bcopy(ctx->gcm_last_input + block_size,
+			    ctx->gcm_last_input, ctx->gcm_last_input_fill);
+			bcopy(data, ctx->gcm_last_input +
+			    ctx->gcm_last_input_fill, length);
+			ctx->gcm_last_input_fill += length;
+			/* No more input left */
+			goto out;
+		}
+		/*
+		 * There is enough data ahead for the auth tag, so crunch
+		 * everything in our buffer now and empty it.
+		 */
+		rv = gcm_process_contiguous_blocks(ctx,
+		    (char *)ctx->gcm_last_input, ctx->gcm_last_input_fill,
+		    out, block_size, B_FALSE, encrypt_block, copy_block,
+		    xor_block, cipher_ctr);
+		if (rv != CRYPTO_SUCCESS)
+			goto out;
+		ctx->gcm_last_input_fill = 0;
 	}
+	/*
+	 * Last input buffer is empty, so what's left ahead is block-aligned.
+	 * Crunch all the blocks up until the near end, which might be our
+	 * auth tag and we must NOT decrypt.
+	 */
+	ASSERT(ctx->gcm_last_input_fill == 0);
+	if (length >= block_size + ctx->gcm_tag_len) {
+		size_t to_decrypt = (length - ctx->gcm_tag_len) &
+		    ~(block_size - 1);
 
-	ctx->gcm_remainder_len = 0;
-	return (CRYPTO_SUCCESS);
+		rv = gcm_process_contiguous_blocks(ctx, data, to_decrypt, out,
+		    block_size, B_FALSE, encrypt_block, copy_block, xor_block,
+		    cipher_ctr);
+		if (rv != CRYPTO_SUCCESS)
+			goto out;
+		data += to_decrypt;
+		length -= to_decrypt;
+	}
+	/*
+	 * Copy the remainder into our input buffer, it's potentially
+	 * the auth tag and a last partial block.
+	 */
+	ASSERT(length < sizeof (ctx->gcm_last_input));
+	bcopy(data, ctx->gcm_last_input, length);
+	ctx->gcm_last_input_fill += length;
+out:
+	GCM_ACCEL_EXIT;
+
+	return (rv);
 }
 
 int
 gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
 {
-	size_t pt_len;
-	size_t remainder;
-	uint8_t *ghash;
-	uint8_t *blockp;
-	uint8_t *cbp;
-	uint64_t counter;
-	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
-	int processed = 0, rv;
+	int rv = CRYPTO_SUCCESS;
 
-	ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len);
+	/* Check there's enough data to at least compute a tag */
+	if (ctx->gcm_last_input_fill < ctx->gcm_tag_len)
+		return (SET_ERROR(CRYPTO_DATA_LEN_RANGE));
 
-	pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
-	ghash = (uint8_t *)ctx->gcm_ghash;
-	blockp = ctx->gcm_pt_buf;
-	remainder = pt_len;
-	while (remainder > 0) {
-		/* Incomplete last block */
-		if (remainder < block_size) {
-			bcopy(blockp, ctx->gcm_remainder, remainder);
-			ctx->gcm_remainder_len = remainder;
-			/*
-			 * not expecting anymore ciphertext, just
-			 * compute plaintext for the remaining input
-			 */
-			gcm_decrypt_incomplete_block(ctx, block_size,
-			    processed, encrypt_block, xor_block);
-			ctx->gcm_remainder_len = 0;
-			goto out;
-		}
-		/* add ciphertext to the hash */
-		GHASH(ctx, blockp, ghash);
+	GCM_ACCEL_ENTER;
 
-		/*
-		 * Increment counter.
-		 * Counter bits are confined to the bottom 32 bits
-		 */
-		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
-		counter = htonll(counter + 1);
-		counter &= counter_mask;
-		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+	/* Finish any unprocessed input */
+	if (ctx->gcm_last_input_fill > ctx->gcm_tag_len) {
+		size_t last_blk_len = MIN(block_size,
+		    ctx->gcm_last_input_fill - ctx->gcm_tag_len);
 
-		cbp = (uint8_t *)ctx->gcm_tmp;
-		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp);
+		/* Finish last full block */
+		if (last_blk_len >= block_size) {
+			rv = gcm_process_contiguous_blocks(ctx,
+			    (char *)ctx->gcm_last_input, block_size, out,
+			    block_size, B_FALSE, encrypt_block, copy_block,
+			    xor_block, cipher_ctr);
+			if (rv != CRYPTO_SUCCESS)
+				goto errout;
 
-		/* XOR with ciphertext */
-		xor_block(cbp, blockp);
+			last_blk_len -= block_size;
+			ctx->gcm_processed_data_len += block_size;
+			ctx->gcm_last_input_fill -= block_size;
 
-		processed += block_size;
-		blockp += block_size;
-		remainder -= block_size;
+			/* Shift what remains in the input buffer forward */
+			bcopy(ctx->gcm_last_input + block_size,
+			    ctx->gcm_last_input, ctx->gcm_last_input_fill);
+		}
+		/* Finish last incomplete block before auth tag */
+		if (last_blk_len > 0) {
+			gcm_decrypt_incomplete_block(ctx, ctx->gcm_last_input,
+			    last_blk_len, block_size, out, encrypt_block,
+			    xor_block);
+
+			ctx->gcm_processed_data_len += last_blk_len;
+			ctx->gcm_last_input_fill -= last_blk_len;
+
+			/* Shift what remains in the input buffer forward */
+			bcopy(ctx->gcm_last_input + last_blk_len,
+			    ctx->gcm_last_input, ctx->gcm_last_input_fill);
+		}
+		/* Now the last_input buffer holds just the auth tag */
 	}
-out:
-	ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len));
-	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
+
+	ASSERT(ctx->gcm_last_input_fill == ctx->gcm_tag_len);
+
+	ctx->gcm_len_a_len_c[1] =
+	    htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len));
+	GHASH(ctx, ctx->gcm_len_a_len_c, ctx->gcm_ghash);
 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
 	    (uint8_t *)ctx->gcm_J0);
-	xor_block((uint8_t *)ctx->gcm_J0, ghash);
+	xor_block((uint8_t *)ctx->gcm_J0, (uint8_t *)ctx->gcm_ghash);
 
+	GCM_ACCEL_EXIT;
+
 	/* compare the input authentication tag with what we calculated */
-	if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) {
-		/* They don't match */
-		return (CRYPTO_INVALID_MAC);
-	} else {
-		rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len);
-		if (rv != CRYPTO_SUCCESS)
-			return (rv);
-		out->cd_offset += pt_len;
-	}
+	if (bcmp(&ctx->gcm_last_input, ctx->gcm_ghash, ctx->gcm_tag_len) != 0)
+		return (SET_ERROR(CRYPTO_INVALID_MAC));
+
 	return (CRYPTO_SUCCESS);
+
+errout:
+	GCM_ACCEL_EXIT;
+	return (rv);
 }
 
 static int
@@ -500,20 +731,21 @@
 	case 128:
 		break;
 	default:
-		return (CRYPTO_MECHANISM_PARAM_INVALID);
+		return (SET_ERROR(CRYPTO_MECHANISM_PARAM_INVALID));
 	}
 
 	if (gcm_param->ulIvLen == 0)
-		return (CRYPTO_MECHANISM_PARAM_INVALID);
+		return (SET_ERROR(CRYPTO_MECHANISM_PARAM_INVALID));
 
 	return (CRYPTO_SUCCESS);
 }
 
+/*ARGSUSED*/
 static void
 gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len,
     gcm_ctx_t *ctx, size_t block_size,
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint8_t *cb;
 	ulong_t remainder = iv_len;
@@ -564,12 +796,14 @@
 gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
     unsigned char *auth_data, size_t auth_data_len, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint8_t *ghash, *datap, *authp;
 	size_t remainder, processed;
 
+	GCM_ACCEL_ENTER;
+
 	/* encrypt zero block to get subkey H */
 	bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H,
@@ -578,6 +812,16 @@
 	gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
 	    copy_block, xor_block);
 
+#ifdef	__amd64
+	if (intel_pclmulqdq_instruction_present()) {
+		uint64_t H_bswap64[2] = {
+		    ntohll(ctx->gcm_H[0]), ntohll(ctx->gcm_H[1])
+		};
+
+		gcm_init_clmul(H_bswap64, ctx->gcm_H_table);
+	}
+#endif
+
 	authp = (uint8_t *)ctx->gcm_tmp;
 	ghash = (uint8_t *)ctx->gcm_ghash;
 	bzero(authp, block_size);
@@ -606,6 +850,8 @@
 
 	} while (remainder > 0);
 
+	GCM_ACCEL_EXIT;
+
 	return (CRYPTO_SUCCESS);
 }
 
@@ -612,9 +858,13 @@
 int
 gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
+	/*
+	 * No GHASH invocations in this function and gcm_init does its own
+	 * FPU saving, so no need to GCM_ACCEL_ENTER/GCM_ACCEL_EXIT here.
+	 */
 	int rv;
 	CK_AES_GCM_PARAMS *gcm_param;
 
@@ -652,9 +902,13 @@
 int
 gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
+	/*
+	 * No GHASH invocations in this function and gcm_init does its own
+	 * FPU saving, so no need to GCM_ACCEL_ENTER/GCM_ACCEL_EXIT here.
+	 */
 	int rv;
 	CK_AES_GMAC_PARAMS *gmac_param;
 
@@ -732,7 +986,7 @@
  * Note: the userland version uses getisax().  The kernel version uses
  * is_x86_featureset().
  */
-static int
+static inline int
 intel_pclmulqdq_instruction_present(void)
 {
 	static int	cached_result = -1;
--- old/usr/src/common/crypto/modes/modes.c	Thu Apr 30 20:52:31 2015
+++ new/usr/src/common/crypto/modes/modes.c	Thu Apr 30 20:52:31 2015
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef _KERNEL
 #include <stdlib.h>
@@ -78,94 +81,7 @@
 	} /* end switch */
 }
 
-/*
- * Get pointers for where in the output to copy a block of encrypted or
- * decrypted data.  The iov_or_mp argument stores a pointer to the current
- * iovec or mp, and offset stores an offset into the current iovec or mp.
- */
 void
-crypto_get_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset,
-    uint8_t **out_data_1, size_t *out_data_1_len, uint8_t **out_data_2,
-    size_t amt)
-{
-	offset_t offset;
-
-	switch (out->cd_format) {
-	case CRYPTO_DATA_RAW: {
-		iovec_t *iov;
-
-		offset = *current_offset;
-		iov = &out->cd_raw;
-		if ((offset + amt) <= iov->iov_len) {
-			/* one block fits */
-			*out_data_1 = (uint8_t *)iov->iov_base + offset;
-			*out_data_1_len = amt;
-			*out_data_2 = NULL;
-			*current_offset = offset + amt;
-		}
-		break;
-	}
-
-	case CRYPTO_DATA_UIO: {
-		uio_t *uio = out->cd_uio;
-		iovec_t *iov;
-		offset_t offset;
-		uintptr_t vec_idx;
-		uint8_t *p;
-
-		offset = *current_offset;
-		vec_idx = (uintptr_t)(*iov_or_mp);
-		iov = &uio->uio_iov[vec_idx];
-		p = (uint8_t *)iov->iov_base + offset;
-		*out_data_1 = p;
-
-		if (offset + amt <= iov->iov_len) {
-			/* can fit one block into this iov */
-			*out_data_1_len = amt;
-			*out_data_2 = NULL;
-			*current_offset = offset + amt;
-		} else {
-			/* one block spans two iovecs */
-			*out_data_1_len = iov->iov_len - offset;
-			if (vec_idx == uio->uio_iovcnt)
-				return;
-			vec_idx++;
-			iov = &uio->uio_iov[vec_idx];
-			*out_data_2 = (uint8_t *)iov->iov_base;
-			*current_offset = amt - *out_data_1_len;
-		}
-		*iov_or_mp = (void *)vec_idx;
-		break;
-	}
-
-	case CRYPTO_DATA_MBLK: {
-		mblk_t *mp;
-		uint8_t *p;
-
-		offset = *current_offset;
-		mp = (mblk_t *)*iov_or_mp;
-		p = mp->b_rptr + offset;
-		*out_data_1 = p;
-		if ((p + amt) <= mp->b_wptr) {
-			/* can fit one block into this mblk */
-			*out_data_1_len = amt;
-			*out_data_2 = NULL;
-			*current_offset = offset + amt;
-		} else {
-			/* one block spans two mblks */
-			*out_data_1_len = _PTRDIFF(mp->b_wptr, p);
-			if ((mp = mp->b_cont) == NULL)
-				return;
-			*out_data_2 = mp->b_rptr;
-			*current_offset = (amt - *out_data_1_len);
-		}
-		*iov_or_mp = mp;
-		break;
-	}
-	} /* end switch */
-}
-
-void
 crypto_free_mode_ctx(void *ctx)
 {
 	common_ctx_t *common_ctx = (common_ctx_t *)ctx;
@@ -213,14 +129,8 @@
 	case GCM_MODE:
 	case GMAC_MODE:
 #ifdef _KERNEL
-		if (((gcm_ctx_t *)ctx)->gcm_pt_buf != NULL)
-			kmem_free(((gcm_ctx_t *)ctx)->gcm_pt_buf,
-			    ((gcm_ctx_t *)ctx)->gcm_pt_buf_len);
-
 		kmem_free(ctx, sizeof (gcm_ctx_t));
 #else
-		if (((gcm_ctx_t *)ctx)->gcm_pt_buf != NULL)
-			free(((gcm_ctx_t *)ctx)->gcm_pt_buf);
 		free(ctx);
 #endif
 	}
--- old/usr/src/common/crypto/modes/modes.h	Thu Apr 30 20:52:31 2015
+++ new/usr/src/common/crypto/modes/modes.h	Thu Apr 30 20:52:31 2015
@@ -22,6 +22,9 @@
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #ifndef	_COMMON_CRYPTO_MODES_H
 #define	_COMMON_CRYPTO_MODES_H
@@ -187,34 +190,44 @@
  *			Length of processed plaintext (encrypt) or
  *			length of processed ciphertext (decrypt).
  *
- * gcm_pt_buf:		Stores the decrypted plaintext returned by
- *			decrypt_final when the computed authentication
- *			tag matches the	user supplied tag.
- *
- * gcm_pt_buf_len:	Length of the plaintext buffer.
- *
  * gcm_H:		Subkey.
  *
+ * gcm_H_table:		Pipelined Karatsuba multipliers.
+ *
  * gcm_J0:		Pre-counter block generated from the IV.
  *
+ * gcm_tmp:		Temp storage for ciphertext when padding is needed.
+ *
  * gcm_len_a_len_c:	64-bit representations of the bit lengths of
  *			AAD and ciphertext.
  *
  * gcm_kmflag:		Current value of kmflag. Used only for allocating
  *			the plaintext buffer during decryption.
+ *
+ * gcm_last_input:	Buffer of (up to) two last blocks. This is used when
+ *			input is not block-aligned and to temporarily hold
+ *			the end of the ciphertext stream during decryption,
+ *			since that could potentially be the GHASH auth tag
+ *			which we must check in the final() call instead of
+ *			decrypting it.
+ *
+ * gcm_last_input_fill:	Number of bytes actually stored in gcm_last_input.
  */
 typedef struct gcm_ctx {
 	struct common_ctx gcm_common;
 	size_t gcm_tag_len;
 	size_t gcm_processed_data_len;
-	size_t gcm_pt_buf_len;
-	uint32_t gcm_tmp[4];
 	uint64_t gcm_ghash[2];
 	uint64_t gcm_H[2];
+#ifdef	__amd64
+	uint8_t gcm_H_table[256];
+#endif
 	uint64_t gcm_J0[2];
+	uint64_t gcm_tmp[2];
 	uint64_t gcm_len_a_len_c[2];
-	uint8_t *gcm_pt_buf;
 	int gcm_kmflag;
+	uint8_t gcm_last_input[32];
+	size_t gcm_last_input_fill;
 } gcm_ctx_t;
 
 #define	gcm_keysched		gcm_common.cc_keysched
@@ -283,99 +296,109 @@
 #define	dc_lastp		dcu.dcu_ecb.ecb_common.cc_lastp
 
 extern int ecb_cipher_contiguous_blocks(ecb_ctx_t *, char *, size_t,
-    crypto_data_t *, size_t, int (*cipher)(const void *, const uint8_t *,
-    uint8_t *));
+    crypto_data_t *, size_t,
+    int (*cipher)(const void *, const uint8_t *, uint8_t *),
+    int (*cipher_ecb)(const void *, const uint8_t *, uint8_t *, uint64_t));
 
 extern int cbc_encrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t,
     crypto_data_t *, size_t,
     int (*encrypt)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*encrypt_cbc)(const void *, const uint8_t *, uint8_t *,
+    const uint8_t *, uint64_t));
 
 extern int cbc_decrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t,
     crypto_data_t *, size_t,
     int (*decrypt)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*decrypt_ecb)(const void *, const uint8_t *, uint8_t *, uint64_t),
+    void (*xor_block_range)(const uint8_t *, uint8_t *, uint64_t));
 
 extern int ctr_mode_contiguous_blocks(ctr_ctx_t *, char *, size_t,
     crypto_data_t *, size_t,
     int (*cipher)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *));
 
 extern int ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t,
     crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 extern int ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t,
     crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 extern int gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t,
     crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *));
 
 extern int gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t,
     crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *));
 
 int ccm_encrypt_final(ccm_ctx_t *, crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 int gcm_encrypt_final(gcm_ctx_t *, crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 extern int ccm_decrypt_final(ccm_ctx_t *, crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 extern int gcm_decrypt_final(gcm_ctx_t *, crypto_data_t *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *));
 
 extern int ctr_mode_final(ctr_ctx_t *, crypto_data_t *,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *));
 
 extern int cbc_init_ctx(cbc_ctx_t *, char *, size_t, size_t,
-    void (*copy_block)(uint8_t *, uint64_t *));
+    void (*copy_block)(const uint8_t *, uint64_t *));
 
 extern int ctr_init_ctx(ctr_ctx_t *, ulong_t, uint8_t *,
-    void (*copy_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *));
 
 extern int ccm_init_ctx(ccm_ctx_t *, char *, int, boolean_t, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 extern int gcm_init_ctx(gcm_ctx_t *, char *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 extern int gmac_init_ctx(gcm_ctx_t *, char *, size_t,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *));
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *));
 
 extern void calculate_ccm_mac(ccm_ctx_t *, uint8_t *,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *));
 
-extern void gcm_mul(uint64_t *, uint64_t *, uint64_t *);
-
 extern void crypto_init_ptrs(crypto_data_t *, void **, offset_t *);
-extern void crypto_get_ptrs(crypto_data_t *, void **, offset_t *,
-    uint8_t **, size_t *, uint8_t **, size_t);
 
 extern void *ecb_alloc_ctx(int);
 extern void *cbc_alloc_ctx(int);
@@ -386,6 +409,115 @@
 extern void crypto_free_mode_ctx(void *);
 extern void gcm_set_kmflag(gcm_ctx_t *, int);
 
+#ifdef	INLINE_CRYPTO_GET_PTRS
+/*
+ * Get pointers for where in the output to copy a block of encrypted or
+ * decrypted data.  The iov_or_mp argument stores a pointer to the current
+ * iovec or mp, and offset stores an offset into the current iovec or mp.
+ */
+static inline void
+crypto_get_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset,
+    uint8_t **out_data_1, size_t *out_data_1_len, uint8_t **out_data_2,
+    size_t amt)
+{
+	offset_t offset;
+
+	switch (out->cd_format) {
+	case CRYPTO_DATA_RAW: {
+		iovec_t *iov;
+
+		offset = *current_offset;
+		iov = &out->cd_raw;
+		if ((offset + amt) <= iov->iov_len) {
+			/* one block fits */
+			*out_data_1 = (uint8_t *)iov->iov_base + offset;
+			*out_data_1_len = amt;
+			*out_data_2 = NULL;
+			*current_offset = offset + amt;
+		}
+		break;
+	}
+
+	case CRYPTO_DATA_UIO: {
+		uio_t *uio = out->cd_uio;
+		iovec_t *iov;
+		offset_t offset;
+		uintptr_t vec_idx;
+		uint8_t *p;
+
+		offset = *current_offset;
+		vec_idx = (uintptr_t)(*iov_or_mp);
+		iov = &uio->uio_iov[vec_idx];
+		p = (uint8_t *)iov->iov_base + offset;
+		*out_data_1 = p;
+
+		if (offset + amt <= iov->iov_len) {
+			/* can fit one block into this iov */
+			*out_data_1_len = amt;
+			*out_data_2 = NULL;
+			*current_offset = offset + amt;
+		} else {
+			/* one block spans two iovecs */
+			*out_data_1_len = iov->iov_len - offset;
+			if (vec_idx == uio->uio_iovcnt)
+				return;
+			vec_idx++;
+			iov = &uio->uio_iov[vec_idx];
+			*out_data_2 = (uint8_t *)iov->iov_base;
+			*current_offset = amt - *out_data_1_len;
+		}
+		*iov_or_mp = (void *)vec_idx;
+		break;
+	}
+
+	case CRYPTO_DATA_MBLK: {
+		mblk_t *mp;
+		uint8_t *p;
+
+		offset = *current_offset;
+		mp = (mblk_t *)*iov_or_mp;
+		p = mp->b_rptr + offset;
+		*out_data_1 = p;
+		if ((p + amt) <= mp->b_wptr) {
+			/* can fit one block into this mblk */
+			*out_data_1_len = amt;
+			*out_data_2 = NULL;
+			*current_offset = offset + amt;
+		} else {
+			/* one block spans two mblks */
+			*out_data_1_len = _PTRDIFF(mp->b_wptr, p);
+			if ((mp = mp->b_cont) == NULL)
+				return;
+			*out_data_2 = mp->b_rptr;
+			*current_offset = (amt - *out_data_1_len);
+		}
+		*iov_or_mp = mp;
+		break;
+	}
+	} /* end switch */
+}
+#endif	/* INLINE_CRYPTO_GET_PTRS */
+
+/*
+ * Checks whether a crypto_data_t object is composed of a single contiguous
+ * buffer. This is used in all fastpath detection code to avoid the
+ * possibility of having to do partial block splicing.
+ */
+#define	CRYPTO_DATA_IS_SINGLE_BLOCK(cd) \
+	(cd != NULL && (cd->cd_format == CRYPTO_DATA_RAW || \
+	(cd->cd_format == CRYPTO_DATA_UIO && cd->cd_uio->uio_iovcnt == 1) || \
+	(cd->cd_format == CRYPTO_DATA_MBLK && cd->cd_mp->b_next == NULL)))
+
+/*
+ * Returns the first contiguous data buffer in a crypto_data_t object.
+ */
+#define	CRYPTO_DATA_FIRST_BLOCK(cd) \
+	(cd->cd_format == CRYPTO_DATA_RAW ? \
+	(void *)(cd->cd_raw.iov_base + cd->cd_offset) : \
+	(cd->cd_format == CRYPTO_DATA_UIO ? \
+	(void *)(cd->cd_uio->uio_iov[0].iov_base + cd->cd_offset) : \
+	(void *)(cd->cd_mp->b_rptr + cd->cd_offset)))
+
 #ifdef	__cplusplus
 }
 #endif
--- old/usr/src/lib/pkcs11/libsoftcrypto/amd64/Makefile	Thu Apr 30 20:52:31 2015
+++ new/usr/src/lib/pkcs11/libsoftcrypto/amd64/Makefile	Thu Apr 30 20:52:31 2015
@@ -21,6 +21,7 @@
 
 #
 # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2015 by Saso Kiselkov. All rights reserved.
 #
 
 LIBRARY =	libsoftcrypto.a
@@ -38,6 +39,7 @@
 CERRWARN +=	-_gcc=-Wno-type-limits
 CERRWARN +=	-_gcc=-Wno-parentheses
 CERRWARN +=	-_gcc=-Wno-uninitialized
+C99MODE =	$(C99_ENABLE)
 
 AES_PSM_SRC =	$(AES_DIR)/$(MACH64)/aes_amd64.s \
 		$(AES_DIR)/$(MACH64)/aes_intel.s \
--- old/usr/src/lib/pkcs11/libsoftcrypto/i386/Makefile	Thu Apr 30 20:52:31 2015
+++ new/usr/src/lib/pkcs11/libsoftcrypto/i386/Makefile	Thu Apr 30 20:52:31 2015
@@ -21,6 +21,7 @@
 
 #
 # Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright 2015 by Saso Kiselkov. All rights reserved.
 #
 
 LIBRARY =	libsoftcrypto.a
@@ -33,6 +34,7 @@
 CERRWARN +=	-_gcc=-Wno-type-limits
 CERRWARN +=	-_gcc=-Wno-parentheses
 CERRWARN +=	-_gcc=-Wno-uninitialized
+C99MODE =	$(C99_ENABLE)
 
 # Sources need to be redefined after Makefile.lib inclusion.
 SRCS =		$(AES_SRC) $(ARCFOUR_SRC) $(BIGNUM_SRC) $(BLOWFISH_SRC) \
--- old/usr/src/uts/common/crypto/core/kcf_prov_lib.c	Thu Apr 30 20:52:32 2015
+++ new/usr/src/uts/common/crypto/core/kcf_prov_lib.c	Thu Apr 30 20:52:32 2015
@@ -22,6 +22,9 @@
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 #include <sys/strsun.h>
 #include <sys/systm.h>
@@ -396,7 +399,7 @@
 int
 crypto_update_iov(void *ctx, crypto_data_t *input, crypto_data_t *output,
     int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
-    void (*copy_block)(uint8_t *, uint64_t *))
+    void (*copy_block)(const uint8_t *, uint64_t *))
 {
 	common_ctx_t *common_ctx = ctx;
 	int rv;
@@ -418,7 +421,7 @@
 int
 crypto_update_uio(void *ctx, crypto_data_t *input, crypto_data_t *output,
     int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
-    void (*copy_block)(uint8_t *, uint64_t *))
+    void (*copy_block)(const uint8_t *, uint64_t *))
 {
 	common_ctx_t *common_ctx = ctx;
 	uio_t *uiop = input->cd_uio;
@@ -483,7 +486,7 @@
 int
 crypto_update_mp(void *ctx, crypto_data_t *input, crypto_data_t *output,
     int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
-    void (*copy_block)(uint8_t *, uint64_t *))
+    void (*copy_block)(const uint8_t *, uint64_t *))
 {
 	common_ctx_t *common_ctx = ctx;
 	off_t offset = input->cd_offset;
--- old/usr/src/uts/common/crypto/io/aes.c	Thu Apr 30 20:52:32 2015
+++ new/usr/src/uts/common/crypto/io/aes.c	Thu Apr 30 20:52:32 2015
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
  */
 
 /*
@@ -386,7 +387,7 @@
 }
 
 static void
-aes_copy_block64(uint8_t *in, uint64_t *out)
+aes_copy_block64(const uint8_t *in, uint64_t *out)
 {
 	if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
 		/* LINTED: pointer alignment */
@@ -396,7 +397,7 @@
 	} else {
 		uint8_t *iv8 = (uint8_t *)&out[0];
 
-		AES_COPY_BLOCK(in, iv8);
+		AES_COPY_BLOCK_UNALIGNED(in, iv8);
 	}
 }
 
@@ -475,7 +476,7 @@
 		ciphertext->cd_offset = ciphertext->cd_length;
 		ciphertext->cd_length = saved_length - ciphertext->cd_length;
 		ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
 		if (ret != CRYPTO_SUCCESS) {
 			return (ret);
 		}
@@ -496,8 +497,8 @@
 		ciphertext->cd_offset = ciphertext->cd_length;
 		ciphertext->cd_length = saved_length - ciphertext->cd_length;
 		ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+		    AES_XOR_BLOCK);
 		if (ret != CRYPTO_SUCCESS) {
 			return (ret);
 		}
@@ -590,8 +591,8 @@
 		plaintext->cd_length = saved_length - plaintext->cd_length;
 
 		ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+		    AES_XOR_BLOCK);
 		if (ret == CRYPTO_SUCCESS) {
 			if (plaintext != ciphertext) {
 				plaintext->cd_length =
@@ -608,7 +609,8 @@
 		plaintext->cd_length = saved_length - plaintext->cd_length;
 
 		ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK,
+		    AES_COPY_BLOCK, aes_ctr_mode);
 		if (ret == CRYPTO_SUCCESS) {
 			if (plaintext != ciphertext) {
 				plaintext->cd_length =
@@ -722,13 +724,15 @@
 
 	/*
 	 * Compute number of bytes that will hold the plaintext.
-	 * This is not necessary for CCM, GCM, and GMAC since these
+	 * This is not necessary for CCM and GMAC since these
 	 * mechanisms never return plaintext for update operations.
 	 */
-	if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
+	if ((aes_ctx->ac_flags & (CCM_MODE|GMAC_MODE)) == 0) {
 		out_len = aes_ctx->ac_remainder_len;
 		out_len += ciphertext->cd_length;
 		out_len &= ~(AES_BLOCK_LEN - 1);
+		if (aes_ctx->ac_flags & GCM_MODE)
+			out_len -= ((gcm_ctx_t *)aes_ctx)->gcm_tag_len;
 
 		/* return length needed to store the output */
 		if (plaintext->cd_length < out_len) {
@@ -818,7 +822,7 @@
 		}
 	} else if (aes_ctx->ac_flags & CCM_MODE) {
 		ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_XOR_BLOCK);
 		if (ret != CRYPTO_SUCCESS) {
 			return (ret);
 		}
@@ -826,8 +830,8 @@
 		size_t saved_offset = data->cd_offset;
 
 		ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+		    AES_XOR_BLOCK);
 		if (ret != CRYPTO_SUCCESS) {
 			return (ret);
 		}
@@ -903,8 +907,8 @@
 		saved_offset = data->cd_offset;
 		saved_length = data->cd_length;
 		ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+		    AES_XOR_BLOCK);
 		if (ret == CRYPTO_SUCCESS) {
 			data->cd_length = data->cd_offset - saved_offset;
 		} else {
@@ -917,21 +921,21 @@
 		}
 	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
 		/*
-		 * This is where all the plaintext is returned, make sure
-		 * the plaintext buffer is big enough
+		 * Check to make sure there is enough space for remaining
+		 * plaintext.
 		 */
 		gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
-		size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
+		size_t pt_len = ctx->gcm_last_input_fill - ctx->gcm_tag_len;
 
 		if (data->cd_length < pt_len) {
 			data->cd_length = pt_len;
 			return (CRYPTO_BUFFER_TOO_SMALL);
 		}
-
 		saved_offset = data->cd_offset;
 		saved_length = data->cd_length;
 		ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+		    AES_XOR_BLOCK, aes_ctr_mode);
 		if (ret == CRYPTO_SUCCESS) {
 			data->cd_length = data->cd_offset - saved_offset;
 		} else {
@@ -944,7 +948,6 @@
 		}
 	}
 
-
 	if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
 		data->cd_length = 0;
 	}
@@ -1043,7 +1046,7 @@
 		if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) {
 			ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
 			    ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
-			    aes_xor_block);
+			    AES_XOR_BLOCK);
 			if (ret != CRYPTO_SUCCESS)
 				goto out;
 			ASSERT(aes_ctx.ac_remainder_len == 0);
@@ -1051,7 +1054,7 @@
 		    mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
 			ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
 			    ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
-			    aes_copy_block, aes_xor_block);
+			    AES_COPY_BLOCK, AES_XOR_BLOCK);
 			if (ret != CRYPTO_SUCCESS)
 				goto out;
 			ASSERT(aes_ctx.ac_remainder_len == 0);
@@ -1182,7 +1185,7 @@
 			    == aes_ctx.ac_mac_len);
 			ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
 			    plaintext, AES_BLOCK_LEN, aes_encrypt_block,
-			    aes_copy_block, aes_xor_block);
+			    AES_COPY_BLOCK, AES_XOR_BLOCK);
 			ASSERT(aes_ctx.ac_remainder_len == 0);
 			if ((ret == CRYPTO_SUCCESS) &&
 			    (ciphertext != plaintext)) {
@@ -1195,7 +1198,7 @@
 		    mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
 			ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
 			    plaintext, AES_BLOCK_LEN, aes_encrypt_block,
-			    aes_xor_block);
+			    AES_COPY_BLOCK, AES_XOR_BLOCK, aes_ctr_mode);
 			ASSERT(aes_ctx.ac_remainder_len == 0);
 			if ((ret == CRYPTO_SUCCESS) &&
 			    (ciphertext != plaintext)) {
@@ -1237,11 +1240,6 @@
 		if (aes_ctx.ac_pt_buf != NULL) {
 			kmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len);
 		}
-	} else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) {
-		if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) {
-			kmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
-			    ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
-		}
 	}
 
 	return (ret);
@@ -1352,7 +1350,7 @@
 		}
 		pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
 		rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
-		    pp->cb, aes_copy_block);
+		    pp->cb, AES_COPY_BLOCK);
 		break;
 	}
 	case AES_CCM_MECH_INFO_TYPE:
@@ -1362,7 +1360,7 @@
 		}
 		rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
 		    kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
-		    aes_xor_block);
+		    AES_XOR_BLOCK);
 		break;
 	case AES_GCM_MECH_INFO_TYPE:
 		if (mechanism->cm_param == NULL ||
@@ -1370,8 +1368,8 @@
 			return (CRYPTO_MECHANISM_PARAM_INVALID);
 		}
 		rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+		    AES_XOR_BLOCK);
 		break;
 	case AES_GMAC_MECH_INFO_TYPE:
 		if (mechanism->cm_param == NULL ||
@@ -1379,8 +1377,8 @@
 			return (CRYPTO_MECHANISM_PARAM_INVALID);
 		}
 		rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
-		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
-		    aes_xor_block);
+		    AES_BLOCK_LEN, aes_encrypt_block, AES_COPY_BLOCK,
+		    AES_XOR_BLOCK);
 		break;
 	case AES_ECB_MECH_INFO_TYPE:
 		aes_ctx->ac_flags |= ECB_MODE;
--- old/usr/src/uts/common/crypto/io/blowfish.c	Thu Apr 30 20:52:32 2015
+++ new/usr/src/uts/common/crypto/io/blowfish.c	Thu Apr 30 20:52:32 2015
@@ -22,6 +22,9 @@
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 /*
  * Blowfish provider for the Kernel Cryptographic Framework (KCF)
@@ -319,7 +322,7 @@
 }
 
 static void
-blowfish_copy_block64(uint8_t *in, uint64_t *out)
+blowfish_copy_block64(const uint8_t *in, uint64_t *out)
 {
 	if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
 		/* LINTED: pointer alignment */
--- old/usr/src/uts/common/des/des_crypt.c	Thu Apr 30 20:52:32 2015
+++ new/usr/src/uts/common/des/des_crypt.c	Thu Apr 30 20:52:32 2015
@@ -23,6 +23,9 @@
  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
+/*
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
+ */
 
 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
 /*	  All Rights Reserved  	*/
@@ -492,7 +495,7 @@
 }
 
 static void
-des_copy_block64(uint8_t *in, uint64_t *out)
+des_copy_block64(const uint8_t *in, uint64_t *out)
 {
 	if (IS_P2ALIGNED(in, sizeof (uint64_t))) {
 		/* LINTED: pointer alignment */
--- old/usr/src/uts/common/sys/crypto/impl.h	Thu Apr 30 20:52:32 2015
+++ new/usr/src/uts/common/sys/crypto/impl.h	Thu Apr 30 20:52:32 2015
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
  */
 
 #ifndef	_SYS_CRYPTO_IMPL_H
@@ -1356,13 +1357,13 @@
     void (*update)(), void (*final)(), uchar_t);
 extern int crypto_update_iov(void *, crypto_data_t *, crypto_data_t *,
     int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
-    void (*copy_block)(uint8_t *, uint64_t *));
+    void (*copy_block)(const uint8_t *, uint64_t *));
 extern int crypto_update_uio(void *, crypto_data_t *, crypto_data_t *,
     int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
-    void (*copy_block)(uint8_t *, uint64_t *));
+    void (*copy_block)(const uint8_t *, uint64_t *));
 extern int crypto_update_mp(void *, crypto_data_t *, crypto_data_t *,
     int (*cipher)(void *, caddr_t, size_t, crypto_data_t *),
-    void (*copy_block)(uint8_t *, uint64_t *));
+    void (*copy_block)(const uint8_t *, uint64_t *));
 extern int crypto_get_key_attr(crypto_key_t *, crypto_attr_type_t, uchar_t **,
     ssize_t *);
 
--- old/usr/src/uts/intel/kcf/Makefile	Thu Apr 30 20:52:32 2015
+++ new/usr/src/uts/intel/kcf/Makefile	Thu Apr 30 20:52:32 2015
@@ -21,6 +21,7 @@
 #
 # Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
+# Copyright 2015 by Saso Kiselkov. All rights reserved.
 #
 #	This makefile drives the production of the Kernel Cryptographic
 #	Framework core module.
@@ -40,7 +41,7 @@
 MODULE		= kcf
 LINTS		= $(KCF_OBJS:%.o=$(LINTS_DIR)/%.ln)
 KCF_OBJS_32	=
-KCF_OBJS_64	= gcm_intel.o
+KCF_OBJS_64	= gcm_intel.o gcm_intel_cryptogams.o
 KCF_OBJS	+= $(KCF_OBJS_$(CLASS))
 OBJECTS		= $(KCF_OBJS:%=$(OBJS_DIR)/%)
 ROOTMODULE	= $(ROOT_MISC_DIR)/$(MODULE)
