--- old/usr/src/common/crypto/modes/gcm.c	Thu Apr 30 20:52:31 2015
+++ new/usr/src/common/crypto/modes/gcm.c	Thu Apr 30 20:52:31 2015
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright 2015 by Saso Kiselkov. All rights reserved.
  */
 
 
@@ -30,14 +31,21 @@
 #include <security/cryptoki.h>
 #endif	/* _KERNEL */
 
-
+#include <sys/cmn_err.h>
 #include <sys/types.h>
 #include <sys/kmem.h>
+#define	INLINE_CRYPTO_GET_PTRS
 #include <modes/modes.h>
 #include <sys/crypto/common.h>
 #include <sys/crypto/impl.h>
 #include <sys/byteorder.h>
 
+#define	COUNTER_MASK	0x00000000ffffffffULL
+
+#ifdef	_KERNEL
+#include <sys/sdt.h>		/* SET_ERROR */
+#endif	/* _KERNEL */
+
 #ifdef __amd64
 
 #ifdef _KERNEL
@@ -45,19 +53,57 @@
 #include <sys/x86_archext.h>	/* x86_featureset, X86FSET_*, CPUID_* */
 #include <sys/disp.h>		/* kpreempt_disable(), kpreempt_enable */
 /* Workaround for no XMM kernel thread save/restore */
-#define	KPREEMPT_DISABLE	kpreempt_disable()
-#define	KPREEMPT_ENABLE		kpreempt_enable()
+extern void gcm_accel_save(void *savestate);
+extern void gcm_accel_restore(void *savestate);
 
+#if	defined(lint) || defined(__lint)
+#define	GCM_ACCEL_SAVESTATE(name)	uint8_t name[16 * 16 + 8]
 #else
+#define	GCM_ACCEL_SAVESTATE(name) \
+	/* stack space for xmm0--xmm15 and cr0 (16 x 128 bits + 64 bits) */ \
+	uint8_t name[16 * 16 + 8] __attribute__((aligned(16)))
+#endif
+
+/*
+ * Disables kernel thread preemption and conditionally gcm_accel_save() iff
+ * Intel PCLMULQDQ support is present. Must be balanced by GCM_ACCEL_EXIT.
+ * This must be present in all externally callable GCM functions which
+ * invoke GHASH operations using FPU-accelerated implementations, or call
+ * static functions which do (such as gcm_encrypt_fastpath128()).
+ */
+#define	GCM_ACCEL_ENTER \
+	GCM_ACCEL_SAVESTATE(savestate); \
+	do { \
+		if (intel_pclmulqdq_instruction_present()) { \
+			kpreempt_disable(); \
+			gcm_accel_save(savestate); \
+		} \
+		_NOTE(CONSTCOND) \
+	} while (0)
+#define	GCM_ACCEL_EXIT \
+	do { \
+		if (intel_pclmulqdq_instruction_present()) { \
+			gcm_accel_restore(savestate); \
+			kpreempt_enable(); \
+		} \
+		_NOTE(CONSTCOND) \
+	} while (0)
+
+#else	/* _KERNEL */
 #include <sys/auxv.h>		/* getisax() */
 #include <sys/auxv_386.h>	/* AV_386_PCLMULQDQ bit */
-#define	KPREEMPT_DISABLE
-#define	KPREEMPT_ENABLE
+#define	SET_ERROR(x)	(x)
 #endif	/* _KERNEL */
 
 extern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res);
-static int intel_pclmulqdq_instruction_present(void);
-#endif	/* __amd64 */
+extern void gcm_init_clmul(const uint64_t hash_init[2], uint8_t Htable[256]);
+extern void gcm_ghash_clmul(uint64_t ghash[2], const uint8_t Htable[256],
+    const uint8_t *inp, size_t length);
+static inline int intel_pclmulqdq_instruction_present(void);
+#else	/* !__amd64 */
+#define	GCM_ACCEL_ENTER
+#define	GCM_ACCEL_EXIT
+#endif	/* !__amd64 */
 
 struct aes_block {
 	uint64_t a;
@@ -75,14 +121,16 @@
  * Note: x_in, y, and res all point to 16-byte numbers (an array of two
  * 64-bit integers).
  */
-void
+static inline void
 gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res)
 {
 #ifdef __amd64
 	if (intel_pclmulqdq_instruction_present()) {
-		KPREEMPT_DISABLE;
+		/*
+		 * FPU context will have been saved and kernel thread
+		 * preemption disabled already.
+		 */
 		gcm_mul_pclmulqdq(x_in, y, res);
-		KPREEMPT_ENABLE;
 	} else
 #endif	/* __amd64 */
 	{
@@ -116,24 +164,93 @@
 	}
 }
 
-
 #define	GHASH(c, d, t) \
-	xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
-	gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
-	(uint64_t *)(void *)(t));
+	do { \
+		xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \
+		gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \
+		    (uint64_t *)(void *)(t)); \
+		_NOTE(CONSTCOND) \
+	} while (0)
 
+boolean_t gcm_fastpath_enabled = B_TRUE;
 
-/*
- * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
- * is done in another function.
- */
-int
-gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
-    crypto_data_t *out, size_t block_size,
+static void
+gcm_fastpath128(gcm_ctx_t *ctx, const uint8_t *data, size_t length,
+    uint8_t *out, boolean_t encrypt,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
 {
+	/* When decrypting, `data' holds the ciphertext we need to GHASH. */
+	if (!encrypt) {
+#ifdef	__amd64
+		if (intel_pclmulqdq_instruction_present())
+			gcm_ghash_clmul(ctx->gcm_ghash, ctx->gcm_H_table,
+			    data, length);
+		else
+#endif	/* __amd64 */
+			for (size_t i = 0; i < length; i += 16)
+				GHASH(ctx, &data[i], ctx->gcm_ghash);
+	}
+
+	if (cipher_ctr != NULL) {
+		/*
+		 * GCM is almost but not quite like CTR. GCM increments the
+		 * counter value *before* processing the first input block,
+		 * whereas CTR does so afterwards. So we need to increment
+		 * the counter before calling CTR and decrement it afterwards.
+		 */
+		uint64_t counter = ntohll(ctx->gcm_cb[1]);
+
+		ctx->gcm_cb[1] = htonll((counter & ~COUNTER_MASK) |
+		    ((counter & COUNTER_MASK) + 1));
+		cipher_ctr(ctx->gcm_keysched, data, out, length, ctx->gcm_cb);
+		counter = ntohll(ctx->gcm_cb[1]);
+		ctx->gcm_cb[1] = htonll((counter & ~COUNTER_MASK) |
+		    ((counter & COUNTER_MASK) - 1));
+	} else {
+		uint64_t counter = ntohll(ctx->gcm_cb[1]);
+
+		for (size_t i = 0; i < length; i += 16) {
+			/*LINTED(E_BAD_PTR_CAST_ALIGN)*/
+			*(uint64_t *)&out[i] = ctx->gcm_cb[0];
+			/*LINTED(E_BAD_PTR_CAST_ALIGN)*/
+			*(uint64_t *)&out[i + 8] = htonll(counter++);
+			encrypt_block(ctx->gcm_keysched, &out[i], &out[i]);
+			xor_block(&data[i], &out[i]);
+		}
+
+		ctx->gcm_cb[1] = htonll(counter);
+	}
+
+	/* When encrypting, `out' holds the ciphertext we need to GHASH. */
+	if (encrypt) {
+#ifdef	__amd64
+		if (intel_pclmulqdq_instruction_present())
+			gcm_ghash_clmul(ctx->gcm_ghash, ctx->gcm_H_table,
+			    out, length);
+		else
+#endif	/* __amd64 */
+			for (size_t i = 0; i < length; i += 16)
+				GHASH(ctx, &out[i], ctx->gcm_ghash);
+
+		/* If no more data comes in, the last block is the auth tag. */
+		bcopy(&out[length - 16], ctx->gcm_tmp, 16);
+	}
+
+	ctx->gcm_processed_data_len += length;
+}
+
+static int
+gcm_process_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size, boolean_t encrypt,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
+{
 	size_t remainder = length;
 	size_t need;
 	uint8_t *datap = (uint8_t *)data;
@@ -146,7 +263,29 @@
 	size_t out_data_1_len;
 	uint64_t counter;
 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
+	int rv = CRYPTO_SUCCESS;
 
+	GCM_ACCEL_ENTER;
+
+	/*
+	 * GCM mode fastpath requirements:
+	 * - fastpath is enabled
+	 * - block size is 128 bits
+	 * - input is block-aligned
+	 * - the counter value won't overflow
+	 * - output is a single contiguous region and doesn't alias input
+	 */
+	if (gcm_fastpath_enabled && block_size == 16 &&
+	    ctx->gcm_remainder_len == 0 && (length & (block_size - 1)) == 0 &&
+	    ntohll(ctx->gcm_cb[1] & counter_mask) <= ntohll(counter_mask) -
+	    length / block_size && CRYPTO_DATA_IS_SINGLE_BLOCK(out)) {
+		gcm_fastpath128(ctx, (uint8_t *)data, length,
+		    CRYPTO_DATA_FIRST_BLOCK(out), encrypt, encrypt_block,
+		    xor_block, cipher_ctr);
+		out->cd_offset += length;
+		goto out;
+	}
+
 	if (length + ctx->gcm_remainder_len < block_size) {
 		/* accumulate bytes here and return */
 		bcopy(datap,
@@ -154,7 +293,7 @@
 		    length);
 		ctx->gcm_remainder_len += length;
 		ctx->gcm_copy_to = datap;
-		return (CRYPTO_SUCCESS);
+		goto out;
 	}
 
 	lastp = (uint8_t *)ctx->gcm_cb;
@@ -166,8 +305,10 @@
 		if (ctx->gcm_remainder_len > 0) {
 			need = block_size - ctx->gcm_remainder_len;
 
-			if (need > remainder)
-				return (CRYPTO_DATA_LEN_RANGE);
+			if (need > remainder) {
+				rv = SET_ERROR(CRYPTO_DATA_LEN_RANGE);
+				goto out;
+			}
 
 			bcopy(datap, &((uint8_t *)ctx->gcm_remainder)
 			    [ctx->gcm_remainder_len], need);
@@ -177,6 +318,10 @@
 			blockp = datap;
 		}
 
+		/* add ciphertext to the hash */
+		if (!encrypt)
+			GHASH(ctx, blockp, ctx->gcm_ghash);
+
 		/*
 		 * Increment counter. Counter bits are confined
 		 * to the bottom 32 bits of the counter block.
@@ -221,7 +366,8 @@
 		}
 
 		/* add ciphertext to the hash */
-		GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+		if (encrypt)
+			GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
 
 		/* Update pointer to next block of data to be processed. */
 		if (ctx->gcm_remainder_len != 0) {
@@ -244,23 +390,47 @@
 
 	} while (remainder > 0);
 out:
-	return (CRYPTO_SUCCESS);
+	GCM_ACCEL_EXIT;
+
+	return (rv);
 }
 
+
+/*
+ * Encrypt multiple blocks of data in GCM mode.  Decrypt for GCM mode
+ * is done in another function.
+ */
+/*ARGSUSED*/
+int
+gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
+    crypto_data_t *out, size_t block_size,
+    int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
+{
+	return (gcm_process_contiguous_blocks(ctx, data, length, out,
+	    block_size, B_TRUE, encrypt_block, copy_block, xor_block,
+	    cipher_ctr));
+}
+
 /* ARGSUSED */
 int
 gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
 	uint8_t *ghash, *macp;
 	int i, rv;
 
-	if (out->cd_length <
-	    (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
-		return (CRYPTO_DATA_LEN_RANGE);
+	GCM_ACCEL_ENTER;
+
+	if (out->cd_length < (ctx->gcm_remainder_len + ctx->gcm_tag_len)) {
+		rv = CRYPTO_DATA_LEN_RANGE;
+		goto out;
 	}
 
 	ghash = (uint8_t *)ctx->gcm_ghash;
@@ -310,16 +480,17 @@
 	if (ctx->gcm_remainder_len > 0) {
 		rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len);
 		if (rv != CRYPTO_SUCCESS)
-			return (rv);
+			goto out;
 	}
 	out->cd_offset += ctx->gcm_remainder_len;
 	ctx->gcm_remainder_len = 0;
 	rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len);
 	if (rv != CRYPTO_SUCCESS)
-		return (rv);
+		goto out;
 	out->cd_offset += ctx->gcm_tag_len;
-
-	return (CRYPTO_SUCCESS);
+out:
+	GCM_ACCEL_EXIT;
+	return (rv);
 }
 
 /*
@@ -326,19 +497,25 @@
  * This will only deal with decrypting the last block of the input that
  * might not be a multiple of block length.
  */
+/*ARGSUSED*/
 static void
-gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index,
+gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, uint8_t *data, size_t length,
+    size_t block_size, crypto_data_t *out,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
-	uint8_t *datap, *outp, *counterp;
 	uint64_t counter;
 	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
-	int i;
 
+	/* padd last block and add to GHASH */
+	bcopy(data, ctx->gcm_tmp, length);
+	bzero(((uint8_t *)ctx->gcm_tmp) + length,
+	    sizeof (ctx->gcm_tmp) - length);
+	GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
+
 	/*
 	 * Increment counter.
-	 * Counter bits are confined to the bottom 32 bits
+	 * Counter bits are confined to the bottom 32 bits.
 	 */
 	counter = ntohll(ctx->gcm_cb[1] & counter_mask);
 	counter = htonll(counter + 1);
@@ -345,23 +522,19 @@
 	counter &= counter_mask;
 	ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
 
-	datap = (uint8_t *)ctx->gcm_remainder;
-	outp = &((ctx->gcm_pt_buf)[index]);
-	counterp = (uint8_t *)ctx->gcm_tmp;
+	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb,
+	    (uint8_t *)ctx->gcm_tmp);
 
-	/* authentication tag */
-	bzero((uint8_t *)ctx->gcm_tmp, block_size);
-	bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len);
-
-	/* add ciphertext to the hash */
-	GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash);
-
-	/* decrypt remaining ciphertext */
-	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp);
-
 	/* XOR with counter block */
-	for (i = 0; i < ctx->gcm_remainder_len; i++) {
-		outp[i] = datap[i] ^ counterp[i];
+	for (size_t i = 0; i < length; i++)
+		((uint8_t *)ctx->gcm_tmp)[i] ^= data[i];
+
+	if (out != NULL) {
+		(void) crypto_put_output_data((uchar_t *)ctx->gcm_tmp, out,
+		    length);
+		out->cd_offset += length;
+	} else {
+		bcopy(ctx->gcm_tmp, data, length);
 	}
 }
 
@@ -370,115 +543,173 @@
 gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length,
     crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
 {
-	size_t new_len;
-	uint8_t *new;
+	int rv = CRYPTO_SUCCESS;
 
+	GCM_ACCEL_ENTER;
+
 	/*
-	 * Copy contiguous ciphertext input blocks to plaintext buffer.
-	 * Ciphertext will be decrypted in the final.
+	 * Previous calls accumulate data in the input buffer to make sure
+	 * we have the auth tag (the last part of the ciphertext) when we
+	 * receive a final() call.
 	 */
-	if (length > 0) {
-		new_len = ctx->gcm_pt_buf_len + length;
-#ifdef _KERNEL
-		new = kmem_alloc(new_len, ctx->gcm_kmflag);
-		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
-		kmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
-#else
-		new = malloc(new_len);
-		bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len);
-		free(ctx->gcm_pt_buf);
-#endif
-		if (new == NULL)
-			return (CRYPTO_HOST_MEMORY);
+	if (ctx->gcm_last_input_fill > 0) {
+		/* Try to complete the input buffer */
+		size_t to_copy = MIN(length,
+		    sizeof (ctx->gcm_last_input) - ctx->gcm_last_input_fill);
 
-		ctx->gcm_pt_buf = new;
-		ctx->gcm_pt_buf_len = new_len;
-		bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len],
-		    length);
-		ctx->gcm_processed_data_len += length;
+		bcopy(data, ctx->gcm_last_input + ctx->gcm_last_input_fill,
+		    to_copy);
+		data += to_copy;
+		ctx->gcm_last_input_fill += to_copy;
+		length -= to_copy;
+
+		if (ctx->gcm_last_input_fill < sizeof (ctx->gcm_last_input))
+			/* Not enough input data to continue */
+			goto out;
+
+		if (length < ctx->gcm_tag_len) {
+			/*
+			 * There isn't enough data ahead to constitute a full
+			 * auth tag, so only crunch one input block and copy
+			 * the remainder of the input into our buffer.
+			 */
+			rv = gcm_process_contiguous_blocks(ctx,
+			    (char *)ctx->gcm_last_input, block_size, out,
+			    block_size, B_FALSE, encrypt_block, copy_block,
+			    xor_block, cipher_ctr);
+			if (rv != CRYPTO_SUCCESS)
+				goto out;
+			ctx->gcm_last_input_fill -= block_size;
+			bcopy(ctx->gcm_last_input + block_size,
+			    ctx->gcm_last_input, ctx->gcm_last_input_fill);
+			bcopy(data, ctx->gcm_last_input +
+			    ctx->gcm_last_input_fill, length);
+			ctx->gcm_last_input_fill += length;
+			/* No more input left */
+			goto out;
+		}
+		/*
+		 * There is enough data ahead for the auth tag, so crunch
+		 * everything in our buffer now and empty it.
+		 */
+		rv = gcm_process_contiguous_blocks(ctx,
+		    (char *)ctx->gcm_last_input, ctx->gcm_last_input_fill,
+		    out, block_size, B_FALSE, encrypt_block, copy_block,
+		    xor_block, cipher_ctr);
+		if (rv != CRYPTO_SUCCESS)
+			goto out;
+		ctx->gcm_last_input_fill = 0;
 	}
+	/*
+	 * Last input buffer is empty, so what's left ahead is block-aligned.
+	 * Crunch all the blocks up until the near end, which might be our
+	 * auth tag and we must NOT decrypt.
+	 */
+	ASSERT(ctx->gcm_last_input_fill == 0);
+	if (length >= block_size + ctx->gcm_tag_len) {
+		size_t to_decrypt = (length - ctx->gcm_tag_len) &
+		    ~(block_size - 1);
 
-	ctx->gcm_remainder_len = 0;
-	return (CRYPTO_SUCCESS);
+		rv = gcm_process_contiguous_blocks(ctx, data, to_decrypt, out,
+		    block_size, B_FALSE, encrypt_block, copy_block, xor_block,
+		    cipher_ctr);
+		if (rv != CRYPTO_SUCCESS)
+			goto out;
+		data += to_decrypt;
+		length -= to_decrypt;
+	}
+	/*
+	 * Copy the remainder into our input buffer, it's potentially
+	 * the auth tag and a last partial block.
+	 */
+	ASSERT(length < sizeof (ctx->gcm_last_input));
+	bcopy(data, ctx->gcm_last_input, length);
+	ctx->gcm_last_input_fill += length;
+out:
+	GCM_ACCEL_EXIT;
+
+	return (rv);
 }
 
 int
 gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *),
+    int (*cipher_ctr)(const void *, const uint8_t *, uint8_t *, uint64_t,
+    uint64_t *))
 {
-	size_t pt_len;
-	size_t remainder;
-	uint8_t *ghash;
-	uint8_t *blockp;
-	uint8_t *cbp;
-	uint64_t counter;
-	uint64_t counter_mask = ntohll(0x00000000ffffffffULL);
-	int processed = 0, rv;
+	int rv = CRYPTO_SUCCESS;
 
-	ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len);
+	/* Check there's enough data to at least compute a tag */
+	if (ctx->gcm_last_input_fill < ctx->gcm_tag_len)
+		return (SET_ERROR(CRYPTO_DATA_LEN_RANGE));
 
-	pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
-	ghash = (uint8_t *)ctx->gcm_ghash;
-	blockp = ctx->gcm_pt_buf;
-	remainder = pt_len;
-	while (remainder > 0) {
-		/* Incomplete last block */
-		if (remainder < block_size) {
-			bcopy(blockp, ctx->gcm_remainder, remainder);
-			ctx->gcm_remainder_len = remainder;
-			/*
-			 * not expecting anymore ciphertext, just
-			 * compute plaintext for the remaining input
-			 */
-			gcm_decrypt_incomplete_block(ctx, block_size,
-			    processed, encrypt_block, xor_block);
-			ctx->gcm_remainder_len = 0;
-			goto out;
-		}
-		/* add ciphertext to the hash */
-		GHASH(ctx, blockp, ghash);
+	GCM_ACCEL_ENTER;
 
-		/*
-		 * Increment counter.
-		 * Counter bits are confined to the bottom 32 bits
-		 */
-		counter = ntohll(ctx->gcm_cb[1] & counter_mask);
-		counter = htonll(counter + 1);
-		counter &= counter_mask;
-		ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter;
+	/* Finish any unprocessed input */
+	if (ctx->gcm_last_input_fill > ctx->gcm_tag_len) {
+		size_t last_blk_len = MIN(block_size,
+		    ctx->gcm_last_input_fill - ctx->gcm_tag_len);
 
-		cbp = (uint8_t *)ctx->gcm_tmp;
-		encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp);
+		/* Finish last full block */
+		if (last_blk_len >= block_size) {
+			rv = gcm_process_contiguous_blocks(ctx,
+			    (char *)ctx->gcm_last_input, block_size, out,
+			    block_size, B_FALSE, encrypt_block, copy_block,
+			    xor_block, cipher_ctr);
+			if (rv != CRYPTO_SUCCESS)
+				goto errout;
 
-		/* XOR with ciphertext */
-		xor_block(cbp, blockp);
+			last_blk_len -= block_size;
+			ctx->gcm_processed_data_len += block_size;
+			ctx->gcm_last_input_fill -= block_size;
 
-		processed += block_size;
-		blockp += block_size;
-		remainder -= block_size;
+			/* Shift what remains in the input buffer forward */
+			bcopy(ctx->gcm_last_input + block_size,
+			    ctx->gcm_last_input, ctx->gcm_last_input_fill);
+		}
+		/* Finish last incomplete block before auth tag */
+		if (last_blk_len > 0) {
+			gcm_decrypt_incomplete_block(ctx, ctx->gcm_last_input,
+			    last_blk_len, block_size, out, encrypt_block,
+			    xor_block);
+
+			ctx->gcm_processed_data_len += last_blk_len;
+			ctx->gcm_last_input_fill -= last_blk_len;
+
+			/* Shift what remains in the input buffer forward */
+			bcopy(ctx->gcm_last_input + last_blk_len,
+			    ctx->gcm_last_input, ctx->gcm_last_input_fill);
+		}
+		/* Now the last_input buffer holds just the auth tag */
 	}
-out:
-	ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len));
-	GHASH(ctx, ctx->gcm_len_a_len_c, ghash);
+
+	ASSERT(ctx->gcm_last_input_fill == ctx->gcm_tag_len);
+
+	ctx->gcm_len_a_len_c[1] =
+	    htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len));
+	GHASH(ctx, ctx->gcm_len_a_len_c, ctx->gcm_ghash);
 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0,
 	    (uint8_t *)ctx->gcm_J0);
-	xor_block((uint8_t *)ctx->gcm_J0, ghash);
+	xor_block((uint8_t *)ctx->gcm_J0, (uint8_t *)ctx->gcm_ghash);
 
+	GCM_ACCEL_EXIT;
+
 	/* compare the input authentication tag with what we calculated */
-	if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) {
-		/* They don't match */
-		return (CRYPTO_INVALID_MAC);
-	} else {
-		rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len);
-		if (rv != CRYPTO_SUCCESS)
-			return (rv);
-		out->cd_offset += pt_len;
-	}
+	if (bcmp(&ctx->gcm_last_input, ctx->gcm_ghash, ctx->gcm_tag_len) != 0)
+		return (SET_ERROR(CRYPTO_INVALID_MAC));
+
 	return (CRYPTO_SUCCESS);
+
+errout:
+	GCM_ACCEL_EXIT;
+	return (rv);
 }
 
 static int
@@ -500,20 +731,21 @@
 	case 128:
 		break;
 	default:
-		return (CRYPTO_MECHANISM_PARAM_INVALID);
+		return (SET_ERROR(CRYPTO_MECHANISM_PARAM_INVALID));
 	}
 
 	if (gcm_param->ulIvLen == 0)
-		return (CRYPTO_MECHANISM_PARAM_INVALID);
+		return (SET_ERROR(CRYPTO_MECHANISM_PARAM_INVALID));
 
 	return (CRYPTO_SUCCESS);
 }
 
+/*ARGSUSED*/
 static void
 gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len,
     gcm_ctx_t *ctx, size_t block_size,
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint8_t *cb;
 	ulong_t remainder = iv_len;
@@ -564,12 +796,14 @@
 gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len,
     unsigned char *auth_data, size_t auth_data_len, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
 	uint8_t *ghash, *datap, *authp;
 	size_t remainder, processed;
 
+	GCM_ACCEL_ENTER;
+
 	/* encrypt zero block to get subkey H */
 	bzero(ctx->gcm_H, sizeof (ctx->gcm_H));
 	encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H,
@@ -578,6 +812,16 @@
 	gcm_format_initial_blocks(iv, iv_len, ctx, block_size,
 	    copy_block, xor_block);
 
+#ifdef	__amd64
+	if (intel_pclmulqdq_instruction_present()) {
+		uint64_t H_bswap64[2] = {
+		    ntohll(ctx->gcm_H[0]), ntohll(ctx->gcm_H[1])
+		};
+
+		gcm_init_clmul(H_bswap64, ctx->gcm_H_table);
+	}
+#endif
+
 	authp = (uint8_t *)ctx->gcm_tmp;
 	ghash = (uint8_t *)ctx->gcm_ghash;
 	bzero(authp, block_size);
@@ -606,6 +850,8 @@
 
 	} while (remainder > 0);
 
+	GCM_ACCEL_EXIT;
+
 	return (CRYPTO_SUCCESS);
 }
 
@@ -612,9 +858,13 @@
 int
 gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
+	/*
+	 * No GHASH invocations in this function and gcm_init does its own
+	 * FPU saving, so no need to GCM_ACCEL_ENTER/GCM_ACCEL_EXIT here.
+	 */
 	int rv;
 	CK_AES_GCM_PARAMS *gcm_param;
 
@@ -652,9 +902,13 @@
 int
 gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size,
     int (*encrypt_block)(const void *, const uint8_t *, uint8_t *),
-    void (*copy_block)(uint8_t *, uint8_t *),
-    void (*xor_block)(uint8_t *, uint8_t *))
+    void (*copy_block)(const uint8_t *, uint8_t *),
+    void (*xor_block)(const uint8_t *, uint8_t *))
 {
+	/*
+	 * No GHASH invocations in this function and gcm_init does its own
+	 * FPU saving, so no need to GCM_ACCEL_ENTER/GCM_ACCEL_EXIT here.
+	 */
 	int rv;
 	CK_AES_GMAC_PARAMS *gmac_param;
 
@@ -732,7 +986,7 @@
  * Note: the userland version uses getisax().  The kernel version uses
  * is_x86_featureset().
  */
-static int
+static inline int
 intel_pclmulqdq_instruction_present(void)
 {
 	static int	cached_result = -1;
