12513 SMB 3.1.1 support for server

@@ -9,10 +9,11 @@
  * http://www.illumos.org/license/CDDL.
  */
 
 /*
  * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2020 RackTop Systems, Inc.
  */
 
 /*
  * Helper functions for SMB3 encryption using the
  * Kernel Cryptographic Framework (KCF)

@@ -26,40 +27,67 @@
 #include <smbsrv/smb_kcrypt.h>
 #include <smbsrv/smb2_kproto.h>
 #include <sys/cmn_err.h>
 
 /*
- * SMB3 encryption helpers:
- * (getmech, init, update, final)
+ * Common function to see if a mech is available.
  */
-
-int
-smb3_encrypt_getmech(smb_crypto_mech_t *mech)
+static int
+find_mech(smb_crypto_mech_t *mech, crypto_mech_name_t name)
 {
         crypto_mech_type_t t;
 
-        t = crypto_mech2id(SUN_CKM_AES_CCM);
+        t = crypto_mech2id(name);
         if (t == CRYPTO_MECH_INVALID) {
-                cmn_err(CE_NOTE, "smb: no kcf mech: %s", SUN_CKM_AES_CCM);
+                cmn_err(CE_NOTE, "smb: no kcf mech: %s", name);
                 return (-1);
         }
         mech->cm_type = t;
-
         return (0);
 }
 
+/*
+ * SMB3 encryption helpers:
+ * (getmech, init, update, final)
+ */
+
+int
+smb3_aes_ccm_getmech(smb_crypto_mech_t *mech)
+{
+        return (find_mech(mech, SUN_CKM_AES_CCM));
+}
+
+int
+smb3_aes_gcm_getmech(smb_crypto_mech_t *mech)
+{
+        return (find_mech(mech, SUN_CKM_AES_GCM));
+}
+
 void
-smb3_crypto_init_param(smb3_crypto_param_t *param,
+smb3_crypto_init_ccm_param(smb3_crypto_param_t *param,
     uint8_t *nonce, size_t noncesize, uint8_t *auth, size_t authsize,
     size_t datasize)
 {
-        param->ulMACSize = SMB2_SIG_SIZE;
-        param->ulNonceSize = noncesize;
-        param->nonce = nonce;
-        param->ulDataSize = datasize;
-        param->ulAuthDataSize = authsize;
-        param->authData = auth;
+        param->ccm.ulMACSize = SMB2_SIG_SIZE;
+        param->ccm.ulNonceSize = noncesize;
+        param->ccm.nonce = nonce;
+        param->ccm.ulDataSize = datasize;
+        param->ccm.ulAuthDataSize = authsize;
+        param->ccm.authData = auth;
+}
+
+void
+smb3_crypto_init_gcm_param(smb3_crypto_param_t *param,
+    uint8_t *nonce, size_t noncesize, uint8_t *auth, size_t authsize)
+{
+        ASSERT3U(noncesize, ==, 12);
+        param->gcm.pIv = nonce;
+        param->gcm.ulIvLen = noncesize;         /* should be 12 bytes */
+        /* tform hdr size - (protcolo id + signing) == 32 bytes */
+        param->gcm.ulTagBits = SMB2_SIG_SIZE << 3; /* convert bytes to bits */
+        param->gcm.pAAD = auth;                 /* auth data */
+        param->gcm.ulAADLen = authsize;         /* auth data len */
 }
 
 /*
  * Start the KCF session, load the key
  */

@@ -197,11 +225,26 @@
         if (rv != CRYPTO_SUCCESS) {
                 cmn_err(CE_WARN, "crypto_encrypt_final failed: 0x%x", rv);
                 return (-1);
         }
 
+        /*
+         * For some reason AES module processes ccm_encrypt_final and
+         * gcm_encrypt_final differently.
+         * For GCM it restores original offset (which is 0) and updates
+         * cd_length to size of residual data + mac len.
+         * For CCM it does nothing, what means offset is updated and cd_length
+         * is decreased by size of residual data + mac len.
+         */
+        if (out.cd_offset == 0) {
+                /* GCM */
+                outlen = out.cd_length - SMB2_SIG_SIZE;
+        } else {
+                /* CCM */
         outlen = out.cd_offset - SMB2_SIG_SIZE;
+        }
+
         if (outlen > 0)
                 bcopy(buf, ctxp->output.cd_raw.iov_base +
                     ctxp->output.cd_offset, outlen);
         bcopy(buf + outlen, digest16, SMB2_SIG_SIZE);