3830 SIGQUEUE_MAX's limit of 32 is too low
Reviewed by: Cedric Blancher <cedric.blancher@gmail.com>
Reviewed by: John Kennedy <john.kennedy@delphix.com>
Reviewed by: Irek Szczesniak <iszczesniak@gmail.com>

@@ -24,12 +24,10 @@
  * Use is subject to license terms.
  */
 
 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
 
-#pragma ident   "%Z%%M% %I%     %E% SMI"
-
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/sysmacros.h>
 #include <sys/systm.h>
 #include <sys/errno.h>

@@ -38,10 +36,12 @@
 #include <sys/fault.h>
 #include <sys/signal.h>
 #include <sys/siginfo.h>
 #include <sys/debug.h>
 
+extern rctl_hndl_t rc_process_sigqueue;
+
 static int
 sigqkill(pid_t pid, sigsend_t *sigsend)
 {
         proc_t *p;
         int error;

@@ -131,12 +131,19 @@
         /* The si_code value must indicate the signal will be queued */
         if (pid <= 0 || !sigwillqueue(sig, si_code))
                 return (set_errno(EINVAL));
 
         if ((sqh = p->p_sigqhdr) == NULL) {
+                rlim64_t sigqsz_max;
+
+                mutex_enter(&p->p_lock);
+                sigqsz_max = rctl_enforced_value(rc_process_sigqueue,
+                    p->p_rctls, p);
+                mutex_exit(&p->p_lock);
+
                 /* Allocate sigqueue pool first time */
-                sqh = sigqhdralloc(sizeof (sigqueue_t), _SIGQUEUE_MAX);
+                sqh = sigqhdralloc(sizeof (sigqueue_t), (uint_t)sigqsz_max);
                 mutex_enter(&p->p_lock);
                 if (p->p_sigqhdr == NULL) {
                         /* hang the pool head on proc */
                         p->p_sigqhdr = sqh;
                 } else {