1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2012 Nexenta Systems, Inc.  All rights reserved.
  24  */
  25 
  26 #ifndef _RDSV3_IMPL_H
  27 #define _RDSV3_IMPL_H
  28 
  29 #include <sys/atomic.h>
  30 
  31 /*
  32  * This file is only present in Solaris
  33  */
  34 
  35 #ifdef __cplusplus
  36 extern "C" {
  37 #endif
  38 
  39 extern dev_info_t       *rdsv3_dev_info;
  40 
  41 #define uint16_be_t     uint16_t
  42 #define uint32_be_t     uint32_t
  43 #define uint64_be_t     uint64_t
  44 
  45 /*
  46  * RDS Well known service id
  47  * Format: 0x1h00144Fhhhhhhhh
  48  *         "00144F" is the Sun OUI
  49  * 'h' can be any hex-decimal digit.
  50  */
  51 #define RDS_SERVICE_ID          0x1000144F00000001ULL
  52 
  53 /*
  54  * Atomic operations
  55  */
  56 typedef unsigned int    atomic_t;
  57 #define ATOMIC_INIT(a)  a
  58 
  59 #define atomic_get(p)   (*(p))
  60 
  61 #define atomic_cmpset_long(p, c, n) \
  62         ((c == atomic_cas_uint(p, c, n)) ? c : -1)
  63 
  64 #define atomic_dec_and_test(a)                  \
  65         (atomic_dec_uint_nv((a)) == 0)
  66 
  67 #define atomic_cmpxchg(a, o, n)                 \
  68         atomic_cas_uint(a, o, n)
  69 
  70 #ifdef _LP64
  71 #define set_bit(b, p) \
  72         atomic_or_ulong(((volatile ulong_t *)(void *)(p)) + ((b) >> 6), \
  73         1ul << ((b) & 0x3f))
  74 
  75 #define clear_bit(b, p) \
  76         atomic_and_ulong(((volatile ulong_t *)(void *)(p)) + ((b) >> 6), \
  77         ~(1ul << ((b) & 0x3f)))
  78 
  79 #define test_bit(b, p) \
  80         (((volatile ulong_t *)(void *)(p))[(b) >> 6] & (1ul << ((b) & 0x3f)))
  81 
  82 #define test_and_set_bit(b, p) \
  83         atomic_set_long_excl(((ulong_t *)(void *)(p)) +         \
  84             ((b) >> 6), ((b) & 0x3f))
  85 #define test_and_clear_bit(b, p) \
  86         !atomic_clear_long_excl(((ulong_t *)(void *)(p)) + ((b) >> 6), \
  87         ((b) & 0x3f))
  88 #else
  89 #define set_bit(b, p) \
  90         atomic_or_uint(((volatile uint_t *)(void *)p) + (b >> 5), \
  91         1ul << (b & 0x1f))
  92 
  93 #define clear_bit(b, p) \
  94         atomic_and_uint(((volatile uint_t *)(void *)p) + (b >> 5), \
  95         ~(1ul << (b & 0x1f)))
  96 
  97 #define test_bit(b, p) \
  98         (((volatile uint_t *)(void *)p)[b >> 5] & (1ul << (b & 0x1f)))
  99 
 100 #define test_and_set_bit(b, p) \
 101         atomic_set_long_excl(((ulong_t *)(void *)p) + (b >> 5), (b & 0x1f))
 102 #define test_and_clear_bit(b, p) \
 103         !atomic_clear_long_excl(((ulong_t *)(void *)p) + (b >> 5), (b & 0x1f))
 104 #endif
 105 
 106 /*
 107  * These macros and/or constants are used instead of Linux
 108  * generic_{test,__{clear,set}}_le_bit().
 109  */
 110 #if defined(sparc)
 111 #define LE_BIT_XOR      ((BITS_PER_LONG-1) & ~0x7)
 112 #else
 113 #define LE_BIT_XOR      0
 114 #endif
 115 
 116 #define set_le_bit(b, p)        set_bit(b ^ LE_BIT_XOR, p)
 117 #define clear_le_bit(b, p)      clear_bit(b ^ LE_BIT_XOR, p)
 118 #define test_le_bit(b, p)       test_bit(b ^ LE_BIT_XOR, p)
 119 
 120 uint_t  rdsv3_one_sec_in_hz;
 121 
 122 #define jiffies 100
 123 #define HZ      (drv_hztousec(1))
 124 /* setting this to PAGESIZE throws build errors */
 125 #define PAGE_SIZE       4096 /* xxx - fix this */
 126 #define BITS_PER_LONG   (sizeof (unsigned long) * 8)
 127 
 128 /* debug */
 129 #define RDSV3_PANIC()           cmn_err(CE_PANIC, "Panic forced by RDSV3");
 130 
 131 /* ERR */
 132 #define MAX_ERRNO       4095
 133 #define ERR_PTR(x)      ((void *)(uintptr_t)x)
 134 #define IS_ERR(ptr)     (((uintptr_t)ptr) >= (uintptr_t)-MAX_ERRNO)
 135 #define PTR_ERR(ptr)    (int)(uintptr_t)ptr
 136 
 137 #define MAX_SCHEDULE_TIMEOUT    (~0UL>>1)
 138 
 139 #define RDMA_CM_EVENT_ADDR_CHANGE       14
 140 
 141 /* list */
 142 /* copied and modified list_remove_node */
 143 #define list_remove_node(node)                                          \
 144         if ((node)->list_next != NULL) {                             \
 145                 (node)->list_prev->list_next = (node)->list_next;      \
 146                 (node)->list_next->list_prev = (node)->list_prev;      \
 147                 (node)->list_next = (node)->list_prev = NULL;             \
 148         }
 149 
 150 #define list_splice(src, dst)   {                               \
 151         list_create(dst, (src)->list_size, (src)->list_offset);   \
 152         list_move_tail(dst, src);                               \
 153         }
 154 
 155 #define RDSV3_FOR_EACH_LIST_NODE(objp, listp, member)   \
 156         for (objp = list_head(listp); objp; objp = list_next(listp, objp))
 157 #define RDSV3_FOR_EACH_LIST_NODE_SAFE(objp, tmp, listp, member) \
 158         for (objp = list_head(listp), tmp = (objp != NULL) ?    \
 159             list_next(listp, objp) : NULL;                      \
 160             objp;                                               \
 161             objp = tmp, tmp = (objp != NULL) ?                  \
 162             list_next(listp, objp) : NULL)
 163 
 164 /* simulate wait_queue_head_t */
 165 typedef struct rdsv3_wait_queue_s {
 166         kmutex_t        waitq_mutex;
 167         kcondvar_t      waitq_cv;
 168         uint_t          waitq_waiters;
 169 } rdsv3_wait_queue_t;
 170 
 171 #define rdsv3_init_waitqueue(waitqp)                                    \
 172         mutex_init(&(waitqp)->waitq_mutex, NULL, MUTEX_DRIVER, NULL);    \
 173         cv_init(&(waitqp)->waitq_cv, NULL, CV_DRIVER, NULL);             \
 174         (waitqp)->waitq_waiters = 0
 175 
 176 #define rdsv3_exit_waitqueue(waitqp)                                    \
 177         ASSERT((waitqp)->waitq_waiters == 0);                                \
 178         mutex_destroy(&(waitqp)->waitq_mutex);                           \
 179         cv_destroy(&(waitqp)->waitq_cv)
 180 
 181 #define rdsv3_wake_up(waitqp)   {                                       \
 182         mutex_enter(&(waitqp)->waitq_mutex);                             \
 183         if ((waitqp)->waitq_waiters)                                 \
 184                 cv_signal(&(waitqp)->waitq_cv);                          \
 185         mutex_exit(&(waitqp)->waitq_mutex);                              \
 186         }
 187 
 188 #define rdsv3_wake_up_all(waitqp)       {                               \
 189         mutex_enter(&(waitqp)->waitq_mutex);                             \
 190         if ((waitqp)->waitq_waiters)                                 \
 191                 cv_broadcast(&(waitqp)->waitq_cv);                       \
 192         mutex_exit(&(waitqp)->waitq_mutex);                              \
 193         }
 194 
 195 /* analogous to cv_wait */
 196 #define rdsv3_wait_event(waitq, condition)                              \
 197 {                                                                       \
 198         mutex_enter(&(waitq)->waitq_mutex);                              \
 199         (waitq)->waitq_waiters++;                                    \
 200         while (!(condition)) {                                          \
 201                 cv_wait(&(waitq)->waitq_cv, &(waitq)->waitq_mutex);       \
 202         }                                                               \
 203         (waitq)->waitq_waiters--;                                    \
 204         mutex_exit(&(waitq)->waitq_mutex);                               \
 205 }
 206 
 207 /* analogous to cv_wait_sig */
 208 #define rdsv3_wait_sig(waitqp, condition)                               \
 209 (                                                                       \
 210 {                                                                       \
 211         int cv_return = 1;                                              \
 212         mutex_enter(&(waitqp)->waitq_mutex);                             \
 213         (waitqp)->waitq_waiters++;                                   \
 214         while (!(condition)) {                                          \
 215                 cv_return = cv_wait_sig(&(waitqp)->waitq_cv,             \
 216                     &(waitqp)->waitq_mutex);                             \
 217                 if (cv_return == 0) {                                   \
 218                         break;                                          \
 219                 }                                                       \
 220         }                                                               \
 221         (waitqp)->waitq_waiters--;                                   \
 222         mutex_exit(&(waitqp)->waitq_mutex);                              \
 223         cv_return;                                                      \
 224 }                                                                       \
 225 )
 226 
 227 #define SOCK_DEAD       1ul
 228 
 229 /* socket */
 230 typedef struct rsock {
 231         sock_upper_handle_t     sk_upper_handle;
 232         sock_upcalls_t          *sk_upcalls;
 233 
 234         kmutex_t                sk_lock;
 235         ulong_t                 sk_flag;
 236         rdsv3_wait_queue_t      *sk_sleep; /* Also protected by rs_recv_lock */
 237         int                     sk_sndbuf;
 238         int                     sk_rcvbuf;
 239         atomic_t                sk_refcount;
 240 
 241         struct rdsv3_sock       *sk_protinfo;
 242 } rsock_t;
 243 
 244 typedef struct rdsv3_conn_info_s {
 245         uint32_be_t  c_laddr;
 246         uint32_be_t  c_faddr;
 247 } rdsv3_conn_info_t;
 248 
 249 /* WQ */
 250 typedef struct rdsv3_workqueue_struct_s {
 251         kmutex_t wq_lock;
 252         uint_t  wq_state;
 253         int     wq_pending;
 254         list_t  wq_queue;
 255 } rdsv3_workqueue_struct_t;
 256 
 257 struct rdsv3_work_s;
 258 typedef void (*rdsv3_work_func_t)(struct rdsv3_work_s *);
 259 typedef struct rdsv3_work_s {
 260         list_node_t     work_item;
 261         rdsv3_work_func_t       func;
 262 } rdsv3_work_t;
 263 
 264 /* simulate delayed_work */
 265 typedef struct rdsv3_delayed_work_s {
 266         kmutex_t                lock;
 267         rdsv3_work_t            work;
 268         timeout_id_t            timeid;
 269         rdsv3_workqueue_struct_t        *wq;
 270 } rdsv3_delayed_work_t;
 271 
 272 #define RDSV3_INIT_WORK(wp, f)  (wp)->func = f
 273 #define RDSV3_INIT_DELAYED_WORK(dwp, f)                         \
 274         (dwp)->work.func = f;                                        \
 275         mutex_init(&(dwp)->lock, NULL, MUTEX_DRIVER, NULL);      \
 276         (dwp)->timeid = 0
 277 
 278 /* simulate scatterlist */
 279 struct rdsv3_scatterlist {
 280         caddr_t         vaddr;
 281         uint_t          length;
 282         ibt_wr_ds_t     *sgl;
 283         ibt_mi_hdl_t    mihdl;
 284 };
 285 #define rdsv3_sg_page(scat)     (scat)->vaddr
 286 #define rdsv3_sg_len(scat)      (scat)->length
 287 #define rdsv3_sg_set_page(scat, pg, len, off)           \
 288         (scat)->vaddr = (caddr_t)(pg + off);         \
 289         (scat)->length = len
 290 #define rdsv3_ib_sg_dma_len(dev, scat)  rdsv3_sg_len(scat)
 291 
 292 /* copied from sys/socket.h */
 293 #if defined(__sparc)
 294 /* To maintain backward compatibility, alignment needs to be 8 on sparc. */
 295 #define _CMSG_HDR_ALIGNMENT     8
 296 #else
 297 /* for __i386 (and other future architectures) */
 298 #define _CMSG_HDR_ALIGNMENT     4
 299 #endif  /* defined(__sparc) */
 300 
 301 /*
 302  * The cmsg headers (and macros dealing with them) were made available as
 303  * part of UNIX95 and hence need to be protected with a _XPG4_2 define.
 304  */
 305 #define _CMSG_DATA_ALIGNMENT    (sizeof (int))
 306 #define _CMSG_HDR_ALIGN(x)      (((uintptr_t)(x) + _CMSG_HDR_ALIGNMENT - 1) & \
 307                                     ~(_CMSG_HDR_ALIGNMENT - 1))
 308 #define _CMSG_DATA_ALIGN(x)     (((uintptr_t)(x) + _CMSG_DATA_ALIGNMENT - 1) & \
 309                                     ~(_CMSG_DATA_ALIGNMENT - 1))
 310 #define CMSG_DATA(c)                                                    \
 311         ((unsigned char *)_CMSG_DATA_ALIGN((struct cmsghdr *)(c) + 1))
 312 
 313 #define CMSG_FIRSTHDR(m)                                                \
 314         (((m)->msg_controllen < sizeof (struct cmsghdr)) ?                \
 315             (struct cmsghdr *)0 : (struct cmsghdr *)((m)->msg_control))
 316 
 317 #define CMSG_NXTHDR(m, c)                                               \
 318         (((c) == 0) ? CMSG_FIRSTHDR(m) :                        \
 319         ((((uintptr_t)_CMSG_HDR_ALIGN((char *)(c) +                     \
 320         ((struct cmsghdr *)(c))->cmsg_len) + sizeof (struct cmsghdr)) >   \
 321         (((uintptr_t)((struct msghdr *)(m))->msg_control) +          \
 322         ((uintptr_t)((struct msghdr *)(m))->msg_controllen))) ?              \
 323         ((struct cmsghdr *)0) :                                         \
 324         ((struct cmsghdr *)_CMSG_HDR_ALIGN((char *)(c) +                \
 325             ((struct cmsghdr *)(c))->cmsg_len))))
 326 
 327 /* Amount of space + padding needed for a message of length l */
 328 #define CMSG_SPACE(l)                                                   \
 329         ((unsigned int)_CMSG_HDR_ALIGN(sizeof (struct cmsghdr) + (l)))
 330 
 331 /* Value to be used in cmsg_len, does not include trailing padding */
 332 #define CMSG_LEN(l)                                                     \
 333         ((unsigned int)_CMSG_DATA_ALIGN(sizeof (struct cmsghdr)) + (l))
 334 
 335 /* OFUV -> IB */
 336 #define RDSV3_IBDEV2HCAHDL(device)      (device)->hca_hdl
 337 #define RDSV3_QP2CHANHDL(qp)            (qp)->ibt_qp
 338 #define RDSV3_PD2PDHDL(pd)              (pd)->ibt_pd
 339 #define RDSV3_CQ2CQHDL(cq)              (cq)->ibt_cq
 340 
 341 struct rdsv3_hdrs_mr {
 342         ibt_lkey_t      lkey;
 343         caddr_t         addr;
 344         size_t          size;
 345         ibt_mr_hdl_t    hdl;
 346 };
 347 
 348 /* rdsv3_impl.c */
 349 void rdsv3_trans_init();
 350 boolean_t rdsv3_capable_interface(struct lifreq *lifrp);
 351 int rdsv3_do_ip_ioctl(ksocket_t so4, void **ipaddrs, int *size, int *nifs);
 352 int rdsv3_do_ip_ioctl_old(ksocket_t so4, void **ipaddrs, int *size, int *nifs);
 353 boolean_t rdsv3_isloopback(ipaddr_t addr);
 354 void rdsv3_cancel_delayed_work(rdsv3_delayed_work_t *dwp);
 355 void rdsv3_flush_workqueue(rdsv3_workqueue_struct_t *wq);
 356 void rdsv3_queue_work(rdsv3_workqueue_struct_t *wq, rdsv3_work_t *wp);
 357 void rdsv3_queue_delayed_work(rdsv3_workqueue_struct_t *wq,
 358     rdsv3_delayed_work_t *dwp, uint_t delay);
 359 struct rsock *rdsv3_sk_alloc();
 360 void rdsv3_sock_init_data(struct rsock *sk);
 361 void rdsv3_sock_exit_data(struct rsock *sk);
 362 void rdsv3_destroy_task_workqueue(rdsv3_workqueue_struct_t *wq);
 363 rdsv3_workqueue_struct_t *rdsv3_create_task_workqueue(char *name);
 364 int rdsv3_conn_constructor(void *buf, void *arg, int kmflags);
 365 void rdsv3_conn_destructor(void *buf, void *arg);
 366 int rdsv3_conn_compare(const void *conn1, const void *conn2);
 367 void rdsv3_loop_init();
 368 int rdsv3_mr_compare(const void *mr1, const void *mr2);
 369 int rdsv3_put_cmsg(struct nmsghdr *msg, int level, int type, size_t size,
 370     void *payload);
 371 int rdsv3_verify_bind_address(ipaddr_t addr);
 372 uint16_t rdsv3_ip_fast_csum(void *buffer, size_t length);
 373 uint_t rdsv3_ib_dma_map_sg(struct ib_device *dev, struct rdsv3_scatterlist
 374         *scat, uint_t num);
 375 void rdsv3_ib_dma_unmap_sg(ib_device_t *dev, struct rdsv3_scatterlist *scat,
 376     uint_t num);
 377 static inline void
 378 rdsv3_sk_sock_hold(struct rsock *sk)
 379 {
 380         atomic_add_32(&sk->sk_refcount, 1);
 381 }
 382 static inline void
 383 rdsv3_sk_sock_put(struct rsock *sk)
 384 {
 385         if (atomic_dec_and_test(&sk->sk_refcount))
 386                 rdsv3_sock_exit_data(sk);
 387 }
 388 static inline int
 389 rdsv3_sk_sock_flag(struct rsock *sk, uint_t flag)
 390 {
 391         return (test_bit(flag, &sk->sk_flag));
 392 }
 393 static inline void
 394 rdsv3_sk_sock_orphan(struct rsock *sk)
 395 {
 396         set_bit(SOCK_DEAD, &sk->sk_flag);
 397 }
 398 
 399 #define rdsv3_sndtimeo(a, b)    b ? 0 : 3600    /* check this value on linux */
 400 #define rdsv3_rcvtimeo(a, b)    b ? 0 : 3600    /* check this value on linux */
 401 
 402 void rdsv3_ib_free_conn(void *arg);
 403 
 404 #ifdef  __cplusplus
 405 }
 406 #endif
 407 
 408 #endif /* _RDSV3_IMPL_H */