Print this page
8368 remove warlock leftovers from usr/src/uts
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/io/ib/clients/rdsv3/message.c
+++ new/usr/src/uts/common/io/ib/clients/rdsv3/message.c
1 1 /*
2 2 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
3 3 */
4 4
5 5 /*
6 6 * This file contains code imported from the OFED rds source file message.c
7 7 * Oracle elects to have and use the contents of message.c under and governed
8 8 * by the OpenIB.org BSD license (see below for full license text). However,
9 9 * the following notice accompanied the original version of this file:
10 10 */
11 11
12 12 /*
13 13 * Copyright (c) 2006 Oracle. All rights reserved.
14 14 *
15 15 * This software is available to you under a choice of one of two
16 16 * licenses. You may choose to be licensed under the terms of the GNU
17 17 * General Public License (GPL) Version 2, available from the file
18 18 * COPYING in the main directory of this source tree, or the
19 19 * OpenIB.org BSD license below:
20 20 *
21 21 * Redistribution and use in source and binary forms, with or
22 22 * without modification, are permitted provided that the following
23 23 * conditions are met:
24 24 *
25 25 * - Redistributions of source code must retain the above
26 26 * copyright notice, this list of conditions and the following
27 27 * disclaimer.
28 28 *
29 29 * - Redistributions in binary form must reproduce the above
30 30 * copyright notice, this list of conditions and the following
31 31 * disclaimer in the documentation and/or other materials
32 32 * provided with the distribution.
33 33 *
34 34 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
35 35 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
36 36 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
37 37 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
38 38 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
39 39 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
↓ open down ↓ |
39 lines elided |
↑ open up ↑ |
40 40 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
41 41 * SOFTWARE.
42 42 *
43 43 */
44 44 #include <sys/rds.h>
45 45
46 46 #include <sys/ib/clients/rdsv3/rdsv3.h>
47 47 #include <sys/ib/clients/rdsv3/rdma.h>
48 48 #include <sys/ib/clients/rdsv3/rdsv3_debug.h>
49 49
50 -#ifndef __lock_lint
51 50 static unsigned int rdsv3_exthdr_size[__RDSV3_EXTHDR_MAX] = {
52 51 [RDSV3_EXTHDR_NONE] = 0,
53 52 [RDSV3_EXTHDR_VERSION] = sizeof (struct rdsv3_ext_header_version),
54 53 [RDSV3_EXTHDR_RDMA] = sizeof (struct rdsv3_ext_header_rdma),
55 54 [RDSV3_EXTHDR_RDMA_DEST] = sizeof (struct rdsv3_ext_header_rdma_dest),
56 55 };
57 -#else
58 -static unsigned int rdsv3_exthdr_size[__RDSV3_EXTHDR_MAX] = {
59 - 0,
60 - sizeof (struct rdsv3_ext_header_version),
61 - sizeof (struct rdsv3_ext_header_rdma),
62 - sizeof (struct rdsv3_ext_header_rdma_dest),
63 -};
64 -#endif
65 56
66 57 void
67 58 rdsv3_message_addref(struct rdsv3_message *rm)
68 59 {
69 60 RDSV3_DPRINTF5("rdsv3_message_addref", "addref rm %p ref %d",
70 61 rm, atomic_get(&rm->m_refcount));
71 62 atomic_inc_32(&rm->m_refcount);
72 63 }
73 64
74 65 /*
75 66 * This relies on dma_map_sg() not touching sg[].page during merging.
76 67 */
77 68 static void
78 69 rdsv3_message_purge(struct rdsv3_message *rm)
79 70 {
80 71 unsigned long i;
81 72
82 73 RDSV3_DPRINTF4("rdsv3_message_purge", "Enter(rm: %p)", rm);
83 74
84 75 if (test_bit(RDSV3_MSG_PAGEVEC, &rm->m_flags))
85 76 return;
86 77
87 78 for (i = 0; i < rm->m_nents; i++) {
88 79 RDSV3_DPRINTF5("rdsv3_message_purge", "putting data page %p\n",
89 80 (void *)rdsv3_sg_page(&rm->m_sg[i]));
90 81 /* XXX will have to put_page for page refs */
91 82 kmem_free(rdsv3_sg_page(&rm->m_sg[i]),
92 83 rdsv3_sg_len(&rm->m_sg[i]));
93 84 }
94 85
95 86 if (rm->m_rdma_op)
96 87 rdsv3_rdma_free_op(rm->m_rdma_op);
97 88 if (rm->m_rdma_mr) {
98 89 struct rdsv3_mr *mr = rm->m_rdma_mr;
99 90 if (mr->r_refcount == 0) {
100 91 RDSV3_DPRINTF4("rdsv3_message_purge ASSERT 0",
101 92 "rm %p mr %p", rm, mr);
102 93 return;
103 94 }
104 95 if (mr->r_refcount == 0xdeadbeef) {
105 96 RDSV3_DPRINTF4("rdsv3_message_purge ASSERT deadbeef",
106 97 "rm %p mr %p", rm, mr);
107 98 return;
108 99 }
109 100 if (atomic_dec_and_test(&mr->r_refcount)) {
110 101 rm->m_rdma_mr = NULL;
111 102 __rdsv3_put_mr_final(mr);
112 103 }
113 104 }
114 105
115 106 RDSV3_DPRINTF4("rdsv3_message_purge", "Return(rm: %p)", rm);
116 107
117 108 }
118 109
119 110 void
120 111 rdsv3_message_put(struct rdsv3_message *rm)
121 112 {
122 113 RDSV3_DPRINTF5("rdsv3_message_put",
123 114 "put rm %p ref %d\n", rm, atomic_get(&rm->m_refcount));
124 115
125 116 if (atomic_dec_and_test(&rm->m_refcount)) {
126 117 ASSERT(!list_link_active(&rm->m_sock_item));
127 118 ASSERT(!list_link_active(&rm->m_conn_item));
128 119 rdsv3_message_purge(rm);
129 120
130 121 kmem_free(rm, sizeof (struct rdsv3_message) +
131 122 (rm->m_nents * sizeof (struct rdsv3_scatterlist)));
132 123 }
133 124 }
134 125
135 126 void
136 127 rdsv3_message_inc_free(struct rdsv3_incoming *inc)
137 128 {
138 129 struct rdsv3_message *rm =
139 130 container_of(inc, struct rdsv3_message, m_inc);
140 131 rdsv3_message_put(rm);
141 132 }
142 133
143 134 void
144 135 rdsv3_message_populate_header(struct rdsv3_header *hdr, uint16_be_t sport,
145 136 uint16_be_t dport, uint64_t seq)
146 137 {
147 138 hdr->h_flags = 0;
148 139 hdr->h_sport = sport;
149 140 hdr->h_dport = dport;
150 141 hdr->h_sequence = htonll(seq);
151 142 hdr->h_exthdr[0] = RDSV3_EXTHDR_NONE;
152 143 }
153 144
154 145 int
155 146 rdsv3_message_add_extension(struct rdsv3_header *hdr,
156 147 unsigned int type, const void *data, unsigned int len)
157 148 {
158 149 unsigned int ext_len = sizeof (uint8_t) + len;
159 150 unsigned char *dst;
160 151
161 152 RDSV3_DPRINTF4("rdsv3_message_add_extension", "Enter");
162 153
163 154 /* For now, refuse to add more than one extension header */
164 155 if (hdr->h_exthdr[0] != RDSV3_EXTHDR_NONE)
165 156 return (0);
166 157
167 158 if (type >= __RDSV3_EXTHDR_MAX ||
168 159 len != rdsv3_exthdr_size[type])
169 160 return (0);
170 161
171 162 if (ext_len >= RDSV3_HEADER_EXT_SPACE)
172 163 return (0);
173 164 dst = hdr->h_exthdr;
174 165
175 166 *dst++ = type;
176 167 (void) memcpy(dst, data, len);
177 168
178 169 dst[len] = RDSV3_EXTHDR_NONE;
179 170
180 171 RDSV3_DPRINTF4("rdsv3_message_add_extension", "Return");
181 172 return (1);
182 173 }
183 174
184 175 /*
185 176 * If a message has extension headers, retrieve them here.
186 177 * Call like this:
187 178 *
188 179 * unsigned int pos = 0;
189 180 *
190 181 * while (1) {
191 182 * buflen = sizeof(buffer);
192 183 * type = rdsv3_message_next_extension(hdr, &pos, buffer, &buflen);
193 184 * if (type == RDSV3_EXTHDR_NONE)
194 185 * break;
195 186 * ...
196 187 * }
197 188 */
198 189 int
199 190 rdsv3_message_next_extension(struct rdsv3_header *hdr,
200 191 unsigned int *pos, void *buf, unsigned int *buflen)
201 192 {
202 193 unsigned int offset, ext_type, ext_len;
203 194 uint8_t *src = hdr->h_exthdr;
204 195
205 196 RDSV3_DPRINTF4("rdsv3_message_next_extension", "Enter");
206 197
207 198 offset = *pos;
208 199 if (offset >= RDSV3_HEADER_EXT_SPACE)
209 200 goto none;
210 201
211 202 /*
212 203 * Get the extension type and length. For now, the
213 204 * length is implied by the extension type.
214 205 */
215 206 ext_type = src[offset++];
216 207
217 208 if (ext_type == RDSV3_EXTHDR_NONE || ext_type >= __RDSV3_EXTHDR_MAX)
218 209 goto none;
219 210 ext_len = rdsv3_exthdr_size[ext_type];
220 211 if (offset + ext_len > RDSV3_HEADER_EXT_SPACE)
221 212 goto none;
222 213
223 214 *pos = offset + ext_len;
224 215 if (ext_len < *buflen)
225 216 *buflen = ext_len;
226 217 (void) memcpy(buf, src + offset, *buflen);
227 218 return (ext_type);
228 219
229 220 none:
230 221 *pos = RDSV3_HEADER_EXT_SPACE;
231 222 *buflen = 0;
232 223 return (RDSV3_EXTHDR_NONE);
233 224 }
234 225
235 226 int
236 227 rdsv3_message_add_version_extension(struct rdsv3_header *hdr,
237 228 unsigned int version)
238 229 {
239 230 struct rdsv3_ext_header_version ext_hdr;
240 231
241 232 ext_hdr.h_version = htonl(version);
242 233 return (rdsv3_message_add_extension(hdr, RDSV3_EXTHDR_VERSION,
243 234 &ext_hdr, sizeof (ext_hdr)));
244 235 }
245 236
246 237 int
247 238 rdsv3_message_get_version_extension(struct rdsv3_header *hdr,
248 239 unsigned int *version)
249 240 {
250 241 struct rdsv3_ext_header_version ext_hdr;
251 242 unsigned int pos = 0, len = sizeof (ext_hdr);
252 243
253 244 RDSV3_DPRINTF4("rdsv3_message_get_version_extension", "Enter");
254 245
255 246 /*
256 247 * We assume the version extension is the only one present
257 248 */
258 249 if (rdsv3_message_next_extension(hdr, &pos, &ext_hdr, &len) !=
259 250 RDSV3_EXTHDR_VERSION)
260 251 return (0);
261 252 *version = ntohl(ext_hdr.h_version);
262 253 return (1);
263 254 }
264 255
265 256 int
266 257 rdsv3_message_add_rdma_dest_extension(struct rdsv3_header *hdr, uint32_t r_key,
267 258 uint32_t offset)
268 259 {
269 260 struct rdsv3_ext_header_rdma_dest ext_hdr;
270 261
271 262 ext_hdr.h_rdma_rkey = htonl(r_key);
272 263 ext_hdr.h_rdma_offset = htonl(offset);
273 264 return (rdsv3_message_add_extension(hdr, RDSV3_EXTHDR_RDMA_DEST,
274 265 &ext_hdr, sizeof (ext_hdr)));
275 266 }
276 267
277 268 struct rdsv3_message *
278 269 rdsv3_message_alloc(unsigned int nents, int gfp)
279 270 {
280 271 struct rdsv3_message *rm;
281 272
282 273 RDSV3_DPRINTF4("rdsv3_message_alloc", "Enter(nents: %d)", nents);
283 274
284 275 rm = kmem_zalloc(sizeof (struct rdsv3_message) +
285 276 (nents * sizeof (struct rdsv3_scatterlist)), gfp);
286 277 if (!rm)
287 278 goto out;
288 279
289 280 rm->m_refcount = 1;
290 281 list_link_init(&rm->m_sock_item);
291 282 list_link_init(&rm->m_conn_item);
292 283 mutex_init(&rm->m_rs_lock, NULL, MUTEX_DRIVER, NULL);
293 284 rdsv3_init_waitqueue(&rm->m_flush_wait);
294 285
295 286 RDSV3_DPRINTF4("rdsv3_message_alloc", "Return(rm: %p)", rm);
296 287 out:
297 288 return (rm);
|
↓ open down ↓ |
223 lines elided |
↑ open up ↑ |
298 289 }
299 290
300 291 struct rdsv3_message *
301 292 rdsv3_message_map_pages(unsigned long *page_addrs, unsigned int total_len)
302 293 {
303 294 struct rdsv3_message *rm;
304 295 unsigned int i;
305 296
306 297 RDSV3_DPRINTF4("rdsv3_message_map_pages", "Enter(len: %d)", total_len);
307 298
308 -#ifndef __lock_lint
309 299 rm = rdsv3_message_alloc(ceil(total_len, PAGE_SIZE), KM_NOSLEEP);
310 -#else
311 - rm = NULL;
312 -#endif
313 300 if (rm == NULL)
314 301 return (ERR_PTR(-ENOMEM));
315 302
316 303 set_bit(RDSV3_MSG_PAGEVEC, &rm->m_flags);
317 304 rm->m_inc.i_hdr.h_len = htonl(total_len);
318 -#ifndef __lock_lint
319 305 rm->m_nents = ceil(total_len, PAGE_SIZE);
320 -#else
321 - rm->m_nents = 0;
322 -#endif
323 -
324 306 for (i = 0; i < rm->m_nents; ++i) {
325 307 rdsv3_sg_set_page(&rm->m_sg[i],
326 308 page_addrs[i],
327 309 PAGE_SIZE, 0);
328 310 }
329 311
330 312 return (rm);
331 313 }
332 314
333 315 struct rdsv3_message *
334 316 rdsv3_message_copy_from_user(struct uio *uiop,
335 317 size_t total_len)
336 318 {
337 319 struct rdsv3_message *rm;
338 320 struct rdsv3_scatterlist *sg;
339 321 int ret;
340 322
341 323 RDSV3_DPRINTF4("rdsv3_message_copy_from_user", "Enter: %d", total_len);
342 324
343 -#ifndef __lock_lint
344 325 rm = rdsv3_message_alloc(ceil(total_len, PAGE_SIZE), KM_NOSLEEP);
345 -#else
346 - rm = NULL;
347 -#endif
348 326 if (rm == NULL) {
349 327 ret = -ENOMEM;
350 328 goto out;
351 329 }
352 330
353 331 rm->m_inc.i_hdr.h_len = htonl(total_len);
354 332
355 333 /*
356 334 * now allocate and copy in the data payload.
357 335 */
358 336 sg = rm->m_sg;
359 337
360 338 while (total_len) {
361 339 if (rdsv3_sg_page(sg) == NULL) {
362 340 ret = rdsv3_page_remainder_alloc(sg, total_len, 0);
363 341 if (ret)
364 342 goto out;
365 343 rm->m_nents++;
366 344 }
367 345
368 346 ret = uiomove(rdsv3_sg_page(sg), rdsv3_sg_len(sg), UIO_WRITE,
369 347 uiop);
370 348 if (ret) {
371 349 RDSV3_DPRINTF2("rdsv3_message_copy_from_user",
372 350 "uiomove failed");
373 351 ret = -ret;
374 352 goto out;
375 353 }
376 354
377 355 total_len -= rdsv3_sg_len(sg);
378 356 sg++;
379 357 }
380 358 ret = 0;
381 359 out:
382 360 if (ret) {
383 361 if (rm)
384 362 rdsv3_message_put(rm);
385 363 rm = ERR_PTR(ret);
386 364 }
387 365 return (rm);
388 366 }
389 367
390 368 int
391 369 rdsv3_message_inc_copy_to_user(struct rdsv3_incoming *inc,
392 370 uio_t *uiop, size_t size)
393 371 {
394 372 struct rdsv3_message *rm;
395 373 struct rdsv3_scatterlist *sg;
396 374 unsigned long to_copy;
397 375 unsigned long vec_off;
398 376 int copied;
399 377 int ret;
400 378 uint32_t len;
401 379
402 380 rm = container_of(inc, struct rdsv3_message, m_inc);
403 381 len = ntohl(rm->m_inc.i_hdr.h_len);
404 382
405 383 RDSV3_DPRINTF4("rdsv3_message_inc_copy_to_user",
406 384 "Enter(rm: %p, len: %d)", rm, len);
407 385
408 386 sg = rm->m_sg;
409 387 vec_off = 0;
410 388 copied = 0;
411 389
412 390 while (copied < size && copied < len) {
413 391
414 392 to_copy = min(len - copied, sg->length - vec_off);
415 393 to_copy = min(size - copied, to_copy);
416 394
417 395 RDSV3_DPRINTF5("rdsv3_message_inc_copy_to_user",
418 396 "copying %lu bytes to user iov %p from sg [%p, %u] + %lu\n",
419 397 to_copy, uiop,
420 398 rdsv3_sg_page(sg), sg->length, vec_off);
421 399
422 400 ret = uiomove(rdsv3_sg_page(sg), to_copy, UIO_READ, uiop);
423 401 if (ret)
424 402 break;
425 403
426 404 vec_off += to_copy;
427 405 copied += to_copy;
428 406
429 407 if (vec_off == sg->length) {
430 408 vec_off = 0;
431 409 sg++;
432 410 }
433 411 }
434 412
435 413 return (copied);
436 414 }
437 415
438 416 /*
439 417 * If the message is still on the send queue, wait until the transport
440 418 * is done with it. This is particularly important for RDMA operations.
441 419 */
442 420 /* ARGSUSED */
443 421 void
444 422 rdsv3_message_wait(struct rdsv3_message *rm)
445 423 {
446 424 rdsv3_wait_event(&rm->m_flush_wait,
447 425 !test_bit(RDSV3_MSG_MAPPED, &rm->m_flags));
448 426 }
449 427
450 428 void
451 429 rdsv3_message_unmapped(struct rdsv3_message *rm)
452 430 {
453 431 clear_bit(RDSV3_MSG_MAPPED, &rm->m_flags);
454 432 rdsv3_wake_up_all(&rm->m_flush_wait);
455 433 }
|
↓ open down ↓ |
98 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX