1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/debug.h>
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/time.h>
31 #include <sys/buf.h>
32 #include <sys/errno.h>
33 #include <sys/systm.h>
34 #include <sys/conf.h>
35 #include <sys/signal.h>
36 #include <sys/file.h>
37 #include <sys/uio.h>
38 #include <sys/ioctl.h>
39 #include <sys/map.h>
40 #include <sys/proc.h>
41 #include <sys/user.h>
42 #include <sys/mman.h>
43 #include <sys/cred.h>
44 #include <sys/open.h>
45 #include <sys/stat.h>
46 #include <sys/utsname.h>
47 #include <sys/kmem.h>
48 #include <sys/cmn_err.h>
49 #include <sys/vnode.h>
50 #include <vm/page.h>
51 #include <vm/as.h>
52 #include <vm/hat.h>
53 #include <vm/seg.h>
54 #include <vm/seg_kmem.h>
55 #include <vm/hat_i86.h>
56 #include <sys/vmsystm.h>
57 #include <sys/ddi.h>
58 #include <sys/devops.h>
59 #include <sys/sunddi.h>
60 #include <sys/ddi_impldefs.h>
61 #include <sys/fs/snode.h>
62 #include <sys/pci.h>
63 #include <sys/modctl.h>
64 #include <sys/uio.h>
65 #include <sys/visual_io.h>
66 #include <sys/fbio.h>
67 #include <sys/ddidmareq.h>
68 #include <sys/tnf_probe.h>
69 #include <sys/kstat.h>
70 #include <sys/callb.h>
71 #include <sys/promif.h>
72 #include <sys/atomic.h>
73 #include <sys/gfx_private.h>
74
75 #ifdef __xpv
76 #include <sys/hypervisor.h>
77 #endif
78
79 /*
80 * Create a kva mapping for a pa (start..start+size) with
81 * the specified cache attributes (mode).
82 */
83 gfxp_kva_t
84 gfxp_map_kernel_space(uint64_t start, size_t size, uint32_t mode)
85 {
86 uint_t pgoffset;
87 uint64_t base;
88 pgcnt_t npages;
89 caddr_t cvaddr;
90 int hat_flags;
91 uint_t hat_attr;
92 pfn_t pfn;
93
94 if (size == 0)
95 return (0);
96
97 #ifdef __xpv
98 /*
99 * The hypervisor doesn't allow r/w mappings to some pages, such as
100 * page tables, gdt, etc. Detect %cr3 to notify users of this interface.
101 */
102 if (start == mmu_ptob(mmu_btop(getcr3())))
103 return (0);
104 #endif
105
106 if (mode == GFXP_MEMORY_CACHED)
107 hat_attr = HAT_STORECACHING_OK;
108 else if (mode == GFXP_MEMORY_WRITECOMBINED)
109 hat_attr = HAT_MERGING_OK | HAT_PLAT_NOCACHE;
110 else /* GFXP_MEMORY_UNCACHED */
111 hat_attr = HAT_STRICTORDER | HAT_PLAT_NOCACHE;
112 hat_flags = HAT_LOAD_LOCK;
113 pgoffset = start & PAGEOFFSET;
114 base = start - pgoffset;
115 npages = btopr(size + pgoffset);
116 cvaddr = vmem_alloc(heap_arena, ptob(npages), VM_NOSLEEP);
117 if (cvaddr == NULL)
118 return (NULL);
119
120 #ifdef __xpv
121 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
122 pfn = xen_assign_pfn(mmu_btop(base));
123 #else
124 pfn = btop(base);
125 #endif
126
127 hat_devload(kas.a_hat, cvaddr, ptob(npages), pfn,
128 PROT_READ|PROT_WRITE|hat_attr, hat_flags);
129 return (cvaddr + pgoffset);
130 }
131
132 /*
133 * Destroy the mapping created by gfxp_map_kernel_space().
134 * Physical memory is not reclaimed.
135 */
136 void
137 gfxp_unmap_kernel_space(gfxp_kva_t address, size_t size)
138 {
139 uint_t pgoffset;
140 caddr_t base;
141 pgcnt_t npages;
142
143 if (size == 0 || address == NULL)
144 return;
145
146 pgoffset = (uintptr_t)address & PAGEOFFSET;
147 base = (caddr_t)address - pgoffset;
148 npages = btopr(size + pgoffset);
149 hat_unload(kas.a_hat, base, ptob(npages), HAT_UNLOAD_UNLOCK);
150 vmem_free(heap_arena, base, ptob(npages));
151 }
152
153 /*
154 * For a VA return the pfn
155 */
156 int
157 gfxp_va2pa(struct as *as, caddr_t addr, uint64_t *pa)
158 {
159 #ifdef __xpv
160 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
161 *pa = pa_to_ma(pfn_to_pa(hat_getpfnum(as->a_hat, addr)));
162 #else
163 *pa = pfn_to_pa(hat_getpfnum(as->a_hat, addr));
164 #endif
165 return (0);
166 }
167
168 /*
169 * NOP now
170 */
171 /* ARGSUSED */
172 void
173 gfxp_fix_mem_cache_attrs(caddr_t kva_start, size_t length, int cache_attr)
174 {
175 }
176
177 int
178 gfxp_ddi_dma_mem_alloc(ddi_dma_handle_t handle, size_t length,
179 ddi_device_acc_attr_t *accattrp, uint_t flags, int (*waitfp) (caddr_t),
180 caddr_t arg, caddr_t *kaddrp, size_t *real_length,
181 ddi_acc_handle_t *handlep)
182 {
183 uint_t l_flags = flags & ~IOMEM_DATA_MASK; /* clear cache attrs */
184 int e;
185
186 /*
187 * Set an appropriate attribute from devacc_attr_dataorder
188 * to keep compatibility. The cache attributes are igonred
189 * if specified.
190 */
191 if (accattrp != NULL) {
192 if (accattrp->devacc_attr_dataorder == DDI_STRICTORDER_ACC) {
193 l_flags |= IOMEM_DATA_UNCACHED;
194 } else if (accattrp->devacc_attr_dataorder ==
195 DDI_MERGING_OK_ACC) {
196 l_flags |= IOMEM_DATA_UC_WR_COMBINE;
197 } else {
198 l_flags |= IOMEM_DATA_CACHED;
199 }
200 }
201
202 e = ddi_dma_mem_alloc(handle, length, accattrp, l_flags, waitfp,
203 arg, kaddrp, real_length, handlep);
204 return (e);
205 }
206
207 int
208 gfxp_mlock_user_memory(caddr_t address, size_t length)
209 {
210 struct as *as = ttoproc(curthread)->p_as;
211 int error = 0;
212
213 if (((uintptr_t)address & PAGEOFFSET) != 0 || length == 0)
214 return (set_errno(EINVAL));
215
216 if (valid_usr_range(address, length, 0, as, as->a_userlimit) !=
217 RANGE_OKAY)
218 return (set_errno(ENOMEM));
219
220 error = as_ctl(as, address, length, MC_LOCK, 0, 0, NULL, 0);
221 if (error)
222 (void) set_errno(error);
223
224 return (error);
225 }
226
227 int
228 gfxp_munlock_user_memory(caddr_t address, size_t length)
229 {
230 struct as *as = ttoproc(curthread)->p_as;
231 int error = 0;
232
233 if (((uintptr_t)address & PAGEOFFSET) != 0 || length == 0)
234 return (set_errno(EINVAL));
235
236 if (valid_usr_range(address, length, 0, as, as->a_userlimit) !=
237 RANGE_OKAY)
238 return (set_errno(ENOMEM));
239
240 error = as_ctl(as, address, length, MC_UNLOCK, 0, 0, NULL, 0);
241 if (error)
242 (void) set_errno(error);
243
244 return (error);
245 }
246
247 gfx_maddr_t
248 gfxp_convert_addr(paddr_t paddr)
249 {
250 #ifdef __xpv
251 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
252 return (pfn_to_pa(xen_assign_pfn(btop(paddr))));
253 #else
254 return ((gfx_maddr_t)paddr);
255 #endif
256 }
257
258 /*
259 * Support getting VA space separately from pages
260 */
261
262 /*
263 * A little like gfxp_map_kernel_space, but
264 * just the vmem_alloc part.
265 */
266 caddr_t
267 gfxp_alloc_kernel_space(size_t size)
268 {
269 caddr_t cvaddr;
270 pgcnt_t npages;
271
272 npages = btopr(size);
273 cvaddr = vmem_alloc(heap_arena, ptob(npages), VM_NOSLEEP);
274 return (cvaddr);
275 }
276
277 /*
278 * Like gfxp_unmap_kernel_space, but
279 * just the vmem_free part.
280 */
281 void
282 gfxp_free_kernel_space(caddr_t address, size_t size)
283 {
284
285 uint_t pgoffset;
286 caddr_t base;
287 pgcnt_t npages;
288
289 if (size == 0 || address == NULL)
290 return;
291
292 pgoffset = (uintptr_t)address & PAGEOFFSET;
293 base = (caddr_t)address - pgoffset;
294 npages = btopr(size + pgoffset);
295 vmem_free(heap_arena, base, ptob(npages));
296 }
297
298 /*
299 * Like gfxp_map_kernel_space, but
300 * just the hat_devload part.
301 */
302 void
303 gfxp_load_kernel_space(uint64_t start, size_t size,
304 uint32_t mode, caddr_t cvaddr)
305 {
306 uint_t pgoffset;
307 uint64_t base;
308 pgcnt_t npages;
309 int hat_flags;
310 uint_t hat_attr;
311 pfn_t pfn;
312
313 if (size == 0)
314 return;
315
316 #ifdef __xpv
317 /*
318 * The hypervisor doesn't allow r/w mappings to some pages, such as
319 * page tables, gdt, etc. Detect %cr3 to notify users of this interface.
320 */
321 if (start == mmu_ptob(mmu_btop(getcr3())))
322 return;
323 #endif
324
325 if (mode == GFXP_MEMORY_CACHED)
326 hat_attr = HAT_STORECACHING_OK;
327 else if (mode == GFXP_MEMORY_WRITECOMBINED)
328 hat_attr = HAT_MERGING_OK | HAT_PLAT_NOCACHE;
329 else /* GFXP_MEMORY_UNCACHED */
330 hat_attr = HAT_STRICTORDER | HAT_PLAT_NOCACHE;
331 hat_flags = HAT_LOAD_LOCK;
332
333 pgoffset = start & PAGEOFFSET;
334 base = start - pgoffset;
335 npages = btopr(size + pgoffset);
336
337 #ifdef __xpv
338 ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
339 pfn = xen_assign_pfn(mmu_btop(base));
340 #else
341 pfn = btop(base);
342 #endif
343
344 hat_devload(kas.a_hat, cvaddr, ptob(npages), pfn,
345 PROT_READ|PROT_WRITE|hat_attr, hat_flags);
346 }
347
348 /*
349 * Like gfxp_unmap_kernel_space, but
350 * just the had_unload part.
351 */
352 void
353 gfxp_unload_kernel_space(caddr_t address, size_t size)
354 {
355 uint_t pgoffset;
356 caddr_t base;
357 pgcnt_t npages;
358
359 if (size == 0 || address == NULL)
360 return;
361
362 pgoffset = (uintptr_t)address & PAGEOFFSET;
363 base = (caddr_t)address - pgoffset;
364 npages = btopr(size + pgoffset);
365 hat_unload(kas.a_hat, base, ptob(npages), HAT_UNLOAD_UNLOCK);
366 }
367
368 /*
369 * Note that "mempool" is optional and normally disabled in drm_gem.c
370 * (see HAS_MEM_POOL). Let's just stub these out so we can reduce
371 * changes from the upstream in the DRM driver code.
372 */
373
374 void
375 gfxp_mempool_init(void)
376 {
377 }
378
379 void
380 gfxp_mempool_destroy(void)
381 {
382 }
383
384 /* ARGSUSED */
385 int
386 gfxp_alloc_from_mempool(struct gfxp_pmem_cookie *cookie, caddr_t *kva,
387 pfn_t *pgarray, pgcnt_t alen, int flags)
388 {
389 return (-1);
390 }
391
392 /* ARGSUSED */
393 void
394 gfxp_free_mempool(struct gfxp_pmem_cookie *cookie, caddr_t kva, size_t len)
395 {
396 }