18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25
26 #include <sys/param.h>
27 #include <sys/errno.h>
28 #include <sys/asm_linkage.h>
29 #include <sys/vtrace.h>
30 #include <sys/machthread.h>
31 #include <sys/clock.h>
32 #include <sys/asi.h>
33 #include <sys/fsr.h>
34 #include <sys/privregs.h>
35 #include <sys/machasi.h>
36 #include <sys/niagaraasi.h>
37
38 #if !defined(lint)
39 #include "assym.h"
40 #endif /* lint */
41
42
43 /*
44 * Pseudo-code to aid in understanding the control flow of the
45 * bcopy/kcopy routine.
46 *
47 * ! WARNING : <Register usage convention>
48 * ! In kcopy() the %o5, holds previous error handler and a flag
49 * ! LOFAULT_SET (low bits). The %o5 is null in bcopy().
50 * ! The %o5 is not available for any other use.
51 *
52 * On entry:
53 * ! Determine whether to use the FP register version or the
54 * ! the leaf routine version depending on the size of the copy.
55 * ! Set up error handling accordingly.
56 * ! The transition point depends on FP_COPY
57 * ! For both versions %o5 is reserved
58 *
59 * kcopy():
60 * if(length > FP_COPY)
357 fmuld %f0, %f2, %f10 ;\
358 faddd %f0, %f2, %f12 ;\
359 fmuld %f0, %f2, %f14 ;\
360 faddd %f0, %f2, %f16 ;\
361 fmuld %f0, %f2, %f18 ;\
362 faddd %f0, %f2, %f20 ;\
363 fmuld %f0, %f2, %f22 ;\
364 faddd %f0, %f2, %f24 ;\
365 fmuld %f0, %f2, %f26 ;\
366 faddd %f0, %f2, %f28 ;\
367 fmuld %f0, %f2, %f30 ;\
368 faddd %f0, %f2, %f48 ;\
369 fmuld %f0, %f2, %f50 ;\
370 faddd %f0, %f2, %f52 ;\
371 fmuld %f0, %f2, %f54 ;\
372 faddd %f0, %f2, %f56 ;\
373 fmuld %f0, %f2, %f58 ;\
374 faddd %f0, %f2, %f60 ;\
375 fmuld %f0, %f2, %f62
376
377 #if !defined(lint)
378
379 /*
380 * Macros to save and restore fp registers to/from the stack.
381 * Used to save and restore in-use fp registers when we want to use FP.
382 */
383 #define BST_FP_TOSTACK(tmp1) \
384 /* membar #Sync */ ;\
385 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
386 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
387 stda %f0, [tmp1]ASI_BLK_P ;\
388 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
389 stda %f16, [tmp1]ASI_BLK_P ;\
390 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
391 stda %f48, [tmp1]ASI_BLK_P ;\
392 membar #Sync
393
394 #define BLD_FP_FROMSTACK(tmp1) \
395 /* membar #Sync - provided at copy completion */ ;\
396 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
397 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
398 ldda [tmp1]ASI_BLK_P, %f0 ;\
399 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
400 ldda [tmp1]ASI_BLK_P, %f16 ;\
401 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
402 ldda [tmp1]ASI_BLK_P, %f48 ;\
403 membar #Sync
404 #endif /* NIAGARA_IMPL */
405
406 #endif /* lint */
407 /*
408 * Copy a block of storage, returning an error code if `from' or
409 * `to' takes a kernel pagefault which cannot be resolved.
410 * Returns errno value on pagefault error, 0 if all ok
411 */
412
413 #if defined(lint)
414
415 /* ARGSUSED */
416 int
417 kcopy(const void *from, void *to, size_t count)
418 { return(0); }
419
420 #else /* lint */
421
422 .seg ".text"
423 .align 4
424
425 ENTRY(kcopy)
426 #if !defined(NIAGARA_IMPL)
427 cmp %o2, FP_COPY ! check for small copy/leaf case
428 bgt,pt %ncc, .kcopy_more !
429 nop
430 .kcopy_small: ! setup error handler
431 sethi %hi(.sm_copyerr), %o4
432 or %o4, %lo(.sm_copyerr), %o4 ! .sm_copyerr is lofault value
433 ldn [THREAD_REG + T_LOFAULT], %o5 ! save existing handler
434 ! Note that we carefully do *not* flag the setting of
435 ! t_lofault.
436 membar #Sync ! sync error barrier
437 b .sm_do_copy ! common code
438 stn %o4, [THREAD_REG + T_LOFAULT] ! set t_lofault
439
440
441 .kcopy_more:
575 membar #Sync ! sync error barrier
576 b .do_copy ! common code
577 stn %l7, [THREAD_REG + T_LOFAULT] ! set t_lofault
578
579 /*
580 * We got here because of a fault during kcopy.
581 * Errno value is in %g1.
582 */
583 .copyerr:
584 ! The kcopy() *always* sets a t_lofault handler and it ORs LOFAULT_SET
585 ! into %o5 to indicate it has set t_lofault handler. Need to clear
586 ! LOFAULT_SET flag before restoring the error handler.
587 andn %o5, LOFAULT_SET, %o5
588 membar #Sync ! sync error barrier
589 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
590 ret
591 restore %g1, 0, %o0
592 #endif /* NIAGARA_IMPL */
593
594 SET_SIZE(kcopy)
595 #endif /* lint */
596
597
598 /*
599 * Copy a block of storage - must not overlap (from + len <= to).
600 */
601 #if defined(lint)
602
603 /* ARGSUSED */
604 void
605 bcopy(const void *from, void *to, size_t count)
606 {}
607
608 #else /* lint */
609
610 ENTRY(bcopy)
611 #if !defined(NIAGARA_IMPL)
612 cmp %o2, FP_COPY ! check for small copy/leaf case
613 bgt,pt %ncc, .bcopy_more !
614 nop
615 .bcopy_small: ! setup error handler
616 ldn [THREAD_REG + T_LOFAULT], %o5 ! save existing handler
617 tst %o5
618 bz,pt %icc, .sm_do_copy
619 sethi %hi(.sm_copyerr), %o4
620 or %o4, %lo(.sm_copyerr), %o4 ! .sm_copyerr is lofault value
621 membar #Sync ! sync error barrier
622 stn %o4, [THREAD_REG + T_LOFAULT] ! set t_lofault
623 or %o5, LOFAULT_SET, %o5 ! Error should trampoline
624 .sm_do_copy:
625 mov %o0, %g1 ! save %o0
626 cmp %o2, SHORTCOPY ! make sure there is enough to align
627 ble,pt %ncc, .bc_smallest
628 andcc %o1, 0x7, %o3 ! is dest long aligned
629 bnz,pn %ncc, .bc_align
2669 * boundaries. Aligns source and destination and returns a count
2670 * of aligned bytes to transfer in %i3
2671 */
2672 1:
2673 inc %i0 ! inc from
2674 stb %o4, [%i1] ! write a byte
2675 inc %i1 ! inc to
2676 dec %i2 ! dec count
2677 .alignit:
2678 btst %o0, %i0 ! %o0 is bit mask to check for alignment
2679 bnz,a 1b
2680 ldub [%i0], %o4 ! read next byte
2681
2682 retl
2683 andn %i2, %o0, %i3 ! return size of aligned bytes
2684
2685 SET_SIZE(bcopy)
2686
2687 #endif /* NIAGARA_IMPL */
2688
2689 #endif /* lint */
2690
2691 /*
2692 * Block copy with possibly overlapped operands.
2693 */
2694
2695 #if defined(lint)
2696
2697 /*ARGSUSED*/
2698 void
2699 ovbcopy(const void *from, void *to, size_t count)
2700 {}
2701
2702 #else /* lint */
2703
2704 ENTRY(ovbcopy)
2705 tst %o2 ! check count
2706 bgu,a %ncc, 1f ! nothing to do or bad arguments
2707 subcc %o0, %o1, %o3 ! difference of from and to address
2708
2709 retl ! return
2710 nop
2711 1:
2712 bneg,a %ncc, 2f
2713 neg %o3 ! if < 0, make it positive
2714 2: cmp %o2, %o3 ! cmp size and abs(from - to)
2715 bleu %ncc, bcopy ! if size <= abs(diff): use bcopy,
2716 .empty ! no overlap
2717 cmp %o0, %o1 ! compare from and to addresses
2718 blu %ncc, .ov_bkwd ! if from < to, copy backwards
2719 nop
2720 !
2721 ! Copy forwards.
2722 !
2723 .ov_fwd:
2726 stb %o3, [%o1] ! write to address
2727 deccc %o2 ! dec count
2728 bgu %ncc, .ov_fwd ! loop till done
2729 inc %o1 ! inc to address
2730
2731 retl ! return
2732 nop
2733 !
2734 ! Copy backwards.
2735 !
2736 .ov_bkwd:
2737 deccc %o2 ! dec count
2738 ldub [%o0 + %o2], %o3 ! get byte at end of src
2739 bgu %ncc, .ov_bkwd ! loop till done
2740 stb %o3, [%o1 + %o2] ! delay slot, store at end of dst
2741
2742 retl ! return
2743 nop
2744 SET_SIZE(ovbcopy)
2745
2746 #endif /* lint */
2747
2748 /*
2749 * hwblkpagecopy()
2750 *
2751 * Copies exactly one page. This routine assumes the caller (ppcopy)
2752 * has already disabled kernel preemption and has checked
2753 * use_hw_bcopy.
2754 */
2755 #ifdef lint
2756 /*ARGSUSED*/
2757 void
2758 hwblkpagecopy(const void *src, void *dst)
2759 { }
2760 #else /* lint */
2761 ENTRY(hwblkpagecopy)
2762 save %sp, -SA(MINFRAME), %sp
2763
2764 ! %i0 - source address (arg)
2765 ! %i1 - destination address (arg)
2766 ! %i2 - length of region (not arg)
2767
2768 set PAGESIZE, %i2
2769
2770 /*
2771 * Copying exactly one page and PAGESIZE is in mutliple of 0x80.
2772 */
2773 mov ASI_BLK_INIT_ST_QUAD_LDD_P, %asi
2774 prefetch [%i0+0x0], #one_read
2775 prefetch [%i0+0x40], #one_read
2776 1:
2777 prefetch [%i0+0x80], #one_read
2778 prefetch [%i0+0xc0], #one_read
2779 ldda [%i0+0x0]%asi, %l0
2780 ldda [%i0+0x10]%asi, %l2
2793 ldda [%i0+0x60]%asi, %l4
2794 ldda [%i0+0x70]%asi, %l6
2795 stxa %l0, [%i1+0x40]%asi
2796 stxa %l1, [%i1+0x48]%asi
2797 stxa %l2, [%i1+0x50]%asi
2798 stxa %l3, [%i1+0x58]%asi
2799 stxa %l4, [%i1+0x60]%asi
2800 stxa %l5, [%i1+0x68]%asi
2801 stxa %l6, [%i1+0x70]%asi
2802 stxa %l7, [%i1+0x78]%asi
2803
2804 add %i0, 0x80, %i0
2805 subcc %i2, 0x80, %i2
2806 bgu,pt %xcc, 1b
2807 add %i1, 0x80, %i1
2808
2809 membar #Sync
2810 ret
2811 restore %g0, 0, %o0
2812 SET_SIZE(hwblkpagecopy)
2813 #endif /* lint */
2814
2815
2816 /*
2817 * Transfer data to and from user space -
2818 * Note that these routines can cause faults
2819 * It is assumed that the kernel has nothing at
2820 * less than KERNELBASE in the virtual address space.
2821 *
2822 * Note that copyin(9F) and copyout(9F) are part of the
2823 * DDI/DKI which specifies that they return '-1' on "errors."
2824 *
2825 * Sigh.
2826 *
2827 * So there's two extremely similar routines - xcopyin() and xcopyout()
2828 * which return the errno that we've faithfully computed. This
2829 * allows other callers (e.g. uiomove(9F)) to work correctly.
2830 * Given that these are used pretty heavily, we expand the calling
2831 * sequences inline for all flavours (rather than making wrappers).
2832 *
2833 * There are also stub routines for xcopyout_little and xcopyin_little,
2887 * is left to be copied by examining %o3. If that is zero, we're
2888 * done and can go home. If not, we figure out what the largest
2889 * chunk size left to be copied is and branch to that copy loop
2890 * unless there's only one byte left. We load that as we're
2891 * branching to code that stores it just before we return.
2892 *
2893 * Fault handlers are invoked if we reference memory that has no
2894 * current mapping. All forms share the same copyio_fault handler.
2895 * This routine handles fixing up the stack and general housecleaning.
2896 * Each copy operation has a simple fault handler that is then called
2897 * to do the work specific to the invidual operation. The handler
2898 * for copyOP and xcopyOP are found at the end of individual function.
2899 * The handlers for xcopyOP_little are found at the end of xcopyin_little.
2900 * The handlers for copyOP_noerr are found at the end of copyin_noerr.
2901 */
2902
2903 /*
2904 * Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
2905 */
2906
2907 #if defined(lint)
2908
2909 /*ARGSUSED*/
2910 int
2911 copyout(const void *kaddr, void *uaddr, size_t count)
2912 { return (0); }
2913
2914 #else /* lint */
2915
2916 /*
2917 * We save the arguments in the following registers in case of a fault:
2918 * kaddr - %g2
2919 * uaddr - %g3
2920 * count - %g4
2921 */
2922 #define SAVE_SRC %g2
2923 #define SAVE_DST %g3
2924 #define SAVE_COUNT %g4
2925
2926 #define REAL_LOFAULT %g5
2927 #define SAVED_LOFAULT %g6
2928
2929 /*
2930 * Generic copyio fault handler. This is the first line of defense when a
2931 * fault occurs in (x)copyin/(x)copyout. In order for this to function
2932 * properly, the value of the 'real' lofault handler should be in REAL_LOFAULT.
2933 * This allows us to share common code for all the flavors of the copy
2934 * operations, including the _noerr versions.
2935 *
5082
5083 .copyout_exit:
5084 membar #Sync
5085 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
5086 ret
5087 restore %g0, 0, %o0
5088
5089 .copyout_err:
5090 ldn [THREAD_REG + T_COPYOPS], %o4
5091 brz %o4, 2f
5092 nop
5093 ldn [%o4 + CP_COPYOUT], %g2
5094 jmp %g2
5095 nop
5096 2:
5097 retl
5098 mov -1, %o0
5099 #endif /* NIAGARA_IMPL */
5100 SET_SIZE(copyout)
5101
5102 #endif /* lint */
5103
5104
5105 #ifdef lint
5106
5107 /*ARGSUSED*/
5108 int
5109 xcopyout(const void *kaddr, void *uaddr, size_t count)
5110 { return (0); }
5111
5112 #else /* lint */
5113
5114 ENTRY(xcopyout)
5115 sethi %hi(.xcopyout_err), REAL_LOFAULT
5116 b .do_copyout
5117 or REAL_LOFAULT, %lo(.xcopyout_err), REAL_LOFAULT
5118 .xcopyout_err:
5119 ldn [THREAD_REG + T_COPYOPS], %o4
5120 brz %o4, 2f
5121 nop
5122 ldn [%o4 + CP_XCOPYOUT], %g2
5123 jmp %g2
5124 nop
5125 2:
5126 retl
5127 mov %g1, %o0
5128 SET_SIZE(xcopyout)
5129
5130 #endif /* lint */
5131
5132 #ifdef lint
5133
5134 /*ARGSUSED*/
5135 int
5136 xcopyout_little(const void *kaddr, void *uaddr, size_t count)
5137 { return (0); }
5138
5139 #else /* lint */
5140
5141 ENTRY(xcopyout_little)
5142 sethi %hi(.little_err), %o4
5143 ldn [THREAD_REG + T_LOFAULT], %o5
5144 or %o4, %lo(.little_err), %o4
5145 membar #Sync ! sync error barrier
5146 stn %o4, [THREAD_REG + T_LOFAULT]
5147
5148 subcc %g0, %o2, %o3
5149 add %o0, %o2, %o0
5150 bz,pn %ncc, 2f ! check for zero bytes
5151 sub %o2, 1, %o4
5152 add %o0, %o4, %o0 ! start w/last byte
5153 add %o1, %o2, %o1
5154 ldub [%o0+%o3], %o4
5155
5156 1: stba %o4, [%o1+%o3]ASI_AIUSL
5157 inccc %o3
5158 sub %o0, 2, %o0 ! get next byte
5159 bcc,a,pt %ncc, 1b
5160 ldub [%o0+%o3], %o4
5161
5162 2: membar #Sync ! sync error barrier
5163 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
5164 retl
5165 mov %g0, %o0 ! return (0)
5166 SET_SIZE(xcopyout_little)
5167
5168 #endif /* lint */
5169
5170 /*
5171 * Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
5172 */
5173
5174 #if defined(lint)
5175
5176 /*ARGSUSED*/
5177 int
5178 copyin(const void *uaddr, void *kaddr, size_t count)
5179 { return (0); }
5180
5181 #else /* lint */
5182
5183 ENTRY(copyin)
5184 sethi %hi(.copyin_err), REAL_LOFAULT
5185 or REAL_LOFAULT, %lo(.copyin_err), REAL_LOFAULT
5186
5187 #if !defined(NIAGARA_IMPL)
5188 .do_copyin:
5189 tst %o2 ! check for zero count; quick exit
5190 bz,pt %ncc, .ci_smallqx
5191 mov %o0, SAVE_SRC
5192 mov %o1, SAVE_DST
5193 mov %o2, SAVE_COUNT
5194 cmp %o2, FP_COPY ! check for small copy/leaf case
5195 bgt,pt %ncc, .ci_copy_more
5196 ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT
5197 /*
5198 * Small copy in code
5199 *
5200 */
5201 sethi %hi(copyio_fault_nowindow), %o3
5202 or %o3, %lo(copyio_fault_nowindow), %o3
7266 inc %i0
7267
7268 .copyin_exit:
7269 membar #Sync
7270 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7271 ret
7272 restore %g0, 0, %o0
7273 .copyin_err:
7274 ldn [THREAD_REG + T_COPYOPS], %o4
7275 brz %o4, 2f
7276 nop
7277 ldn [%o4 + CP_COPYIN], %g2
7278 jmp %g2
7279 nop
7280 2:
7281 retl
7282 mov -1, %o0
7283 #endif /* NIAGARA_IMPL */
7284 SET_SIZE(copyin)
7285
7286 #endif /* lint */
7287
7288 #ifdef lint
7289
7290 /*ARGSUSED*/
7291 int
7292 xcopyin(const void *uaddr, void *kaddr, size_t count)
7293 { return (0); }
7294
7295 #else /* lint */
7296
7297 ENTRY(xcopyin)
7298 sethi %hi(.xcopyin_err), REAL_LOFAULT
7299 b .do_copyin
7300 or REAL_LOFAULT, %lo(.xcopyin_err), REAL_LOFAULT
7301 .xcopyin_err:
7302 ldn [THREAD_REG + T_COPYOPS], %o4
7303 brz %o4, 2f
7304 nop
7305 ldn [%o4 + CP_XCOPYIN], %g2
7306 jmp %g2
7307 nop
7308 2:
7309 retl
7310 mov %g1, %o0
7311 SET_SIZE(xcopyin)
7312
7313 #endif /* lint */
7314
7315 #ifdef lint
7316
7317 /*ARGSUSED*/
7318 int
7319 xcopyin_little(const void *uaddr, void *kaddr, size_t count)
7320 { return (0); }
7321
7322 #else /* lint */
7323
7324 ENTRY(xcopyin_little)
7325 sethi %hi(.little_err), %o4
7326 ldn [THREAD_REG + T_LOFAULT], %o5
7327 or %o4, %lo(.little_err), %o4
7328 membar #Sync ! sync error barrier
7329 stn %o4, [THREAD_REG + T_LOFAULT]
7330
7331 subcc %g0, %o2, %o3
7332 add %o0, %o2, %o0
7333 bz,pn %ncc, 2f ! check for zero bytes
7334 sub %o2, 1, %o4
7335 add %o0, %o4, %o0 ! start w/last byte
7336 add %o1, %o2, %o1
7337 lduba [%o0+%o3]ASI_AIUSL, %o4
7338
7339 1: stb %o4, [%o1+%o3]
7340 inccc %o3
7341 sub %o0, 2, %o0 ! get next byte
7342 bcc,a,pt %ncc, 1b
7343 lduba [%o0+%o3]ASI_AIUSL, %o4
7344
7345 2: membar #Sync ! sync error barrier
7346 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7347 retl
7348 mov %g0, %o0 ! return (0)
7349
7350 .little_err:
7351 membar #Sync ! sync error barrier
7352 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7353 retl
7354 mov %g1, %o0
7355 SET_SIZE(xcopyin_little)
7356
7357 #endif /* lint */
7358
7359
7360 /*
7361 * Copy a block of storage - must not overlap (from + len <= to).
7362 * No fault handler installed (to be called under on_fault())
7363 */
7364 #if defined(lint)
7365
7366 /* ARGSUSED */
7367 void
7368 copyin_noerr(const void *ufrom, void *kto, size_t count)
7369 {}
7370
7371 #else /* lint */
7372
7373 ENTRY(copyin_noerr)
7374 sethi %hi(.copyio_noerr), REAL_LOFAULT
7375 b .do_copyin
7376 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
7377 .copyio_noerr:
7378 jmp SAVED_LOFAULT
7379 nop
7380 SET_SIZE(copyin_noerr)
7381
7382 #endif /* lint */
7383
7384 /*
7385 * Copy a block of storage - must not overlap (from + len <= to).
7386 * No fault handler installed (to be called under on_fault())
7387 */
7388
7389 #if defined(lint)
7390
7391 /* ARGSUSED */
7392 void
7393 copyout_noerr(const void *kfrom, void *uto, size_t count)
7394 {}
7395
7396 #else /* lint */
7397
7398 ENTRY(copyout_noerr)
7399 sethi %hi(.copyio_noerr), REAL_LOFAULT
7400 b .do_copyout
7401 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
7402 SET_SIZE(copyout_noerr)
7403
7404 #endif /* lint */
7405
7406 #if defined(lint)
7407
7408 int use_hw_bcopy = 1;
7409 int use_hw_bzero = 1;
7410 uint_t hw_copy_limit_1 = 0x100;
7411 uint_t hw_copy_limit_2 = 0x200;
7412 uint_t hw_copy_limit_4 = 0x400;
7413 uint_t hw_copy_limit_8 = 0x400;
7414
7415 #else /* !lint */
7416
7417 .align 4
7418 DGDEF(use_hw_bcopy)
7419 .word 1
7420 DGDEF(use_hw_bzero)
7421 .word 1
7422 DGDEF(hw_copy_limit_1)
7423 .word 0x100
7424 DGDEF(hw_copy_limit_2)
7425 .word 0x200
7426 DGDEF(hw_copy_limit_4)
7427 .word 0x400
7428 DGDEF(hw_copy_limit_8)
7429 .word 0x400
7430
7431 .align 64
7432 .section ".text"
7433 #endif /* !lint */
7434
7435 /*
7436 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
7437 * longer than 256 bytes in length using Niagara's block stores/quad store.
7438 * If the criteria for using this routine are not met then it calls bzero
7439 * and returns 1. Otherwise 0 is returned indicating success.
7440 * Caller is responsible for ensuring use_hw_bzero is true and that
7441 * kpreempt_disable() has been called.
7442 */
7443 #ifdef lint
7444 /*ARGSUSED*/
7445 int
7446 hwblkclr(void *addr, size_t len)
7447 {
7448 return(0);
7449 }
7450 #else /* lint */
7451 ! %i0 - start address
7452 ! %i1 - length of region (multiple of 64)
7453
7454 ENTRY(hwblkclr)
7455 save %sp, -SA(MINFRAME), %sp
7456
7457 ! Must be block-aligned
7458 andcc %i0, 0x3f, %g0
7459 bnz,pn %ncc, 1f
7460 nop
7461
7462 ! ... and must be 256 bytes or more
7463 cmp %i1, 0x100
7464 blu,pn %ncc, 1f
7465 nop
7466
7467 ! ... and length must be a multiple of 64
7468 andcc %i1, 0x3f, %g0
7469 bz,pn %ncc, .pz_doblock
7470 mov ASI_BLK_INIT_ST_QUAD_LDD_P, %asi
7528
7529 3:
7530 stxa %g0, [%i0+0x0]%asi
7531 stxa %g0, [%i0+0x8]%asi
7532 stxa %g0, [%i0+0x10]%asi
7533 stxa %g0, [%i0+0x18]%asi
7534 stxa %g0, [%i0+0x20]%asi
7535 stxa %g0, [%i0+0x28]%asi
7536 stxa %g0, [%i0+0x30]%asi
7537 stxa %g0, [%i0+0x38]%asi
7538
7539 subcc %i1, 0x40, %i1
7540 bgu,pt %ncc, 3b
7541 add %i0, 0x40, %i0
7542
7543 .pz_finish:
7544 membar #Sync
7545 ret
7546 restore %g0, 0, %o0 ! return (bzero or not)
7547 SET_SIZE(hwblkclr)
7548 #endif /* lint */
7549
7550 #ifdef lint
7551 /* Copy 32 bytes of data from src to dst using physical addresses */
7552 /*ARGSUSED*/
7553 void
7554 hw_pa_bcopy32(uint64_t src, uint64_t dst)
7555 {}
7556 #else /*!lint */
7557
7558 /*
7559 * Copy 32 bytes of data from src (%o0) to dst (%o1)
7560 * using physical addresses.
7561 */
7562 ENTRY_NP(hw_pa_bcopy32)
7563 rdpr %pstate, %g1
7564 andn %g1, PSTATE_IE, %g2
7565 wrpr %g0, %g2, %pstate
7566
7567 ldxa [%o0]ASI_MEM, %o2
7568 add %o0, 8, %o0
7569 ldxa [%o0]ASI_MEM, %o3
7570 add %o0, 8, %o0
7571 ldxa [%o0]ASI_MEM, %o4
7572 add %o0, 8, %o0
7573 ldxa [%o0]ASI_MEM, %o5
7574 stxa %o2, [%o1]ASI_MEM
7575 add %o1, 8, %o1
7576 stxa %o3, [%o1]ASI_MEM
7577 add %o1, 8, %o1
7578 stxa %o4, [%o1]ASI_MEM
7579 add %o1, 8, %o1
7580 stxa %o5, [%o1]ASI_MEM
7581
7582 membar #Sync
7583 retl
7584 wrpr %g0, %g1, %pstate
7585 SET_SIZE(hw_pa_bcopy32)
7586 #endif /* lint */
7587
7588 /*
7589 * Zero a block of storage.
7590 *
7591 * uzero is used by the kernel to zero a block in user address space.
7592 */
7593
7594 /*
7595 * Control flow of the bzero/kzero/uzero routine.
7596 *
7597 * For fewer than 7 bytes stores, bytes will be zeroed.
7598 *
7599 * For less than 15 bytes stores, align the address on 4 byte boundary.
7600 * Then store as many 4-byte chunks, followed by trailing bytes.
7601 *
7602 * For sizes greater than 15 bytes, align the address on 8 byte boundary.
7603 * if (count > 128) {
7604 * store as many 8-bytes chunks to block align the address
7605 * store using ASI_BLK_INIT_ST_QUAD_LDD_P (bzero/kzero) OR
7606 * store using ASI_BLK_INIT_QUAD_LDD_AIUS (uzero)
7607 * }
7608 * Store as many 8-byte chunks, followed by trailing bytes.
7609 */
7610
7611 #if defined(lint)
7612
7613 /* ARGSUSED */
7614 int
7615 kzero(void *addr, size_t count)
7616 { return(0); }
7617
7618 /* ARGSUSED */
7619 void
7620 uzero(void *addr, size_t count)
7621 {}
7622
7623 #else /* lint */
7624
7625 ENTRY(uzero)
7626 !
7627 ! Set a new lo_fault handler only if we came in with one
7628 ! already specified.
7629 !
7630 wr %g0, ASI_USER, %asi
7631 ldn [THREAD_REG + T_LOFAULT], %o5
7632 tst %o5
7633 bz,pt %ncc, .do_zero
7634 sethi %hi(.zeroerr), %o2
7635 or %o2, %lo(.zeroerr), %o2
7636 membar #Sync
7637 ba,pt %ncc, .do_zero
7638 stn %o2, [THREAD_REG + T_LOFAULT]
7639
7640 ENTRY(kzero)
7641 !
7642 ! Always set a lo_fault handler
7643 !
7644 wr %g0, ASI_P, %asi
7677 ! Old handler was zero. Just return the error.
7678 !
7679 retl ! return
7680 mov %g1, %o0 ! error code from %g1
7681 3:
7682 !
7683 ! We're here because %o5 was non-zero. It was non-zero
7684 ! because either LOFAULT_SET was present, a previous fault
7685 ! handler was present or both. In all cases we need to reset
7686 ! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
7687 ! before we either simply return the error or we invoke the
7688 ! previously specified handler.
7689 !
7690 be %ncc, 2b
7691 stn %o5, [THREAD_REG + T_LOFAULT]
7692 jmp %o5 ! goto real handler
7693 nop
7694 SET_SIZE(kzero)
7695 SET_SIZE(uzero)
7696
7697 #endif /* lint */
7698
7699 /*
7700 * Zero a block of storage.
7701 */
7702
7703 #if defined(lint)
7704
7705 /* ARGSUSED */
7706 void
7707 bzero(void *addr, size_t count)
7708 {}
7709
7710 #else /* lint */
7711
7712 ENTRY(bzero)
7713 wr %g0, ASI_P, %asi
7714
7715 ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector
7716 tst %o5
7717 bz,pt %ncc, .do_zero
7718 sethi %hi(.zeroerr), %o2
7719 or %o2, %lo(.zeroerr), %o2
7720 membar #Sync ! sync error barrier
7721 stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector
7722
7723 .do_zero:
7724 cmp %o1, 7
7725 blu,pn %ncc, .byteclr
7726 nop
7727
7728 cmp %o1, 15
7729 blu,pn %ncc, .wdalign
7730 nop
7731
7902 ! We're just concerned with whether t_lofault was set
7903 ! when we came in. We end up here from either kzero()
7904 ! or bzero(). kzero() *always* sets a lofault handler.
7905 ! It ors LOFAULT_SET into %o5 to indicate it has done
7906 ! this even if the value of %o5 is otherwise zero.
7907 ! bzero() sets a lofault handler *only* if one was
7908 ! previously set. Accordingly we need to examine
7909 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET
7910 ! before resetting the error handler.
7911 !
7912 tst %o5
7913 bz %ncc, 1f
7914 andn %o5, LOFAULT_SET, %o5
7915 membar #Sync ! sync error barrier
7916 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7917 1:
7918 retl
7919 clr %o0 ! return (0)
7920
7921 SET_SIZE(bzero)
7922 #endif /* lint */
|
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25
26 #include <sys/param.h>
27 #include <sys/errno.h>
28 #include <sys/asm_linkage.h>
29 #include <sys/vtrace.h>
30 #include <sys/machthread.h>
31 #include <sys/clock.h>
32 #include <sys/asi.h>
33 #include <sys/fsr.h>
34 #include <sys/privregs.h>
35 #include <sys/machasi.h>
36 #include <sys/niagaraasi.h>
37
38 #include "assym.h"
39
40
41 /*
42 * Pseudo-code to aid in understanding the control flow of the
43 * bcopy/kcopy routine.
44 *
45 * ! WARNING : <Register usage convention>
46 * ! In kcopy() the %o5, holds previous error handler and a flag
47 * ! LOFAULT_SET (low bits). The %o5 is null in bcopy().
48 * ! The %o5 is not available for any other use.
49 *
50 * On entry:
51 * ! Determine whether to use the FP register version or the
52 * ! the leaf routine version depending on the size of the copy.
53 * ! Set up error handling accordingly.
54 * ! The transition point depends on FP_COPY
55 * ! For both versions %o5 is reserved
56 *
57 * kcopy():
58 * if(length > FP_COPY)
355 fmuld %f0, %f2, %f10 ;\
356 faddd %f0, %f2, %f12 ;\
357 fmuld %f0, %f2, %f14 ;\
358 faddd %f0, %f2, %f16 ;\
359 fmuld %f0, %f2, %f18 ;\
360 faddd %f0, %f2, %f20 ;\
361 fmuld %f0, %f2, %f22 ;\
362 faddd %f0, %f2, %f24 ;\
363 fmuld %f0, %f2, %f26 ;\
364 faddd %f0, %f2, %f28 ;\
365 fmuld %f0, %f2, %f30 ;\
366 faddd %f0, %f2, %f48 ;\
367 fmuld %f0, %f2, %f50 ;\
368 faddd %f0, %f2, %f52 ;\
369 fmuld %f0, %f2, %f54 ;\
370 faddd %f0, %f2, %f56 ;\
371 fmuld %f0, %f2, %f58 ;\
372 faddd %f0, %f2, %f60 ;\
373 fmuld %f0, %f2, %f62
374
375 /*
376 * Macros to save and restore fp registers to/from the stack.
377 * Used to save and restore in-use fp registers when we want to use FP.
378 */
379 #define BST_FP_TOSTACK(tmp1) \
380 /* membar #Sync */ ;\
381 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
382 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
383 stda %f0, [tmp1]ASI_BLK_P ;\
384 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
385 stda %f16, [tmp1]ASI_BLK_P ;\
386 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
387 stda %f48, [tmp1]ASI_BLK_P ;\
388 membar #Sync
389
390 #define BLD_FP_FROMSTACK(tmp1) \
391 /* membar #Sync - provided at copy completion */ ;\
392 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
393 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
394 ldda [tmp1]ASI_BLK_P, %f0 ;\
395 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
396 ldda [tmp1]ASI_BLK_P, %f16 ;\
397 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
398 ldda [tmp1]ASI_BLK_P, %f48 ;\
399 membar #Sync
400
401 #endif
402 /*
403 * Copy a block of storage, returning an error code if `from' or
404 * `to' takes a kernel pagefault which cannot be resolved.
405 * Returns errno value on pagefault error, 0 if all ok
406 */
407
408 .seg ".text"
409 .align 4
410
411 ENTRY(kcopy)
412 #if !defined(NIAGARA_IMPL)
413 cmp %o2, FP_COPY ! check for small copy/leaf case
414 bgt,pt %ncc, .kcopy_more !
415 nop
416 .kcopy_small: ! setup error handler
417 sethi %hi(.sm_copyerr), %o4
418 or %o4, %lo(.sm_copyerr), %o4 ! .sm_copyerr is lofault value
419 ldn [THREAD_REG + T_LOFAULT], %o5 ! save existing handler
420 ! Note that we carefully do *not* flag the setting of
421 ! t_lofault.
422 membar #Sync ! sync error barrier
423 b .sm_do_copy ! common code
424 stn %o4, [THREAD_REG + T_LOFAULT] ! set t_lofault
425
426
427 .kcopy_more:
561 membar #Sync ! sync error barrier
562 b .do_copy ! common code
563 stn %l7, [THREAD_REG + T_LOFAULT] ! set t_lofault
564
565 /*
566 * We got here because of a fault during kcopy.
567 * Errno value is in %g1.
568 */
569 .copyerr:
570 ! The kcopy() *always* sets a t_lofault handler and it ORs LOFAULT_SET
571 ! into %o5 to indicate it has set t_lofault handler. Need to clear
572 ! LOFAULT_SET flag before restoring the error handler.
573 andn %o5, LOFAULT_SET, %o5
574 membar #Sync ! sync error barrier
575 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
576 ret
577 restore %g1, 0, %o0
578 #endif /* NIAGARA_IMPL */
579
580 SET_SIZE(kcopy)
581
582
583 /*
584 * Copy a block of storage - must not overlap (from + len <= to).
585 */
586
587 ENTRY(bcopy)
588 #if !defined(NIAGARA_IMPL)
589 cmp %o2, FP_COPY ! check for small copy/leaf case
590 bgt,pt %ncc, .bcopy_more !
591 nop
592 .bcopy_small: ! setup error handler
593 ldn [THREAD_REG + T_LOFAULT], %o5 ! save existing handler
594 tst %o5
595 bz,pt %icc, .sm_do_copy
596 sethi %hi(.sm_copyerr), %o4
597 or %o4, %lo(.sm_copyerr), %o4 ! .sm_copyerr is lofault value
598 membar #Sync ! sync error barrier
599 stn %o4, [THREAD_REG + T_LOFAULT] ! set t_lofault
600 or %o5, LOFAULT_SET, %o5 ! Error should trampoline
601 .sm_do_copy:
602 mov %o0, %g1 ! save %o0
603 cmp %o2, SHORTCOPY ! make sure there is enough to align
604 ble,pt %ncc, .bc_smallest
605 andcc %o1, 0x7, %o3 ! is dest long aligned
606 bnz,pn %ncc, .bc_align
2646 * boundaries. Aligns source and destination and returns a count
2647 * of aligned bytes to transfer in %i3
2648 */
2649 1:
2650 inc %i0 ! inc from
2651 stb %o4, [%i1] ! write a byte
2652 inc %i1 ! inc to
2653 dec %i2 ! dec count
2654 .alignit:
2655 btst %o0, %i0 ! %o0 is bit mask to check for alignment
2656 bnz,a 1b
2657 ldub [%i0], %o4 ! read next byte
2658
2659 retl
2660 andn %i2, %o0, %i3 ! return size of aligned bytes
2661
2662 SET_SIZE(bcopy)
2663
2664 #endif /* NIAGARA_IMPL */
2665
2666 /*
2667 * Block copy with possibly overlapped operands.
2668 */
2669
2670 ENTRY(ovbcopy)
2671 tst %o2 ! check count
2672 bgu,a %ncc, 1f ! nothing to do or bad arguments
2673 subcc %o0, %o1, %o3 ! difference of from and to address
2674
2675 retl ! return
2676 nop
2677 1:
2678 bneg,a %ncc, 2f
2679 neg %o3 ! if < 0, make it positive
2680 2: cmp %o2, %o3 ! cmp size and abs(from - to)
2681 bleu %ncc, bcopy ! if size <= abs(diff): use bcopy,
2682 .empty ! no overlap
2683 cmp %o0, %o1 ! compare from and to addresses
2684 blu %ncc, .ov_bkwd ! if from < to, copy backwards
2685 nop
2686 !
2687 ! Copy forwards.
2688 !
2689 .ov_fwd:
2692 stb %o3, [%o1] ! write to address
2693 deccc %o2 ! dec count
2694 bgu %ncc, .ov_fwd ! loop till done
2695 inc %o1 ! inc to address
2696
2697 retl ! return
2698 nop
2699 !
2700 ! Copy backwards.
2701 !
2702 .ov_bkwd:
2703 deccc %o2 ! dec count
2704 ldub [%o0 + %o2], %o3 ! get byte at end of src
2705 bgu %ncc, .ov_bkwd ! loop till done
2706 stb %o3, [%o1 + %o2] ! delay slot, store at end of dst
2707
2708 retl ! return
2709 nop
2710 SET_SIZE(ovbcopy)
2711
2712 /*
2713 * hwblkpagecopy()
2714 *
2715 * Copies exactly one page. This routine assumes the caller (ppcopy)
2716 * has already disabled kernel preemption and has checked
2717 * use_hw_bcopy.
2718 */
2719 ENTRY(hwblkpagecopy)
2720 save %sp, -SA(MINFRAME), %sp
2721
2722 ! %i0 - source address (arg)
2723 ! %i1 - destination address (arg)
2724 ! %i2 - length of region (not arg)
2725
2726 set PAGESIZE, %i2
2727
2728 /*
2729 * Copying exactly one page and PAGESIZE is in mutliple of 0x80.
2730 */
2731 mov ASI_BLK_INIT_ST_QUAD_LDD_P, %asi
2732 prefetch [%i0+0x0], #one_read
2733 prefetch [%i0+0x40], #one_read
2734 1:
2735 prefetch [%i0+0x80], #one_read
2736 prefetch [%i0+0xc0], #one_read
2737 ldda [%i0+0x0]%asi, %l0
2738 ldda [%i0+0x10]%asi, %l2
2751 ldda [%i0+0x60]%asi, %l4
2752 ldda [%i0+0x70]%asi, %l6
2753 stxa %l0, [%i1+0x40]%asi
2754 stxa %l1, [%i1+0x48]%asi
2755 stxa %l2, [%i1+0x50]%asi
2756 stxa %l3, [%i1+0x58]%asi
2757 stxa %l4, [%i1+0x60]%asi
2758 stxa %l5, [%i1+0x68]%asi
2759 stxa %l6, [%i1+0x70]%asi
2760 stxa %l7, [%i1+0x78]%asi
2761
2762 add %i0, 0x80, %i0
2763 subcc %i2, 0x80, %i2
2764 bgu,pt %xcc, 1b
2765 add %i1, 0x80, %i1
2766
2767 membar #Sync
2768 ret
2769 restore %g0, 0, %o0
2770 SET_SIZE(hwblkpagecopy)
2771
2772
2773 /*
2774 * Transfer data to and from user space -
2775 * Note that these routines can cause faults
2776 * It is assumed that the kernel has nothing at
2777 * less than KERNELBASE in the virtual address space.
2778 *
2779 * Note that copyin(9F) and copyout(9F) are part of the
2780 * DDI/DKI which specifies that they return '-1' on "errors."
2781 *
2782 * Sigh.
2783 *
2784 * So there's two extremely similar routines - xcopyin() and xcopyout()
2785 * which return the errno that we've faithfully computed. This
2786 * allows other callers (e.g. uiomove(9F)) to work correctly.
2787 * Given that these are used pretty heavily, we expand the calling
2788 * sequences inline for all flavours (rather than making wrappers).
2789 *
2790 * There are also stub routines for xcopyout_little and xcopyin_little,
2844 * is left to be copied by examining %o3. If that is zero, we're
2845 * done and can go home. If not, we figure out what the largest
2846 * chunk size left to be copied is and branch to that copy loop
2847 * unless there's only one byte left. We load that as we're
2848 * branching to code that stores it just before we return.
2849 *
2850 * Fault handlers are invoked if we reference memory that has no
2851 * current mapping. All forms share the same copyio_fault handler.
2852 * This routine handles fixing up the stack and general housecleaning.
2853 * Each copy operation has a simple fault handler that is then called
2854 * to do the work specific to the invidual operation. The handler
2855 * for copyOP and xcopyOP are found at the end of individual function.
2856 * The handlers for xcopyOP_little are found at the end of xcopyin_little.
2857 * The handlers for copyOP_noerr are found at the end of copyin_noerr.
2858 */
2859
2860 /*
2861 * Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
2862 */
2863
2864 /*
2865 * We save the arguments in the following registers in case of a fault:
2866 * kaddr - %g2
2867 * uaddr - %g3
2868 * count - %g4
2869 */
2870 #define SAVE_SRC %g2
2871 #define SAVE_DST %g3
2872 #define SAVE_COUNT %g4
2873
2874 #define REAL_LOFAULT %g5
2875 #define SAVED_LOFAULT %g6
2876
2877 /*
2878 * Generic copyio fault handler. This is the first line of defense when a
2879 * fault occurs in (x)copyin/(x)copyout. In order for this to function
2880 * properly, the value of the 'real' lofault handler should be in REAL_LOFAULT.
2881 * This allows us to share common code for all the flavors of the copy
2882 * operations, including the _noerr versions.
2883 *
5030
5031 .copyout_exit:
5032 membar #Sync
5033 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
5034 ret
5035 restore %g0, 0, %o0
5036
5037 .copyout_err:
5038 ldn [THREAD_REG + T_COPYOPS], %o4
5039 brz %o4, 2f
5040 nop
5041 ldn [%o4 + CP_COPYOUT], %g2
5042 jmp %g2
5043 nop
5044 2:
5045 retl
5046 mov -1, %o0
5047 #endif /* NIAGARA_IMPL */
5048 SET_SIZE(copyout)
5049
5050
5051 ENTRY(xcopyout)
5052 sethi %hi(.xcopyout_err), REAL_LOFAULT
5053 b .do_copyout
5054 or REAL_LOFAULT, %lo(.xcopyout_err), REAL_LOFAULT
5055 .xcopyout_err:
5056 ldn [THREAD_REG + T_COPYOPS], %o4
5057 brz %o4, 2f
5058 nop
5059 ldn [%o4 + CP_XCOPYOUT], %g2
5060 jmp %g2
5061 nop
5062 2:
5063 retl
5064 mov %g1, %o0
5065 SET_SIZE(xcopyout)
5066
5067 ENTRY(xcopyout_little)
5068 sethi %hi(.little_err), %o4
5069 ldn [THREAD_REG + T_LOFAULT], %o5
5070 or %o4, %lo(.little_err), %o4
5071 membar #Sync ! sync error barrier
5072 stn %o4, [THREAD_REG + T_LOFAULT]
5073
5074 subcc %g0, %o2, %o3
5075 add %o0, %o2, %o0
5076 bz,pn %ncc, 2f ! check for zero bytes
5077 sub %o2, 1, %o4
5078 add %o0, %o4, %o0 ! start w/last byte
5079 add %o1, %o2, %o1
5080 ldub [%o0+%o3], %o4
5081
5082 1: stba %o4, [%o1+%o3]ASI_AIUSL
5083 inccc %o3
5084 sub %o0, 2, %o0 ! get next byte
5085 bcc,a,pt %ncc, 1b
5086 ldub [%o0+%o3], %o4
5087
5088 2: membar #Sync ! sync error barrier
5089 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
5090 retl
5091 mov %g0, %o0 ! return (0)
5092 SET_SIZE(xcopyout_little)
5093
5094 /*
5095 * Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
5096 */
5097
5098 ENTRY(copyin)
5099 sethi %hi(.copyin_err), REAL_LOFAULT
5100 or REAL_LOFAULT, %lo(.copyin_err), REAL_LOFAULT
5101
5102 #if !defined(NIAGARA_IMPL)
5103 .do_copyin:
5104 tst %o2 ! check for zero count; quick exit
5105 bz,pt %ncc, .ci_smallqx
5106 mov %o0, SAVE_SRC
5107 mov %o1, SAVE_DST
5108 mov %o2, SAVE_COUNT
5109 cmp %o2, FP_COPY ! check for small copy/leaf case
5110 bgt,pt %ncc, .ci_copy_more
5111 ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT
5112 /*
5113 * Small copy in code
5114 *
5115 */
5116 sethi %hi(copyio_fault_nowindow), %o3
5117 or %o3, %lo(copyio_fault_nowindow), %o3
7181 inc %i0
7182
7183 .copyin_exit:
7184 membar #Sync
7185 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7186 ret
7187 restore %g0, 0, %o0
7188 .copyin_err:
7189 ldn [THREAD_REG + T_COPYOPS], %o4
7190 brz %o4, 2f
7191 nop
7192 ldn [%o4 + CP_COPYIN], %g2
7193 jmp %g2
7194 nop
7195 2:
7196 retl
7197 mov -1, %o0
7198 #endif /* NIAGARA_IMPL */
7199 SET_SIZE(copyin)
7200
7201 ENTRY(xcopyin)
7202 sethi %hi(.xcopyin_err), REAL_LOFAULT
7203 b .do_copyin
7204 or REAL_LOFAULT, %lo(.xcopyin_err), REAL_LOFAULT
7205 .xcopyin_err:
7206 ldn [THREAD_REG + T_COPYOPS], %o4
7207 brz %o4, 2f
7208 nop
7209 ldn [%o4 + CP_XCOPYIN], %g2
7210 jmp %g2
7211 nop
7212 2:
7213 retl
7214 mov %g1, %o0
7215 SET_SIZE(xcopyin)
7216
7217 ENTRY(xcopyin_little)
7218 sethi %hi(.little_err), %o4
7219 ldn [THREAD_REG + T_LOFAULT], %o5
7220 or %o4, %lo(.little_err), %o4
7221 membar #Sync ! sync error barrier
7222 stn %o4, [THREAD_REG + T_LOFAULT]
7223
7224 subcc %g0, %o2, %o3
7225 add %o0, %o2, %o0
7226 bz,pn %ncc, 2f ! check for zero bytes
7227 sub %o2, 1, %o4
7228 add %o0, %o4, %o0 ! start w/last byte
7229 add %o1, %o2, %o1
7230 lduba [%o0+%o3]ASI_AIUSL, %o4
7231
7232 1: stb %o4, [%o1+%o3]
7233 inccc %o3
7234 sub %o0, 2, %o0 ! get next byte
7235 bcc,a,pt %ncc, 1b
7236 lduba [%o0+%o3]ASI_AIUSL, %o4
7237
7238 2: membar #Sync ! sync error barrier
7239 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7240 retl
7241 mov %g0, %o0 ! return (0)
7242
7243 .little_err:
7244 membar #Sync ! sync error barrier
7245 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7246 retl
7247 mov %g1, %o0
7248 SET_SIZE(xcopyin_little)
7249
7250
7251 /*
7252 * Copy a block of storage - must not overlap (from + len <= to).
7253 * No fault handler installed (to be called under on_fault())
7254 */
7255
7256 ENTRY(copyin_noerr)
7257 sethi %hi(.copyio_noerr), REAL_LOFAULT
7258 b .do_copyin
7259 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
7260 .copyio_noerr:
7261 jmp SAVED_LOFAULT
7262 nop
7263 SET_SIZE(copyin_noerr)
7264
7265 /*
7266 * Copy a block of storage - must not overlap (from + len <= to).
7267 * No fault handler installed (to be called under on_fault())
7268 */
7269
7270 ENTRY(copyout_noerr)
7271 sethi %hi(.copyio_noerr), REAL_LOFAULT
7272 b .do_copyout
7273 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
7274 SET_SIZE(copyout_noerr)
7275
7276 .align 4
7277 DGDEF(use_hw_bcopy)
7278 .word 1
7279 DGDEF(use_hw_bzero)
7280 .word 1
7281 DGDEF(hw_copy_limit_1)
7282 .word 0x100
7283 DGDEF(hw_copy_limit_2)
7284 .word 0x200
7285 DGDEF(hw_copy_limit_4)
7286 .word 0x400
7287 DGDEF(hw_copy_limit_8)
7288 .word 0x400
7289
7290 .align 64
7291 .section ".text"
7292
7293 /*
7294 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
7295 * longer than 256 bytes in length using Niagara's block stores/quad store.
7296 * If the criteria for using this routine are not met then it calls bzero
7297 * and returns 1. Otherwise 0 is returned indicating success.
7298 * Caller is responsible for ensuring use_hw_bzero is true and that
7299 * kpreempt_disable() has been called.
7300 */
7301 ! %i0 - start address
7302 ! %i1 - length of region (multiple of 64)
7303
7304 ENTRY(hwblkclr)
7305 save %sp, -SA(MINFRAME), %sp
7306
7307 ! Must be block-aligned
7308 andcc %i0, 0x3f, %g0
7309 bnz,pn %ncc, 1f
7310 nop
7311
7312 ! ... and must be 256 bytes or more
7313 cmp %i1, 0x100
7314 blu,pn %ncc, 1f
7315 nop
7316
7317 ! ... and length must be a multiple of 64
7318 andcc %i1, 0x3f, %g0
7319 bz,pn %ncc, .pz_doblock
7320 mov ASI_BLK_INIT_ST_QUAD_LDD_P, %asi
7378
7379 3:
7380 stxa %g0, [%i0+0x0]%asi
7381 stxa %g0, [%i0+0x8]%asi
7382 stxa %g0, [%i0+0x10]%asi
7383 stxa %g0, [%i0+0x18]%asi
7384 stxa %g0, [%i0+0x20]%asi
7385 stxa %g0, [%i0+0x28]%asi
7386 stxa %g0, [%i0+0x30]%asi
7387 stxa %g0, [%i0+0x38]%asi
7388
7389 subcc %i1, 0x40, %i1
7390 bgu,pt %ncc, 3b
7391 add %i0, 0x40, %i0
7392
7393 .pz_finish:
7394 membar #Sync
7395 ret
7396 restore %g0, 0, %o0 ! return (bzero or not)
7397 SET_SIZE(hwblkclr)
7398
7399 /*
7400 * Copy 32 bytes of data from src (%o0) to dst (%o1)
7401 * using physical addresses.
7402 */
7403 ENTRY_NP(hw_pa_bcopy32)
7404 rdpr %pstate, %g1
7405 andn %g1, PSTATE_IE, %g2
7406 wrpr %g0, %g2, %pstate
7407
7408 ldxa [%o0]ASI_MEM, %o2
7409 add %o0, 8, %o0
7410 ldxa [%o0]ASI_MEM, %o3
7411 add %o0, 8, %o0
7412 ldxa [%o0]ASI_MEM, %o4
7413 add %o0, 8, %o0
7414 ldxa [%o0]ASI_MEM, %o5
7415 stxa %o2, [%o1]ASI_MEM
7416 add %o1, 8, %o1
7417 stxa %o3, [%o1]ASI_MEM
7418 add %o1, 8, %o1
7419 stxa %o4, [%o1]ASI_MEM
7420 add %o1, 8, %o1
7421 stxa %o5, [%o1]ASI_MEM
7422
7423 membar #Sync
7424 retl
7425 wrpr %g0, %g1, %pstate
7426 SET_SIZE(hw_pa_bcopy32)
7427
7428 /*
7429 * Zero a block of storage.
7430 *
7431 * uzero is used by the kernel to zero a block in user address space.
7432 */
7433
7434 /*
7435 * Control flow of the bzero/kzero/uzero routine.
7436 *
7437 * For fewer than 7 bytes stores, bytes will be zeroed.
7438 *
7439 * For less than 15 bytes stores, align the address on 4 byte boundary.
7440 * Then store as many 4-byte chunks, followed by trailing bytes.
7441 *
7442 * For sizes greater than 15 bytes, align the address on 8 byte boundary.
7443 * if (count > 128) {
7444 * store as many 8-bytes chunks to block align the address
7445 * store using ASI_BLK_INIT_ST_QUAD_LDD_P (bzero/kzero) OR
7446 * store using ASI_BLK_INIT_QUAD_LDD_AIUS (uzero)
7447 * }
7448 * Store as many 8-byte chunks, followed by trailing bytes.
7449 */
7450
7451 ENTRY(uzero)
7452 !
7453 ! Set a new lo_fault handler only if we came in with one
7454 ! already specified.
7455 !
7456 wr %g0, ASI_USER, %asi
7457 ldn [THREAD_REG + T_LOFAULT], %o5
7458 tst %o5
7459 bz,pt %ncc, .do_zero
7460 sethi %hi(.zeroerr), %o2
7461 or %o2, %lo(.zeroerr), %o2
7462 membar #Sync
7463 ba,pt %ncc, .do_zero
7464 stn %o2, [THREAD_REG + T_LOFAULT]
7465
7466 ENTRY(kzero)
7467 !
7468 ! Always set a lo_fault handler
7469 !
7470 wr %g0, ASI_P, %asi
7503 ! Old handler was zero. Just return the error.
7504 !
7505 retl ! return
7506 mov %g1, %o0 ! error code from %g1
7507 3:
7508 !
7509 ! We're here because %o5 was non-zero. It was non-zero
7510 ! because either LOFAULT_SET was present, a previous fault
7511 ! handler was present or both. In all cases we need to reset
7512 ! T_LOFAULT to the value of %o5 after clearing LOFAULT_SET
7513 ! before we either simply return the error or we invoke the
7514 ! previously specified handler.
7515 !
7516 be %ncc, 2b
7517 stn %o5, [THREAD_REG + T_LOFAULT]
7518 jmp %o5 ! goto real handler
7519 nop
7520 SET_SIZE(kzero)
7521 SET_SIZE(uzero)
7522
7523 /*
7524 * Zero a block of storage.
7525 */
7526
7527 ENTRY(bzero)
7528 wr %g0, ASI_P, %asi
7529
7530 ldn [THREAD_REG + T_LOFAULT], %o5 ! save old vector
7531 tst %o5
7532 bz,pt %ncc, .do_zero
7533 sethi %hi(.zeroerr), %o2
7534 or %o2, %lo(.zeroerr), %o2
7535 membar #Sync ! sync error barrier
7536 stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector
7537
7538 .do_zero:
7539 cmp %o1, 7
7540 blu,pn %ncc, .byteclr
7541 nop
7542
7543 cmp %o1, 15
7544 blu,pn %ncc, .wdalign
7545 nop
7546
7717 ! We're just concerned with whether t_lofault was set
7718 ! when we came in. We end up here from either kzero()
7719 ! or bzero(). kzero() *always* sets a lofault handler.
7720 ! It ors LOFAULT_SET into %o5 to indicate it has done
7721 ! this even if the value of %o5 is otherwise zero.
7722 ! bzero() sets a lofault handler *only* if one was
7723 ! previously set. Accordingly we need to examine
7724 ! %o5 and if it is non-zero be sure to clear LOFAULT_SET
7725 ! before resetting the error handler.
7726 !
7727 tst %o5
7728 bz %ncc, 1f
7729 andn %o5, LOFAULT_SET, %o5
7730 membar #Sync ! sync error barrier
7731 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
7732 1:
7733 retl
7734 clr %o0 ! return (0)
7735
7736 SET_SIZE(bzero)
|