7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #pragma ident "%Z%%M% %I% %E% SMI"
28
29 #include <sys/param.h>
30 #include <sys/errno.h>
31 #include <sys/asm_linkage.h>
32 #include <sys/vtrace.h>
33 #include <sys/machthread.h>
34 #include <sys/clock.h>
35 #include <sys/asi.h>
36 #include <sys/fsr.h>
37 #include <sys/privregs.h>
38
39 #if !defined(lint)
40 #include "assym.h"
41 #endif /* lint */
42
43
44 /*
45 * Pseudo-code to aid in understanding the control flow of the
46 * bcopy routine.
47 *
48 * On entry to bcopy:
49 *
50 * %l6 = curthread->t_lofault;
51 * used_block_copy = FALSE; ! %l6 |= 1
52 * if (%l6 != NULL) {
53 * curthread->t_lofault = .copyerr;
54 * caller_error_handler = TRUE ! %l6 |= 2
55 * }
56 *
57 * if (length < VIS_COPY)
58 * goto regular_copy;
59 *
60 * if (!use_vis)
61 * goto_regular_copy;
465
466 #define FALIGN_D46 \
467 faligndata %d46, %d0, %d48 ;\
468 faligndata %d0, %d2, %d50 ;\
469 faligndata %d2, %d4, %d52 ;\
470 faligndata %d4, %d6, %d54 ;\
471 faligndata %d6, %d8, %d56 ;\
472 faligndata %d8, %d10, %d58 ;\
473 faligndata %d10, %d12, %d60 ;\
474 faligndata %d12, %d14, %d62
475
476
477 /*
478 * Copy a block of storage, returning an error code if `from' or
479 * `to' takes a kernel pagefault which cannot be resolved.
480 * Returns errno value on pagefault error, 0 if all ok
481 */
482
483
484
485 #if defined(lint)
486
487 /* ARGSUSED */
488 int
489 kcopy(const void *from, void *to, size_t count)
490 { return(0); }
491
492 #else /* lint */
493
494 .seg ".text"
495 .align 4
496
497 ENTRY(kcopy)
498
499 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
500 set .copyerr, %l6 ! copyerr is lofault value
501 ldn [THREAD_REG + T_LOFAULT], %l7 ! save existing handler
502 membar #Sync ! sync error barrier (see copy.s)
503 stn %l6, [THREAD_REG + T_LOFAULT] ! set t_lofault
504 !
505 ! Note that we carefully do *not* flag the setting of
506 ! t_lofault.
507 !
508 ba,pt %ncc, .do_copy ! common code
509 mov %l7, %l6
510
511 /*
512 * We got here because of a fault during kcopy or bcopy if a fault
513 * handler existed when bcopy was called.
600 !
601 ! We're here via bcopy. There *must* have been an error handler
602 ! in place otheerwise we would have died a nasty death already.
603 !
604 jmp %l6 ! goto real handler
605 restore %g0, 0, %o0 ! dispose of copy window
606
607 /*
608 * We got here because of a fault in .copyerr. We can't safely restore fp
609 * state, so we panic.
610 */
611 fp_panic_msg:
612 .asciz "Unable to restore fp state after copy operation"
613
614 .align 4
615 .copyerr2:
616 set fp_panic_msg, %o0
617 call panic
618 nop
619 SET_SIZE(kcopy)
620 #endif /* lint */
621
622
623 /*
624 * Copy a block of storage - must not overlap (from + len <= to).
625 * Registers: l6 - saved t_lofault
626 *
627 * Copy a page of memory.
628 * Assumes double word alignment and a count >= 256.
629 */
630 #if defined(lint)
631
632 /* ARGSUSED */
633 void
634 bcopy(const void *from, void *to, size_t count)
635 {}
636
637 #else /* lint */
638
639 ENTRY(bcopy)
640
641 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
642 ldn [THREAD_REG + T_LOFAULT], %l6 ! save t_lofault
643 tst %l6
644 !
645 ! We've already captured whether t_lofault was zero on entry.
646 ! We need to mark ourselves as being from bcopy since both
647 ! kcopy and bcopy use the same code path. If BCOPY_FLAG is set
648 ! and the saved lofault was zero, we won't reset lofault on
649 ! returning.
650 !
651 or %l6, BCOPY_FLAG, %l6
652 bz,pt %ncc, .do_copy
653 sethi %hi(.copyerr), %o2
654 or %o2, %lo(.copyerr), %o2
655 membar #Sync ! sync error barrier
656 stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector
657
658 .do_copy:
1853
1854 /*
1855 * Common code used to align transfers on word and doubleword
1856 * boudaries. Aligns source and destination and returns a count
1857 * of aligned bytes to transfer in %i3
1858 */
1859 1:
1860 inc %i0 ! inc from
1861 stb %o4, [%i1] ! write a byte
1862 inc %i1 ! inc to
1863 dec %i2 ! dec count
1864 .alignit:
1865 btst %o0, %i0 ! %o0 is bit mask to check for alignment
1866 bnz,a 1b
1867 ldub [%i0], %o4 ! read next byte
1868
1869 retl
1870 andn %i2, %o0, %i3 ! return size of aligned bytes
1871 SET_SIZE(bcopy)
1872
1873 #endif /* lint */
1874
1875 /*
1876 * Block copy with possibly overlapped operands.
1877 */
1878
1879 #if defined(lint)
1880
1881 /*ARGSUSED*/
1882 void
1883 ovbcopy(const void *from, void *to, size_t count)
1884 {}
1885
1886 #else /* lint */
1887
1888 ENTRY(ovbcopy)
1889 tst %o2 ! check count
1890 bgu,a %ncc, 1f ! nothing to do or bad arguments
1891 subcc %o0, %o1, %o3 ! difference of from and to address
1892
1893 retl ! return
1894 nop
1895 1:
1896 bneg,a %ncc, 2f
1897 neg %o3 ! if < 0, make it positive
1898 2: cmp %o2, %o3 ! cmp size and abs(from - to)
1899 bleu %ncc, bcopy ! if size <= abs(diff): use bcopy,
1900 .empty ! no overlap
1901 cmp %o0, %o1 ! compare from and to addresses
1902 blu %ncc, .ov_bkwd ! if from < to, copy backwards
1903 nop
1904 !
1905 ! Copy forwards.
1906 !
1907 .ov_fwd:
1910 stb %o3, [%o1] ! write to address
1911 deccc %o2 ! dec count
1912 bgu %ncc, .ov_fwd ! loop till done
1913 inc %o1 ! inc to address
1914
1915 retl ! return
1916 nop
1917 !
1918 ! Copy backwards.
1919 !
1920 .ov_bkwd:
1921 deccc %o2 ! dec count
1922 ldub [%o0 + %o2], %o3 ! get byte at end of src
1923 bgu %ncc, .ov_bkwd ! loop till done
1924 stb %o3, [%o1 + %o2] ! delay slot, store at end of dst
1925
1926 retl ! return
1927 nop
1928 SET_SIZE(ovbcopy)
1929
1930 #endif /* lint */
1931
1932 /*
1933 * hwblkpagecopy()
1934 *
1935 * Copies exactly one page. This routine assumes the caller (ppcopy)
1936 * has already disabled kernel preemption and has checked
1937 * use_hw_bcopy.
1938 */
1939 #ifdef lint
1940 /*ARGSUSED*/
1941 void
1942 hwblkpagecopy(const void *src, void *dst)
1943 { }
1944 #else /* lint */
1945 ENTRY(hwblkpagecopy)
1946 ! get another window w/space for three aligned blocks of saved fpregs
1947 save %sp, -SA(MINFRAME + 4*64), %sp
1948
1949 ! %i0 - source address (arg)
1950 ! %i1 - destination address (arg)
1951 ! %i2 - length of region (not arg)
1952 ! %l0 - saved fprs
1953 ! %l1 - pointer to saved fpregs
1954
1955 rd %fprs, %l0 ! check for unused fp
1956 btst FPRS_FEF, %l0
1957 bz 1f
1958 membar #Sync
1959
1960 ! save in-use fpregs on stack
1961 add %fp, STACK_BIAS - 193, %l1
1962 and %l1, -64, %l1
1963 stda %d0, [%l1]ASI_BLK_P
1964 add %l1, 64, %l3
2002 add %i1, 64, %i1
2003
2004 3: membar #Sync
2005 btst FPRS_FEF, %l0
2006 bz 4f
2007 stda %d16, [%i1]ASI_BLK_P
2008
2009 ! restore fpregs from stack
2010 membar #Sync
2011 ldda [%l1]ASI_BLK_P, %d0
2012 add %l1, 64, %l3
2013 ldda [%l3]ASI_BLK_P, %d16
2014 add %l3, 64, %l3
2015 ldda [%l3]ASI_BLK_P, %d32
2016
2017 4: wr %l0, 0, %fprs ! restore fprs
2018 membar #Sync
2019 ret
2020 restore %g0, 0, %o0
2021 SET_SIZE(hwblkpagecopy)
2022 #endif /* lint */
2023
2024
2025 /*
2026 * Transfer data to and from user space -
2027 * Note that these routines can cause faults
2028 * It is assumed that the kernel has nothing at
2029 * less than KERNELBASE in the virtual address space.
2030 *
2031 * Note that copyin(9F) and copyout(9F) are part of the
2032 * DDI/DKI which specifies that they return '-1' on "errors."
2033 *
2034 * Sigh.
2035 *
2036 * So there's two extremely similar routines - xcopyin() and xcopyout()
2037 * which return the errno that we've faithfully computed. This
2038 * allows other callers (e.g. uiomove(9F)) to work correctly.
2039 * Given that these are used pretty heavily, we expand the calling
2040 * sequences inline for all flavours (rather than making wrappers).
2041 *
2042 * There are also stub routines for xcopyout_little and xcopyin_little,
2114 * (in the default configuration) a window of 256 bytes between
2115 * the single byte aligned copy limit and what VIS treats as its
2116 * minimum if floating point is in use in the calling app. We need
2117 * to be prepared to handle this. See the .small_copyOP label for
2118 * details.
2119 *
2120 * Fault handlers are invoked if we reference memory that has no
2121 * current mapping. All forms share the same copyio_fault handler.
2122 * This routine handles fixing up the stack and general housecleaning.
2123 * Each copy operation has a simple fault handler that is then called
2124 * to do the work specific to the invidual operation. The handlers
2125 * for default_copyOP and copyOP_noerr are found at the end of
2126 * default_copyout. The handlers for default_xcopyOP are found at the
2127 * end of xdefault_copyin.
2128 */
2129
2130 /*
2131 * Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
2132 */
2133
2134 #if defined(lint)
2135
2136 /*ARGSUSED*/
2137 int
2138 copyout(const void *kaddr, void *uaddr, size_t count)
2139 { return (0); }
2140
2141 #else /* lint */
2142
2143 /*
2144 * We save the arguments in the following registers in case of a fault:
2145 * kaddr - %g2
2146 * uaddr - %g3
2147 * count - %g4
2148 */
2149 #define SAVE_SRC %g2
2150 #define SAVE_DST %g3
2151 #define SAVE_COUNT %g4
2152
2153 #define REAL_LOFAULT %g5
2154 #define SAVED_LOFAULT %g6
2155
2156 /*
2157 * Generic copyio fault handler. This is the first line of defense when a
2158 * fault occurs in (x)copyin/(x)copyout. In order for this to function
2159 * properly, the value of the 'real' lofault handler should be in REAL_LOFAULT.
2160 * This allows us to share common code for all the flavors of the copy
2161 * operations, including the _noerr versions.
2162 *
3375
3376 1:
3377 andn SAVED_LOFAULT, FPUSED_FLAG, SAVED_LOFAULT
3378 membar #Sync ! sync error barrier
3379 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3380 ret
3381 restore %g0, 0, %o0
3382
3383 .copyout_err:
3384 ldn [THREAD_REG + T_COPYOPS], %o4
3385 brz %o4, 2f
3386 nop
3387 ldn [%o4 + CP_COPYOUT], %g2
3388 jmp %g2
3389 nop
3390 2:
3391 retl
3392 mov -1, %o0
3393 SET_SIZE(copyout)
3394
3395 #endif /* lint */
3396
3397
3398 #ifdef lint
3399
3400 /*ARGSUSED*/
3401 int
3402 xcopyout(const void *kaddr, void *uaddr, size_t count)
3403 { return (0); }
3404
3405 #else /* lint */
3406
3407 ENTRY(xcopyout)
3408 sethi %hi(.xcopyout_err), REAL_LOFAULT
3409 b .do_copyout
3410 or REAL_LOFAULT, %lo(.xcopyout_err), REAL_LOFAULT
3411 .xcopyout_err:
3412 ldn [THREAD_REG + T_COPYOPS], %o4
3413 brz %o4, 2f
3414 nop
3415 ldn [%o4 + CP_XCOPYOUT], %g2
3416 jmp %g2
3417 nop
3418 2:
3419 retl
3420 mov %g1, %o0
3421 SET_SIZE(xcopyout)
3422
3423 #endif /* lint */
3424
3425 #ifdef lint
3426
3427 /*ARGSUSED*/
3428 int
3429 xcopyout_little(const void *kaddr, void *uaddr, size_t count)
3430 { return (0); }
3431
3432 #else /* lint */
3433
3434 ENTRY(xcopyout_little)
3435 sethi %hi(.little_err), %o4
3436 ldn [THREAD_REG + T_LOFAULT], %o5
3437 or %o4, %lo(.little_err), %o4
3438 membar #Sync ! sync error barrier
3439 stn %o4, [THREAD_REG + T_LOFAULT]
3440
3441 subcc %g0, %o2, %o3
3442 add %o0, %o2, %o0
3443 bz,pn %ncc, 2f ! check for zero bytes
3444 sub %o2, 1, %o4
3445 add %o0, %o4, %o0 ! start w/last byte
3446 add %o1, %o2, %o1
3447 ldub [%o0+%o3], %o4
3448
3449 1: stba %o4, [%o1+%o3]ASI_AIUSL
3450 inccc %o3
3451 sub %o0, 2, %o0 ! get next byte
3452 bcc,a,pt %ncc, 1b
3453 ldub [%o0+%o3], %o4
3454
3455 2: membar #Sync ! sync error barrier
3456 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3457 retl
3458 mov %g0, %o0 ! return (0)
3459 SET_SIZE(xcopyout_little)
3460
3461 #endif /* lint */
3462
3463 /*
3464 * Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
3465 */
3466
3467 #if defined(lint)
3468
3469 /*ARGSUSED*/
3470 int
3471 copyin(const void *uaddr, void *kaddr, size_t count)
3472 { return (0); }
3473
3474 #else /* lint */
3475
3476 ENTRY(copyin)
3477 sethi %hi(.copyin_err), REAL_LOFAULT
3478 or REAL_LOFAULT, %lo(.copyin_err), REAL_LOFAULT
3479
3480 .do_copyin:
3481 !
3482 ! Check the length and bail if zero.
3483 !
3484 tst %o2
3485 bnz,pt %ncc, 1f
3486 nop
3487 retl
3488 clr %o0
3489 1:
3490 sethi %hi(copyio_fault), %o4
3491 or %o4, %lo(copyio_fault), %o4
3492 sethi %hi(copyio_fault_nowindow), %o3
3493 ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT
3494 or %o3, %lo(copyio_fault_nowindow), %o3
3495 membar #Sync
4613 wr %o3, 0, %fprs ! restore fprs
4614
4615 1:
4616 andn SAVED_LOFAULT, FPUSED_FLAG, SAVED_LOFAULT
4617 membar #Sync ! sync error barrier
4618 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
4619 ret
4620 restore %g0, 0, %o0
4621 .copyin_err:
4622 ldn [THREAD_REG + T_COPYOPS], %o4
4623 brz %o4, 2f
4624 nop
4625 ldn [%o4 + CP_COPYIN], %g2
4626 jmp %g2
4627 nop
4628 2:
4629 retl
4630 mov -1, %o0
4631 SET_SIZE(copyin)
4632
4633 #endif /* lint */
4634
4635 #ifdef lint
4636
4637 /*ARGSUSED*/
4638 int
4639 xcopyin(const void *uaddr, void *kaddr, size_t count)
4640 { return (0); }
4641
4642 #else /* lint */
4643
4644 ENTRY(xcopyin)
4645 sethi %hi(.xcopyin_err), REAL_LOFAULT
4646 b .do_copyin
4647 or REAL_LOFAULT, %lo(.xcopyin_err), REAL_LOFAULT
4648 .xcopyin_err:
4649 ldn [THREAD_REG + T_COPYOPS], %o4
4650 brz %o4, 2f
4651 nop
4652 ldn [%o4 + CP_XCOPYIN], %g2
4653 jmp %g2
4654 nop
4655 2:
4656 retl
4657 mov %g1, %o0
4658 SET_SIZE(xcopyin)
4659
4660 #endif /* lint */
4661
4662 #ifdef lint
4663
4664 /*ARGSUSED*/
4665 int
4666 xcopyin_little(const void *uaddr, void *kaddr, size_t count)
4667 { return (0); }
4668
4669 #else /* lint */
4670
4671 ENTRY(xcopyin_little)
4672 sethi %hi(.little_err), %o4
4673 ldn [THREAD_REG + T_LOFAULT], %o5
4674 or %o4, %lo(.little_err), %o4
4675 membar #Sync ! sync error barrier
4676 stn %o4, [THREAD_REG + T_LOFAULT]
4677
4678 subcc %g0, %o2, %o3
4679 add %o0, %o2, %o0
4680 bz,pn %ncc, 2f ! check for zero bytes
4681 sub %o2, 1, %o4
4682 add %o0, %o4, %o0 ! start w/last byte
4683 add %o1, %o2, %o1
4684 lduba [%o0+%o3]ASI_AIUSL, %o4
4685
4686 1: stb %o4, [%o1+%o3]
4687 inccc %o3
4688 sub %o0, 2, %o0 ! get next byte
4689 bcc,a,pt %ncc, 1b
4690 lduba [%o0+%o3]ASI_AIUSL, %o4
4691
4692 2: membar #Sync ! sync error barrier
4693 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
4694 retl
4695 mov %g0, %o0 ! return (0)
4696
4697 .little_err:
4698 membar #Sync ! sync error barrier
4699 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
4700 retl
4701 mov %g1, %o0
4702 SET_SIZE(xcopyin_little)
4703
4704 #endif /* lint */
4705
4706
4707 /*
4708 * Copy a block of storage - must not overlap (from + len <= to).
4709 * No fault handler installed (to be called under on_fault())
4710 */
4711 #if defined(lint)
4712
4713 /* ARGSUSED */
4714 void
4715 copyin_noerr(const void *ufrom, void *kto, size_t count)
4716 {}
4717
4718 #else /* lint */
4719
4720 ENTRY(copyin_noerr)
4721 sethi %hi(.copyio_noerr), REAL_LOFAULT
4722 b .do_copyin
4723 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
4724 .copyio_noerr:
4725 jmp SAVED_LOFAULT
4726 nop
4727 SET_SIZE(copyin_noerr)
4728
4729 #endif /* lint */
4730
4731 /*
4732 * Copy a block of storage - must not overlap (from + len <= to).
4733 * No fault handler installed (to be called under on_fault())
4734 */
4735
4736 #if defined(lint)
4737
4738 /* ARGSUSED */
4739 void
4740 copyout_noerr(const void *kfrom, void *uto, size_t count)
4741 {}
4742
4743 #else /* lint */
4744
4745 ENTRY(copyout_noerr)
4746 sethi %hi(.copyio_noerr), REAL_LOFAULT
4747 b .do_copyout
4748 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
4749 SET_SIZE(copyout_noerr)
4750
4751 #endif /* lint */
4752
4753 #if defined(lint)
4754
4755 int use_hw_bcopy = 1;
4756 int use_hw_copyio = 1;
4757 int use_hw_bzero = 1;
4758 uint_t hw_copy_limit_1 = 0;
4759 uint_t hw_copy_limit_2 = 0;
4760 uint_t hw_copy_limit_4 = 0;
4761 uint_t hw_copy_limit_8 = 0;
4762
4763 #else /* !lint */
4764
4765 .align 4
4766 DGDEF(use_hw_bcopy)
4767 .word 1
4768 DGDEF(use_hw_copyio)
4769 .word 1
4770 DGDEF(use_hw_bzero)
4771 .word 1
4772 DGDEF(hw_copy_limit_1)
4773 .word 0
4774 DGDEF(hw_copy_limit_2)
4775 .word 0
4776 DGDEF(hw_copy_limit_4)
4777 .word 0
4778 DGDEF(hw_copy_limit_8)
4779 .word 0
4780
4781 .align 64
4782 .section ".text"
4783 #endif /* !lint */
4784
4785
4786 /*
4787 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
4788 * longer than 256 bytes in length using spitfire's block stores. If
4789 * the criteria for using this routine are not met then it calls bzero
4790 * and returns 1. Otherwise 0 is returned indicating success.
4791 * Caller is responsible for ensuring use_hw_bzero is true and that
4792 * kpreempt_disable() has been called.
4793 */
4794 #ifdef lint
4795 /*ARGSUSED*/
4796 int
4797 hwblkclr(void *addr, size_t len)
4798 {
4799 return(0);
4800 }
4801 #else /* lint */
4802 ! %i0 - start address
4803 ! %i1 - length of region (multiple of 64)
4804 ! %l0 - saved fprs
4805 ! %l1 - pointer to saved %d0 block
4806 ! %l2 - saved curthread->t_lwp
4807
4808 ENTRY(hwblkclr)
4809 ! get another window w/space for one aligned block of saved fpregs
4810 save %sp, -SA(MINFRAME + 2*64), %sp
4811
4812 ! Must be block-aligned
4813 andcc %i0, (64-1), %g0
4814 bnz,pn %ncc, 1f
4815 nop
4816
4817 ! ... and must be 256 bytes or more
4818 cmp %i1, 256
4819 blu,pn %ncc, 1f
4820 nop
4821
4881 set .pz_zinst, %i4
4882 sub %i4, %i2, %i4
4883 jmp %i4
4884 nop
4885
4886 .pz_finish:
4887 membar #Sync
4888 btst FPRS_FEF, %l0
4889 bz,a .pz_finished
4890 wr %l0, 0, %fprs ! restore fprs
4891
4892 ! restore fpregs from stack
4893 ldda [%l1]ASI_BLK_P, %d0
4894 membar #Sync
4895 wr %l0, 0, %fprs ! restore fprs
4896
4897 .pz_finished:
4898 ret
4899 restore %g0, 0, %o0 ! return (bzero or not)
4900 SET_SIZE(hwblkclr)
4901 #endif /* lint */
4902
4903 #ifdef lint
4904 /* Copy 32 bytes of data from src to dst using physical addresses */
4905 /*ARGSUSED*/
4906 void
4907 hw_pa_bcopy32(uint64_t src, uint64_t dst)
4908 {}
4909 #else /*!lint */
4910
4911 /*
4912 * Copy 32 bytes of data from src (%o0) to dst (%o1)
4913 * using physical addresses.
4914 */
4915 ENTRY_NP(hw_pa_bcopy32)
4916 rdpr %pstate, %g1
4917 andn %g1, PSTATE_IE, %g2
4918 wrpr %g0, %g2, %pstate
4919
4920 ldxa [%o0]ASI_MEM, %o2
4921 add %o0, 8, %o0
4922 ldxa [%o0]ASI_MEM, %o3
4923 add %o0, 8, %o0
4924 ldxa [%o0]ASI_MEM, %o4
4925 add %o0, 8, %o0
4926 ldxa [%o0]ASI_MEM, %o5
4927 stxa %o2, [%o1]ASI_MEM
4928 add %o1, 8, %o1
4929 stxa %o3, [%o1]ASI_MEM
4930 add %o1, 8, %o1
4931 stxa %o4, [%o1]ASI_MEM
4932 add %o1, 8, %o1
4933 stxa %o5, [%o1]ASI_MEM
4934
4935 membar #Sync
4936 retl
4937 wrpr %g0, %g1, %pstate
4938 SET_SIZE(hw_pa_bcopy32)
4939 #endif /* lint */
|
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2004 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <sys/param.h>
28 #include <sys/errno.h>
29 #include <sys/asm_linkage.h>
30 #include <sys/vtrace.h>
31 #include <sys/machthread.h>
32 #include <sys/clock.h>
33 #include <sys/asi.h>
34 #include <sys/fsr.h>
35 #include <sys/privregs.h>
36
37 #include "assym.h"
38
39
40 /*
41 * Pseudo-code to aid in understanding the control flow of the
42 * bcopy routine.
43 *
44 * On entry to bcopy:
45 *
46 * %l6 = curthread->t_lofault;
47 * used_block_copy = FALSE; ! %l6 |= 1
48 * if (%l6 != NULL) {
49 * curthread->t_lofault = .copyerr;
50 * caller_error_handler = TRUE ! %l6 |= 2
51 * }
52 *
53 * if (length < VIS_COPY)
54 * goto regular_copy;
55 *
56 * if (!use_vis)
57 * goto_regular_copy;
461
462 #define FALIGN_D46 \
463 faligndata %d46, %d0, %d48 ;\
464 faligndata %d0, %d2, %d50 ;\
465 faligndata %d2, %d4, %d52 ;\
466 faligndata %d4, %d6, %d54 ;\
467 faligndata %d6, %d8, %d56 ;\
468 faligndata %d8, %d10, %d58 ;\
469 faligndata %d10, %d12, %d60 ;\
470 faligndata %d12, %d14, %d62
471
472
473 /*
474 * Copy a block of storage, returning an error code if `from' or
475 * `to' takes a kernel pagefault which cannot be resolved.
476 * Returns errno value on pagefault error, 0 if all ok
477 */
478
479
480
481 .seg ".text"
482 .align 4
483
484 ENTRY(kcopy)
485
486 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
487 set .copyerr, %l6 ! copyerr is lofault value
488 ldn [THREAD_REG + T_LOFAULT], %l7 ! save existing handler
489 membar #Sync ! sync error barrier (see copy.s)
490 stn %l6, [THREAD_REG + T_LOFAULT] ! set t_lofault
491 !
492 ! Note that we carefully do *not* flag the setting of
493 ! t_lofault.
494 !
495 ba,pt %ncc, .do_copy ! common code
496 mov %l7, %l6
497
498 /*
499 * We got here because of a fault during kcopy or bcopy if a fault
500 * handler existed when bcopy was called.
587 !
588 ! We're here via bcopy. There *must* have been an error handler
589 ! in place otheerwise we would have died a nasty death already.
590 !
591 jmp %l6 ! goto real handler
592 restore %g0, 0, %o0 ! dispose of copy window
593
594 /*
595 * We got here because of a fault in .copyerr. We can't safely restore fp
596 * state, so we panic.
597 */
598 fp_panic_msg:
599 .asciz "Unable to restore fp state after copy operation"
600
601 .align 4
602 .copyerr2:
603 set fp_panic_msg, %o0
604 call panic
605 nop
606 SET_SIZE(kcopy)
607
608
609 /*
610 * Copy a block of storage - must not overlap (from + len <= to).
611 * Registers: l6 - saved t_lofault
612 *
613 * Copy a page of memory.
614 * Assumes double word alignment and a count >= 256.
615 */
616
617 ENTRY(bcopy)
618
619 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
620 ldn [THREAD_REG + T_LOFAULT], %l6 ! save t_lofault
621 tst %l6
622 !
623 ! We've already captured whether t_lofault was zero on entry.
624 ! We need to mark ourselves as being from bcopy since both
625 ! kcopy and bcopy use the same code path. If BCOPY_FLAG is set
626 ! and the saved lofault was zero, we won't reset lofault on
627 ! returning.
628 !
629 or %l6, BCOPY_FLAG, %l6
630 bz,pt %ncc, .do_copy
631 sethi %hi(.copyerr), %o2
632 or %o2, %lo(.copyerr), %o2
633 membar #Sync ! sync error barrier
634 stn %o2, [THREAD_REG + T_LOFAULT] ! install new vector
635
636 .do_copy:
1831
1832 /*
1833 * Common code used to align transfers on word and doubleword
1834 * boudaries. Aligns source and destination and returns a count
1835 * of aligned bytes to transfer in %i3
1836 */
1837 1:
1838 inc %i0 ! inc from
1839 stb %o4, [%i1] ! write a byte
1840 inc %i1 ! inc to
1841 dec %i2 ! dec count
1842 .alignit:
1843 btst %o0, %i0 ! %o0 is bit mask to check for alignment
1844 bnz,a 1b
1845 ldub [%i0], %o4 ! read next byte
1846
1847 retl
1848 andn %i2, %o0, %i3 ! return size of aligned bytes
1849 SET_SIZE(bcopy)
1850
1851 /*
1852 * Block copy with possibly overlapped operands.
1853 */
1854
1855 ENTRY(ovbcopy)
1856 tst %o2 ! check count
1857 bgu,a %ncc, 1f ! nothing to do or bad arguments
1858 subcc %o0, %o1, %o3 ! difference of from and to address
1859
1860 retl ! return
1861 nop
1862 1:
1863 bneg,a %ncc, 2f
1864 neg %o3 ! if < 0, make it positive
1865 2: cmp %o2, %o3 ! cmp size and abs(from - to)
1866 bleu %ncc, bcopy ! if size <= abs(diff): use bcopy,
1867 .empty ! no overlap
1868 cmp %o0, %o1 ! compare from and to addresses
1869 blu %ncc, .ov_bkwd ! if from < to, copy backwards
1870 nop
1871 !
1872 ! Copy forwards.
1873 !
1874 .ov_fwd:
1877 stb %o3, [%o1] ! write to address
1878 deccc %o2 ! dec count
1879 bgu %ncc, .ov_fwd ! loop till done
1880 inc %o1 ! inc to address
1881
1882 retl ! return
1883 nop
1884 !
1885 ! Copy backwards.
1886 !
1887 .ov_bkwd:
1888 deccc %o2 ! dec count
1889 ldub [%o0 + %o2], %o3 ! get byte at end of src
1890 bgu %ncc, .ov_bkwd ! loop till done
1891 stb %o3, [%o1 + %o2] ! delay slot, store at end of dst
1892
1893 retl ! return
1894 nop
1895 SET_SIZE(ovbcopy)
1896
1897 /*
1898 * hwblkpagecopy()
1899 *
1900 * Copies exactly one page. This routine assumes the caller (ppcopy)
1901 * has already disabled kernel preemption and has checked
1902 * use_hw_bcopy.
1903 */
1904 ENTRY(hwblkpagecopy)
1905 ! get another window w/space for three aligned blocks of saved fpregs
1906 save %sp, -SA(MINFRAME + 4*64), %sp
1907
1908 ! %i0 - source address (arg)
1909 ! %i1 - destination address (arg)
1910 ! %i2 - length of region (not arg)
1911 ! %l0 - saved fprs
1912 ! %l1 - pointer to saved fpregs
1913
1914 rd %fprs, %l0 ! check for unused fp
1915 btst FPRS_FEF, %l0
1916 bz 1f
1917 membar #Sync
1918
1919 ! save in-use fpregs on stack
1920 add %fp, STACK_BIAS - 193, %l1
1921 and %l1, -64, %l1
1922 stda %d0, [%l1]ASI_BLK_P
1923 add %l1, 64, %l3
1961 add %i1, 64, %i1
1962
1963 3: membar #Sync
1964 btst FPRS_FEF, %l0
1965 bz 4f
1966 stda %d16, [%i1]ASI_BLK_P
1967
1968 ! restore fpregs from stack
1969 membar #Sync
1970 ldda [%l1]ASI_BLK_P, %d0
1971 add %l1, 64, %l3
1972 ldda [%l3]ASI_BLK_P, %d16
1973 add %l3, 64, %l3
1974 ldda [%l3]ASI_BLK_P, %d32
1975
1976 4: wr %l0, 0, %fprs ! restore fprs
1977 membar #Sync
1978 ret
1979 restore %g0, 0, %o0
1980 SET_SIZE(hwblkpagecopy)
1981
1982
1983 /*
1984 * Transfer data to and from user space -
1985 * Note that these routines can cause faults
1986 * It is assumed that the kernel has nothing at
1987 * less than KERNELBASE in the virtual address space.
1988 *
1989 * Note that copyin(9F) and copyout(9F) are part of the
1990 * DDI/DKI which specifies that they return '-1' on "errors."
1991 *
1992 * Sigh.
1993 *
1994 * So there's two extremely similar routines - xcopyin() and xcopyout()
1995 * which return the errno that we've faithfully computed. This
1996 * allows other callers (e.g. uiomove(9F)) to work correctly.
1997 * Given that these are used pretty heavily, we expand the calling
1998 * sequences inline for all flavours (rather than making wrappers).
1999 *
2000 * There are also stub routines for xcopyout_little and xcopyin_little,
2072 * (in the default configuration) a window of 256 bytes between
2073 * the single byte aligned copy limit and what VIS treats as its
2074 * minimum if floating point is in use in the calling app. We need
2075 * to be prepared to handle this. See the .small_copyOP label for
2076 * details.
2077 *
2078 * Fault handlers are invoked if we reference memory that has no
2079 * current mapping. All forms share the same copyio_fault handler.
2080 * This routine handles fixing up the stack and general housecleaning.
2081 * Each copy operation has a simple fault handler that is then called
2082 * to do the work specific to the invidual operation. The handlers
2083 * for default_copyOP and copyOP_noerr are found at the end of
2084 * default_copyout. The handlers for default_xcopyOP are found at the
2085 * end of xdefault_copyin.
2086 */
2087
2088 /*
2089 * Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
2090 */
2091
2092 /*
2093 * We save the arguments in the following registers in case of a fault:
2094 * kaddr - %g2
2095 * uaddr - %g3
2096 * count - %g4
2097 */
2098 #define SAVE_SRC %g2
2099 #define SAVE_DST %g3
2100 #define SAVE_COUNT %g4
2101
2102 #define REAL_LOFAULT %g5
2103 #define SAVED_LOFAULT %g6
2104
2105 /*
2106 * Generic copyio fault handler. This is the first line of defense when a
2107 * fault occurs in (x)copyin/(x)copyout. In order for this to function
2108 * properly, the value of the 'real' lofault handler should be in REAL_LOFAULT.
2109 * This allows us to share common code for all the flavors of the copy
2110 * operations, including the _noerr versions.
2111 *
3324
3325 1:
3326 andn SAVED_LOFAULT, FPUSED_FLAG, SAVED_LOFAULT
3327 membar #Sync ! sync error barrier
3328 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3329 ret
3330 restore %g0, 0, %o0
3331
3332 .copyout_err:
3333 ldn [THREAD_REG + T_COPYOPS], %o4
3334 brz %o4, 2f
3335 nop
3336 ldn [%o4 + CP_COPYOUT], %g2
3337 jmp %g2
3338 nop
3339 2:
3340 retl
3341 mov -1, %o0
3342 SET_SIZE(copyout)
3343
3344
3345 ENTRY(xcopyout)
3346 sethi %hi(.xcopyout_err), REAL_LOFAULT
3347 b .do_copyout
3348 or REAL_LOFAULT, %lo(.xcopyout_err), REAL_LOFAULT
3349 .xcopyout_err:
3350 ldn [THREAD_REG + T_COPYOPS], %o4
3351 brz %o4, 2f
3352 nop
3353 ldn [%o4 + CP_XCOPYOUT], %g2
3354 jmp %g2
3355 nop
3356 2:
3357 retl
3358 mov %g1, %o0
3359 SET_SIZE(xcopyout)
3360
3361 ENTRY(xcopyout_little)
3362 sethi %hi(.little_err), %o4
3363 ldn [THREAD_REG + T_LOFAULT], %o5
3364 or %o4, %lo(.little_err), %o4
3365 membar #Sync ! sync error barrier
3366 stn %o4, [THREAD_REG + T_LOFAULT]
3367
3368 subcc %g0, %o2, %o3
3369 add %o0, %o2, %o0
3370 bz,pn %ncc, 2f ! check for zero bytes
3371 sub %o2, 1, %o4
3372 add %o0, %o4, %o0 ! start w/last byte
3373 add %o1, %o2, %o1
3374 ldub [%o0+%o3], %o4
3375
3376 1: stba %o4, [%o1+%o3]ASI_AIUSL
3377 inccc %o3
3378 sub %o0, 2, %o0 ! get next byte
3379 bcc,a,pt %ncc, 1b
3380 ldub [%o0+%o3], %o4
3381
3382 2: membar #Sync ! sync error barrier
3383 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3384 retl
3385 mov %g0, %o0 ! return (0)
3386 SET_SIZE(xcopyout_little)
3387
3388 /*
3389 * Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
3390 */
3391
3392 ENTRY(copyin)
3393 sethi %hi(.copyin_err), REAL_LOFAULT
3394 or REAL_LOFAULT, %lo(.copyin_err), REAL_LOFAULT
3395
3396 .do_copyin:
3397 !
3398 ! Check the length and bail if zero.
3399 !
3400 tst %o2
3401 bnz,pt %ncc, 1f
3402 nop
3403 retl
3404 clr %o0
3405 1:
3406 sethi %hi(copyio_fault), %o4
3407 or %o4, %lo(copyio_fault), %o4
3408 sethi %hi(copyio_fault_nowindow), %o3
3409 ldn [THREAD_REG + T_LOFAULT], SAVED_LOFAULT
3410 or %o3, %lo(copyio_fault_nowindow), %o3
3411 membar #Sync
4529 wr %o3, 0, %fprs ! restore fprs
4530
4531 1:
4532 andn SAVED_LOFAULT, FPUSED_FLAG, SAVED_LOFAULT
4533 membar #Sync ! sync error barrier
4534 stn SAVED_LOFAULT, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
4535 ret
4536 restore %g0, 0, %o0
4537 .copyin_err:
4538 ldn [THREAD_REG + T_COPYOPS], %o4
4539 brz %o4, 2f
4540 nop
4541 ldn [%o4 + CP_COPYIN], %g2
4542 jmp %g2
4543 nop
4544 2:
4545 retl
4546 mov -1, %o0
4547 SET_SIZE(copyin)
4548
4549 ENTRY(xcopyin)
4550 sethi %hi(.xcopyin_err), REAL_LOFAULT
4551 b .do_copyin
4552 or REAL_LOFAULT, %lo(.xcopyin_err), REAL_LOFAULT
4553 .xcopyin_err:
4554 ldn [THREAD_REG + T_COPYOPS], %o4
4555 brz %o4, 2f
4556 nop
4557 ldn [%o4 + CP_XCOPYIN], %g2
4558 jmp %g2
4559 nop
4560 2:
4561 retl
4562 mov %g1, %o0
4563 SET_SIZE(xcopyin)
4564
4565 ENTRY(xcopyin_little)
4566 sethi %hi(.little_err), %o4
4567 ldn [THREAD_REG + T_LOFAULT], %o5
4568 or %o4, %lo(.little_err), %o4
4569 membar #Sync ! sync error barrier
4570 stn %o4, [THREAD_REG + T_LOFAULT]
4571
4572 subcc %g0, %o2, %o3
4573 add %o0, %o2, %o0
4574 bz,pn %ncc, 2f ! check for zero bytes
4575 sub %o2, 1, %o4
4576 add %o0, %o4, %o0 ! start w/last byte
4577 add %o1, %o2, %o1
4578 lduba [%o0+%o3]ASI_AIUSL, %o4
4579
4580 1: stb %o4, [%o1+%o3]
4581 inccc %o3
4582 sub %o0, 2, %o0 ! get next byte
4583 bcc,a,pt %ncc, 1b
4584 lduba [%o0+%o3]ASI_AIUSL, %o4
4585
4586 2: membar #Sync ! sync error barrier
4587 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
4588 retl
4589 mov %g0, %o0 ! return (0)
4590
4591 .little_err:
4592 membar #Sync ! sync error barrier
4593 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
4594 retl
4595 mov %g1, %o0
4596 SET_SIZE(xcopyin_little)
4597
4598
4599 /*
4600 * Copy a block of storage - must not overlap (from + len <= to).
4601 * No fault handler installed (to be called under on_fault())
4602 */
4603
4604 ENTRY(copyin_noerr)
4605 sethi %hi(.copyio_noerr), REAL_LOFAULT
4606 b .do_copyin
4607 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
4608 .copyio_noerr:
4609 jmp SAVED_LOFAULT
4610 nop
4611 SET_SIZE(copyin_noerr)
4612
4613 /*
4614 * Copy a block of storage - must not overlap (from + len <= to).
4615 * No fault handler installed (to be called under on_fault())
4616 */
4617
4618 ENTRY(copyout_noerr)
4619 sethi %hi(.copyio_noerr), REAL_LOFAULT
4620 b .do_copyout
4621 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
4622 SET_SIZE(copyout_noerr)
4623
4624 .align 4
4625 DGDEF(use_hw_bcopy)
4626 .word 1
4627 DGDEF(use_hw_copyio)
4628 .word 1
4629 DGDEF(use_hw_bzero)
4630 .word 1
4631 DGDEF(hw_copy_limit_1)
4632 .word 0
4633 DGDEF(hw_copy_limit_2)
4634 .word 0
4635 DGDEF(hw_copy_limit_4)
4636 .word 0
4637 DGDEF(hw_copy_limit_8)
4638 .word 0
4639
4640 .align 64
4641 .section ".text"
4642
4643
4644 /*
4645 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
4646 * longer than 256 bytes in length using spitfire's block stores. If
4647 * the criteria for using this routine are not met then it calls bzero
4648 * and returns 1. Otherwise 0 is returned indicating success.
4649 * Caller is responsible for ensuring use_hw_bzero is true and that
4650 * kpreempt_disable() has been called.
4651 */
4652 ! %i0 - start address
4653 ! %i1 - length of region (multiple of 64)
4654 ! %l0 - saved fprs
4655 ! %l1 - pointer to saved %d0 block
4656 ! %l2 - saved curthread->t_lwp
4657
4658 ENTRY(hwblkclr)
4659 ! get another window w/space for one aligned block of saved fpregs
4660 save %sp, -SA(MINFRAME + 2*64), %sp
4661
4662 ! Must be block-aligned
4663 andcc %i0, (64-1), %g0
4664 bnz,pn %ncc, 1f
4665 nop
4666
4667 ! ... and must be 256 bytes or more
4668 cmp %i1, 256
4669 blu,pn %ncc, 1f
4670 nop
4671
4731 set .pz_zinst, %i4
4732 sub %i4, %i2, %i4
4733 jmp %i4
4734 nop
4735
4736 .pz_finish:
4737 membar #Sync
4738 btst FPRS_FEF, %l0
4739 bz,a .pz_finished
4740 wr %l0, 0, %fprs ! restore fprs
4741
4742 ! restore fpregs from stack
4743 ldda [%l1]ASI_BLK_P, %d0
4744 membar #Sync
4745 wr %l0, 0, %fprs ! restore fprs
4746
4747 .pz_finished:
4748 ret
4749 restore %g0, 0, %o0 ! return (bzero or not)
4750 SET_SIZE(hwblkclr)
4751
4752 /*
4753 * Copy 32 bytes of data from src (%o0) to dst (%o1)
4754 * using physical addresses.
4755 */
4756 ENTRY_NP(hw_pa_bcopy32)
4757 rdpr %pstate, %g1
4758 andn %g1, PSTATE_IE, %g2
4759 wrpr %g0, %g2, %pstate
4760
4761 ldxa [%o0]ASI_MEM, %o2
4762 add %o0, 8, %o0
4763 ldxa [%o0]ASI_MEM, %o3
4764 add %o0, 8, %o0
4765 ldxa [%o0]ASI_MEM, %o4
4766 add %o0, 8, %o0
4767 ldxa [%o0]ASI_MEM, %o5
4768 stxa %o2, [%o1]ASI_MEM
4769 add %o1, 8, %o1
4770 stxa %o3, [%o1]ASI_MEM
4771 add %o1, 8, %o1
4772 stxa %o4, [%o1]ASI_MEM
4773 add %o1, 8, %o1
4774 stxa %o5, [%o1]ASI_MEM
4775
4776 membar #Sync
4777 retl
4778 wrpr %g0, %g1, %pstate
4779 SET_SIZE(hw_pa_bcopy32)
|