16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/param.h>
27 #include <sys/errno.h>
28 #include <sys/asm_linkage.h>
29 #include <sys/vtrace.h>
30 #include <sys/machthread.h>
31 #include <sys/clock.h>
32 #include <sys/asi.h>
33 #include <sys/fsr.h>
34 #include <sys/privregs.h>
35
36 #if !defined(lint)
37 #include "assym.h"
38 #endif /* lint */
39
40 /*
41 * Pseudo-code to aid in understanding the control flow of the
42 * bcopy/copyin/copyout routines.
43 *
44 * On entry:
45 *
46 * ! Determine whether to use the FP register version
47 * ! or the leaf routine version depending on size
48 * ! of copy and flags. Set up error handling accordingly.
49 * ! The transition point depends on whether the src and
50 * ! dst addresses can be aligned to long word, word,
51 * ! half word, or byte boundaries.
52 * !
53 * ! WARNING: <Register usage convention>
54 * ! For FP version, %l6 holds previous error handling and
55 * ! a flag: TRAMP_FLAG (low bits)
56 * ! for leaf routine version, %o4 holds those values.
57 * ! So either %l6 or %o4 is reserved and not available for
58 * ! any other use.
569 * Macros to save and restore quadrants 1 and 3 or 2 and 4 to/from the stack.
570 * Used to save and restore in-use fp registers when we want to use FP
571 * and find fp already in use and copy size still large enough to justify
572 * the additional overhead of this save and restore.
573 *
574 * A membar #Sync is needed before save to sync fp ops initiated before
575 * the call to the copy function (by whoever has fp in use); for example
576 * an earlier block load to the quadrant we are about to save may still be
577 * "in flight". A membar #Sync is required at the end of the save to
578 * sync our block store (the copy code is about to begin ldd's to the
579 * first quadrant).
580 *
581 * Similarly: a membar #Sync before restore allows the block stores of
582 * the copy operation to complete before we fill the quadrants with their
583 * original data, and a membar #Sync after restore lets the block loads
584 * of the restore complete before we return to whoever has the fp regs
585 * in use. To avoid repeated membar #Sync we make it the responsibility
586 * of the copy code to membar #Sync immediately after copy is complete
587 * and before using the BLD_*_FROMSTACK macro.
588 */
589 #if !defined(lint)
590 #define BST_FPQ1Q3_TOSTACK(tmp1) \
591 /* membar #Sync */ ;\
592 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
593 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
594 stda %f0, [tmp1]ASI_BLK_P ;\
595 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
596 stda %f32, [tmp1]ASI_BLK_P ;\
597 membar #Sync
598
599 #define BLD_FPQ1Q3_FROMSTACK(tmp1) \
600 /* membar #Sync - provided at copy completion */ ;\
601 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
602 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
603 ldda [tmp1]ASI_BLK_P, %f0 ;\
604 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
605 ldda [tmp1]ASI_BLK_P, %f32 ;\
606 membar #Sync
607
608 #define BST_FPQ2Q4_TOSTACK(tmp1) \
609 /* membar #Sync */ ;\
610 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
611 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
612 stda %f16, [tmp1]ASI_BLK_P ;\
613 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
614 stda %f48, [tmp1]ASI_BLK_P ;\
615 membar #Sync
616
617 #define BLD_FPQ2Q4_FROMSTACK(tmp1) \
618 /* membar #Sync - provided at copy completion */ ;\
619 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
620 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
621 ldda [tmp1]ASI_BLK_P, %f16 ;\
622 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
623 ldda [tmp1]ASI_BLK_P, %f48 ;\
624 membar #Sync
625 #endif
626
627 /*
628 * FP_NOMIGRATE and FP_ALLOWMIGRATE. Prevent migration (or, stronger,
629 * prevent preemption if there is no t_lwp to save FP state to on context
630 * switch) before commencing a FP copy, and reallow it on completion or
631 * in error trampoline paths when we were using FP copy.
632 *
633 * Both macros may call other functions, so be aware that all outputs are
634 * forfeit after using these macros. For this reason we do not pass registers
635 * to use - we just use any outputs we want.
636 *
637 * Pseudo code:
638 *
639 * FP_NOMIGRATE:
640 *
641 * if (curthread->t_lwp) {
642 * thread_nomigrate();
643 * } else {
644 * kpreempt_disable();
645 * }
675 ba label2/**/f ;\
676 nop ;\
677 label1: ;\
678 dec %o1 ;\
679 brnz,pn %o1, label2/**/f ;\
680 stb %o1, [THREAD_REG + T_PREEMPT] ;\
681 ldn [THREAD_REG + T_CPU], %o0 ;\
682 ldub [%o0 + CPU_KPRUNRUN], %o0 ;\
683 brz,pt %o0, label2/**/f ;\
684 nop ;\
685 call kpreempt ;\
686 rdpr %pil, %o0 ;\
687 label2:
688
689 /*
690 * Copy a block of storage, returning an error code if `from' or
691 * `to' takes a kernel pagefault which cannot be resolved.
692 * Returns errno value on pagefault error, 0 if all ok
693 */
694
695 #if defined(lint)
696
697 /* ARGSUSED */
698 int
699 kcopy(const void *from, void *to, size_t count)
700 { return(0); }
701
702 #else /* lint */
703
704 .seg ".text"
705 .align 4
706
707 ENTRY(kcopy)
708
709 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
710 bleu,pt %ncc, .kcopy_small ! go to larger cases
711 xor %o0, %o1, %o3 ! are src, dst alignable?
712 btst 7, %o3 !
713 bz,pt %ncc, .kcopy_8 ! check for longword alignment
714 nop
715 btst 1, %o3 !
716 bz,pt %ncc, .kcopy_2 ! check for half-word
717 nop
718 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
719 ld [%o3 + %lo(hw_copy_limit_1)], %o3
720 tst %o3
721 bz,pn %icc, .kcopy_small ! if zero, disable HW copy
722 cmp %o2, %o3 ! if length <= limit
723 bleu,pt %ncc, .kcopy_small ! go to small copy
851
852 /*
853 * We got here because of a fault during a small kcopy or bcopy.
854 * No floating point registers are used by the small copies.
855 * Errno value is in %g1.
856 */
857 .sm_copyerr:
858 1:
859 btst TRAMP_FLAG, %o4
860 membar #Sync
861 andn %o4, TRAMP_FLAG, %o4
862 bnz,pn %ncc, 3f
863 stn %o4, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
864 retl
865 mov %g1, %o0
866 3:
867 jmp %o4 ! goto real handler
868 mov %g0, %o0 !
869
870 SET_SIZE(kcopy)
871 #endif /* lint */
872
873
874 /*
875 * Copy a block of storage - must not overlap (from + len <= to).
876 * Registers: l6 - saved t_lofault
877 * (for short copies, o4 - saved t_lofault)
878 *
879 * Copy a page of memory.
880 * Assumes double word alignment and a count >= 256.
881 */
882 #if defined(lint)
883
884 /* ARGSUSED */
885 void
886 bcopy(const void *from, void *to, size_t count)
887 {}
888
889 #else /* lint */
890
891 ENTRY(bcopy)
892
893 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
894 bleu,pt %ncc, .bcopy_small ! go to larger cases
895 xor %o0, %o1, %o3 ! are src, dst alignable?
896 btst 7, %o3 !
897 bz,pt %ncc, .bcopy_8 ! check for longword alignment
898 nop
899 btst 1, %o3 !
900 bz,pt %ncc, .bcopy_2 ! check for half-word
901 nop
902 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
903 ld [%o3 + %lo(hw_copy_limit_1)], %o3
904 tst %o3
905 bz,pn %icc, .bcopy_small ! if zero, disable HW copy
906 cmp %o2, %o3 ! if length <= limit
907 bleu,pt %ncc, .bcopy_small ! go to small copy
908 nop
909 ba,pt %ncc, .bcopy_more ! otherwise go to large copy
910 nop
1427 bz,pt %icc, 4f
1428 nop
1429
1430 BLD_FPQ1Q3_FROMSTACK(%o2)
1431
1432 ba,pt %ncc, 2f
1433 wr %o3, 0, %fprs ! restore fprs
1434 4:
1435 FZEROQ1Q3
1436 wr %o3, 0, %fprs ! restore fprs
1437 2:
1438 membar #Sync ! sync error barrier
1439 andn %l6, MASK_FLAGS, %l6
1440 stn %l6, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
1441 FP_ALLOWMIGRATE(5, 6)
1442 ret
1443 restore %g0, 0, %o0
1444
1445 SET_SIZE(bcopy_more)
1446
1447 #endif /* lint */
1448
1449 /*
1450 * Block copy with possibly overlapped operands.
1451 */
1452
1453 #if defined(lint)
1454
1455 /*ARGSUSED*/
1456 void
1457 ovbcopy(const void *from, void *to, size_t count)
1458 {}
1459
1460 #else /* lint */
1461
1462 ENTRY(ovbcopy)
1463 tst %o2 ! check count
1464 bgu,a %ncc, 1f ! nothing to do or bad arguments
1465 subcc %o0, %o1, %o3 ! difference of from and to address
1466
1467 retl ! return
1468 nop
1469 1:
1470 bneg,a %ncc, 2f
1471 neg %o3 ! if < 0, make it positive
1472 2: cmp %o2, %o3 ! cmp size and abs(from - to)
1473 bleu %ncc, bcopy ! if size <= abs(diff): use bcopy,
1474 .empty ! no overlap
1475 cmp %o0, %o1 ! compare from and to addresses
1476 blu %ncc, .ov_bkwd ! if from < to, copy backwards
1477 nop
1478 !
1479 ! Copy forwards.
1480 !
1481 .ov_fwd:
1485 deccc %o2 ! dec count
1486 bgu %ncc, .ov_fwd ! loop till done
1487 inc %o1 ! inc to address
1488
1489 retl ! return
1490 nop
1491 !
1492 ! Copy backwards.
1493 !
1494 .ov_bkwd:
1495 deccc %o2 ! dec count
1496 ldub [%o0 + %o2], %o3 ! get byte at end of src
1497 bgu %ncc, .ov_bkwd ! loop till done
1498 stb %o3, [%o1 + %o2] ! delay slot, store at end of dst
1499
1500 retl ! return
1501 nop
1502
1503 SET_SIZE(ovbcopy)
1504
1505 #endif /* lint */
1506
1507
1508 /*
1509 * hwblkpagecopy()
1510 *
1511 * Copies exactly one page. This routine assumes the caller (ppcopy)
1512 * has already disabled kernel preemption and has checked
1513 * use_hw_bcopy. Preventing preemption also prevents cpu migration.
1514 */
1515 #ifdef lint
1516 /*ARGSUSED*/
1517 void
1518 hwblkpagecopy(const void *src, void *dst)
1519 { }
1520 #else /* lint */
1521 ENTRY(hwblkpagecopy)
1522 ! get another window w/space for three aligned blocks of saved fpregs
1523 prefetch [%o0], #n_reads
1524 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
1525
1526 ! %i0 - source address (arg)
1527 ! %i1 - destination address (arg)
1528 ! %i2 - length of region (not arg)
1529 ! %l0 - saved fprs
1530 ! %l1 - pointer to saved fpregs
1531
1532 rd %fprs, %l0 ! check for unused fp
1533 btst FPRS_FEF, %l0
1534 bz,a,pt %icc, 1f
1535 wr %g0, FPRS_FEF, %fprs
1536
1537 BST_FPQ1Q3_TOSTACK(%l1)
1538
1539 1: set PAGESIZE, CNT
1540 mov REALSRC, SRC
1618 fsrc1 %f14, %f46
1619 stda %f32, [DST]ASI_BLK_P
1620
1621 membar #Sync
1622
1623 btst FPRS_FEF, %l0
1624 bz,pt %icc, 2f
1625 nop
1626
1627 BLD_FPQ1Q3_FROMSTACK(%l3)
1628 ba 3f
1629 nop
1630
1631 2: FZEROQ1Q3
1632
1633 3: wr %l0, 0, %fprs ! restore fprs
1634 ret
1635 restore %g0, 0, %o0
1636
1637 SET_SIZE(hwblkpagecopy)
1638 #endif /* lint */
1639
1640
1641 /*
1642 * Transfer data to and from user space -
1643 * Note that these routines can cause faults
1644 * It is assumed that the kernel has nothing at
1645 * less than KERNELBASE in the virtual address space.
1646 *
1647 * Note that copyin(9F) and copyout(9F) are part of the
1648 * DDI/DKI which specifies that they return '-1' on "errors."
1649 *
1650 * Sigh.
1651 *
1652 * So there's two extremely similar routines - xcopyin() and xcopyout()
1653 * which return the errno that we've faithfully computed. This
1654 * allows other callers (e.g. uiomove(9F)) to work correctly.
1655 * Given that these are used pretty heavily, we expand the calling
1656 * sequences inline for all flavours (rather than making wrappers).
1657 *
1658 * There are also stub routines for xcopyout_little and xcopyin_little,
1677 *
1678 * None of the copyops routines grab a window until it's decided that
1679 * we need to do a HW block copy operation. This saves a window
1680 * spill/fill when we're called during socket ops. The typical IO
1681 * path won't cause spill/fill traps.
1682 *
1683 * This code uses a set of 4 limits for the maximum size that will
1684 * be copied given a particular input/output address alignment.
1685 * If the value for a particular limit is zero, the copy will be performed
1686 * by the plain copy loops rather than FPBLK.
1687 *
1688 * See the description of bcopy above for more details of the
1689 * data copying algorithm and the default limits.
1690 *
1691 */
1692
1693 /*
1694 * Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
1695 */
1696
1697 #if defined(lint)
1698
1699
1700 #else /* lint */
1701 /*
1702 * We save the arguments in the following registers in case of a fault:
1703 * kaddr - %l1
1704 * uaddr - %l2
1705 * count - %l3
1706 */
1707 #define SAVE_SRC %l1
1708 #define SAVE_DST %l2
1709 #define SAVE_COUNT %l3
1710
1711 #define SM_SAVE_SRC %g4
1712 #define SM_SAVE_DST %g5
1713 #define SM_SAVE_COUNT %o5
1714 #define ERRNO %l5
1715
1716
1717 #define REAL_LOFAULT %l4
1718 /*
1719 * Generic copyio fault handler. This is the first line of defense when a
1720 * fault occurs in (x)copyin/(x)copyout. In order for this to function
1747 wr %o3, 0, %fprs ! restore fprs
1748
1749 4:
1750 FZEROQ2Q4
1751 wr %o3, 0, %fprs ! restore fprs
1752
1753 1:
1754 andn %l6, FPUSED_FLAG, %l6
1755 membar #Sync
1756 stn %l6, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
1757 FP_ALLOWMIGRATE(5, 6)
1758
1759 mov SAVE_SRC, %i0
1760 mov SAVE_DST, %i1
1761 jmp REAL_LOFAULT
1762 mov SAVE_COUNT, %i2
1763
1764 SET_SIZE(copyio_fault)
1765
1766
1767 #endif
1768
1769 #if defined(lint)
1770
1771 /*ARGSUSED*/
1772 int
1773 copyout(const void *kaddr, void *uaddr, size_t count)
1774 { return (0); }
1775
1776 #else /* lint */
1777
1778 ENTRY(copyout)
1779
1780 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
1781 bleu,pt %ncc, .copyout_small ! go to larger cases
1782 xor %o0, %o1, %o3 ! are src, dst alignable?
1783 btst 7, %o3 !
1784 bz,pt %ncc, .copyout_8 ! check for longword alignment
1785 nop
1786 btst 1, %o3 !
1787 bz,pt %ncc, .copyout_2 ! check for half-word
1788 nop
1789 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
1790 ld [%o3 + %lo(hw_copy_limit_1)], %o3
1791 tst %o3
1792 bz,pn %icc, .copyout_small ! if zero, disable HW copy
1793 cmp %o2, %o3 ! if length <= limit
1794 bleu,pt %ncc, .copyout_small ! go to small copy
1795 nop
1796 ba,pt %ncc, .copyout_more ! otherwise go to large copy
1797 nop
2373
2374 /*
2375 * We got here because of a fault during copyout.
2376 * Errno value is in ERRNO, but DDI/DKI says return -1 (sigh).
2377 */
2378 .copyout_err:
2379 ldn [THREAD_REG + T_COPYOPS], %o4 ! check for copyop handler
2380 tst %o4
2381 bz,pt %ncc, 2f ! if not, return error
2382 nop
2383 ldn [%o4 + CP_COPYOUT], %g2 ! if handler, invoke it with
2384 jmp %g2 ! original arguments
2385 restore %g0, 0, %g0 ! dispose of copy window
2386 2:
2387 ret
2388 restore %g0, -1, %o0 ! return error value
2389
2390
2391 SET_SIZE(copyout_more)
2392
2393 #endif /* lint */
2394
2395
2396 #ifdef lint
2397
2398 /*ARGSUSED*/
2399 int
2400 xcopyout(const void *kaddr, void *uaddr, size_t count)
2401 { return (0); }
2402
2403 #else /* lint */
2404
2405 ENTRY(xcopyout)
2406 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
2407 bleu,pt %ncc, .xcopyout_small ! go to larger cases
2408 xor %o0, %o1, %o3 ! are src, dst alignable?
2409 btst 7, %o3 !
2410 bz,pt %ncc, .xcopyout_8 !
2411 nop
2412 btst 1, %o3 !
2413 bz,pt %ncc, .xcopyout_2 ! check for half-word
2414 nop
2415 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
2416 ld [%o3 + %lo(hw_copy_limit_1)], %o3
2417 tst %o3
2418 bz,pn %icc, .xcopyout_small ! if zero, disable HW copy
2419 cmp %o2, %o3 ! if length <= limit
2420 bleu,pt %ncc, .xcopyout_small ! go to small copy
2421 nop
2422 ba,pt %ncc, .xcopyout_more ! otherwise go to large copy
2423 nop
2424 .xcopyout_2:
2489 .sm_xcopyout_err:
2490
2491 membar #Sync
2492 stn %o4, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
2493 mov SM_SAVE_SRC, %o0
2494 mov SM_SAVE_DST, %o1
2495 mov SM_SAVE_COUNT, %o2
2496 ldn [THREAD_REG + T_COPYOPS], %o3 ! check for copyop handler
2497 tst %o3
2498 bz,pt %ncc, 3f ! if not, return error
2499 nop
2500 ldn [%o3 + CP_XCOPYOUT], %o5 ! if handler, invoke it with
2501 jmp %o5 ! original arguments
2502 nop
2503 3:
2504 retl
2505 or %g1, 0, %o0 ! return errno value
2506
2507 SET_SIZE(xcopyout)
2508
2509 #endif /* lint */
2510
2511 #ifdef lint
2512
2513 /*ARGSUSED*/
2514 int
2515 xcopyout_little(const void *kaddr, void *uaddr, size_t count)
2516 { return (0); }
2517
2518 #else /* lint */
2519
2520 ENTRY(xcopyout_little)
2521 sethi %hi(.xcopyio_err), %o5
2522 or %o5, %lo(.xcopyio_err), %o5
2523 ldn [THREAD_REG + T_LOFAULT], %o4
2524 membar #Sync ! sync error barrier
2525 stn %o5, [THREAD_REG + T_LOFAULT]
2526 mov %o4, %o5
2527
2528 subcc %g0, %o2, %o3
2529 add %o0, %o2, %o0
2530 bz,pn %ncc, 2f ! check for zero bytes
2531 sub %o2, 1, %o4
2532 add %o0, %o4, %o0 ! start w/last byte
2533 add %o1, %o2, %o1
2534 ldub [%o0 + %o3], %o4
2535
2536 1: stba %o4, [%o1 + %o3]ASI_AIUSL
2537 inccc %o3
2538 sub %o0, 2, %o0 ! get next byte
2539 bcc,a,pt %ncc, 1b
2540 ldub [%o0 + %o3], %o4
2541
2542 2:
2543 membar #Sync ! sync error barrier
2544 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
2545 retl
2546 mov %g0, %o0 ! return (0)
2547
2548 SET_SIZE(xcopyout_little)
2549
2550 #endif /* lint */
2551
2552 /*
2553 * Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
2554 */
2555
2556 #if defined(lint)
2557
2558 /*ARGSUSED*/
2559 int
2560 copyin(const void *uaddr, void *kaddr, size_t count)
2561 { return (0); }
2562
2563 #else /* lint */
2564
2565 ENTRY(copyin)
2566 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
2567 bleu,pt %ncc, .copyin_small ! go to larger cases
2568 xor %o0, %o1, %o3 ! are src, dst alignable?
2569 btst 7, %o3 !
2570 bz,pt %ncc, .copyin_8 ! check for longword alignment
2571 nop
2572 btst 1, %o3 !
2573 bz,pt %ncc, .copyin_2 ! check for half-word
2574 nop
2575 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
2576 ld [%o3 + %lo(hw_copy_limit_1)], %o3
2577 tst %o3
2578 bz,pn %icc, .copyin_small ! if zero, disable HW copy
2579 cmp %o2, %o3 ! if length <= limit
2580 bleu,pt %ncc, .copyin_small ! go to small copy
2581 nop
2582 ba,pt %ncc, .copyin_more ! otherwise go to large copy
2583 nop
2584 .copyin_2:
3151 restore %g0, 0, %o0
3152 /*
3153 * We got here because of a fault during copyin
3154 * Errno value is in ERRNO, but DDI/DKI says return -1 (sigh).
3155 */
3156 .copyin_err:
3157 ldn [THREAD_REG + T_COPYOPS], %o4 ! check for copyop handler
3158 tst %o4
3159 bz,pt %ncc, 2f ! if not, return error
3160 nop
3161 ldn [%o4 + CP_COPYIN], %g2 ! if handler, invoke it with
3162 jmp %g2 ! original arguments
3163 restore %g0, 0, %g0 ! dispose of copy window
3164 2:
3165 ret
3166 restore %g0, -1, %o0 ! return error value
3167
3168
3169 SET_SIZE(copyin_more)
3170
3171 #endif /* lint */
3172
3173 #ifdef lint
3174
3175 /*ARGSUSED*/
3176 int
3177 xcopyin(const void *uaddr, void *kaddr, size_t count)
3178 { return (0); }
3179
3180 #else /* lint */
3181
3182 ENTRY(xcopyin)
3183
3184 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
3185 bleu,pt %ncc, .xcopyin_small ! go to larger cases
3186 xor %o0, %o1, %o3 ! are src, dst alignable?
3187 btst 7, %o3 !
3188 bz,pt %ncc, .xcopyin_8 ! check for longword alignment
3189 nop
3190 btst 1, %o3 !
3191 bz,pt %ncc, .xcopyin_2 ! check for half-word
3192 nop
3193 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
3194 ld [%o3 + %lo(hw_copy_limit_1)], %o3
3195 tst %o3
3196 bz,pn %icc, .xcopyin_small ! if zero, disable HW copy
3197 cmp %o2, %o3 ! if length <= limit
3198 bleu,pt %ncc, .xcopyin_small ! go to small copy
3199 nop
3200 ba,pt %ncc, .xcopyin_more ! otherwise go to large copy
3201 nop
3267 .sm_xcopyin_err:
3268
3269 membar #Sync
3270 stn %o4, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3271 mov SM_SAVE_SRC, %o0
3272 mov SM_SAVE_DST, %o1
3273 mov SM_SAVE_COUNT, %o2
3274 ldn [THREAD_REG + T_COPYOPS], %o3 ! check for copyop handler
3275 tst %o3
3276 bz,pt %ncc, 3f ! if not, return error
3277 nop
3278 ldn [%o3 + CP_XCOPYIN], %o5 ! if handler, invoke it with
3279 jmp %o5 ! original arguments
3280 nop
3281 3:
3282 retl
3283 or %g1, 0, %o0 ! return errno value
3284
3285 SET_SIZE(xcopyin)
3286
3287 #endif /* lint */
3288
3289 #ifdef lint
3290
3291 /*ARGSUSED*/
3292 int
3293 xcopyin_little(const void *uaddr, void *kaddr, size_t count)
3294 { return (0); }
3295
3296 #else /* lint */
3297
3298 ENTRY(xcopyin_little)
3299 sethi %hi(.xcopyio_err), %o5
3300 or %o5, %lo(.xcopyio_err), %o5
3301 ldn [THREAD_REG + T_LOFAULT], %o4
3302 membar #Sync ! sync error barrier
3303 stn %o5, [THREAD_REG + T_LOFAULT]
3304 mov %o4, %o5
3305
3306 subcc %g0, %o2, %o3
3307 add %o0, %o2, %o0
3308 bz,pn %ncc, 2f ! check for zero bytes
3309 sub %o2, 1, %o4
3310 add %o0, %o4, %o0 ! start w/last byte
3311 add %o1, %o2, %o1
3312 lduba [%o0 + %o3]ASI_AIUSL, %o4
3313
3314 1: stb %o4, [%o1 + %o3]
3315 inccc %o3
3316 sub %o0, 2, %o0 ! get next byte
3317 bcc,a,pt %ncc, 1b
3318 lduba [%o0 + %o3]ASI_AIUSL, %o4
3319
3320 2:
3321 membar #Sync ! sync error barrier
3322 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3323 retl
3324 mov %g0, %o0 ! return (0)
3325
3326 .xcopyio_err:
3327 membar #Sync ! sync error barrier
3328 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3329 retl
3330 mov %g1, %o0
3331
3332 SET_SIZE(xcopyin_little)
3333
3334 #endif /* lint */
3335
3336
3337 /*
3338 * Copy a block of storage - must not overlap (from + len <= to).
3339 * No fault handler installed (to be called under on_fault())
3340 */
3341 #if defined(lint)
3342
3343 /* ARGSUSED */
3344 void
3345 copyin_noerr(const void *ufrom, void *kto, size_t count)
3346 {}
3347
3348 #else /* lint */
3349 ENTRY(copyin_noerr)
3350
3351 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
3352 bleu,pt %ncc, .copyin_ne_small ! go to larger cases
3353 xor %o0, %o1, %o3 ! are src, dst alignable?
3354 btst 7, %o3 !
3355 bz,pt %ncc, .copyin_ne_8 ! check for longword alignment
3356 nop
3357 btst 1, %o3 !
3358 bz,pt %ncc, .copyin_ne_2 ! check for half-word
3359 nop
3360 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
3361 ld [%o3 + %lo(hw_copy_limit_1)], %o3
3362 tst %o3
3363 bz,pn %icc, .copyin_ne_small ! if zero, disable HW copy
3364 cmp %o2, %o3 ! if length <= limit
3365 bleu,pt %ncc, .copyin_ne_small ! go to small copy
3366 nop
3367 ba,pt %ncc, .copyin_noerr_more ! otherwise go to large copy
3368 nop
3412 ba,pt %ncc, .sm_do_copyin
3413 stn %o5, [THREAD_REG + T_LOFAULT] ! set/save t_lofault
3414
3415 .copyin_noerr_more:
3416 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3417 sethi %hi(.copyio_noerr), REAL_LOFAULT
3418 ba,pt %ncc, .do_copyin
3419 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
3420
3421 .copyio_noerr:
3422 jmp %l6
3423 restore %g0,0,%g0
3424
3425 .sm_copyio_noerr:
3426 membar #Sync
3427 stn %o4, [THREAD_REG + T_LOFAULT] ! restore t_lofault
3428 jmp %o4
3429 nop
3430
3431 SET_SIZE(copyin_noerr)
3432 #endif /* lint */
3433
3434 /*
3435 * Copy a block of storage - must not overlap (from + len <= to).
3436 * No fault handler installed (to be called under on_fault())
3437 */
3438
3439 #if defined(lint)
3440
3441 /* ARGSUSED */
3442 void
3443 copyout_noerr(const void *kfrom, void *uto, size_t count)
3444 {}
3445
3446 #else /* lint */
3447 ENTRY(copyout_noerr)
3448
3449 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
3450 bleu,pt %ncc, .copyout_ne_small ! go to larger cases
3451 xor %o0, %o1, %o3 ! are src, dst alignable?
3452 btst 7, %o3 !
3453 bz,pt %ncc, .copyout_ne_8 ! check for longword alignment
3454 nop
3455 btst 1, %o3 !
3456 bz,pt %ncc, .copyout_ne_2 ! check for half-word
3457 nop
3458 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
3459 ld [%o3 + %lo(hw_copy_limit_1)], %o3
3460 tst %o3
3461 bz,pn %icc, .copyout_ne_small ! if zero, disable HW copy
3462 cmp %o2, %o3 ! if length <= limit
3463 bleu,pt %ncc, .copyout_ne_small ! go to small copy
3464 nop
3465 ba,pt %ncc, .copyout_noerr_more ! otherwise go to large copy
3466 nop
3500 nop
3501
3502 .copyout_ne_small:
3503 ldn [THREAD_REG + T_LOFAULT], %o4
3504 tst %o4
3505 bz,pn %ncc, .sm_do_copyout
3506 nop
3507 sethi %hi(.sm_copyio_noerr), %o5
3508 or %o5, %lo(.sm_copyio_noerr), %o5
3509 membar #Sync ! sync error barrier
3510 ba,pt %ncc, .sm_do_copyout
3511 stn %o5, [THREAD_REG + T_LOFAULT] ! set/save t_lofault
3512
3513 .copyout_noerr_more:
3514 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3515 sethi %hi(.copyio_noerr), REAL_LOFAULT
3516 ba,pt %ncc, .do_copyout
3517 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
3518
3519 SET_SIZE(copyout_noerr)
3520 #endif /* lint */
3521
3522
3523 /*
3524 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
3525 * longer than 256 bytes in length using spitfire's block stores. If
3526 * the criteria for using this routine are not met then it calls bzero
3527 * and returns 1. Otherwise 0 is returned indicating success.
3528 * Caller is responsible for ensuring use_hw_bzero is true and that
3529 * kpreempt_disable() has been called.
3530 */
3531 #ifdef lint
3532 /*ARGSUSED*/
3533 int
3534 hwblkclr(void *addr, size_t len)
3535 {
3536 return(0);
3537 }
3538 #else /* lint */
3539 ! %i0 - start address
3540 ! %i1 - length of region (multiple of 64)
3541 ! %l0 - saved fprs
3542 ! %l1 - pointer to saved %d0 block
3543 ! %l2 - saved curthread->t_lwp
3544
3545 ENTRY(hwblkclr)
3546 ! get another window w/space for one aligned block of saved fpregs
3547 save %sp, -SA(MINFRAME + 2*VIS_BLOCKSIZE), %sp
3548
3549 ! Must be block-aligned
3550 andcc %i0, (VIS_BLOCKSIZE-1), %g0
3551 bnz,pn %ncc, 1f
3552 nop
3553
3554 ! ... and must be 256 bytes or more
3555 cmp %i1, 256
3556 blu,pn %ncc, 1f
3557 nop
3558
3619 sub %i4, %i2, %i4
3620 jmp %i4
3621 nop
3622
3623 .pz_finish:
3624 membar #Sync
3625 btst FPRS_FEF, %l0
3626 bz,a .pz_finished
3627 wr %l0, 0, %fprs ! restore fprs
3628
3629 ! restore fpregs from stack
3630 ldda [%l1]ASI_BLK_P, %d0
3631 membar #Sync
3632 wr %l0, 0, %fprs ! restore fprs
3633
3634 .pz_finished:
3635 ret
3636 restore %g0, 0, %o0 ! return (bzero or not)
3637
3638 SET_SIZE(hwblkclr)
3639 #endif /* lint */
3640
3641 #ifdef lint
3642 /*ARGSUSED*/
3643 void
3644 hw_pa_bcopy32(uint64_t src, uint64_t dst)
3645 {}
3646 #else /*!lint */
3647 /*
3648 * Copy 32 bytes of data from src (%o0) to dst (%o1)
3649 * using physical addresses.
3650 */
3651 ENTRY_NP(hw_pa_bcopy32)
3652 rdpr %pstate, %g1
3653 andn %g1, PSTATE_IE, %g2
3654 wrpr %g0, %g2, %pstate
3655
3656 rdpr %pstate, %g0
3657 ldxa [%o0]ASI_MEM, %o2
3658 add %o0, 8, %o0
3659 ldxa [%o0]ASI_MEM, %o3
3660 add %o0, 8, %o0
3661 ldxa [%o0]ASI_MEM, %o4
3662 add %o0, 8, %o0
3663 ldxa [%o0]ASI_MEM, %o5
3664 membar #Sync
3665
3666 stxa %o2, [%o1]ASI_MEM
3667 add %o1, 8, %o1
3668 stxa %o3, [%o1]ASI_MEM
3669 add %o1, 8, %o1
3670 stxa %o4, [%o1]ASI_MEM
3671 add %o1, 8, %o1
3672 stxa %o5, [%o1]ASI_MEM
3673
3674 retl
3675 wrpr %g0, %g1, %pstate
3676
3677 SET_SIZE(hw_pa_bcopy32)
3678
3679 #endif /* lint */
3680
3681 #if defined(lint)
3682
3683 int use_hw_bcopy = 1;
3684 int use_hw_bzero = 1;
3685 uint_t hw_copy_limit_1 = 0;
3686 uint_t hw_copy_limit_2 = 0;
3687 uint_t hw_copy_limit_4 = 0;
3688 uint_t hw_copy_limit_8 = 0;
3689
3690 #else /* !lint */
3691
3692 DGDEF(use_hw_bcopy)
3693 .word 1
3694 DGDEF(use_hw_bzero)
3695 .word 1
3696 DGDEF(hw_copy_limit_1)
3697 .word 0
3698 DGDEF(hw_copy_limit_2)
3699 .word 0
3700 DGDEF(hw_copy_limit_4)
3701 .word 0
3702 DGDEF(hw_copy_limit_8)
3703 .word 0
3704
3705 .align 64
3706 .section ".text"
3707 #endif /* !lint */
|
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/param.h>
27 #include <sys/errno.h>
28 #include <sys/asm_linkage.h>
29 #include <sys/vtrace.h>
30 #include <sys/machthread.h>
31 #include <sys/clock.h>
32 #include <sys/asi.h>
33 #include <sys/fsr.h>
34 #include <sys/privregs.h>
35
36 #include "assym.h"
37
38 /*
39 * Pseudo-code to aid in understanding the control flow of the
40 * bcopy/copyin/copyout routines.
41 *
42 * On entry:
43 *
44 * ! Determine whether to use the FP register version
45 * ! or the leaf routine version depending on size
46 * ! of copy and flags. Set up error handling accordingly.
47 * ! The transition point depends on whether the src and
48 * ! dst addresses can be aligned to long word, word,
49 * ! half word, or byte boundaries.
50 * !
51 * ! WARNING: <Register usage convention>
52 * ! For FP version, %l6 holds previous error handling and
53 * ! a flag: TRAMP_FLAG (low bits)
54 * ! for leaf routine version, %o4 holds those values.
55 * ! So either %l6 or %o4 is reserved and not available for
56 * ! any other use.
567 * Macros to save and restore quadrants 1 and 3 or 2 and 4 to/from the stack.
568 * Used to save and restore in-use fp registers when we want to use FP
569 * and find fp already in use and copy size still large enough to justify
570 * the additional overhead of this save and restore.
571 *
572 * A membar #Sync is needed before save to sync fp ops initiated before
573 * the call to the copy function (by whoever has fp in use); for example
574 * an earlier block load to the quadrant we are about to save may still be
575 * "in flight". A membar #Sync is required at the end of the save to
576 * sync our block store (the copy code is about to begin ldd's to the
577 * first quadrant).
578 *
579 * Similarly: a membar #Sync before restore allows the block stores of
580 * the copy operation to complete before we fill the quadrants with their
581 * original data, and a membar #Sync after restore lets the block loads
582 * of the restore complete before we return to whoever has the fp regs
583 * in use. To avoid repeated membar #Sync we make it the responsibility
584 * of the copy code to membar #Sync immediately after copy is complete
585 * and before using the BLD_*_FROMSTACK macro.
586 */
587 #define BST_FPQ1Q3_TOSTACK(tmp1) \
588 /* membar #Sync */ ;\
589 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
590 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
591 stda %f0, [tmp1]ASI_BLK_P ;\
592 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
593 stda %f32, [tmp1]ASI_BLK_P ;\
594 membar #Sync
595
596 #define BLD_FPQ1Q3_FROMSTACK(tmp1) \
597 /* membar #Sync - provided at copy completion */ ;\
598 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
599 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
600 ldda [tmp1]ASI_BLK_P, %f0 ;\
601 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
602 ldda [tmp1]ASI_BLK_P, %f32 ;\
603 membar #Sync
604
605 #define BST_FPQ2Q4_TOSTACK(tmp1) \
606 /* membar #Sync */ ;\
607 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
608 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
609 stda %f16, [tmp1]ASI_BLK_P ;\
610 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
611 stda %f48, [tmp1]ASI_BLK_P ;\
612 membar #Sync
613
614 #define BLD_FPQ2Q4_FROMSTACK(tmp1) \
615 /* membar #Sync - provided at copy completion */ ;\
616 add %fp, STACK_BIAS - SAVED_FPREGS_ADJUST, tmp1 ;\
617 and tmp1, -VIS_BLOCKSIZE, tmp1 /* block align */ ;\
618 ldda [tmp1]ASI_BLK_P, %f16 ;\
619 add tmp1, VIS_BLOCKSIZE, tmp1 ;\
620 ldda [tmp1]ASI_BLK_P, %f48 ;\
621 membar #Sync
622
623 /*
624 * FP_NOMIGRATE and FP_ALLOWMIGRATE. Prevent migration (or, stronger,
625 * prevent preemption if there is no t_lwp to save FP state to on context
626 * switch) before commencing a FP copy, and reallow it on completion or
627 * in error trampoline paths when we were using FP copy.
628 *
629 * Both macros may call other functions, so be aware that all outputs are
630 * forfeit after using these macros. For this reason we do not pass registers
631 * to use - we just use any outputs we want.
632 *
633 * Pseudo code:
634 *
635 * FP_NOMIGRATE:
636 *
637 * if (curthread->t_lwp) {
638 * thread_nomigrate();
639 * } else {
640 * kpreempt_disable();
641 * }
671 ba label2/**/f ;\
672 nop ;\
673 label1: ;\
674 dec %o1 ;\
675 brnz,pn %o1, label2/**/f ;\
676 stb %o1, [THREAD_REG + T_PREEMPT] ;\
677 ldn [THREAD_REG + T_CPU], %o0 ;\
678 ldub [%o0 + CPU_KPRUNRUN], %o0 ;\
679 brz,pt %o0, label2/**/f ;\
680 nop ;\
681 call kpreempt ;\
682 rdpr %pil, %o0 ;\
683 label2:
684
685 /*
686 * Copy a block of storage, returning an error code if `from' or
687 * `to' takes a kernel pagefault which cannot be resolved.
688 * Returns errno value on pagefault error, 0 if all ok
689 */
690
691 .seg ".text"
692 .align 4
693
694 ENTRY(kcopy)
695
696 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
697 bleu,pt %ncc, .kcopy_small ! go to larger cases
698 xor %o0, %o1, %o3 ! are src, dst alignable?
699 btst 7, %o3 !
700 bz,pt %ncc, .kcopy_8 ! check for longword alignment
701 nop
702 btst 1, %o3 !
703 bz,pt %ncc, .kcopy_2 ! check for half-word
704 nop
705 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
706 ld [%o3 + %lo(hw_copy_limit_1)], %o3
707 tst %o3
708 bz,pn %icc, .kcopy_small ! if zero, disable HW copy
709 cmp %o2, %o3 ! if length <= limit
710 bleu,pt %ncc, .kcopy_small ! go to small copy
838
839 /*
840 * We got here because of a fault during a small kcopy or bcopy.
841 * No floating point registers are used by the small copies.
842 * Errno value is in %g1.
843 */
844 .sm_copyerr:
845 1:
846 btst TRAMP_FLAG, %o4
847 membar #Sync
848 andn %o4, TRAMP_FLAG, %o4
849 bnz,pn %ncc, 3f
850 stn %o4, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
851 retl
852 mov %g1, %o0
853 3:
854 jmp %o4 ! goto real handler
855 mov %g0, %o0 !
856
857 SET_SIZE(kcopy)
858
859
860 /*
861 * Copy a block of storage - must not overlap (from + len <= to).
862 * Registers: l6 - saved t_lofault
863 * (for short copies, o4 - saved t_lofault)
864 *
865 * Copy a page of memory.
866 * Assumes double word alignment and a count >= 256.
867 */
868
869 ENTRY(bcopy)
870
871 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
872 bleu,pt %ncc, .bcopy_small ! go to larger cases
873 xor %o0, %o1, %o3 ! are src, dst alignable?
874 btst 7, %o3 !
875 bz,pt %ncc, .bcopy_8 ! check for longword alignment
876 nop
877 btst 1, %o3 !
878 bz,pt %ncc, .bcopy_2 ! check for half-word
879 nop
880 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
881 ld [%o3 + %lo(hw_copy_limit_1)], %o3
882 tst %o3
883 bz,pn %icc, .bcopy_small ! if zero, disable HW copy
884 cmp %o2, %o3 ! if length <= limit
885 bleu,pt %ncc, .bcopy_small ! go to small copy
886 nop
887 ba,pt %ncc, .bcopy_more ! otherwise go to large copy
888 nop
1405 bz,pt %icc, 4f
1406 nop
1407
1408 BLD_FPQ1Q3_FROMSTACK(%o2)
1409
1410 ba,pt %ncc, 2f
1411 wr %o3, 0, %fprs ! restore fprs
1412 4:
1413 FZEROQ1Q3
1414 wr %o3, 0, %fprs ! restore fprs
1415 2:
1416 membar #Sync ! sync error barrier
1417 andn %l6, MASK_FLAGS, %l6
1418 stn %l6, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
1419 FP_ALLOWMIGRATE(5, 6)
1420 ret
1421 restore %g0, 0, %o0
1422
1423 SET_SIZE(bcopy_more)
1424
1425 /*
1426 * Block copy with possibly overlapped operands.
1427 */
1428
1429 ENTRY(ovbcopy)
1430 tst %o2 ! check count
1431 bgu,a %ncc, 1f ! nothing to do or bad arguments
1432 subcc %o0, %o1, %o3 ! difference of from and to address
1433
1434 retl ! return
1435 nop
1436 1:
1437 bneg,a %ncc, 2f
1438 neg %o3 ! if < 0, make it positive
1439 2: cmp %o2, %o3 ! cmp size and abs(from - to)
1440 bleu %ncc, bcopy ! if size <= abs(diff): use bcopy,
1441 .empty ! no overlap
1442 cmp %o0, %o1 ! compare from and to addresses
1443 blu %ncc, .ov_bkwd ! if from < to, copy backwards
1444 nop
1445 !
1446 ! Copy forwards.
1447 !
1448 .ov_fwd:
1452 deccc %o2 ! dec count
1453 bgu %ncc, .ov_fwd ! loop till done
1454 inc %o1 ! inc to address
1455
1456 retl ! return
1457 nop
1458 !
1459 ! Copy backwards.
1460 !
1461 .ov_bkwd:
1462 deccc %o2 ! dec count
1463 ldub [%o0 + %o2], %o3 ! get byte at end of src
1464 bgu %ncc, .ov_bkwd ! loop till done
1465 stb %o3, [%o1 + %o2] ! delay slot, store at end of dst
1466
1467 retl ! return
1468 nop
1469
1470 SET_SIZE(ovbcopy)
1471
1472
1473 /*
1474 * hwblkpagecopy()
1475 *
1476 * Copies exactly one page. This routine assumes the caller (ppcopy)
1477 * has already disabled kernel preemption and has checked
1478 * use_hw_bcopy. Preventing preemption also prevents cpu migration.
1479 */
1480 ENTRY(hwblkpagecopy)
1481 ! get another window w/space for three aligned blocks of saved fpregs
1482 prefetch [%o0], #n_reads
1483 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
1484
1485 ! %i0 - source address (arg)
1486 ! %i1 - destination address (arg)
1487 ! %i2 - length of region (not arg)
1488 ! %l0 - saved fprs
1489 ! %l1 - pointer to saved fpregs
1490
1491 rd %fprs, %l0 ! check for unused fp
1492 btst FPRS_FEF, %l0
1493 bz,a,pt %icc, 1f
1494 wr %g0, FPRS_FEF, %fprs
1495
1496 BST_FPQ1Q3_TOSTACK(%l1)
1497
1498 1: set PAGESIZE, CNT
1499 mov REALSRC, SRC
1577 fsrc1 %f14, %f46
1578 stda %f32, [DST]ASI_BLK_P
1579
1580 membar #Sync
1581
1582 btst FPRS_FEF, %l0
1583 bz,pt %icc, 2f
1584 nop
1585
1586 BLD_FPQ1Q3_FROMSTACK(%l3)
1587 ba 3f
1588 nop
1589
1590 2: FZEROQ1Q3
1591
1592 3: wr %l0, 0, %fprs ! restore fprs
1593 ret
1594 restore %g0, 0, %o0
1595
1596 SET_SIZE(hwblkpagecopy)
1597
1598
1599 /*
1600 * Transfer data to and from user space -
1601 * Note that these routines can cause faults
1602 * It is assumed that the kernel has nothing at
1603 * less than KERNELBASE in the virtual address space.
1604 *
1605 * Note that copyin(9F) and copyout(9F) are part of the
1606 * DDI/DKI which specifies that they return '-1' on "errors."
1607 *
1608 * Sigh.
1609 *
1610 * So there's two extremely similar routines - xcopyin() and xcopyout()
1611 * which return the errno that we've faithfully computed. This
1612 * allows other callers (e.g. uiomove(9F)) to work correctly.
1613 * Given that these are used pretty heavily, we expand the calling
1614 * sequences inline for all flavours (rather than making wrappers).
1615 *
1616 * There are also stub routines for xcopyout_little and xcopyin_little,
1635 *
1636 * None of the copyops routines grab a window until it's decided that
1637 * we need to do a HW block copy operation. This saves a window
1638 * spill/fill when we're called during socket ops. The typical IO
1639 * path won't cause spill/fill traps.
1640 *
1641 * This code uses a set of 4 limits for the maximum size that will
1642 * be copied given a particular input/output address alignment.
1643 * If the value for a particular limit is zero, the copy will be performed
1644 * by the plain copy loops rather than FPBLK.
1645 *
1646 * See the description of bcopy above for more details of the
1647 * data copying algorithm and the default limits.
1648 *
1649 */
1650
1651 /*
1652 * Copy kernel data to user space (copyout/xcopyout/xcopyout_little).
1653 */
1654
1655 /*
1656 * We save the arguments in the following registers in case of a fault:
1657 * kaddr - %l1
1658 * uaddr - %l2
1659 * count - %l3
1660 */
1661 #define SAVE_SRC %l1
1662 #define SAVE_DST %l2
1663 #define SAVE_COUNT %l3
1664
1665 #define SM_SAVE_SRC %g4
1666 #define SM_SAVE_DST %g5
1667 #define SM_SAVE_COUNT %o5
1668 #define ERRNO %l5
1669
1670
1671 #define REAL_LOFAULT %l4
1672 /*
1673 * Generic copyio fault handler. This is the first line of defense when a
1674 * fault occurs in (x)copyin/(x)copyout. In order for this to function
1701 wr %o3, 0, %fprs ! restore fprs
1702
1703 4:
1704 FZEROQ2Q4
1705 wr %o3, 0, %fprs ! restore fprs
1706
1707 1:
1708 andn %l6, FPUSED_FLAG, %l6
1709 membar #Sync
1710 stn %l6, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
1711 FP_ALLOWMIGRATE(5, 6)
1712
1713 mov SAVE_SRC, %i0
1714 mov SAVE_DST, %i1
1715 jmp REAL_LOFAULT
1716 mov SAVE_COUNT, %i2
1717
1718 SET_SIZE(copyio_fault)
1719
1720
1721 ENTRY(copyout)
1722
1723 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
1724 bleu,pt %ncc, .copyout_small ! go to larger cases
1725 xor %o0, %o1, %o3 ! are src, dst alignable?
1726 btst 7, %o3 !
1727 bz,pt %ncc, .copyout_8 ! check for longword alignment
1728 nop
1729 btst 1, %o3 !
1730 bz,pt %ncc, .copyout_2 ! check for half-word
1731 nop
1732 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
1733 ld [%o3 + %lo(hw_copy_limit_1)], %o3
1734 tst %o3
1735 bz,pn %icc, .copyout_small ! if zero, disable HW copy
1736 cmp %o2, %o3 ! if length <= limit
1737 bleu,pt %ncc, .copyout_small ! go to small copy
1738 nop
1739 ba,pt %ncc, .copyout_more ! otherwise go to large copy
1740 nop
2316
2317 /*
2318 * We got here because of a fault during copyout.
2319 * Errno value is in ERRNO, but DDI/DKI says return -1 (sigh).
2320 */
2321 .copyout_err:
2322 ldn [THREAD_REG + T_COPYOPS], %o4 ! check for copyop handler
2323 tst %o4
2324 bz,pt %ncc, 2f ! if not, return error
2325 nop
2326 ldn [%o4 + CP_COPYOUT], %g2 ! if handler, invoke it with
2327 jmp %g2 ! original arguments
2328 restore %g0, 0, %g0 ! dispose of copy window
2329 2:
2330 ret
2331 restore %g0, -1, %o0 ! return error value
2332
2333
2334 SET_SIZE(copyout_more)
2335
2336
2337 ENTRY(xcopyout)
2338 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
2339 bleu,pt %ncc, .xcopyout_small ! go to larger cases
2340 xor %o0, %o1, %o3 ! are src, dst alignable?
2341 btst 7, %o3 !
2342 bz,pt %ncc, .xcopyout_8 !
2343 nop
2344 btst 1, %o3 !
2345 bz,pt %ncc, .xcopyout_2 ! check for half-word
2346 nop
2347 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
2348 ld [%o3 + %lo(hw_copy_limit_1)], %o3
2349 tst %o3
2350 bz,pn %icc, .xcopyout_small ! if zero, disable HW copy
2351 cmp %o2, %o3 ! if length <= limit
2352 bleu,pt %ncc, .xcopyout_small ! go to small copy
2353 nop
2354 ba,pt %ncc, .xcopyout_more ! otherwise go to large copy
2355 nop
2356 .xcopyout_2:
2421 .sm_xcopyout_err:
2422
2423 membar #Sync
2424 stn %o4, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
2425 mov SM_SAVE_SRC, %o0
2426 mov SM_SAVE_DST, %o1
2427 mov SM_SAVE_COUNT, %o2
2428 ldn [THREAD_REG + T_COPYOPS], %o3 ! check for copyop handler
2429 tst %o3
2430 bz,pt %ncc, 3f ! if not, return error
2431 nop
2432 ldn [%o3 + CP_XCOPYOUT], %o5 ! if handler, invoke it with
2433 jmp %o5 ! original arguments
2434 nop
2435 3:
2436 retl
2437 or %g1, 0, %o0 ! return errno value
2438
2439 SET_SIZE(xcopyout)
2440
2441 ENTRY(xcopyout_little)
2442 sethi %hi(.xcopyio_err), %o5
2443 or %o5, %lo(.xcopyio_err), %o5
2444 ldn [THREAD_REG + T_LOFAULT], %o4
2445 membar #Sync ! sync error barrier
2446 stn %o5, [THREAD_REG + T_LOFAULT]
2447 mov %o4, %o5
2448
2449 subcc %g0, %o2, %o3
2450 add %o0, %o2, %o0
2451 bz,pn %ncc, 2f ! check for zero bytes
2452 sub %o2, 1, %o4
2453 add %o0, %o4, %o0 ! start w/last byte
2454 add %o1, %o2, %o1
2455 ldub [%o0 + %o3], %o4
2456
2457 1: stba %o4, [%o1 + %o3]ASI_AIUSL
2458 inccc %o3
2459 sub %o0, 2, %o0 ! get next byte
2460 bcc,a,pt %ncc, 1b
2461 ldub [%o0 + %o3], %o4
2462
2463 2:
2464 membar #Sync ! sync error barrier
2465 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
2466 retl
2467 mov %g0, %o0 ! return (0)
2468
2469 SET_SIZE(xcopyout_little)
2470
2471 /*
2472 * Copy user data to kernel space (copyin/xcopyin/xcopyin_little)
2473 */
2474
2475 ENTRY(copyin)
2476 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
2477 bleu,pt %ncc, .copyin_small ! go to larger cases
2478 xor %o0, %o1, %o3 ! are src, dst alignable?
2479 btst 7, %o3 !
2480 bz,pt %ncc, .copyin_8 ! check for longword alignment
2481 nop
2482 btst 1, %o3 !
2483 bz,pt %ncc, .copyin_2 ! check for half-word
2484 nop
2485 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
2486 ld [%o3 + %lo(hw_copy_limit_1)], %o3
2487 tst %o3
2488 bz,pn %icc, .copyin_small ! if zero, disable HW copy
2489 cmp %o2, %o3 ! if length <= limit
2490 bleu,pt %ncc, .copyin_small ! go to small copy
2491 nop
2492 ba,pt %ncc, .copyin_more ! otherwise go to large copy
2493 nop
2494 .copyin_2:
3061 restore %g0, 0, %o0
3062 /*
3063 * We got here because of a fault during copyin
3064 * Errno value is in ERRNO, but DDI/DKI says return -1 (sigh).
3065 */
3066 .copyin_err:
3067 ldn [THREAD_REG + T_COPYOPS], %o4 ! check for copyop handler
3068 tst %o4
3069 bz,pt %ncc, 2f ! if not, return error
3070 nop
3071 ldn [%o4 + CP_COPYIN], %g2 ! if handler, invoke it with
3072 jmp %g2 ! original arguments
3073 restore %g0, 0, %g0 ! dispose of copy window
3074 2:
3075 ret
3076 restore %g0, -1, %o0 ! return error value
3077
3078
3079 SET_SIZE(copyin_more)
3080
3081 ENTRY(xcopyin)
3082
3083 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
3084 bleu,pt %ncc, .xcopyin_small ! go to larger cases
3085 xor %o0, %o1, %o3 ! are src, dst alignable?
3086 btst 7, %o3 !
3087 bz,pt %ncc, .xcopyin_8 ! check for longword alignment
3088 nop
3089 btst 1, %o3 !
3090 bz,pt %ncc, .xcopyin_2 ! check for half-word
3091 nop
3092 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
3093 ld [%o3 + %lo(hw_copy_limit_1)], %o3
3094 tst %o3
3095 bz,pn %icc, .xcopyin_small ! if zero, disable HW copy
3096 cmp %o2, %o3 ! if length <= limit
3097 bleu,pt %ncc, .xcopyin_small ! go to small copy
3098 nop
3099 ba,pt %ncc, .xcopyin_more ! otherwise go to large copy
3100 nop
3166 .sm_xcopyin_err:
3167
3168 membar #Sync
3169 stn %o4, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3170 mov SM_SAVE_SRC, %o0
3171 mov SM_SAVE_DST, %o1
3172 mov SM_SAVE_COUNT, %o2
3173 ldn [THREAD_REG + T_COPYOPS], %o3 ! check for copyop handler
3174 tst %o3
3175 bz,pt %ncc, 3f ! if not, return error
3176 nop
3177 ldn [%o3 + CP_XCOPYIN], %o5 ! if handler, invoke it with
3178 jmp %o5 ! original arguments
3179 nop
3180 3:
3181 retl
3182 or %g1, 0, %o0 ! return errno value
3183
3184 SET_SIZE(xcopyin)
3185
3186 ENTRY(xcopyin_little)
3187 sethi %hi(.xcopyio_err), %o5
3188 or %o5, %lo(.xcopyio_err), %o5
3189 ldn [THREAD_REG + T_LOFAULT], %o4
3190 membar #Sync ! sync error barrier
3191 stn %o5, [THREAD_REG + T_LOFAULT]
3192 mov %o4, %o5
3193
3194 subcc %g0, %o2, %o3
3195 add %o0, %o2, %o0
3196 bz,pn %ncc, 2f ! check for zero bytes
3197 sub %o2, 1, %o4
3198 add %o0, %o4, %o0 ! start w/last byte
3199 add %o1, %o2, %o1
3200 lduba [%o0 + %o3]ASI_AIUSL, %o4
3201
3202 1: stb %o4, [%o1 + %o3]
3203 inccc %o3
3204 sub %o0, 2, %o0 ! get next byte
3205 bcc,a,pt %ncc, 1b
3206 lduba [%o0 + %o3]ASI_AIUSL, %o4
3207
3208 2:
3209 membar #Sync ! sync error barrier
3210 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3211 retl
3212 mov %g0, %o0 ! return (0)
3213
3214 .xcopyio_err:
3215 membar #Sync ! sync error barrier
3216 stn %o5, [THREAD_REG + T_LOFAULT] ! restore old t_lofault
3217 retl
3218 mov %g1, %o0
3219
3220 SET_SIZE(xcopyin_little)
3221
3222
3223 /*
3224 * Copy a block of storage - must not overlap (from + len <= to).
3225 * No fault handler installed (to be called under on_fault())
3226 */
3227 ENTRY(copyin_noerr)
3228
3229 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
3230 bleu,pt %ncc, .copyin_ne_small ! go to larger cases
3231 xor %o0, %o1, %o3 ! are src, dst alignable?
3232 btst 7, %o3 !
3233 bz,pt %ncc, .copyin_ne_8 ! check for longword alignment
3234 nop
3235 btst 1, %o3 !
3236 bz,pt %ncc, .copyin_ne_2 ! check for half-word
3237 nop
3238 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
3239 ld [%o3 + %lo(hw_copy_limit_1)], %o3
3240 tst %o3
3241 bz,pn %icc, .copyin_ne_small ! if zero, disable HW copy
3242 cmp %o2, %o3 ! if length <= limit
3243 bleu,pt %ncc, .copyin_ne_small ! go to small copy
3244 nop
3245 ba,pt %ncc, .copyin_noerr_more ! otherwise go to large copy
3246 nop
3290 ba,pt %ncc, .sm_do_copyin
3291 stn %o5, [THREAD_REG + T_LOFAULT] ! set/save t_lofault
3292
3293 .copyin_noerr_more:
3294 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3295 sethi %hi(.copyio_noerr), REAL_LOFAULT
3296 ba,pt %ncc, .do_copyin
3297 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
3298
3299 .copyio_noerr:
3300 jmp %l6
3301 restore %g0,0,%g0
3302
3303 .sm_copyio_noerr:
3304 membar #Sync
3305 stn %o4, [THREAD_REG + T_LOFAULT] ! restore t_lofault
3306 jmp %o4
3307 nop
3308
3309 SET_SIZE(copyin_noerr)
3310
3311 /*
3312 * Copy a block of storage - must not overlap (from + len <= to).
3313 * No fault handler installed (to be called under on_fault())
3314 */
3315
3316 ENTRY(copyout_noerr)
3317
3318 cmp %o2, VIS_COPY_THRESHOLD ! check for leaf rtn case
3319 bleu,pt %ncc, .copyout_ne_small ! go to larger cases
3320 xor %o0, %o1, %o3 ! are src, dst alignable?
3321 btst 7, %o3 !
3322 bz,pt %ncc, .copyout_ne_8 ! check for longword alignment
3323 nop
3324 btst 1, %o3 !
3325 bz,pt %ncc, .copyout_ne_2 ! check for half-word
3326 nop
3327 sethi %hi(hw_copy_limit_1), %o3 ! Check copy limit
3328 ld [%o3 + %lo(hw_copy_limit_1)], %o3
3329 tst %o3
3330 bz,pn %icc, .copyout_ne_small ! if zero, disable HW copy
3331 cmp %o2, %o3 ! if length <= limit
3332 bleu,pt %ncc, .copyout_ne_small ! go to small copy
3333 nop
3334 ba,pt %ncc, .copyout_noerr_more ! otherwise go to large copy
3335 nop
3369 nop
3370
3371 .copyout_ne_small:
3372 ldn [THREAD_REG + T_LOFAULT], %o4
3373 tst %o4
3374 bz,pn %ncc, .sm_do_copyout
3375 nop
3376 sethi %hi(.sm_copyio_noerr), %o5
3377 or %o5, %lo(.sm_copyio_noerr), %o5
3378 membar #Sync ! sync error barrier
3379 ba,pt %ncc, .sm_do_copyout
3380 stn %o5, [THREAD_REG + T_LOFAULT] ! set/save t_lofault
3381
3382 .copyout_noerr_more:
3383 save %sp, -SA(MINFRAME + HWCOPYFRAMESIZE), %sp
3384 sethi %hi(.copyio_noerr), REAL_LOFAULT
3385 ba,pt %ncc, .do_copyout
3386 or REAL_LOFAULT, %lo(.copyio_noerr), REAL_LOFAULT
3387
3388 SET_SIZE(copyout_noerr)
3389
3390
3391 /*
3392 * hwblkclr - clears block-aligned, block-multiple-sized regions that are
3393 * longer than 256 bytes in length using spitfire's block stores. If
3394 * the criteria for using this routine are not met then it calls bzero
3395 * and returns 1. Otherwise 0 is returned indicating success.
3396 * Caller is responsible for ensuring use_hw_bzero is true and that
3397 * kpreempt_disable() has been called.
3398 */
3399 ! %i0 - start address
3400 ! %i1 - length of region (multiple of 64)
3401 ! %l0 - saved fprs
3402 ! %l1 - pointer to saved %d0 block
3403 ! %l2 - saved curthread->t_lwp
3404
3405 ENTRY(hwblkclr)
3406 ! get another window w/space for one aligned block of saved fpregs
3407 save %sp, -SA(MINFRAME + 2*VIS_BLOCKSIZE), %sp
3408
3409 ! Must be block-aligned
3410 andcc %i0, (VIS_BLOCKSIZE-1), %g0
3411 bnz,pn %ncc, 1f
3412 nop
3413
3414 ! ... and must be 256 bytes or more
3415 cmp %i1, 256
3416 blu,pn %ncc, 1f
3417 nop
3418
3479 sub %i4, %i2, %i4
3480 jmp %i4
3481 nop
3482
3483 .pz_finish:
3484 membar #Sync
3485 btst FPRS_FEF, %l0
3486 bz,a .pz_finished
3487 wr %l0, 0, %fprs ! restore fprs
3488
3489 ! restore fpregs from stack
3490 ldda [%l1]ASI_BLK_P, %d0
3491 membar #Sync
3492 wr %l0, 0, %fprs ! restore fprs
3493
3494 .pz_finished:
3495 ret
3496 restore %g0, 0, %o0 ! return (bzero or not)
3497
3498 SET_SIZE(hwblkclr)
3499
3500 /*
3501 * Copy 32 bytes of data from src (%o0) to dst (%o1)
3502 * using physical addresses.
3503 */
3504 ENTRY_NP(hw_pa_bcopy32)
3505 rdpr %pstate, %g1
3506 andn %g1, PSTATE_IE, %g2
3507 wrpr %g0, %g2, %pstate
3508
3509 rdpr %pstate, %g0
3510 ldxa [%o0]ASI_MEM, %o2
3511 add %o0, 8, %o0
3512 ldxa [%o0]ASI_MEM, %o3
3513 add %o0, 8, %o0
3514 ldxa [%o0]ASI_MEM, %o4
3515 add %o0, 8, %o0
3516 ldxa [%o0]ASI_MEM, %o5
3517 membar #Sync
3518
3519 stxa %o2, [%o1]ASI_MEM
3520 add %o1, 8, %o1
3521 stxa %o3, [%o1]ASI_MEM
3522 add %o1, 8, %o1
3523 stxa %o4, [%o1]ASI_MEM
3524 add %o1, 8, %o1
3525 stxa %o5, [%o1]ASI_MEM
3526
3527 retl
3528 wrpr %g0, %g1, %pstate
3529
3530 SET_SIZE(hw_pa_bcopy32)
3531
3532 DGDEF(use_hw_bcopy)
3533 .word 1
3534 DGDEF(use_hw_bzero)
3535 .word 1
3536 DGDEF(hw_copy_limit_1)
3537 .word 0
3538 DGDEF(hw_copy_limit_2)
3539 .word 0
3540 DGDEF(hw_copy_limit_4)
3541 .word 0
3542 DGDEF(hw_copy_limit_8)
3543 .word 0
3544
3545 .align 64
3546 .section ".text"
|