/* SPDX-License-Identifier: GPL-2.0-or-later */ /* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * IP/TCP/UDP checksumming routines * * Authors: Jorge Cwik, <jorge@laser.satlink.net> * Arnt Gulbrandsen, <agulbra@nvg.unit.no> * Tom May, <ftom@netcom.com> * Pentium Pro/II routines: * Alexander Kjeldaas <astor@guardian.no> * Finn Arne Gangstad <finnag@guardian.no> * Lots of code moved from tcp.c and ip.c; see those files * for more names. * * Changes: Ingo Molnar, converted csum_partial_copy() to 2.1 exception * handling. * Andi Kleen, add zeroing on error * converted to pure assembler */ #include <linux/linkage.h> #include <asm/errno.h> #include <asm/asm.h> #include <asm/export.h> #include <asm/nospec-branch.h> /* * computes a partial checksum, e.g. for TCP/UDP fragments */ /* unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum) */ .text #ifndef CONFIG_X86_USE_PPRO_CHECKSUM /* * Experiments with Ethernet and SLIP connections show that buff * is aligned on either a 2-byte or 4-byte boundary. We get at * least a twofold speedup on 486 and Pentium if it is 4-byte aligned. * Fortunately, it is easy to convert 2-byte alignment to 4-byte * alignment for the unrolled loop. */ SYM_FUNC_START(csum_partial) pushl %esi pushl %ebx movl 20(%esp),%eax # Function arg: unsigned int sum movl 16(%esp),%ecx # Function arg: int len movl 12(%esp),%esi # Function arg: unsigned char *buff testl $3, %esi # Check alignment. jz 2f # Jump if alignment is ok. testl $1, %esi # Check alignment. jz 10f # Jump if alignment is boundary of 2 bytes. # buf is odd dec %ecx jl 8f movzbl (%esi), %ebx adcl %ebx, %eax roll $8, %eax inc %esi testl $2, %esi jz 2f 10: subl $2, %ecx # Alignment uses up two bytes. jae 1f # Jump if we had at least two bytes. addl $2, %ecx # ecx was < 2. Deal with it. jmp 4f 1: movw (%esi), %bx addl $2, %esi addw %bx, %ax adcl $0, %eax 2: movl %ecx, %edx shrl $5, %ecx jz 2f testl %esi, %esi 1: movl (%esi), %ebx adcl %ebx, %eax movl 4(%esi), %ebx adcl %ebx, %eax movl 8(%esi), %ebx adcl %ebx, %eax movl 12(%esi), %ebx adcl %ebx, %eax movl 16(%esi), %ebx adcl %ebx, %eax movl 20(%esi), %ebx adcl %ebx, %eax movl 24(%esi), %ebx adcl %ebx, %eax movl 28(%esi), %ebx adcl %ebx, %eax lea 32(%esi), %esi dec %ecx jne 1b adcl $0, %eax 2: movl %edx, %ecx andl $0x1c, %edx je 4f shrl $2, %edx # This clears CF 3: adcl (%esi), %eax lea 4(%esi), %esi dec %edx jne 3b adcl $0, %eax 4: andl $3, %ecx jz 7f cmpl $2, %ecx jb 5f movw (%esi),%cx leal 2(%esi),%esi je 6f shll $16,%ecx 5: movb (%esi),%cl 6: addl %ecx,%eax adcl $0, %eax 7: testb $1, 12(%esp) jz 8f roll $8, %eax 8: popl %ebx popl %esi RET SYM_FUNC_END(csum_partial) #else /* Version for PentiumII/PPro */ SYM_FUNC_START(csum_partial) pushl %esi pushl %ebx movl 20(%esp),%eax # Function arg: unsigned int sum movl 16(%esp),%ecx # Function arg: int len movl 12(%esp),%esi # Function arg: const unsigned char *buf testl $3, %esi jnz 25f 10: movl %ecx, %edx movl %ecx, %ebx andl $0x7c, %ebx shrl $7, %ecx addl %ebx,%esi shrl $2, %ebx negl %ebx lea 45f(%ebx,%ebx,2), %ebx testl %esi, %esi JMP_NOSPEC ebx # Handle 2-byte-aligned regions 20: addw (%esi), %ax lea 2(%esi), %esi adcl $0, %eax jmp 10b 25: testl $1, %esi jz 30f # buf is odd dec %ecx jl 90f movzbl (%esi), %ebx addl %ebx, %eax adcl $0, %eax roll $8, %eax inc %esi testl $2, %esi jz 10b 30: subl $2, %ecx ja 20b je 32f addl $2, %ecx jz 80f movzbl (%esi),%ebx # csumming 1 byte, 2-aligned addl %ebx, %eax adcl $0, %eax jmp 80f 32: addw (%esi), %ax # csumming 2 bytes, 2-aligned adcl $0, %eax jmp 80f 40: addl -128(%esi), %eax adcl -124(%esi), %eax adcl -120(%esi), %eax adcl -116(%esi), %eax adcl -112(%esi), %eax adcl -108(%esi), %eax adcl -104(%esi), %eax adcl -100(%esi), %eax adcl -96(%esi), %eax adcl -92(%esi), %eax adcl -88(%esi), %eax adcl -84(%esi), %eax adcl -80(%esi), %eax adcl -76(%esi), %eax adcl -72(%esi), %eax adcl -68(%esi), %eax adcl -64(%esi), %eax adcl -60(%esi), %eax adcl -56(%esi), %eax adcl -52(%esi), %eax adcl -48(%esi), %eax adcl -44(%esi), %eax adcl -40(%esi), %eax adcl -36(%esi), %eax adcl -32(%esi), %eax adcl -28(%esi), %eax adcl -24(%esi), %eax adcl -20(%esi), %eax adcl -16(%esi), %eax adcl -12(%esi), %eax adcl -8(%esi), %eax adcl -4(%esi), %eax 45: lea 128(%esi), %esi adcl $0, %eax dec %ecx jge 40b movl %edx, %ecx 50: andl $3, %ecx jz 80f # Handle the last 1-3 bytes without jumping notl %ecx # 1->2, 2->1, 3->0, higher bits are masked movl $0xffffff,%ebx # by the shll and shrl instructions shll $3,%ecx shrl %cl,%ebx andl -128(%esi),%ebx # esi is 4-aligned so should be ok addl %ebx,%eax adcl $0,%eax 80: testb $1, 12(%esp) jz 90f roll $8, %eax 90: popl %ebx popl %esi RET SYM_FUNC_END(csum_partial) #endif EXPORT_SYMBOL(csum_partial) /* unsigned int csum_partial_copy_generic (const char *src, char *dst, int len) */ /* * Copy from ds while checksumming, otherwise like csum_partial */ #define EXC(y...) \ 9999: y; \ _ASM_EXTABLE_TYPE(9999b, 7f, EX_TYPE_UACCESS | EX_FLAG_CLEAR_AX) #ifndef CONFIG_X86_USE_PPRO_CHECKSUM #define ARGBASE 16 #define FP 12 SYM_FUNC_START(csum_partial_copy_generic) subl $4,%esp pushl %edi pushl %esi pushl %ebx movl ARGBASE+12(%esp),%ecx # len movl ARGBASE+4(%esp),%esi # src movl ARGBASE+8(%esp),%edi # dst movl $-1, %eax # sum testl $2, %edi # Check alignment. jz 2f # Jump if alignment is ok. subl $2, %ecx # Alignment uses up two bytes. jae 1f # Jump if we had at least two bytes. addl $2, %ecx # ecx was < 2. Deal with it. jmp 4f EXC(1: movw (%esi), %bx ) addl $2, %esi EXC( movw %bx, (%edi) ) addl $2, %edi addw %bx, %ax adcl $0, %eax 2: movl %ecx, FP(%esp) shrl $5, %ecx jz 2f testl %esi, %esi # what's wrong with clc? EXC(1: movl (%esi), %ebx ) EXC( movl 4(%esi), %edx ) adcl %ebx, %eax EXC( movl %ebx, (%edi) ) adcl %edx, %eax EXC( movl %edx, 4(%edi) ) EXC( movl 8(%esi), %ebx ) EXC( movl 12(%esi), %edx ) adcl %ebx, %eax EXC( movl %ebx, 8(%edi) ) adcl %edx, %eax EXC( movl %edx, 12(%edi) ) EXC( movl 16(%esi), %ebx ) EXC( movl 20(%esi), %edx ) adcl %ebx, %eax EXC( movl %ebx, 16(%edi) ) adcl %edx, %eax EXC( movl %edx, 20(%edi) ) EXC( movl 24(%esi), %ebx ) EXC( movl 28(%esi), %edx ) adcl %ebx, %eax EXC( movl %ebx, 24(%edi) ) adcl %edx, %eax EXC( movl %edx, 28(%edi) ) lea 32(%esi), %esi lea 32(%edi), %edi dec %ecx jne 1b adcl $0, %eax 2: movl FP(%esp), %edx movl %edx, %ecx andl $0x1c, %edx je 4f shrl $2, %edx # This clears CF EXC(3: movl (%esi), %ebx ) adcl %ebx, %eax EXC( movl %ebx, (%edi) ) lea 4(%esi), %esi lea 4(%edi), %edi dec %edx jne 3b adcl $0, %eax 4: andl $3, %ecx jz 7f cmpl $2, %ecx jb 5f EXC( movw (%esi), %cx ) leal 2(%esi), %esi EXC( movw %cx, (%edi) ) leal 2(%edi), %edi je 6f shll $16,%ecx EXC(5: movb (%esi), %cl ) EXC( movb %cl, (%edi) ) 6: addl %ecx, %eax adcl $0, %eax 7: popl %ebx popl %esi popl %edi popl %ecx # equivalent to addl $4,%esp RET SYM_FUNC_END(csum_partial_copy_generic) #else /* Version for PentiumII/PPro */ #define ROUND1(x) \ EXC(movl x(%esi), %ebx ) ; \ addl %ebx, %eax ; \ EXC(movl %ebx, x(%edi) ) ; #define ROUND(x) \ EXC(movl x(%esi), %ebx ) ; \ adcl %ebx, %eax ; \ EXC(movl %ebx, x(%edi) ) ; #define ARGBASE 12 SYM_FUNC_START(csum_partial_copy_generic) pushl %ebx pushl %edi pushl %esi movl ARGBASE+4(%esp),%esi #src movl ARGBASE+8(%esp),%edi #dst movl ARGBASE+12(%esp),%ecx #len movl $-1, %eax #sum # movl %ecx, %edx movl %ecx, %ebx movl %esi, %edx shrl $6, %ecx andl $0x3c, %ebx negl %ebx subl %ebx, %esi subl %ebx, %edi lea -1(%esi),%edx andl $-32,%edx lea 3f(%ebx,%ebx), %ebx testl %esi, %esi JMP_NOSPEC ebx 1: addl $64,%esi addl $64,%edi EXC(movb -32(%edx),%bl) ; EXC(movb (%edx),%bl) ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52) ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36) ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20) ROUND (-16) ROUND(-12) ROUND(-8) ROUND(-4) 3: adcl $0,%eax addl $64, %edx dec %ecx jge 1b 4: movl ARGBASE+12(%esp),%edx #len andl $3, %edx jz 7f cmpl $2, %edx jb 5f EXC( movw (%esi), %dx ) leal 2(%esi), %esi EXC( movw %dx, (%edi) ) leal 2(%edi), %edi je 6f shll $16,%edx 5: EXC( movb (%esi), %dl ) EXC( movb %dl, (%edi) ) 6: addl %edx, %eax adcl $0, %eax 7: popl %esi popl %edi popl %ebx RET SYM_FUNC_END(csum_partial_copy_generic) #undef ROUND #undef ROUND1 #endif EXPORT_SYMBOL(csum_partial_copy_generic)