xref: /linux/lib/raid/xor/x86/xor-mmx.c (revision 440d6635b20037bc9ad46b20817d7b61cef0fc1b)

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Optimized XOR parity functions for MMX.
 *
 * Copyright (C) 1998 Ingo Molnar.
 */
#include <asm/fpu/api.h>
#include "xor_impl.h"
#include "xor_arch.h"

#define LD(x, y)	"       movq   8*("#x")(%1), %%mm"#y"   ;\n"
#define ST(x, y)	"       movq %%mm"#y",   8*("#x")(%1)   ;\n"
#define XO1(x, y)	"       pxor   8*("#x")(%2), %%mm"#y"   ;\n"
#define XO2(x, y)	"       pxor   8*("#x")(%3), %%mm"#y"   ;\n"
#define XO3(x, y)	"       pxor   8*("#x")(%4), %%mm"#y"   ;\n"
#define XO4(x, y)	"       pxor   8*("#x")(%5), %%mm"#y"   ;\n"

static void
xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
	      const unsigned long * __restrict p2)
{
	unsigned long lines = bytes >> 7;

	asm volatile(
#undef BLOCK
#define BLOCK(i)				\
	LD(i, 0)				\
		LD(i + 1, 1)			\
			LD(i + 2, 2)		\
				LD(i + 3, 3)	\
	XO1(i, 0)				\
	ST(i, 0)				\
		XO1(i+1, 1)			\
		ST(i+1, 1)			\
			XO1(i + 2, 2)		\
			ST(i + 2, 2)		\
				XO1(i + 3, 3)	\
				ST(i + 3, 3)

	" .align 32			;\n"
	" 1:                            ;\n"

	BLOCK(0)
	BLOCK(4)
	BLOCK(8)
	BLOCK(12)

	"       addl $128, %1         ;\n"
	"       addl $128, %2         ;\n"
	"       decl %0               ;\n"
	"       jnz 1b                ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2)
	:
	: "memory");
}

static void
xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
	      const unsigned long * __restrict p2,
	      const unsigned long * __restrict p3)
{
	unsigned long lines = bytes >> 7;

	asm volatile(
#undef BLOCK
#define BLOCK(i)				\
	LD(i, 0)				\
		LD(i + 1, 1)			\
			LD(i + 2, 2)		\
				LD(i + 3, 3)	\
	XO1(i, 0)				\
		XO1(i + 1, 1)			\
			XO1(i + 2, 2)		\
				XO1(i + 3, 3)	\
	XO2(i, 0)				\
	ST(i, 0)				\
		XO2(i + 1, 1)			\
		ST(i + 1, 1)			\
			XO2(i + 2, 2)		\
			ST(i + 2, 2)		\
				XO2(i + 3, 3)	\
				ST(i + 3, 3)

	" .align 32			;\n"
	" 1:                            ;\n"

	BLOCK(0)
	BLOCK(4)
	BLOCK(8)
	BLOCK(12)

	"       addl $128, %1         ;\n"
	"       addl $128, %2         ;\n"
	"       addl $128, %3         ;\n"
	"       decl %0               ;\n"
	"       jnz 1b                ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2), "+r" (p3)
	:
	: "memory");
}

static void
xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
	      const unsigned long * __restrict p2,
	      const unsigned long * __restrict p3,
	      const unsigned long * __restrict p4)
{
	unsigned long lines = bytes >> 7;

	asm volatile(
#undef BLOCK
#define BLOCK(i)				\
	LD(i, 0)				\
		LD(i + 1, 1)			\
			LD(i + 2, 2)		\
				LD(i + 3, 3)	\
	XO1(i, 0)				\
		XO1(i + 1, 1)			\
			XO1(i + 2, 2)		\
				XO1(i + 3, 3)	\
	XO2(i, 0)				\
		XO2(i + 1, 1)			\
			XO2(i + 2, 2)		\
				XO2(i + 3, 3)	\
	XO3(i, 0)				\
	ST(i, 0)				\
		XO3(i + 1, 1)			\
		ST(i + 1, 1)			\
			XO3(i + 2, 2)		\
			ST(i + 2, 2)		\
				XO3(i + 3, 3)	\
				ST(i + 3, 3)

	" .align 32			;\n"
	" 1:                            ;\n"

	BLOCK(0)
	BLOCK(4)
	BLOCK(8)
	BLOCK(12)

	"       addl $128, %1         ;\n"
	"       addl $128, %2         ;\n"
	"       addl $128, %3         ;\n"
	"       addl $128, %4         ;\n"
	"       decl %0               ;\n"
	"       jnz 1b                ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
	:
	: "memory");
}


static void
xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
	      const unsigned long * __restrict p2,
	      const unsigned long * __restrict p3,
	      const unsigned long * __restrict p4,
	      const unsigned long * __restrict p5)
{
	unsigned long lines = bytes >> 7;

	/* Make sure GCC forgets anything it knows about p4 or p5,
	   such that it won't pass to the asm volatile below a
	   register that is shared with any other variable.  That's
	   because we modify p4 and p5 there, but we can't mark them
	   as read/write, otherwise we'd overflow the 10-asm-operands
	   limit of GCC < 3.1.  */
	asm("" : "+r" (p4), "+r" (p5));

	asm volatile(
#undef BLOCK
#define BLOCK(i)				\
	LD(i, 0)				\
		LD(i + 1, 1)			\
			LD(i + 2, 2)		\
				LD(i + 3, 3)	\
	XO1(i, 0)				\
		XO1(i + 1, 1)			\
			XO1(i + 2, 2)		\
				XO1(i + 3, 3)	\
	XO2(i, 0)				\
		XO2(i + 1, 1)			\
			XO2(i + 2, 2)		\
				XO2(i + 3, 3)	\
	XO3(i, 0)				\
		XO3(i + 1, 1)			\
			XO3(i + 2, 2)		\
				XO3(i + 3, 3)	\
	XO4(i, 0)				\
	ST(i, 0)				\
		XO4(i + 1, 1)			\
		ST(i + 1, 1)			\
			XO4(i + 2, 2)		\
			ST(i + 2, 2)		\
				XO4(i + 3, 3)	\
				ST(i + 3, 3)

	" .align 32			;\n"
	" 1:                            ;\n"

	BLOCK(0)
	BLOCK(4)
	BLOCK(8)
	BLOCK(12)

	"       addl $128, %1         ;\n"
	"       addl $128, %2         ;\n"
	"       addl $128, %3         ;\n"
	"       addl $128, %4         ;\n"
	"       addl $128, %5         ;\n"
	"       decl %0               ;\n"
	"       jnz 1b                ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2), "+r" (p3)
	: "r" (p4), "r" (p5)
	: "memory");

	/* p4 and p5 were modified, and now the variables are dead.
	   Clobber them just to be sure nobody does something stupid
	   like assuming they have some legal value.  */
	asm("" : "=r" (p4), "=r" (p5));
}

#undef LD
#undef XO1
#undef XO2
#undef XO3
#undef XO4
#undef ST
#undef BLOCK

static void
xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
	     const unsigned long * __restrict p2)
{
	unsigned long lines = bytes >> 6;

	asm volatile(
	" .align 32	             ;\n"
	" 1:                         ;\n"
	"       movq   (%1), %%mm0   ;\n"
	"       movq  8(%1), %%mm1   ;\n"
	"       pxor   (%2), %%mm0   ;\n"
	"       movq 16(%1), %%mm2   ;\n"
	"       movq %%mm0,   (%1)   ;\n"
	"       pxor  8(%2), %%mm1   ;\n"
	"       movq 24(%1), %%mm3   ;\n"
	"       movq %%mm1,  8(%1)   ;\n"
	"       pxor 16(%2), %%mm2   ;\n"
	"       movq 32(%1), %%mm4   ;\n"
	"       movq %%mm2, 16(%1)   ;\n"
	"       pxor 24(%2), %%mm3   ;\n"
	"       movq 40(%1), %%mm5   ;\n"
	"       movq %%mm3, 24(%1)   ;\n"
	"       pxor 32(%2), %%mm4   ;\n"
	"       movq 48(%1), %%mm6   ;\n"
	"       movq %%mm4, 32(%1)   ;\n"
	"       pxor 40(%2), %%mm5   ;\n"
	"       movq 56(%1), %%mm7   ;\n"
	"       movq %%mm5, 40(%1)   ;\n"
	"       pxor 48(%2), %%mm6   ;\n"
	"       pxor 56(%2), %%mm7   ;\n"
	"       movq %%mm6, 48(%1)   ;\n"
	"       movq %%mm7, 56(%1)   ;\n"

	"       addl $64, %1         ;\n"
	"       addl $64, %2         ;\n"
	"       decl %0              ;\n"
	"       jnz 1b               ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2)
	:
	: "memory");
}

static void
xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
	     const unsigned long * __restrict p2,
	     const unsigned long * __restrict p3)
{
	unsigned long lines = bytes >> 6;

	asm volatile(
	" .align 32,0x90             ;\n"
	" 1:                         ;\n"
	"       movq   (%1), %%mm0   ;\n"
	"       movq  8(%1), %%mm1   ;\n"
	"       pxor   (%2), %%mm0   ;\n"
	"       movq 16(%1), %%mm2   ;\n"
	"       pxor  8(%2), %%mm1   ;\n"
	"       pxor   (%3), %%mm0   ;\n"
	"       pxor 16(%2), %%mm2   ;\n"
	"       movq %%mm0,   (%1)   ;\n"
	"       pxor  8(%3), %%mm1   ;\n"
	"       pxor 16(%3), %%mm2   ;\n"
	"       movq 24(%1), %%mm3   ;\n"
	"       movq %%mm1,  8(%1)   ;\n"
	"       movq 32(%1), %%mm4   ;\n"
	"       movq 40(%1), %%mm5   ;\n"
	"       pxor 24(%2), %%mm3   ;\n"
	"       movq %%mm2, 16(%1)   ;\n"
	"       pxor 32(%2), %%mm4   ;\n"
	"       pxor 24(%3), %%mm3   ;\n"
	"       pxor 40(%2), %%mm5   ;\n"
	"       movq %%mm3, 24(%1)   ;\n"
	"       pxor 32(%3), %%mm4   ;\n"
	"       pxor 40(%3), %%mm5   ;\n"
	"       movq 48(%1), %%mm6   ;\n"
	"       movq %%mm4, 32(%1)   ;\n"
	"       movq 56(%1), %%mm7   ;\n"
	"       pxor 48(%2), %%mm6   ;\n"
	"       movq %%mm5, 40(%1)   ;\n"
	"       pxor 56(%2), %%mm7   ;\n"
	"       pxor 48(%3), %%mm6   ;\n"
	"       pxor 56(%3), %%mm7   ;\n"
	"       movq %%mm6, 48(%1)   ;\n"
	"       movq %%mm7, 56(%1)   ;\n"

	"       addl $64, %1         ;\n"
	"       addl $64, %2         ;\n"
	"       addl $64, %3         ;\n"
	"       decl %0              ;\n"
	"       jnz 1b               ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2), "+r" (p3)
	:
	: "memory" );
}

static void
xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
	     const unsigned long * __restrict p2,
	     const unsigned long * __restrict p3,
	     const unsigned long * __restrict p4)
{
	unsigned long lines = bytes >> 6;

	asm volatile(
	" .align 32,0x90             ;\n"
	" 1:                         ;\n"
	"       movq   (%1), %%mm0   ;\n"
	"       movq  8(%1), %%mm1   ;\n"
	"       pxor   (%2), %%mm0   ;\n"
	"       movq 16(%1), %%mm2   ;\n"
	"       pxor  8(%2), %%mm1   ;\n"
	"       pxor   (%3), %%mm0   ;\n"
	"       pxor 16(%2), %%mm2   ;\n"
	"       pxor  8(%3), %%mm1   ;\n"
	"       pxor   (%4), %%mm0   ;\n"
	"       movq 24(%1), %%mm3   ;\n"
	"       pxor 16(%3), %%mm2   ;\n"
	"       pxor  8(%4), %%mm1   ;\n"
	"       movq %%mm0,   (%1)   ;\n"
	"       movq 32(%1), %%mm4   ;\n"
	"       pxor 24(%2), %%mm3   ;\n"
	"       pxor 16(%4), %%mm2   ;\n"
	"       movq %%mm1,  8(%1)   ;\n"
	"       movq 40(%1), %%mm5   ;\n"
	"       pxor 32(%2), %%mm4   ;\n"
	"       pxor 24(%3), %%mm3   ;\n"
	"       movq %%mm2, 16(%1)   ;\n"
	"       pxor 40(%2), %%mm5   ;\n"
	"       pxor 32(%3), %%mm4   ;\n"
	"       pxor 24(%4), %%mm3   ;\n"
	"       movq %%mm3, 24(%1)   ;\n"
	"       movq 56(%1), %%mm7   ;\n"
	"       movq 48(%1), %%mm6   ;\n"
	"       pxor 40(%3), %%mm5   ;\n"
	"       pxor 32(%4), %%mm4   ;\n"
	"       pxor 48(%2), %%mm6   ;\n"
	"       movq %%mm4, 32(%1)   ;\n"
	"       pxor 56(%2), %%mm7   ;\n"
	"       pxor 40(%4), %%mm5   ;\n"
	"       pxor 48(%3), %%mm6   ;\n"
	"       pxor 56(%3), %%mm7   ;\n"
	"       movq %%mm5, 40(%1)   ;\n"
	"       pxor 48(%4), %%mm6   ;\n"
	"       pxor 56(%4), %%mm7   ;\n"
	"       movq %%mm6, 48(%1)   ;\n"
	"       movq %%mm7, 56(%1)   ;\n"

	"       addl $64, %1         ;\n"
	"       addl $64, %2         ;\n"
	"       addl $64, %3         ;\n"
	"       addl $64, %4         ;\n"
	"       decl %0              ;\n"
	"       jnz 1b               ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
	:
	: "memory");
}

static void
xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
	     const unsigned long * __restrict p2,
	     const unsigned long * __restrict p3,
	     const unsigned long * __restrict p4,
	     const unsigned long * __restrict p5)
{
	unsigned long lines = bytes >> 6;

	/* Make sure GCC forgets anything it knows about p4 or p5,
	   such that it won't pass to the asm volatile below a
	   register that is shared with any other variable.  That's
	   because we modify p4 and p5 there, but we can't mark them
	   as read/write, otherwise we'd overflow the 10-asm-operands
	   limit of GCC < 3.1.  */
	asm("" : "+r" (p4), "+r" (p5));

	asm volatile(
	" .align 32,0x90             ;\n"
	" 1:                         ;\n"
	"       movq   (%1), %%mm0   ;\n"
	"       movq  8(%1), %%mm1   ;\n"
	"       pxor   (%2), %%mm0   ;\n"
	"       pxor  8(%2), %%mm1   ;\n"
	"       movq 16(%1), %%mm2   ;\n"
	"       pxor   (%3), %%mm0   ;\n"
	"       pxor  8(%3), %%mm1   ;\n"
	"       pxor 16(%2), %%mm2   ;\n"
	"       pxor   (%4), %%mm0   ;\n"
	"       pxor  8(%4), %%mm1   ;\n"
	"       pxor 16(%3), %%mm2   ;\n"
	"       movq 24(%1), %%mm3   ;\n"
	"       pxor   (%5), %%mm0   ;\n"
	"       pxor  8(%5), %%mm1   ;\n"
	"       movq %%mm0,   (%1)   ;\n"
	"       pxor 16(%4), %%mm2   ;\n"
	"       pxor 24(%2), %%mm3   ;\n"
	"       movq %%mm1,  8(%1)   ;\n"
	"       pxor 16(%5), %%mm2   ;\n"
	"       pxor 24(%3), %%mm3   ;\n"
	"       movq 32(%1), %%mm4   ;\n"
	"       movq %%mm2, 16(%1)   ;\n"
	"       pxor 24(%4), %%mm3   ;\n"
	"       pxor 32(%2), %%mm4   ;\n"
	"       movq 40(%1), %%mm5   ;\n"
	"       pxor 24(%5), %%mm3   ;\n"
	"       pxor 32(%3), %%mm4   ;\n"
	"       pxor 40(%2), %%mm5   ;\n"
	"       movq %%mm3, 24(%1)   ;\n"
	"       pxor 32(%4), %%mm4   ;\n"
	"       pxor 40(%3), %%mm5   ;\n"
	"       movq 48(%1), %%mm6   ;\n"
	"       movq 56(%1), %%mm7   ;\n"
	"       pxor 32(%5), %%mm4   ;\n"
	"       pxor 40(%4), %%mm5   ;\n"
	"       pxor 48(%2), %%mm6   ;\n"
	"       pxor 56(%2), %%mm7   ;\n"
	"       movq %%mm4, 32(%1)   ;\n"
	"       pxor 48(%3), %%mm6   ;\n"
	"       pxor 56(%3), %%mm7   ;\n"
	"       pxor 40(%5), %%mm5   ;\n"
	"       pxor 48(%4), %%mm6   ;\n"
	"       pxor 56(%4), %%mm7   ;\n"
	"       movq %%mm5, 40(%1)   ;\n"
	"       pxor 48(%5), %%mm6   ;\n"
	"       pxor 56(%5), %%mm7   ;\n"
	"       movq %%mm6, 48(%1)   ;\n"
	"       movq %%mm7, 56(%1)   ;\n"

	"       addl $64, %1         ;\n"
	"       addl $64, %2         ;\n"
	"       addl $64, %3         ;\n"
	"       addl $64, %4         ;\n"
	"       addl $64, %5         ;\n"
	"       decl %0              ;\n"
	"       jnz 1b               ;\n"
	: "+r" (lines),
	  "+r" (p1), "+r" (p2), "+r" (p3)
	: "r" (p4), "r" (p5)
	: "memory");

	/* p4 and p5 were modified, and now the variables are dead.
	   Clobber them just to be sure nobody does something stupid
	   like assuming they have some legal value.  */
	asm("" : "=r" (p4), "=r" (p5));
}

DO_XOR_BLOCKS(pII_mmx_inner, xor_pII_mmx_2, xor_pII_mmx_3, xor_pII_mmx_4,
		xor_pII_mmx_5);

static void xor_gen_pII_mmx(void *dest, void **srcs, unsigned int src_cnt,
		unsigned int bytes)
{
	kernel_fpu_begin();
	xor_gen_pII_mmx_inner(dest, srcs, src_cnt, bytes);
	kernel_fpu_end();
}

struct xor_block_template xor_block_pII_mmx = {
	.name		= "pII_mmx",
	.xor_gen	= xor_gen_pII_mmx,
};

DO_XOR_BLOCKS(p5_mmx_inner, xor_p5_mmx_2, xor_p5_mmx_3, xor_p5_mmx_4,
		xor_p5_mmx_5);

static void xor_gen_p5_mmx(void *dest, void **srcs, unsigned int src_cnt,
		unsigned int bytes)
{
	kernel_fpu_begin();
	xor_gen_p5_mmx_inner(dest, srcs, src_cnt, bytes);
	kernel_fpu_end();
}

struct xor_block_template xor_block_p5_mmx = {
	.name		= "p5_mmx",
	.xor_gen	= xor_gen_p5_mmx,
};