powerpc: Improve 64bit copy_tofrom_user

Here is a patch from Paul Mackerras that improves the ppc64 copy_tofrom_user.
The loop now does 32 bytes at a time and as well as pairing loads and stores.

A quick test case that reads 8kB over and over shows the improvement:

POWER6: 53% faster
POWER7: 51% faster

#define _XOPEN_SOURCE 500
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>

#define BUFSIZE (8 * 1024)
#define ITERATIONS 10000000

int main()
{
	char tmpfile[] = "/tmp/copy_to_user_testXXXXXX";
	int fd;
	char *buf[BUFSIZE];
	unsigned long i;

	fd = mkstemp(tmpfile);
	if (fd < 0) {
		perror("open");
		exit(1);
	}

	if (write(fd, buf, BUFSIZE) != BUFSIZE) {
		perror("open");
		exit(1);
	}

	for (i = 0; i < 10000000; i++) {
		if (pread(fd, buf, BUFSIZE, 0) != BUFSIZE) {
			perror("pread");
			exit(1);
		}
	}

	unlink(tmpfile);

	return 0;
}

Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>

1 files changed, 57 insertions, 23 deletions

diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S
index 693b14a778fa..578b625d6a3c 100644
--- a/arch/powerpc/lib/copyuser_64.S
+++ b/arch/powerpc/lib/copyuser_64.S
@@ -44,37 +44,55 @@ BEGIN_FTR_SECTION
 	andi.	r0,r4,7
 	bne	.Lsrc_unaligned
 END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
-	srdi	r7,r5,4
-20:	ld	r9,0(r4)
-	addi	r4,r4,-8
-	mtctr	r7
-	andi.	r5,r5,7
-	bf	cr7*4+0,22f
-	addi	r3,r3,8
-	addi	r4,r4,8
-	mr	r8,r9
-	blt	cr1,72f
-21:	ld	r9,8(r4)
-70:	std	r8,8(r3)
-22:	ldu	r8,16(r4)
-71:	stdu	r9,16(r3)
+	blt	cr1,.Ldo_tail		/* if < 16 bytes to copy */
+	srdi	r0,r5,5
+	cmpdi	cr1,r0,0
+20:	ld	r7,0(r4)
+220:	ld	r6,8(r4)
+	addi	r4,r4,16
+	mtctr	r0
+	andi.	r0,r5,0x10
+	beq	22f
+	addi	r3,r3,16
+	addi	r4,r4,-16
+	mr	r9,r7
+	mr	r8,r6
+	beq	cr1,72f
+21:	ld	r7,16(r4)
+221:	ld	r6,24(r4)
+	addi	r4,r4,32
+70:	std	r9,0(r3)
+270:	std	r8,8(r3)
+22:	ld	r9,0(r4)
+222:	ld	r8,8(r4)
+71:	std	r7,16(r3)
+271:	std	r6,24(r3)
+	addi	r3,r3,32
 	bdnz	21b
-72:	std	r8,8(r3)
+72:	std	r9,0(r3)
+272:	std	r8,8(r3)
+	andi.	r5,r5,0xf
 	beq+	3f
-	addi	r3,r3,16
+	addi	r4,r4,16
 .Ldo_tail:
-	bf	cr7*4+1,1f
-23:	lwz	r9,8(r4)
+	addi	r3,r3,16
+	bf	cr7*4+0,246f
+244:	ld	r9,0(r4)
+	addi	r4,r4,8
+245:	std	r9,0(r3)
+	addi	r3,r3,8
+246:	bf	cr7*4+1,1f
+23:	lwz	r9,0(r4)
 	addi	r4,r4,4
 73:	stw	r9,0(r3)
 	addi	r3,r3,4
 1:	bf	cr7*4+2,2f
-44:	lhz	r9,8(r4)
+44:	lhz	r9,0(r4)
 	addi	r4,r4,2
 74:	sth	r9,0(r3)
 	addi	r3,r3,2
 2:	bf	cr7*4+3,3f
-45:	lbz	r9,8(r4)
+45:	lbz	r9,0(r4)
 75:	stb	r9,0(r3)
 3:	li	r3,0
 	blr
@@ -220,7 +238,9 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
 131:
 	addi	r3,r3,8
 120:
+320:
 122:
+322:
 124:
 125:
 126:
@@ -229,9 +249,11 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
 129:
 133:
 	addi	r3,r3,8
-121:
 132:
 	addi	r3,r3,8
+121:
+321:
+344:
 134:
 135:
 138:
@@ -303,18 +325,22 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
 183:
 	add	r3,r3,r7
 	b	1f
+371:
 180:
 	addi	r3,r3,8
 171:
 177:
 	addi	r3,r3,8
-170:
-172:
+370:
+372:
 176:
 178:
 	addi	r3,r3,4
 185:
 	addi	r3,r3,4
+170:
+172:
+345:
 173:
 174:
 175:
@@ -341,11 +367,19 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD)
 	.section __ex_table,"a"
 	.align	3
 	.llong	20b,120b
+	.llong	220b,320b
 	.llong	21b,121b
+	.llong	221b,321b
 	.llong	70b,170b
+	.llong	270b,370b
 	.llong	22b,122b
+	.llong	222b,322b
 	.llong	71b,171b
+	.llong	271b,371b
 	.llong	72b,172b
+	.llong	272b,372b
+	.llong	244b,344b
+	.llong	245b,345b
 	.llong	23b,123b
 	.llong	73b,173b
 	.llong	44b,144b