397 lines
7.8 KiB
ArmAsm
397 lines
7.8 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0-only */
|
|
.Global memcpy_arm
|
|
.type memcpy_arm, %function
|
|
.align 8
|
|
memcpy_arm:
|
|
mov x4, x0
|
|
subs x2, x2, #8
|
|
b.mi 2f
|
|
1: ldr x3, [x1], #8
|
|
subs x2, x2, #8
|
|
str x3, [x4], #8
|
|
b.pl 1b
|
|
2: adds x2, x2, #4
|
|
b.mi 3f
|
|
ldr w3, [x1], #4
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
3: adds x2, x2, #2
|
|
b.mi 4f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
4: adds x2, x2, #1
|
|
b.mi 5f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
5: ret
|
|
|
|
|
|
.Global memcpy_a
|
|
.type memcpy_a, %function
|
|
.align 8
|
|
|
|
memcpy_a :
|
|
|
|
mov x6, x0
|
|
cmp x2, #64
|
|
b.ge .Lcpy_not_short
|
|
cmp x2, #15
|
|
b.le .Ltail15tiny
|
|
|
|
/* Deal with small copies quickly by dropping straight into the
|
|
* exit block. */
|
|
.Ltail63:
|
|
/* Copy up to 48 bytes of data. At this point we only need the
|
|
* bottom 6 bits of x2 to be accurate. */
|
|
ands x3, x2, #0x30
|
|
b.eq .Ltail15
|
|
add x6, x6, x3
|
|
add x1, x1, x3
|
|
cmp w3, #0x20
|
|
b.eq 1f
|
|
b.lt 2f
|
|
ldp x7, x8, [x1, #-48]
|
|
stp x7, x8, [x6, #-48]
|
|
1:
|
|
ldp x7, x8, [x1, #-32]
|
|
stp x7, x8, [x6, #-32]
|
|
2:
|
|
ldp x7, x8, [x1, #-16]
|
|
stp x7, x8, [x6, #-16]
|
|
|
|
.Ltail15:
|
|
ands x2, x2, #15
|
|
beq 1f
|
|
add x1, x1, x2
|
|
ldp x7, x8, [x1, #-16]
|
|
add x6, x6, x2
|
|
stp x7, x8, [x6, #-16]
|
|
1:
|
|
ret
|
|
|
|
.Ltail15tiny:
|
|
/* Copy up to 15 bytes of data. Does not assume additional data
|
|
being copied. */
|
|
tbz x2, #3, 1f
|
|
ldr x3, [x1], #8
|
|
str x3, [x6], #8
|
|
1:
|
|
tbz x2, #2, 1f
|
|
ldr w3, [x1], #4
|
|
str w3, [x6], #4
|
|
1:
|
|
tbz x2, #1, 1f
|
|
ldrh w3, [x1], #2
|
|
strh w3, [x6], #2
|
|
1:
|
|
tbz x2, #0, 1f
|
|
ldrb w3, [x1]
|
|
strb w3, [x6]
|
|
1:
|
|
ret
|
|
|
|
.Lcpy_not_short:
|
|
/* We don't much care about the alignment of DST, but we want SRC
|
|
* to be 128-bit (16 byte) aligned so that we don't cross cache line
|
|
* boundaries on both loads and stores. */
|
|
neg x4, x1
|
|
ands x4, x4, #15 /* Bytes to reach alignment. */
|
|
b.eq 2f
|
|
sub x2, x2, x4
|
|
/* Copy more data than needed; it's faster than jumping
|
|
* around copying sub-Quadword quantities. We know that
|
|
* it can't overrun. */
|
|
ldp x7, x8, [x1]
|
|
add x1, x1, x4
|
|
stp x7, x8, [x6]
|
|
add x6, x6, x4
|
|
/* There may be less than 63 bytes to go now. */
|
|
cmp x2, #63
|
|
b.le .Ltail63
|
|
2:
|
|
subs x2, x2, #128
|
|
b.ge .Lcpy_body_large
|
|
/* Less than 128 bytes to copy, so handle 64 here and then jump
|
|
* to the tail. */
|
|
ldp x7, x8, [x1]
|
|
ldp x9, x10, [x1, #16]
|
|
ldp x11, x12, [x1, #32]
|
|
ldp x13, x14, [x1, #48]
|
|
stp x7, x8, [x6]
|
|
stp x9, x10, [x6, #16]
|
|
stp x11, x12, [x6, #32]
|
|
stp x13, x14, [x6, #48]
|
|
tst x2, #0x3f
|
|
add x1, x1, #64
|
|
add x6, x6, #64
|
|
b.ne .Ltail63
|
|
ret
|
|
|
|
/* Critical loop. Start at a new cache line boundary. Assuming
|
|
* 64 bytes per line this ensures the entire loop is in one line. */
|
|
.p2align 6
|
|
.Lcpy_body_large:
|
|
/* There are at least 128 bytes to copy. */
|
|
ldp x7, x8, [x1, #0]
|
|
sub x6, x6, #16 /* Pre-bias. */
|
|
ldp x9, x10, [x1, #16]
|
|
ldp x11, x12, [x1, #32]
|
|
ldp x13, x14, [x1, #48]! /* x1 += 64 - Pre-bias. */
|
|
1:
|
|
stp x7, x8, [x6, #16]
|
|
ldp x7, x8, [x1, #16]
|
|
stp x9, x10, [x6, #32]
|
|
ldp x9, x10, [x1, #32]
|
|
stp x11, x12, [x6, #48]
|
|
ldp x11, x12, [x1, #48]
|
|
stp x13, x14, [x6, #64]!
|
|
ldp x13, x14, [x1, #64]!
|
|
subs x2, x2, #64
|
|
b.ge 1b
|
|
stp x7, x8, [x6, #16]
|
|
stp x9, x10, [x6, #32]
|
|
stp x11, x12, [x6, #48]
|
|
stp x13, x14, [x6, #64]
|
|
add x1, x1, #16
|
|
add x6, x6, #64 + 16
|
|
tst x2, #0x3f
|
|
b.ne .Ltail63
|
|
ret
|
|
|
|
|
|
|
|
|
|
|
|
.Global memcpy_n
|
|
.type memcpy_n, %function
|
|
.align 8
|
|
memcpy_n :
|
|
|
|
mov x6, x0
|
|
cmp x2, #64
|
|
b.ge .Lcpy_not_short_n
|
|
cmp x2, #15
|
|
b.le .Ltail15tiny_n
|
|
|
|
/* Deal with small copies quickly by dropping straight into the
|
|
* exit block. */
|
|
.Ltail63_n:
|
|
/* Copy up to 48 bytes of data. At this point we only need the
|
|
* bottom 6 bits of x2 to be accurate. */
|
|
ands x3, x2, #0x30
|
|
b.eq .Ltail15_n
|
|
add x6, x6, x3
|
|
add x1, x1, x3
|
|
cmp w3, #0x20
|
|
b.eq 1f
|
|
b.lt 2f
|
|
ldp x7, x8, [x1, #-48]
|
|
stp x7, x8, [x6, #-48]
|
|
1:
|
|
ldp x7, x8, [x1, #-32]
|
|
stp x7, x8, [x6, #-32]
|
|
2:
|
|
ldp x7, x8, [x1, #-16]
|
|
stp x7, x8, [x6, #-16]
|
|
|
|
.Ltail15_n:
|
|
ands x2, x2, #15
|
|
beq 1f
|
|
add x1, x1, x2
|
|
ldp x7, x8, [x1, #-16]
|
|
add x6, x6, x2
|
|
stp x7, x8, [x6, #-16]
|
|
1:
|
|
ret
|
|
|
|
.Ltail15tiny_n:
|
|
/* Copy up to 15 bytes of data. Does not assume additional data
|
|
being copied. */
|
|
tbz x2, #3, 1f
|
|
ldr x3, [x1], #8
|
|
str x3, [x6], #8
|
|
1:
|
|
tbz x2, #2, 1f
|
|
ldr w3, [x1], #4
|
|
str w3, [x6], #4
|
|
1:
|
|
tbz x2, #1, 1f
|
|
ldrh w3, [x1], #2
|
|
strh w3, [x6], #2
|
|
1:
|
|
tbz x2, #0, 1f
|
|
ldrb w3, [x1]
|
|
strb w3, [x6]
|
|
1:
|
|
ret
|
|
|
|
.Lcpy_not_short_n:
|
|
/* We don't much care about the alignment of DST, but we want SRC
|
|
* to be 128-bit (16 byte) aligned so that we don't cross cache line
|
|
* boundaries on both loads and stores. */
|
|
neg x4, x1
|
|
ands x4, x4, #15 /* Bytes to reach alignment. */
|
|
b.eq 2f
|
|
sub x2, x2, x4
|
|
/* Copy more data than needed; it's faster than jumping
|
|
* around copying sub-Quadword quantities. We know that
|
|
* it can't overrun. */
|
|
ldp x7, x8, [x1]
|
|
add x1, x1, x4
|
|
stp x7, x8, [x6]
|
|
add x6, x6, x4
|
|
/* There may be less than 63 bytes to go now. */
|
|
cmp x2, #63
|
|
b.le .Ltail63_n
|
|
2:
|
|
subs x2, x2, #128
|
|
b.ge .Lcpy_body_large_n
|
|
/* Less than 128 bytes to copy, so handle 64 here and then jump
|
|
* to the tail. */
|
|
ldp q0, q1, [x1]
|
|
ldp q2, q3, [x1, #32]
|
|
stp q0, q1, [x6]
|
|
stp q2, q3, [x6, #32]
|
|
tst x2, #0x3f
|
|
add x1, x1, #64
|
|
add x6, x6, #64
|
|
b.ne .Ltail63_n
|
|
ret
|
|
|
|
/* Critical loop. Start at a new cache line boundary. Assuming
|
|
* 64 bytes per line this ensures the entire loop is in one line. */
|
|
.p2align 6
|
|
.Lcpy_body_large_n:
|
|
/* There are at least 128 bytes to copy. */
|
|
ldp q0, q1, [x1, #0]
|
|
sub x6, x6, #32 /* Pre-bias. */
|
|
ldp q2, q3, [x1, #32]! /* x1 += 64 - Pre-bias. */
|
|
1:
|
|
stp q0, q1, [x6, #32]
|
|
ldp q0, q1, [x1, #32]
|
|
stp q2, q3, [x6, #64]!
|
|
ldp q2, q3, [x1, #64]!
|
|
|
|
subs x2, x2, #64
|
|
b.ge 1b
|
|
|
|
stp q0, q1, [x6, #32]
|
|
stp q2, q3, [x6, #64]
|
|
add x1, x1, #32
|
|
add x6, x6, #64 + 32
|
|
tst x2, #0x3f
|
|
b.ne .Ltail63_n
|
|
ret
|
|
|
|
.Global memcpy_pld
|
|
.type memcpy_pld, %function
|
|
.align 8
|
|
memcpy_pld:
|
|
mov x4, x0
|
|
subs x2, x2, #8
|
|
b.mi 2f
|
|
1: ldr x3, [x1], #8
|
|
subs x2, x2, #8
|
|
str x3, [x4], #8
|
|
prfm PLDL1KEEP, [x1, #504]
|
|
b.pl 1b
|
|
|
|
2: adds x2, x2, #4
|
|
b.mi 3f
|
|
ldr w3, [x1], #4
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
3: adds x2, x2, #2
|
|
b.mi 4f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
4: adds x2, x2, #1
|
|
b.mi 5f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
5: ret
|
|
|
|
.Global memcpy_pld4
|
|
.type memcpy_pld4, %function
|
|
.align 8
|
|
memcpy_pld4:
|
|
mov x4, x0
|
|
subs x2, x2, #4
|
|
b.mi 3f
|
|
1: ldr w3, [x1], #4
|
|
subs x2, x2, #4
|
|
str w3, [x4], #4
|
|
prfm PLDL1KEEP, [x1, #504]
|
|
b.pl 1b
|
|
3: adds x2, x2, #2
|
|
b.mi 4f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
4: adds x2, x2, #1
|
|
b.mi 5f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
5: ret
|
|
|
|
.Global memcpy_pld_512
|
|
.type memcpy_pld_512, %function
|
|
.align 8
|
|
memcpy_pld_512:
|
|
mov x4, x0
|
|
subs x2, x2, #8
|
|
b.mi 2f
|
|
1: ldr x3, [x1], #8
|
|
subs x2, x2, #8
|
|
str x3, [x4], #8
|
|
prfm PLDL1KEEP, [x1, #504]
|
|
b.pl 1b
|
|
2: adds x2, x2, #4
|
|
b.mi 3f
|
|
ldr w3, [x1], #4
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
3: adds x2, x2, #2
|
|
b.mi 4f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
4: adds x2, x2, #1
|
|
b.mi 5f
|
|
ldr w3, [x1], #2
|
|
sub x2, x2, #4
|
|
str w3, [x4], #4
|
|
5: ret
|
|
|
|
.global test_memwrite, test_memread, test_memread_simple
|
|
|
|
.align 8
|
|
test_memwrite:
|
|
1:
|
|
subs x2, x2, #8
|
|
str x1, [x0], #8
|
|
b.pl 1b
|
|
ret
|
|
|
|
.align 8
|
|
test_memread:
|
|
1:
|
|
prfm PLDL1KEEP, [x0, #128]
|
|
subs x1, x1, #8
|
|
ldr q0, [x0], #8
|
|
b.pl 1b
|
|
ret
|
|
|
|
.align 8
|
|
test_memread_simple:
|
|
1:
|
|
subs x1, x1, #8
|
|
ldr q0, [x0], #8
|
|
b.pl 1b
|
|
ret
|