From 59049a3139c89135a49850c1a8ac27369fe2f44f Mon Sep 17 00:00:00 2001 From: Jeff Liu Date: Sat, 5 Jan 2013 04:53:18 +0100 Subject: [PATCH 01/10] binfmt_elf.c: use get_random_int() to fix entropy depleting Changes: -------- v4->v3: - s/random_stack_user()/get_atrandom_bytes()/ - Move this function to ahead of its use to avoid the predeclaration. v3->v2: - Tweak code comments of random_stack_user(). - Remove redundant bits mask and shift upon the random variable. v2->v1: - Fix random copy to check up buffer length that are not 4-byte multiples. v3 can be found at: http://www.spinics.net/lists/linux-fsdevel/msg59597.html v2 can be found at: http://www.spinics.net/lists/linux-fsdevel/msg59418.html v1 can be found at: http://www.spinics.net/lists/linux-fsdevel/msg59128.html Thanks, -Jeff Entropy is quickly depleted under normal operations like ls(1), cat(1), etc... between 2.6.30 to current mainline, for instance: $ cat /proc/sys/kernel/random/entropy_avail 3428 $ cat /proc/sys/kernel/random/entropy_avail 2911 $cat /proc/sys/kernel/random/entropy_avail 2620 We observed this problem has been occurring since 2.6.30 with fs/binfmt_elf.c: create_elf_tables()->get_random_bytes(), introduced by f06295b44c296c8f ("ELF: implement AT_RANDOM for glibc PRNG seeding"). /* * Generate 16 random bytes for userspace PRNG seeding. */ get_random_bytes(k_rand_bytes, sizeof(k_rand_bytes)); The patch introduces a wrapper around get_random_int() which has lower overhead than calling get_random_bytes() directly. With this patch applied: $ cat /proc/sys/kernel/random/entropy_avail 2731 $ cat /proc/sys/kernel/random/entropy_avail 2802 $ cat /proc/sys/kernel/random/entropy_avail 2878 Analyzed by John Sobecki. Signed-off-by: Jie Liu Cc: Andrew Morton Cc: Al Viro Cc: Andreas Dilger Cc: Alan Cox Cc: Arnd Bergmann Cc: John Sobecki Cc: James Morris Cc: Jakub Jelinek Cc: Ted Ts'o Cc: Greg Kroah-Hartman Cc: Kees Cook Cc: Ulrich Drepper Signed-off-by: Francisco Franco --- fs/binfmt_elf.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c index 1cf96193f5bb..5527172c109f 100644 --- a/fs/binfmt_elf.c +++ b/fs/binfmt_elf.c @@ -146,6 +146,25 @@ static int padzero(unsigned long elf_bss) #define ELF_BASE_PLATFORM NULL #endif +/* + * Use get_random_int() to implement AT_RANDOM while avoiding depletion + * of the entropy pool. + */ +static void get_atrandom_bytes(unsigned char *buf, size_t nbytes) +{ + unsigned char *p = buf; + + while (nbytes) { + unsigned int random_variable; + size_t chunk = min(nbytes, sizeof(random_variable)); + + random_variable = get_random_int(); + memcpy(p, &random_variable, chunk); + p += chunk; + nbytes -= chunk; + } +} + static int create_elf_tables(struct linux_binprm *bprm, struct elfhdr *exec, unsigned long load_addr, unsigned long interp_load_addr) @@ -207,7 +226,7 @@ create_elf_tables(struct linux_binprm *bprm, struct elfhdr *exec, /* * Generate 16 random bytes for userspace PRNG seeding. */ - get_random_bytes(k_rand_bytes, sizeof(k_rand_bytes)); + get_atrandom_bytes(k_rand_bytes, sizeof(k_rand_bytes)); u_rand_bytes = (elf_addr_t __user *) STACK_ALLOC(p, sizeof(k_rand_bytes)); if (__copy_to_user(u_rand_bytes, k_rand_bytes, sizeof(k_rand_bytes))) From 05f55cb57a50446d55d9570b637869663b88e9ec Mon Sep 17 00:00:00 2001 From: Bongkyu Kim Date: Wed, 20 Jan 2016 15:01:08 -0800 Subject: [PATCH 02/10] UPSTREAM: lz4: fix wrong compress buffer size for 64-bits (cherry picked from commit 06af1c52c9ea234e0b1266cc0b52c3e0c6c8fe9f) The current lz4 compress buffer is 16kb on 32-bits, 32kb on 64-bits system. But, lz4 needs only 16kb on both. On 64-bits, this causes wasted cpu cycles for additional memset during every compression. In case of lz4hc, the current buffer size is (256kb + 8) on 32-bits, (512kb + 16) on 64-bits. But, lz4hc needs only (256kb + 2 * pointer) on both. This patch fixes these wrong compress buffer sizes for 64-bits. Signed-off-by: Bongkyu Kim Cc: Chanho Min Cc: Yann Collet Cc: Kyungsik Lee Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Signed-off-by: Francisco Franco --- include/linux/lz4.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/linux/lz4.h b/include/linux/lz4.h index 4356686b0a39..6b784c59f321 100644 --- a/include/linux/lz4.h +++ b/include/linux/lz4.h @@ -9,8 +9,8 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ -#define LZ4_MEM_COMPRESS (4096 * sizeof(unsigned char *)) -#define LZ4HC_MEM_COMPRESS (65538 * sizeof(unsigned char *)) +#define LZ4_MEM_COMPRESS (16384) +#define LZ4HC_MEM_COMPRESS (262144 + (2 * sizeof(unsigned char *))) /* * lz4_compressbound() From 1b54673b66379f715157a0415a2419f6b4d98b68 Mon Sep 17 00:00:00 2001 From: Joe Maples Date: Wed, 20 Jul 2016 16:06:43 +0000 Subject: [PATCH 03/10] mm: Set vm_swappiness to 0 Swap is disabled, so theres no reason to want to place memory into swap Signed-off-by: Joe Maples Signed-off-by: engstk --- mm/vmscan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index e3809cddc2e7..6caddc5fe7b0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -144,7 +144,7 @@ struct scan_control { /* * From 0 .. 100. Higher means more swappy. */ -int vm_swappiness = 60; +int vm_swappiness = 0; /* * The total number of pages which are beyond the high watermark within all * zones. From c2b62655c808c844c4f96a07c89dc074a105a4c0 Mon Sep 17 00:00:00 2001 From: Park Ju Hyung Date: Mon, 21 Nov 2016 21:57:51 +0900 Subject: [PATCH 04/10] input: synaptics: s3320: elevate workqueues to high-priority Measured latency within the kernel driver has been fluctuating between 2~7ms. Elevating the workqueues has made it more consistently in the 2~4ms range. alloc_ordered_workqueue() still makes the workqueue singlethreaded (internal __WQ_ORDERED flag), so this commit only adds WQ_HIGHPRI additionally. Signed-off-by: Park Ju Hyung Signed-off-by: engstk --- drivers/input/touchscreen/synaptics_driver_s3320.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/input/touchscreen/synaptics_driver_s3320.c b/drivers/input/touchscreen/synaptics_driver_s3320.c index b6ed10047249..2f6f3b619bff 100644 --- a/drivers/input/touchscreen/synaptics_driver_s3320.c +++ b/drivers/input/touchscreen/synaptics_driver_s3320.c @@ -3575,20 +3575,20 @@ static int synaptics_ts_probe(struct i2c_client *client, const struct i2c_device //push_component_info(TP, ts->fw_id, ts->manu_name); - synaptics_wq = create_singlethread_workqueue("synaptics_wq"); + synaptics_wq = alloc_ordered_workqueue("synaptics_wq", WQ_HIGHPRI); if( !synaptics_wq ){ ret = -ENOMEM; goto exit_createworkqueue_failed; } INIT_DELAYED_WORK(&ts->speed_up_work,speedup_synaptics_resume); - synaptics_report = create_singlethread_workqueue("synaptics_report"); + synaptics_report = alloc_ordered_workqueue("synaptics_report", WQ_HIGHPRI); if( !synaptics_report ){ ret = -ENOMEM; goto exit_createworkqueue_failed; } - get_base_report = create_singlethread_workqueue("get_base_report"); + get_base_report = alloc_ordered_workqueue("get_base_report", WQ_HIGHPRI); if( !get_base_report ){ ret = -ENOMEM; goto exit_createworkqueue_failed; From ba9c70cd517efeffdf6747465e08da222aa58622 Mon Sep 17 00:00:00 2001 From: Tim Murray Date: Tue, 19 Jan 2016 16:33:27 -0800 Subject: [PATCH 05/10] dm-crypt: run in a WQ_HIGHPRI workqueue Running dm-crypt in a standard workqueue results in IO competing for CPU time with standard user apps, which can lead to pipeline bubbles and seriously degraded performance. Move to a WQ_HIGHPRI workqueue to protect against that. bug 25392275 Change-Id: I589149a31c7b5d322fe2ed5b2476b1f6e3d5ee6f Signed-off-by: Francisco Franco --- drivers/md/dm-crypt.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index 5503e43e5f28..9c7a083b2497 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -1841,16 +1841,16 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) } ret = -ENOMEM; - cc->io_queue = alloc_workqueue("kcryptd_io", WQ_MEM_RECLAIM, 1); + cc->io_queue = alloc_workqueue("kcryptd_io", WQ_HIGHPRI | WQ_MEM_RECLAIM, 1); if (!cc->io_queue) { ti->error = "Couldn't create kcryptd io queue"; goto bad; } if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) - cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 1); + cc->crypt_queue = alloc_workqueue("kcryptd", WQ_HIGHPRI | WQ_MEM_RECLAIM, 1); else - cc->crypt_queue = alloc_workqueue("kcryptd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, + cc->crypt_queue = alloc_workqueue("kcryptd", WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus()); if (!cc->crypt_queue) { ti->error = "Couldn't create kcryptd queue"; From 3c83594fc1afc8e61f197abba94a5eba2fe2dbb2 Mon Sep 17 00:00:00 2001 From: Tim Murray Date: Tue, 19 Jan 2016 16:35:17 -0800 Subject: [PATCH 06/10] dm-verity: run in a WQ_HIGHPRI workqueue Running dm-verity in a standard workqueue results in IO competing for CPU time with standard user apps, which can lead to pipeline bubbles and seriously degraded performance. Move to a WQ_HIGHPRI workqueue to protect against that. bug 25392275 Change-Id: Ic65d7bd6f04e4d77780119e926a50e71323575f0 Signed-off-by: Francisco Franco --- drivers/md/dm-verity.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/dm-verity.c b/drivers/md/dm-verity.c index 1288eaf06ab8..71a98d48f3e4 100644 --- a/drivers/md/dm-verity.c +++ b/drivers/md/dm-verity.c @@ -928,7 +928,7 @@ static int verity_ctr(struct dm_target *ti, unsigned argc, char **argv) } /* WQ_UNBOUND greatly improves performance when running on ramdisk */ - v->verify_wq = alloc_workqueue("kverityd", WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus()); + v->verify_wq = alloc_workqueue("kverityd", WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_UNBOUND, num_online_cpus()); if (!v->verify_wq) { ti->error = "Cannot allocate workqueue"; r = -ENOMEM; From c5ef5dc41bea3a2461c09f6418ef21c4ad9831c8 Mon Sep 17 00:00:00 2001 From: Francisco Franco Date: Fri, 21 Oct 2016 04:50:05 +0000 Subject: [PATCH 07/10] proc: much faster /proc/vmstat Every current KDE system has process named ksysguardd polling files below once in several seconds: $ strace -e trace=open -p $(pidof ksysguardd) Process 1812 attached open("/etc/mtab", O_RDONLY|O_CLOEXEC) = 8 open("/etc/mtab", O_RDONLY|O_CLOEXEC) = 8 open("/proc/net/dev", O_RDONLY) = 8 open("/proc/net/wireless", O_RDONLY) = -1 ENOENT (No such file or directory) open("/proc/stat", O_RDONLY) = 8 open("/proc/vmstat", O_RDONLY) = 8 Hell knows what it is doing but speed up reading /proc/vmstat by 33%! Benchmark is open+read+close 1.000.000 times. BEFORE $ perf stat -r 10 taskset -c 3 ./proc-vmstat Performance counter stats for 'taskset -c 3 ./proc-vmstat' (10 runs): 13146.768464 task-clock (msec) # 0.960 CPUs utilized ( +- 0.60% ) 15 context-switches # 0.001 K/sec ( +- 1.41% ) 1 cpu-migrations # 0.000 K/sec ( +- 11.11% ) 104 page-faults # 0.008 K/sec ( +- 0.57% ) 45,489,799,349 cycles # 3.460 GHz ( +- 0.03% ) 9,970,175,743 stalled-cycles-frontend # 21.92% frontend cycles idle ( +- 0.10% ) 2,800,298,015 stalled-cycles-backend # 6.16% backend cycles idle ( +- 0.32% ) 79,241,190,850 instructions # 1.74 insn per cycle # 0.13 stalled cycles per insn ( +- 0.00% ) 17,616,096,146 branches # 1339.956 M/sec ( +- 0.00% ) 176,106,232 branch-misses # 1.00% of all branches ( +- 0.18% ) 13.691078109 seconds time elapsed ( +- 0.03% ) ^^^^^^^^^^^^ AFTER $ perf stat -r 10 taskset -c 3 ./proc-vmstat Performance counter stats for 'taskset -c 3 ./proc-vmstat' (10 runs): 8688.353749 task-clock (msec) # 0.950 CPUs utilized ( +- 1.25% ) 10 context-switches # 0.001 K/sec ( +- 2.13% ) 1 cpu-migrations # 0.000 K/sec 104 page-faults # 0.012 K/sec ( +- 0.56% ) 30,384,010,730 cycles # 3.497 GHz ( +- 0.07% ) 12,296,259,407 stalled-cycles-frontend # 40.47% frontend cycles idle ( +- 0.13% ) 3,370,668,651 stalled-cycles-backend # 11.09% backend cycles idle ( +- 0.69% ) 28,969,052,879 instructions # 0.95 insn per cycle # 0.42 stalled cycles per insn ( +- 0.01% ) 6,308,245,891 branches # 726.058 M/sec ( +- 0.00% ) 214,685,502 branch-misses # 3.40% of all branches ( +- 0.26% ) 9.146081052 seconds time elapsed ( +- 0.07% ) ^^^^^^^^^^^ vsnprintf() is slow because: 1. format_decode() is busy looking for format specifier: 2 branches per character (not in this case, but in others) 2. approximately million branches while parsing format mini language and everywhere 3. just look at what string() does /proc/vmstat is good case because most of its content are strings Link: http://lkml.kernel.org/r/20160806125455.GA1187@p183.telecom.by Signed-off-by: Alexey Dobriyan Cc: Joe Perches Cc: Andi Kleen Cc: Al Viro Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds Signed-off-by: Francisco Franco --- mm/vmstat.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm/vmstat.c b/mm/vmstat.c index f623007c076a..47c307bafbd7 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1318,7 +1318,9 @@ static int vmstat_show(struct seq_file *m, void *arg) unsigned long *l = arg; unsigned long off = l - (unsigned long *)m->private; - seq_printf(m, "%s %lu\n", vmstat_text[off], *l); + seq_puts(m, vmstat_text[off]); + seq_put_decimal_ull(m, ' ', *l); + seq_putc(m, '\n'); return 0; } From 94c58c8515ad2ce9c01707bd81833d74ac63c93b Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 25 Nov 2014 15:44:18 +0000 Subject: [PATCH 08/10] UPSTREAM: arm64: jump labels: NOP out NOP -> NOP replacement In the arm64 arch_static_branch implementation we place an A64 NOP into the instruction stream and log relevant details to a jump_entry in a __jump_table section. Later this may be replaced with an immediate branch without link to the code for the unlikely case. At init time, the core calls arch_jump_label_transform_static to initialise the NOPs. On x86 this involves inserting the optimal NOP for a given microarchitecture, but on arm64 we only use the architectural NOP, and hence replace each NOP with the exact same NOP. This is somewhat pointless. Additionally, at module load time we don't call jump_label_apply_nops to patch the optimal NOPs in, unlike other architectures, but get away with this because we only use the architectural NOP anyway. A later notifier will patch NOPs with branches as required. Similarly to x86 commit 11570da1c5b1dee1 (x86/jump-label: Do not bother updating NOPs if they are correct), we can avoid patching NOPs with identical NOPs. Given that we only use a single NOP encoding, this means we can NOP-out the body of arch_jump_label_transform_static entirely. As the default __weak arch_jump_label_transform_static implementation performs a patch, we must use an empty function to achieve this. Cc: Catalin Marinas Cc: Jiang Liu Cc: Laura Abbott Acked-by: Will Deacon Signed-off-by: Mark Rutland Signed-off-by: Will Deacon (cherry picked from commit 6ddae4186886a81e22ad78ad7c6936ed36bc8225) Signed-off-by: Tomasz Figa Bug: 24475017 Change-Id: I9ef07c3a5a7f91418b30006850fcc0c421e147e2 Signed-off-by: Kees Cook Signed-off-by: Francisco Franco --- arch/arm64/kernel/jump_label.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c index 263a166291fb..4f1fec7a46db 100644 --- a/arch/arm64/kernel/jump_label.c +++ b/arch/arm64/kernel/jump_label.c @@ -22,9 +22,8 @@ #ifdef HAVE_JUMP_LABEL -static void __arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type, - bool is_static) +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) { void *addr = (void *)entry->code; u32 insn; @@ -37,22 +36,18 @@ static void __arch_jump_label_transform(struct jump_entry *entry, insn = aarch64_insn_gen_nop(); } - if (is_static) - aarch64_insn_patch_text_nosync(addr, insn); - else - aarch64_insn_patch_text(&addr, &insn, 1); -} - -void arch_jump_label_transform(struct jump_entry *entry, - enum jump_label_type type) -{ - __arch_jump_label_transform(entry, type, false); + aarch64_insn_patch_text(&addr, &insn, 1); } void arch_jump_label_transform_static(struct jump_entry *entry, enum jump_label_type type) { - __arch_jump_label_transform(entry, type, true); + /* + * We use the architected A64 NOP in arch_static_branch, so there's no + * need to patch an identical A64 NOP over the top of it here. The core + * will call arch_jump_label_transform from a module notifier if the + * NOP needs to be replaced by a branch. + */ } #endif /* HAVE_JUMP_LABEL */ From 6ea1d3844d24e401dd49781f4bdf47ef78c9c890 Mon Sep 17 00:00:00 2001 From: Hong-Mei Li Date: Tue, 21 Apr 2015 14:53:35 -0700 Subject: [PATCH 09/10] arm64: lib: memory utilities optimization Optimize memcpy and memmove, to prefetch several cache lines. We can achieve 15% memcpy speed improvement with the preload method. Change-Id: I2259b98a33eba0b7466920b3f270f953e609cf13 Signed-off-by: Hong-Mei Li Reviewed-on: http://gerrit.mot.com/740766 SLTApproved: Slta Waiver SME-Granted: SME Approvals Granted Tested-by: Jira Key Reviewed-by: Zhi-Ming Yuan Submit-Approved: Jira Key Signed-off-by: engstk --- arch/arm64/lib/memcpy.S | 2 ++ arch/arm64/lib/memmove.S | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S index 42cc4b782034..fb37d2fff9e7 100644 --- a/arch/arm64/lib/memcpy.S +++ b/arch/arm64/lib/memcpy.S @@ -59,6 +59,7 @@ D_h .req x14 .weak memcpy ENTRY(__memcpy) ENTRY(memcpy) + prfm pldl1strm, [src, #(1*L1_CACHE_BYTES)] mov dst, dstin cmp count, #16 /*When memory length is less than 16, the accessed are not aligned.*/ @@ -190,6 +191,7 @@ ENTRY(memcpy) ldp C_l, C_h, [src],#16 stp D_l, D_h, [dst],#16 ldp D_l, D_h, [src],#16 + prfm pldl1strm, [src, #(4*L1_CACHE_BYTES)] subs count, count, #64 b.ge 1b stp A_l, A_h, [dst],#16 diff --git a/arch/arm64/lib/memmove.S b/arch/arm64/lib/memmove.S index 88194332d31d..531ed58f4f5d 100644 --- a/arch/arm64/lib/memmove.S +++ b/arch/arm64/lib/memmove.S @@ -60,6 +60,7 @@ D_h .req x14 .weak memmove ENTRY(__memmove) ENTRY(memmove) + prfm pldl1strm, [src, #L1_CACHE_BYTES] cmp dstin, src b.lo __memcpy add tmp1, src, count @@ -186,6 +187,7 @@ ENTRY(memmove) ldp C_l, C_h, [src, #-48] stp D_l, D_h, [dst, #-64]! ldp D_l, D_h, [src, #-64]! + prfm pldl1strm, [src, #(4*L1_CACHE_BYTES)] subs count, count, #64 b.ge 1b stp A_l, A_h, [dst, #-16] From 1956ea2c68e160030ade8ed3d544167d4ed9899a Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 2 Feb 2016 12:46:25 +0000 Subject: [PATCH 10/10] arm64: lib: improve copy_page to deal with 128 bytes at a time We want to avoid lots of different copy_page implementations, settling for something that is "good enough" everywhere and hopefully easy to understand and maintain whilst we're at it. This patch reworks our copy_page implementation based on discussions with Cavium on the list and benchmarking on Cortex-A processors so that: - The loop is unrolled to copy 128 bytes per iteration - The reads are offset so that we read from the next 128-byte block in the same iteration that we store the previous block - Explicit prefetch instructions are removed for now, since they hurt performance on CPUs with hardware prefetching - The loop exit condition is calculated at the start of the loop Change-Id: I0d9f3bbe4efa2751f41432a3b4b299fbb0e494be Signed-off-by: Will Deacon Tested-by: Andrew Pinski Signed-off-by: Catalin Marinas Signed-off-by: engstk --- arch/arm64/lib/copy_page.S | 46 +++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S index 512b9a7b980e..2534533ceb1d 100644 --- a/arch/arm64/lib/copy_page.S +++ b/arch/arm64/lib/copy_page.S @@ -27,20 +27,50 @@ * x1 - src */ ENTRY(copy_page) - /* Assume cache line size is 64 bytes. */ - prfm pldl1strm, [x1, #64] -1: ldp x2, x3, [x1] + ldp x2, x3, [x1] ldp x4, x5, [x1, #16] ldp x6, x7, [x1, #32] ldp x8, x9, [x1, #48] - add x1, x1, #64 - prfm pldl1strm, [x1, #64] + ldp x10, x11, [x1, #64] + ldp x12, x13, [x1, #80] + ldp x14, x15, [x1, #96] + ldp x16, x17, [x1, #112] + + mov x18, #(PAGE_SIZE - 128) + add x1, x1, #128 +1: + subs x18, x18, #128 + stnp x2, x3, [x0] + ldp x2, x3, [x1] stnp x4, x5, [x0, #16] + ldp x4, x5, [x1, #16] stnp x6, x7, [x0, #32] + ldp x6, x7, [x1, #32] stnp x8, x9, [x0, #48] - add x0, x0, #64 - tst x1, #(PAGE_SIZE - 1) - b.ne 1b + ldp x8, x9, [x1, #48] + stnp x10, x11, [x0, #64] + ldp x10, x11, [x1, #64] + stnp x12, x13, [x0, #80] + ldp x12, x13, [x1, #80] + stnp x14, x15, [x0, #96] + ldp x14, x15, [x1, #96] + stnp x16, x17, [x0, #112] + ldp x16, x17, [x1, #112] + + add x0, x0, #128 + add x1, x1, #128 + + b.gt 1b + + stnp x2, x3, [x0] + stnp x4, x5, [x0, #16] + stnp x6, x7, [x0, #32] + stnp x8, x9, [x0, #48] + stnp x10, x11, [x0, #64] + stnp x12, x13, [x0, #80] + stnp x14, x15, [x0, #96] + stnp x16, x17, [x0, #112] + ret ENDPROC(copy_page)