diff --git a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp index 0eedb6d..a095234 100644 --- a/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp +++ b/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp @@ -23,6 +23,17 @@ namespace ams::kern::arch::arm64::init { + inline void ClearPhysicalMemory(KPhysicalAddress address, size_t size) { + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, sizeof(u64))); + + /* This Physical Address -> void * conversion is valid, because this is init page table code. */ + /* The MMU is necessarily not yet turned on, if we are creating an initial page table. */ + volatile u64 *ptr = reinterpret_cast(GetInteger(address)); + for (size_t i = 0; i < size / sizeof(u64); ++i) { + ptr[i] = 0; + } + } + class KInitialPageTable { public: class IPageAllocator { @@ -61,9 +72,7 @@ } static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address) { - /* This Physical Address -> void * conversion is valid, because this is page table code. */ - /* The MMU is necessarily not yet turned on, if we are creating an initial page table. */ - std::memset(reinterpret_cast(GetInteger(address)), 0, PageSize); + ClearPhysicalMemory(address, PageSize); } private: size_t NOINLINE GetBlockCount(KVirtualAddress virt_addr, size_t size, size_t block_size) { @@ -705,7 +714,7 @@ this->state.next_address += PageSize; } - std::memset(reinterpret_cast(allocated), 0, PageSize); + ClearPhysicalMemory(allocated, PageSize); return allocated; } diff --git a/libraries/libmesosphere/source/libc/arch/arm64/memset.arch.arm64-broken.s b/libraries/libmesosphere/source/libc/arch/arm64/memset.arch.arm64-broken.s deleted file mode 100644 index d8d2727..0000000 --- a/libraries/libmesosphere/source/libc/arch/arm64/memset.arch.arm64-broken.s +++ /dev/null @@ -1,170 +0,0 @@ -/* - * memset - fill memory with a constant byte - * - * Copyright (c) 2012-2020, Arm Limited. - * SPDX-License-Identifier: MIT - */ - -/* Assumptions: - * - * ARMv8-a, AArch64, Advanced SIMD, unaligned accesses. - * - */ - -#include "asmdefs.h" - -#define DC_ZVA_THRESHOLD 512 - -#define dstin x0 -#define val x1 -#define valw w1 -#define count x2 -#define dst x3 -#define dstend x4 -#define zva_val x5 - -ENTRY (memset) - - bfi valw, valw, 8, 8 - bfi valw, valw, 16, 16 - bfi val, val, 32, 32 - - add dstend, dstin, count - - cmp count, 96 - b.hi L(set_long) - cmp count, 16 - b.hs L(set_medium) - - /* Set 0..15 bytes. */ - tbz count, 3, 1f - str val, [dstin] - str val, [dstend, -8] - ret -1: tbz count, 2, 2f - str valw, [dstin] - str valw, [dstend, -4] - ret -2: cbz count, 3f - strb valw, [dstin] - tbz count, 1, 3f - strh valw, [dstend, -2] -3: ret - - /* Set 16..96 bytes. */ - .p2align 4 -L(set_medium): - stp val, val, [dstin] - tbnz count, 6, L(set96) - stp val, val, [dstend, -16] - tbz count, 5, 1f - stp val, val, [dstin, 16] - stp val, val, [dstend, -32] -1: ret - - .p2align 4 - /* Set 64..96 bytes. Write 64 bytes from the start and - 32 bytes from the end. */ -L(set96): - stp val, val, [dstin, 16] - stp val, val, [dstin, 32] - stp val, val, [dstin, 48] - stp val, val, [dstend, -32] - stp val, val, [dstend, -16] - ret - - .p2align 4 -L(set_long): - stp val, val, [dstin] - bic dst, dstin, 15 -#if DC_ZVA_THRESHOLD - cmp count, DC_ZVA_THRESHOLD - ccmp val, 0, 0, cs - b.eq L(zva_64) -#endif - /* Small-size or non-zero memset does not use DC ZVA. */ - sub count, dstend, dst - - /* - * Adjust count and bias for loop. By substracting extra 1 from count, - * it is easy to use tbz instruction to check whether loop tailing - * count is less than 33 bytes, so as to bypass 2 unneccesary stps. - */ - sub count, count, 64+16+1 - -#if DC_ZVA_THRESHOLD - /* Align loop on 16-byte boundary, this might be friendly to i-cache. */ - nop -#endif - -1: stp val, val, [dst, 16] - stp val, val, [dst, 32] - stp val, val, [dst, 48] - stp val, val, [dst, 64]! - subs count, count, 64 - b.hs 1b - - tbz count, 5, 1f /* Remaining count is less than 33 bytes? */ - stp val, val, [dst, 16] - stp val, val, [dst, 32] -1: stp val, val, [dstend, -32] - stp val, val, [dstend, -16] - ret - -#if DC_ZVA_THRESHOLD - .p2align 4 -L(zva_64): - stp val, val, [dst, 16] - stp val, val, [dst, 32] - stp val, val, [dst, 48] - bic dst, dst, 63 - - /* - * Previous memory writes might cross cache line boundary, and cause - * cache line partially dirty. Zeroing this kind of cache line using - * DC ZVA will incur extra cost, for it requires loading untouched - * part of the line from memory before zeoring. - * - * So, write the first 64 byte aligned block using stp to force - * fully dirty cache line. - */ - stp val, val, [dst, 64] - stp val, val, [dst, 80] - stp val, val, [dst, 96] - stp val, val, [dst, 112] - - sub count, dstend, dst - /* - * Adjust count and bias for loop. By substracting extra 1 from count, - * it is easy to use tbz instruction to check whether loop tailing - * count is less than 33 bytes, so as to bypass 2 unneccesary stps. - */ - sub count, count, 128+64+64+1 - add dst, dst, 128 - nop - - /* DC ZVA sets 64 bytes each time. */ -1: dc zva, dst - add dst, dst, 64 - subs count, count, 64 - b.hs 1b - - /* - * Write the last 64 byte aligned block using stp to force fully - * dirty cache line. - */ - stp val, val, [dst, 0] - stp val, val, [dst, 16] - stp val, val, [dst, 32] - stp val, val, [dst, 48] - - tbz count, 5, 1f /* Remaining count is less than 33 bytes? */ - stp val, val, [dst, 64] - stp val, val, [dst, 80] -1: stp val, val, [dstend, -32] - stp val, val, [dstend, -16] - ret -#endif - - -END (memset) diff --git a/libraries/libmesosphere/source/libc/arch/arm64/memset.arch.arm64.s b/libraries/libmesosphere/source/libc/arch/arm64/memset.arch.arm64.s new file mode 100644 index 0000000..700f0e8 --- /dev/null +++ b/libraries/libmesosphere/source/libc/arch/arm64/memset.arch.arm64.s @@ -0,0 +1,172 @@ +/* + * memset - fill memory with a constant byte + * + * Copyright (c) 2012-2020, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, Advanced SIMD, unaligned accesses. + * + */ + +#include "asmdefs.h" + +#define DC_ZVA_THRESHOLD 512 + +#define dstin x0 +#define val x1 +#define valw w1 +#define count x2 +#define dst x3 +#define dstend x4 +#define zva_val x5 + +ENTRY (memset) + + bfi valw, valw, 8, 8 + bfi valw, valw, 16, 16 + bfi val, val, 32, 32 + + add dstend, dstin, count + + cmp count, 96 + b.hi L(set_long) + cmp count, 16 + b.hs L(set_medium) + + /* Set 0..15 bytes. */ + tbz count, 3, 1f + str val, [dstin] + str val, [dstend, -8] + ret +1: tbz count, 2, 2f + str valw, [dstin] + str valw, [dstend, -4] + ret +2: cbz count, 3f + strb valw, [dstin] + tbz count, 1, 3f + strh valw, [dstend, -2] +3: ret + + /* Set 16..96 bytes. */ + .p2align 4 +L(set_medium): + stp val, val, [dstin] + tbnz count, 6, L(set96) + stp val, val, [dstend, -16] + tbz count, 5, 1f + stp val, val, [dstin, 16] + stp val, val, [dstend, -32] +1: ret + + .p2align 4 + /* Set 64..96 bytes. Write 64 bytes from the start and + 32 bytes from the end. */ +L(set96): + stp val, val, [dstin, 16] + stp val, val, [dstin, 32] + stp val, val, [dstin, 48] + stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret + + .p2align 4 +L(set_long): + stp val, val, [dstin] +#if DC_ZVA_THRESHOLD + cmp count, DC_ZVA_THRESHOLD + ccmp val, 0, 0, cs + bic dst, dstin, 15 + b.eq L(zva_64) +#else + bic dst, dstin, 15 +#endif + /* Small-size or non-zero memset does not use DC ZVA. */ + sub count, dstend, dst + + /* + * Adjust count and bias for loop. By substracting extra 1 from count, + * it is easy to use tbz instruction to check whether loop tailing + * count is less than 33 bytes, so as to bypass 2 unneccesary stps. + */ + sub count, count, 64+16+1 + +#if DC_ZVA_THRESHOLD + /* Align loop on 16-byte boundary, this might be friendly to i-cache. */ + nop +#endif + +1: stp val, val, [dst, 16] + stp val, val, [dst, 32] + stp val, val, [dst, 48] + stp val, val, [dst, 64]! + subs count, count, 64 + b.hs 1b + + tbz count, 5, 1f /* Remaining count is less than 33 bytes? */ + stp val, val, [dst, 16] + stp val, val, [dst, 32] +1: stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret + +#if DC_ZVA_THRESHOLD + .p2align 4 +L(zva_64): + stp val, val, [dst, 16] + stp val, val, [dst, 32] + stp val, val, [dst, 48] + bic dst, dst, 63 + + /* + * Previous memory writes might cross cache line boundary, and cause + * cache line partially dirty. Zeroing this kind of cache line using + * DC ZVA will incur extra cost, for it requires loading untouched + * part of the line from memory before zeoring. + * + * So, write the first 64 byte aligned block using stp to force + * fully dirty cache line. + */ + stp val, val, [dst, 64] + stp val, val, [dst, 80] + stp val, val, [dst, 96] + stp val, val, [dst, 112] + + sub count, dstend, dst + /* + * Adjust count and bias for loop. By substracting extra 1 from count, + * it is easy to use tbz instruction to check whether loop tailing + * count is less than 33 bytes, so as to bypass 2 unneccesary stps. + */ + sub count, count, 128+64+64+1 + add dst, dst, 128 + nop + + /* DC ZVA sets 64 bytes each time. */ +1: dc zva, dst + add dst, dst, 64 + subs count, count, 64 + b.hs 1b + + /* + * Write the last 64 byte aligned block using stp to force fully + * dirty cache line. + */ + stp val, val, [dst, 0] + stp val, val, [dst, 16] + stp val, val, [dst, 32] + stp val, val, [dst, 48] + + tbz count, 5, 1f /* Remaining count is less than 33 bytes? */ + stp val, val, [dst, 64] + stp val, val, [dst, 80] +1: stp val, val, [dstend, -32] + stp val, val, [dstend, -16] + ret +#endif + + +END (memset) diff --git a/libraries/libmesosphere/source/libc/kern_libc_config.arch.arm64.h b/libraries/libmesosphere/source/libc/kern_libc_config.arch.arm64.h index 155b013..5d01bc7 100644 --- a/libraries/libmesosphere/source/libc/kern_libc_config.arch.arm64.h +++ b/libraries/libmesosphere/source/libc/kern_libc_config.arch.arm64.h @@ -19,6 +19,6 @@ #define MESOSPHERE_LIBC_MEMCPY_GENERIC 0 #define MESOSPHERE_LIBC_MEMCMP_GENERIC 0 #define MESOSPHERE_LIBC_MEMMOVE_GENERIC 0 -#define MESOSPHERE_LIBC_MEMSET_GENERIC 1 +#define MESOSPHERE_LIBC_MEMSET_GENERIC 0 #define MESOSPHERE_LIBC_STRNCPY_GENERIC 1 -#define MESOSPHERE_LIBC_STRNCMP_GENERIC 1 \ No newline at end of file +#define MESOSPHERE_LIBC_STRNCMP_GENERIC 1 diff --git a/libraries/libmesosphere/source/libc/kern_libc_config.h b/libraries/libmesosphere/source/libc/kern_libc_config.h index b391b99..f79fb75 100644 --- a/libraries/libmesosphere/source/libc/kern_libc_config.h +++ b/libraries/libmesosphere/source/libc/kern_libc_config.h @@ -23,4 +23,4 @@ #error "Unknown architecture for libc" -#endif \ No newline at end of file +#endif diff --git a/mesosphere/kernel_ldr/source/kern_init_loader.cpp b/mesosphere/kernel_ldr/source/kern_init_loader.cpp index bd4fe4d..504a046 100644 --- a/mesosphere/kernel_ldr/source/kern_init_loader.cpp +++ b/mesosphere/kernel_ldr/source/kern_init_loader.cpp @@ -241,9 +241,6 @@ RelocateKernelPhysically(base_address, layout); /* Validate kernel layout. */ - /* TODO: constexpr 0x1000 definition somewhere. */ - /* In stratosphere, this is os::MemoryPageSize. */ - /* We don't have ams::os, this may go in hw:: or something. */ const uintptr_t rx_offset = layout->rx_offset; const uintptr_t rx_end_offset = layout->rx_end_offset; const uintptr_t ro_offset = layout->ro_offset; @@ -251,12 +248,12 @@ const uintptr_t rw_offset = layout->rw_offset; /* UNUSED: const uintptr_t rw_end_offset = layout->rw_end_offset; */ const uintptr_t bss_end_offset = layout->bss_end_offset; - MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rx_offset, 0x1000)); - MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rx_end_offset, 0x1000)); - MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(ro_offset, 0x1000)); - MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(ro_end_offset, 0x1000)); - MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rw_offset, 0x1000)); - MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(bss_end_offset, 0x1000)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rx_offset, PageSize)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rx_end_offset, PageSize)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(ro_offset, PageSize)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(ro_end_offset, PageSize)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rw_offset, PageSize)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(bss_end_offset, PageSize)); const uintptr_t bss_offset = layout->bss_offset; const uintptr_t ini_load_offset = layout->ini_load_offset; const uintptr_t dynamic_offset = layout->dynamic_offset;