diff --git a/libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp b/libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp index 1611e68..bb4d4e9 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_debug_log.hpp @@ -33,7 +33,7 @@ #ifndef MESOSPHERE_DEBUG_LOG_SELECTED #ifdef ATMOSPHERE_BOARD_NINTENDO_SWITCH - #define MESOSPHERE_DEBUG_LOG_USE_UART_A + #define MESOSPHERE_DEBUG_LOG_USE_UART_C #else #error "Unknown board for Default Debug Log Source" #endif diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp index 1f5f42e..4903b4f 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp @@ -50,9 +50,9 @@ private: friend class KScopedSchedulerLock; friend class KScopedSchedulerLockAndSleep; - static inline bool s_scheduler_update_needed; - static inline LockType s_scheduler_lock; - static inline KSchedulerPriorityQueue s_priority_queue; + static bool s_scheduler_update_needed; + static LockType s_scheduler_lock; + static KSchedulerPriorityQueue s_priority_queue; private: SchedulingState state; bool is_active; diff --git a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp index c720d7f..16b0c91 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -173,7 +173,21 @@ s8 priority_inheritance_count; bool resource_limit_release_hint; public: - explicit KThread() /* TODO: : ? */ { MESOSPHERE_ASSERT_THIS(); } + constexpr KThread() : + thread_context(), affinity_mask(), thread_id(), cpu_time(), synced_object(), waiting_lock(), + condvar_key(), entrypoint(), arbiter_key(), parent(), kernel_stack_top(), light_ipc_data(), + tls_address(), tls_heap_address(), activity_pause_lock(), sync_object_buffer(), schedule_count(), + last_scheduled_tick(), per_core_priority_queue_entry(), sleeping_queue_entry(), sleeping_queue(), waiter_list_node(), + condvar_arbiter_tree_node(), process_list_node(), waiter_list(), paused_waiter_list(), lock_owner(), + cond_var_tree(), debug_params(), arbiter_value(), suspend_request_flags(), suspend_allowed_flags(), + wait_result(ResultSuccess()), debug_exception_result(ResultSuccess()), priority(), core_id(), base_priority(), + ideal_core_id(), num_kernel_waiters(), original_affinity_mask(), original_ideal_core_id(), num_core_migration_disables(), + thread_state(), termination_requested(), ipc_cancelled(), wait_cancelled(), cancellable(), + registered(), signaled(), initialized(), debug_attached(), priority_inheritance_count(), + resource_limit_release_hint() + { + /* ... */ + } virtual ~KThread() { /* ... */ } /* TODO: Is a constexpr KThread() possible? */ diff --git a/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp b/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp index 4592529..2ba8599 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_kernel.hpp @@ -62,8 +62,6 @@ static constexpr size_t BlockInfoSlabHeapSize = 4000; private: static State s_state; - static KThread s_main_threads[cpu::NumCores]; - static KThread s_idle_threads[cpu::NumCores]; static KResourceLimit s_system_resource_limit; static KMemoryManager s_memory_manager; static KPageTableManager s_page_table_manager; @@ -87,13 +85,8 @@ static ALWAYS_INLINE State GetState() { return s_state; } static ALWAYS_INLINE void SetState(State state) { s_state = state; } - static ALWAYS_INLINE KThread &GetMainThread(s32 core_id) { - return s_main_threads[core_id]; - } - - static ALWAYS_INLINE KThread &GetIdleThread(s32 core_id) { - return s_idle_threads[core_id]; - } + static KThread &GetMainThread(s32 core_id); + static KThread &GetIdleThread(s32 core_id); static ALWAYS_INLINE KScheduler &GetScheduler() { return GetCoreLocalContext().scheduler; diff --git a/libraries/libmesosphere/include/mesosphere/kern_panic.hpp b/libraries/libmesosphere/include/mesosphere/kern_panic.hpp index 2402b7d..4e1717c 100644 --- a/libraries/libmesosphere/include/mesosphere/kern_panic.hpp +++ b/libraries/libmesosphere/include/mesosphere/kern_panic.hpp @@ -42,8 +42,8 @@ #define MESOSPHERE_ASSERT_IMPL(expr, ...) do { static_cast(expr); } while (0) #endif -#define MESOSPHERE_ASSERT(expr) MESOSPHERE_ASSERT_IMPL(expr, "Assertion failed: %s", #expr) -#define MESOSPHERE_R_ASSERT(expr) MESOSPHERE_ASSERT_IMPL(R_SUCCEEDED(expr), "Result assertion failed: %s", #expr) +#define MESOSPHERE_ASSERT(expr) MESOSPHERE_ASSERT_IMPL(expr, "Assertion failed: %s\n", #expr) +#define MESOSPHERE_R_ASSERT(expr) MESOSPHERE_ASSERT_IMPL(R_SUCCEEDED(expr), "Result assertion failed: %s\n", #expr) #define MESOSPHERE_UNREACHABLE_DEFAULT_CASE() default: MESOSPHERE_PANIC("Unreachable default case entered") #ifdef MESOSPHERE_ENABLE_THIS_ASSERT @@ -58,10 +58,10 @@ #define MESOSPHERE_AUDIT(expr) do { static_cast(expr); } while (0) #endif -#define MESOSPHERE_TODO(arg) ({ constexpr const char *__mesosphere_todo = arg; MESOSPHERE_PANIC("TODO (%s): %s", __PRETTY_FUNCTION__, __mesosphere_todo); }) +#define MESOSPHERE_TODO(arg) ({ constexpr const char *__mesosphere_todo = arg; MESOSPHERE_PANIC("TODO (%s): %s\n", __PRETTY_FUNCTION__, __mesosphere_todo); }) #define MESOSPHERE_TODO_IMPLEMENT() MESOSPHERE_TODO("Implement") -#define MESOSPHERE_ABORT() MESOSPHERE_PANIC("Abort()"); +#define MESOSPHERE_ABORT() MESOSPHERE_PANIC("Abort()\n"); #define MESOSPHERE_INIT_ABORT() do { /* ... */ } while (true) #define MESOSPHERE_ABORT_UNLESS(expr) \ @@ -80,10 +80,10 @@ } \ }) -#define MESOSPHERE_R_ABORT_UNLESS(expr) \ - ({ \ - const ::ams::Result _tmp_meso_r_abort_res = static_cast<::ams::Result>((expr)); \ - if (AMS_UNLIKELY((R_FAILED(_tmp_meso_r_abort_res)))) { \ - MESOSPHERE_PANIC("Result Abort(): %s 2%03d-%04d", #expr, _tmp_meso_r_abort_res.GetModule(), _tmp_meso_r_abort_res.GetDescription()); \ - } \ +#define MESOSPHERE_R_ABORT_UNLESS(expr) \ + ({ \ + const ::ams::Result _tmp_meso_r_abort_res = static_cast<::ams::Result>((expr)); \ + if (AMS_UNLIKELY((R_FAILED(_tmp_meso_r_abort_res)))) { \ + MESOSPHERE_PANIC("Result Abort(): %s 2%03d-%04d\n", #expr, _tmp_meso_r_abort_res.GetModule(), _tmp_meso_r_abort_res.GetDescription()); \ + } \ }) diff --git a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s b/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s deleted file mode 100644 index 68983e5..0000000 --- a/libraries/libmesosphere/source/arch/arm64/kern_exception_handlers_asm.s +++ /dev/null @@ -1,610 +0,0 @@ -/* - * Copyright (c) 2018-2020 Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -/* ams::kern::arm64::EL1IrqExceptionHandler() */ -.section .text._ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, "ax", %progbits -.global _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv -.type _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, %function -_ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv: - /* Save registers that need saving. */ - sub sp, sp, #(8 * 24) - - stp x0, x1, [sp, #(8 * 0)] - stp x2, x3, [sp, #(8 * 2)] - stp x4, x5, [sp, #(8 * 4)] - stp x6, x7, [sp, #(8 * 6)] - stp x8, x9, [sp, #(8 * 8)] - stp x10, x11, [sp, #(8 * 10)] - stp x12, x13, [sp, #(8 * 12)] - stp x14, x15, [sp, #(8 * 14)] - stp x16, x17, [sp, #(8 * 16)] - stp x18, x19, [sp, #(8 * 18)] - stp x20, x21, [sp, #(8 * 20)] - stp x22, x30, [sp, #(8 * 22)] - - mrs x19, sp_el0 - mrs x20, elr_el1 - mrs x21, spsr_el1 - mov w21, w21 - - /* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */ - mrs x18, tpidr_el1 - mov x0, #0 - bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb - - /* Restore registers that we saved. */ - msr sp_el0, x19 - msr elr_el1, x20 - msr spsr_el1, x21 - - ldp x0, x1, [sp, #(8 * 0)] - ldp x2, x3, [sp, #(8 * 2)] - ldp x4, x5, [sp, #(8 * 4)] - ldp x6, x7, [sp, #(8 * 6)] - ldp x8, x9, [sp, #(8 * 8)] - ldp x10, x11, [sp, #(8 * 10)] - ldp x12, x13, [sp, #(8 * 12)] - ldp x14, x15, [sp, #(8 * 14)] - ldp x16, x17, [sp, #(8 * 16)] - ldp x18, x19, [sp, #(8 * 18)] - ldp x20, x21, [sp, #(8 * 20)] - ldp x22, x30, [sp, #(8 * 22)] - - add sp, sp, #(8 * 24) - - /* Return from the exception. */ - eret - -/* ams::kern::arm64::EL0IrqExceptionHandler() */ -.section .text._ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, "ax", %progbits -.global _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv -.type _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, %function -_ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv: - /* Save registers that need saving. */ - sub sp, sp, #(8 * 36) - - stp x0, x1, [sp, #(8 * 0)] - stp x2, x3, [sp, #(8 * 2)] - stp x4, x5, [sp, #(8 * 4)] - stp x6, x7, [sp, #(8 * 6)] - stp x8, x9, [sp, #(8 * 8)] - stp x10, x11, [sp, #(8 * 10)] - stp x12, x13, [sp, #(8 * 12)] - stp x14, x15, [sp, #(8 * 14)] - stp x16, x17, [sp, #(8 * 16)] - stp x18, x19, [sp, #(8 * 18)] - stp x20, x21, [sp, #(8 * 20)] - stp x22, x23, [sp, #(8 * 22)] - stp x24, x25, [sp, #(8 * 24)] - stp x26, x27, [sp, #(8 * 26)] - stp x28, x29, [sp, #(8 * 28)] - - mrs x20, sp_el0 - mrs x21, elr_el1 - mrs x22, spsr_el1 - mrs x23, tpidr_el0 - mov w22, w22 - stp x30, x20, [sp, #(8 * 30)] - stp x21, x22, [sp, #(8 * 32)] - str x23, [sp, #(8 * 34)] - - /* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */ - mrs x18, tpidr_el1 - mov x0, #1 - bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb - - /* Restore state from the context. */ - ldp x30, x20, [sp, #(8 * 30)] - ldp x21, x22, [sp, #(8 * 32)] - ldr x23, [sp, #(8 * 34)] - msr sp_el0, x20 - msr elr_el1, x21 - msr spsr_el1, x22 - msr tpidr_el0, x23 - ldp x0, x1, [sp, #(8 * 0)] - ldp x2, x3, [sp, #(8 * 2)] - ldp x4, x5, [sp, #(8 * 4)] - ldp x6, x7, [sp, #(8 * 6)] - ldp x8, x9, [sp, #(8 * 8)] - ldp x10, x11, [sp, #(8 * 10)] - ldp x12, x13, [sp, #(8 * 12)] - ldp x14, x15, [sp, #(8 * 14)] - ldp x16, x17, [sp, #(8 * 16)] - ldp x18, x19, [sp, #(8 * 18)] - ldp x20, x21, [sp, #(8 * 20)] - ldp x22, x23, [sp, #(8 * 22)] - ldp x24, x25, [sp, #(8 * 24)] - ldp x26, x27, [sp, #(8 * 26)] - ldp x28, x29, [sp, #(8 * 28)] - add sp, sp, #0x120 - - /* Return from the exception. */ - eret - -/* ams::kern::arm64::EL0SynchronousExceptionHandler() */ -.section .text._ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, "ax", %progbits -.global _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv -.type _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, %function -_ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv: - /* Save x16 and x17, so that we can use them as scratch. */ - stp x16, x17, [sp, #-16]! - - /* Get and parse the exception syndrome register. */ - mrs x16, esr_el1 - lsr x17, x16, #0x1a - - /* Is this an aarch32 SVC? */ - cmp x17, #0x11 - b.eq 2f - - /* Is this an aarch64 SVC? */ - cmp x17, #0x15 - b.eq 3f - - /* Is this an FPU error? */ - cmp x17, #0x7 - b.eq 4f - - /* Is this an instruction abort? */ - cmp x17, #0x21 - b.eq 5f - - /* Is this a data abort? */ - cmp x17, #0x25 - b.eq 5f - -1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */ - /* It is also not an SVC or an FPU exception. Handle it generically! */ - - /* Restore x16 and x17. */ - ldp x16, x17, [sp], 16 - - /* Create a KExceptionContext to pass to HandleException. */ - sub sp, sp, #0x120 - stp x0, x1, [sp, #(8 * 0)] - stp x2, x3, [sp, #(8 * 2)] - stp x4, x5, [sp, #(8 * 4)] - stp x6, x7, [sp, #(8 * 6)] - stp x8, x9, [sp, #(8 * 8)] - stp x10, x11, [sp, #(8 * 10)] - stp x12, x13, [sp, #(8 * 12)] - stp x14, x15, [sp, #(8 * 14)] - stp x16, x17, [sp, #(8 * 16)] - stp x18, x19, [sp, #(8 * 18)] - stp x20, x21, [sp, #(8 * 20)] - stp x22, x23, [sp, #(8 * 22)] - stp x24, x25, [sp, #(8 * 24)] - stp x26, x27, [sp, #(8 * 26)] - stp x28, x29, [sp, #(8 * 28)] - mrs x20, sp_el0 - mrs x21, elr_el1 - mrs x22, spsr_el1 - mrs x23, tpidr_el0 - mov w22, w22 - stp x30, x20, [sp, #(8 * 30)] - stp x21, x22, [sp, #(8 * 32)] - str x23, [sp, #(8 * 34)] - - /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ - mrs x18, tpidr_el1 - mov x0, sp - bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE - - /* Restore state from the context. */ - ldp x30, x20, [sp, #(8 * 30)] - ldp x21, x22, [sp, #(8 * 32)] - ldr x23, [sp, #(8 * 34)] - msr sp_el0, x20 - msr elr_el1, x21 - msr spsr_el1, x22 - msr tpidr_el0, x23 - ldp x0, x1, [sp, #(8 * 0)] - ldp x2, x3, [sp, #(8 * 2)] - ldp x4, x5, [sp, #(8 * 4)] - ldp x6, x7, [sp, #(8 * 6)] - ldp x8, x9, [sp, #(8 * 8)] - ldp x10, x11, [sp, #(8 * 10)] - ldp x12, x13, [sp, #(8 * 12)] - ldp x14, x15, [sp, #(8 * 14)] - ldp x16, x17, [sp, #(8 * 16)] - ldp x18, x19, [sp, #(8 * 18)] - ldp x20, x21, [sp, #(8 * 20)] - ldp x22, x23, [sp, #(8 * 22)] - ldp x24, x25, [sp, #(8 * 24)] - ldp x26, x27, [sp, #(8 * 26)] - ldp x28, x29, [sp, #(8 * 28)] - add sp, sp, #0x120 - - /* Return from the exception. */ - eret - -2: /* SVC from aarch32. */ - ldp x16, x17, [sp], 16 - b _ZN3ams4kern5arm6412SvcHandler32Ev - -3: /* SVC from aarch64. */ - ldp x16, x17, [sp], 16 - b _ZN3ams4kern5arm6412SvcHandler64Ev - -4: /* FPU exception. */ - ldp x16, x17, [sp], 16 - b _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv - -5: /* Check if there's a TLB conflict that caused the abort. */ - and x17, x16, #0x3F - cmp x17, #0x30 - b.ne 1b - - /* Get the ASID in x17. */ - mrs x17, ttbr0_el1 - and x17, x17, #(0xFFFF << 48) - - /* Check if FAR is valid by examining the FnV bit. */ - tbnz x16, #10, 6f - - /* FAR is valid, so we can invalidate the address it holds. */ - mrs x16, far_el1 - lsr x16, x16, #12 - orr x17, x16, x17 - tlbi vaae1, x17 - b 7f - -6: /* There's a TLB conflict and FAR isn't valid. */ - /* Invalidate the entire TLB. */ - tlbi vmalle1 - -7: /* Return from a TLB conflict. */ - /* Ensure instruction consistency. */ - dsb ish - isb - - /* Restore x16 and x17. */ - ldp x16, x17, [sp], 16 - - /* Return from the exception. */ - eret - - -/* ams::kern::arm64::EL1SynchronousExceptionHandler() */ -.section .text._ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, "ax", %progbits -.global _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv -.type _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, %function -_ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv: - /* Nintendo uses the "unused" virtual timer compare value as a scratch register. */ - msr cntv_cval_el0, x0 - - /* Get and parse the exception syndrome register. */ - mrs x0, esr_el1 - lsr x0, x0, #0x1a - - /* Is this an instruction abort? */ - cmp x0, #0x21 - b.eq 5f - - /* Is this a data abort? */ - cmp x0, #0x25 - b.eq 5f - -1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */ - /* Load the CoreLocalContext into x0. */ - mrs x0, tpidr_el1 - cbz x0, 2f - - /* Load the exception stack top from the context. */ - ldr x0, [x0, #0x28] - - /* Setup the stack for a generic exception handle */ - sub x0, x0, #0x20 - str x1, [x0, #16] - mov x1, sp - str x1, [x0] - mov sp, x0 - ldr x1, [x0, #16] - mrs x0, cntv_cval_el0 - str x0, [sp, #8] - - /* Check again if this is a data abort from EL1. */ - mrs x0, esr_el1 - lsr x1, x0, #0x1a - cmp x1, #0x25 - b.ne 3f - - /* Data abort. Check if it was from trying to access userspace memory. */ - mrs x1, elr_el1 - adr x0, _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv - cmp x1, x0 - b.lo 3f - adr x0, _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv - cmp x1, x0 - b.hs 3f - - /* We aborted trying to access userspace memory. */ - /* All functions that access user memory return a boolean for whether they succeeded. */ - /* With that in mind, we can simply restore the stack pointer and return false directly. */ - ldr x0, [sp] - mov sp, x0 - - /* Return false. */ - mov x0, #0x0 - msr elr_el1, x30 - eret - -2: /* The CoreLocalContext is nullptr. */ - /* Setup the stack for a generic exception handle. */ - /* NOTE: Nintendo does not restore X0 here, and thus saves nullptr. */ - /* This is probably not their intention, so we'll fix it. */ - /* NOTE: Nintendo also does not really save SP correctly, and so we */ - /* will also fix that. */ - mov x0, sp - sub x0, x0, #0x20 - str x1, [x0, #16] - mov x1, sp - str x1, [x0] - mov sp, x0 - mrs x0, cntv_cval_el0 - str x0, [sp, #8] - -3: /* The exception wasn't an triggered by copying memory from userspace. */ - ldr x0, [sp, #8] - ldr x1, [sp, #16] - - /* Create a KExceptionContext to pass to HandleException. */ - sub sp, sp, #0x120 - stp x0, x1, [sp, #(8 * 0)] - stp x2, x3, [sp, #(8 * 2)] - stp x4, x5, [sp, #(8 * 4)] - stp x6, x7, [sp, #(8 * 6)] - stp x8, x9, [sp, #(8 * 8)] - stp x10, x11, [sp, #(8 * 10)] - stp x12, x13, [sp, #(8 * 12)] - stp x14, x15, [sp, #(8 * 14)] - stp x16, x17, [sp, #(8 * 16)] - stp x18, x19, [sp, #(8 * 18)] - stp x20, x21, [sp, #(8 * 20)] - stp x22, x23, [sp, #(8 * 22)] - stp x24, x25, [sp, #(8 * 24)] - stp x26, x27, [sp, #(8 * 26)] - stp x28, x29, [sp, #(8 * 28)] - mrs x20, sp_el0 - mrs x21, elr_el1 - mrs x22, spsr_el1 - mrs x23, tpidr_el0 - mov w22, w22 - stp x30, x20, [sp, #(8 * 30)] - stp x21, x22, [sp, #(8 * 32)] - str x23, [sp, #(8 * 34)] - - /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ - mrs x18, tpidr_el1 - mov x0, sp - bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE - -4: /* HandleException should never return. The best we can do is infinite loop. */ - b 4b - -5: /* Check if there's a TLB conflict that caused the abort. */ - /* NOTE: There is a Nintendo bug in this code that we correct. */ - /* Nintendo compares the low 6 bits of x0 without restoring the value. */ - /* They intend to check the DFSC/IFSC bits of esr_el1, but because they */ - /* shifted esr earlier, the check is invalid and always fails. */ - mrs x0, esr_el1 - and x0, x0, #0x3F - cmp x0, #0x30 - b.ne 1b - - /* Check if FAR is valid by examining the FnV bit. */ - /* NOTE: Nintendo again has a bug here, the same as above. */ - /* They do not refresh the value of x0, and again compare with */ - /* the relevant bit already masked out of x0. */ - mrs x0, esr_el1 - tbnz x0, #10, 6f - - /* FAR is valid, so we can invalidate the address it holds. */ - mrs x0, far_el1 - lsr x0, x0, #12 - tlbi vaae1, x0 - b 7f - -6: /* There's a TLB conflict and FAR isn't valid. */ - /* Invalidate the entire TLB. */ - tlbi vmalle1 - -7: /* Return from a TLB conflict. */ - /* Ensure instruction consistency. */ - dsb ish - isb - - /* Restore x0 from scratch. */ - mrs x0, cntv_cval_el0 - - /* Return from the exception. */ - eret - - -/* ams::kern::arm64::FpuAccessExceptionHandler() */ -.section .text._ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, "ax", %progbits -.global _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv -.type _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, %function -_ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv: - /* Save registers that need saving. */ - sub sp, sp, #(8 * 24) - - stp x0, x1, [sp, #(8 * 0)] - stp x2, x3, [sp, #(8 * 2)] - stp x4, x5, [sp, #(8 * 4)] - stp x6, x7, [sp, #(8 * 6)] - stp x8, x9, [sp, #(8 * 8)] - stp x10, x11, [sp, #(8 * 10)] - stp x12, x13, [sp, #(8 * 12)] - stp x14, x15, [sp, #(8 * 14)] - stp x16, x17, [sp, #(8 * 16)] - stp x18, x19, [sp, #(8 * 18)] - stp x20, x21, [sp, #(8 * 20)] - stp x22, x30, [sp, #(8 * 22)] - - mrs x18, tpidr_el1 - mrs x19, sp_el0 - mrs x20, elr_el1 - mrs x21, spsr_el1 - - /* Invoke the FPU context switch handler. */ - bl _ZN3ams4kern5arm6423FpuContextSwitchHandlerEv - - /* Restore registers that we saved. */ - msr sp_el0, x19 - msr elr_el1, x20 - msr spsr_el1, x21 - - ldp x0, x1, [sp, #(8 * 0)] - ldp x2, x3, [sp, #(8 * 2)] - ldp x4, x5, [sp, #(8 * 4)] - ldp x6, x7, [sp, #(8 * 6)] - ldp x8, x9, [sp, #(8 * 8)] - ldp x10, x11, [sp, #(8 * 10)] - ldp x12, x13, [sp, #(8 * 12)] - ldp x14, x15, [sp, #(8 * 14)] - ldp x16, x17, [sp, #(8 * 16)] - ldp x18, x19, [sp, #(8 * 18)] - ldp x20, x21, [sp, #(8 * 20)] - ldp x22, x30, [sp, #(8 * 22)] - - add sp, sp, #(8 * 24) - - /* Return from the exception. */ - eret - -/* ams::kern::arm64::EL1SystemErrorHandler() */ -.section .text._ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, "ax", %progbits -.global _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv -.type _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, %function -_ZN3ams4kern5arm6421EL1SystemErrorHandlerEv: - /* Nintendo uses the "unused" virtual timer compare value as a scratch register. */ - msr cntv_cval_el0, x0 - - /* Load the exception stack top from the context. */ - ldr x0, [x0, #0x28] - - /* Setup the stack for a generic exception handle */ - sub x0, x0, #0x20 - str x1, [x0, #16] - mov x1, sp - str x1, [x0] - mov sp, x0 - ldr x1, [x0, #16] - mrs x0, cntv_cval_el0 - str x0, [sp, #8] - - /* Create a KExceptionContext to pass to HandleException. */ - sub sp, sp, #0x120 - stp x0, x1, [sp, #(8 * 0)] - stp x2, x3, [sp, #(8 * 2)] - stp x4, x5, [sp, #(8 * 4)] - stp x6, x7, [sp, #(8 * 6)] - stp x8, x9, [sp, #(8 * 8)] - stp x10, x11, [sp, #(8 * 10)] - stp x12, x13, [sp, #(8 * 12)] - stp x14, x15, [sp, #(8 * 14)] - stp x16, x17, [sp, #(8 * 16)] - stp x18, x19, [sp, #(8 * 18)] - stp x20, x21, [sp, #(8 * 20)] - stp x22, x23, [sp, #(8 * 22)] - stp x24, x25, [sp, #(8 * 24)] - stp x26, x27, [sp, #(8 * 26)] - stp x28, x29, [sp, #(8 * 28)] - mrs x20, sp_el0 - mrs x21, elr_el1 - mrs x22, spsr_el1 - mrs x23, tpidr_el0 - mov w22, w22 - stp x30, x20, [sp, #(8 * 30)] - stp x21, x22, [sp, #(8 * 32)] - str x23, [sp, #(8 * 34)] - - /* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */ - mrs x18, tpidr_el1 - mov x0, sp - bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE - -1: /* HandleException should never return. The best we can do is infinite loop. */ - b 1b - - /* Return from the exception. */ - eret - -/* ams::kern::arm64::EL0SystemErrorHandler() */ -.section .text._ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, "ax", %progbits -.global _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv -.type _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, %function -_ZN3ams4kern5arm6421EL0SystemErrorHandlerEv: - /* Create a KExceptionContext to pass to HandleException. */ - sub sp, sp, #0x120 - stp x0, x1, [sp, #(8 * 0)] - stp x2, x3, [sp, #(8 * 2)] - stp x4, x5, [sp, #(8 * 4)] - stp x6, x7, [sp, #(8 * 6)] - stp x8, x9, [sp, #(8 * 8)] - stp x10, x11, [sp, #(8 * 10)] - stp x12, x13, [sp, #(8 * 12)] - stp x14, x15, [sp, #(8 * 14)] - stp x16, x17, [sp, #(8 * 16)] - stp x18, x19, [sp, #(8 * 18)] - stp x20, x21, [sp, #(8 * 20)] - stp x22, x23, [sp, #(8 * 22)] - stp x24, x25, [sp, #(8 * 24)] - stp x26, x27, [sp, #(8 * 26)] - stp x28, x29, [sp, #(8 * 28)] - mrs x20, sp_el0 - mrs x21, elr_el1 - mrs x22, spsr_el1 - mrs x23, tpidr_el0 - mov w22, w22 - stp x30, x20, [sp, #(8 * 30)] - stp x21, x22, [sp, #(8 * 32)] - str x23, [sp, #(8 * 34)] - - /* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */ - mrs x18, tpidr_el1 - mov x0, sp - bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE - - /* Restore state from the context. */ - ldp x30, x20, [sp, #(8 * 30)] - ldp x21, x22, [sp, #(8 * 32)] - ldr x23, [sp, #(8 * 34)] - msr sp_el0, x20 - msr elr_el1, x21 - msr spsr_el1, x22 - msr tpidr_el0, x23 - ldp x0, x1, [sp, #(8 * 0)] - ldp x2, x3, [sp, #(8 * 2)] - ldp x4, x5, [sp, #(8 * 4)] - ldp x6, x7, [sp, #(8 * 6)] - ldp x8, x9, [sp, #(8 * 8)] - ldp x10, x11, [sp, #(8 * 10)] - ldp x12, x13, [sp, #(8 * 12)] - ldp x14, x15, [sp, #(8 * 14)] - ldp x16, x17, [sp, #(8 * 16)] - ldp x18, x19, [sp, #(8 * 18)] - ldp x20, x21, [sp, #(8 * 20)] - ldp x22, x23, [sp, #(8 * 22)] - ldp x24, x25, [sp, #(8 * 24)] - ldp x26, x27, [sp, #(8 * 26)] - ldp x28, x29, [sp, #(8 * 28)] - add sp, sp, #0x120 - - /* Return from the exception. */ - eret - diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp index d2bd5e1..5add50a 100644 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp +++ b/libraries/libmesosphere/source/arch/arm64/kern_k_page_table.cpp @@ -181,7 +181,6 @@ Result KPageTable::MapContiguous(KProcessAddress virt_addr, KPhysicalAddress phys_addr, size_t num_pages, PageTableEntry entry_template, PageLinkedList *page_list, bool reuse_ll) { MESOSPHERE_ASSERT(this->IsLockedByCurrentThread()); - MESOSPHERE_LOG("KPageTable::MapContiguous(%016lx, %016lx, %zu)\n", GetInteger(virt_addr), GetInteger(phys_addr), num_pages); /* Cache initial addresses for use on cleanup. */ const KProcessAddress orig_virt_addr = virt_addr; diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_scheduler_asm.s b/libraries/libmesosphere/source/arch/arm64/kern_k_scheduler_asm.s deleted file mode 100644 index ce23648..0000000 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_scheduler_asm.s +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright (c) 2018-2020 Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#define SAVE_THREAD_CONTEXT(ctx, tmp0, tmp1, done_label) \ - /* Save the callee save registers + SP and cpacr. */ \ - mov tmp0, sp; \ - mrs tmp1, cpacr_el1; \ - stp x19, x20, [ctx, #(8 * 0)]; \ - stp x21, x22, [ctx, #(8 * 2)]; \ - stp x23, x24, [ctx, #(8 * 4)]; \ - stp x25, x26, [ctx, #(8 * 6)]; \ - stp x27, x28, [ctx, #(8 * 8)]; \ - stp x29, x30, [ctx, #(8 * 10)]; \ - \ - stp tmp0, tmp1, [ctx, #0x60]; \ - \ - /* Check whether the FPU is enabled. */ \ - /* If it isn't, skip saving FPU state. */ \ - and tmp1, tmp1, #0x300000; \ - cbz tmp1, done_label; \ - \ - /* Save fpcr and fpsr. */ \ - mrs tmp0, fpcr; \ - mrs tmp1, fpsr; \ - stp tmp0, tmp1, [ctx, #0x70]; \ - \ - /* Save the FPU registers. */ \ - stp q0, q1, [ctx, #(16 * 0 + 0x80)]; \ - stp q2, q3, [ctx, #(16 * 2 + 0x80)]; \ - stp q4, q5, [ctx, #(16 * 4 + 0x80)]; \ - stp q6, q7, [ctx, #(16 * 6 + 0x80)]; \ - stp q8, q9, [ctx, #(16 * 8 + 0x80)]; \ - stp q10, q11, [ctx, #(16 * 10 + 0x80)]; \ - stp q12, q13, [ctx, #(16 * 12 + 0x80)]; \ - stp q14, q15, [ctx, #(16 * 14 + 0x80)]; \ - stp q16, q17, [ctx, #(16 * 16 + 0x80)]; \ - stp q18, q19, [ctx, #(16 * 18 + 0x80)]; \ - stp q20, q21, [ctx, #(16 * 20 + 0x80)]; \ - stp q22, q23, [ctx, #(16 * 22 + 0x80)]; \ - stp q24, q25, [ctx, #(16 * 24 + 0x80)]; \ - stp q26, q27, [ctx, #(16 * 26 + 0x80)]; \ - stp q28, q29, [ctx, #(16 * 28 + 0x80)]; \ - stp q30, q31, [ctx, #(16 * 30 + 0x80)]; - -#define RESTORE_THREAD_CONTEXT(ctx, tmp0, tmp1, done_label) \ - /* Restore the callee save registers + SP and cpacr. */ \ - ldp tmp0, tmp1, [ctx, #0x60]; \ - mov sp, tmp0; \ - ldp x19, x20, [ctx, #(8 * 0)]; \ - ldp x21, x22, [ctx, #(8 * 2)]; \ - ldp x23, x24, [ctx, #(8 * 4)]; \ - ldp x25, x26, [ctx, #(8 * 6)]; \ - ldp x27, x28, [ctx, #(8 * 8)]; \ - ldp x29, x30, [ctx, #(8 * 10)]; \ - \ - msr cpacr_el1, tmp1; \ - isb; \ - \ - /* Check whether the FPU is enabled. */ \ - /* If it isn't, skip saving FPU state. */ \ - and tmp1, tmp1, #0x300000; \ - cbz tmp1, done_label; \ - \ - /* Save fpcr and fpsr. */ \ - ldp tmp0, tmp1, [ctx, #0x70]; \ - msr fpcr, tmp0; \ - msr fpsr, tmp1; \ - \ - /* Save the FPU registers. */ \ - ldp q0, q1, [ctx, #(16 * 0 + 0x80)]; \ - ldp q2, q3, [ctx, #(16 * 2 + 0x80)]; \ - ldp q4, q5, [ctx, #(16 * 4 + 0x80)]; \ - ldp q6, q7, [ctx, #(16 * 6 + 0x80)]; \ - ldp q8, q9, [ctx, #(16 * 8 + 0x80)]; \ - ldp q10, q11, [ctx, #(16 * 10 + 0x80)]; \ - ldp q12, q13, [ctx, #(16 * 12 + 0x80)]; \ - ldp q14, q15, [ctx, #(16 * 14 + 0x80)]; \ - ldp q16, q17, [ctx, #(16 * 16 + 0x80)]; \ - ldp q18, q19, [ctx, #(16 * 18 + 0x80)]; \ - ldp q20, q21, [ctx, #(16 * 20 + 0x80)]; \ - ldp q22, q23, [ctx, #(16 * 22 + 0x80)]; \ - ldp q24, q25, [ctx, #(16 * 24 + 0x80)]; \ - ldp q26, q27, [ctx, #(16 * 26 + 0x80)]; \ - ldp q28, q29, [ctx, #(16 * 28 + 0x80)]; \ - ldp q30, q31, [ctx, #(16 * 30 + 0x80)]; - - -/* ams::kern::KScheduler::ScheduleImpl() */ -.section .text._ZN3ams4kern10KScheduler12ScheduleImplEv, "ax", %progbits -.global _ZN3ams4kern10KScheduler12ScheduleImplEv -.type _ZN3ams4kern10KScheduler12ScheduleImplEv, %function - -/* Ensure ScheduleImpl is aligned to 0x40 bytes. */ -.balign 0x40 - -_ZN3ams4kern10KScheduler12ScheduleImplEv: - /* Right now, x0 contains (this). We want x1 to point to the scheduling state, */ - /* Current KScheduler layout has state at +0x0. */ - mov x1, x0 - - /* First thing we want to do is check whether the interrupt task thread is runnable. */ - ldrb w3, [x1, #1] - cbz w3, 0f - - /* If it is, we want to call KScheduler::SetInterruptTaskThreadRunnable() to note it runnable. */ - stp x0, x1, [sp, #-16]! - stp x30, xzr, [sp, #-16]! - bl _ZN3ams4kern10KScheduler30SetInterruptTaskThreadRunnableEv - ldp x30, xzr, [sp], 16 - ldp x0, x1, [sp], 16 - - /* Clear the interrupt task thread as runnable. */ - strb wzr, [x1, #1] - -0: /* Interrupt task thread runnable checked. */ - /* Now we want to check if there's any scheduling to do. */ - - /* First, clear the need's scheduling bool (and dmb ish after, as it's an atomic). */ - /* TODO: Should this be a stlrb? Nintendo does not do one. */ - strb wzr, [x1] - dmb ish - - /* Check if the highest priority thread is the same as the current thread. */ - ldr x7, [x1, 16] - ldr x2, [x18] - cmp x7, x2 - b.ne 1f - - /* If they're the same, then we can just return as there's nothing to do. */ - ret - -1: /* The highest priority thread is not the same as the current thread. */ - /* Get a reference to the current thread's stack parameters. */ - add x2, sp, #0x1000 - and x2, x2, #~(0x1000-1) - - /* Check if the thread has terminated. We can do this by checking the DPC flags for DpcFlag_Terminated. */ - ldurb w3, [x2, #-0x20] - tbnz w3, #1, 3f - - /* The current thread hasn't terminated, so we want to save its context. */ - ldur x2, [x2, #-0x10] - SAVE_THREAD_CONTEXT(x2, x4, x5, 2f) - -2: /* We're done saving this thread's context, so we need to unlock it. */ - /* We can just do an atomic write to the relevant KThreadContext member. */ - add x2, x2, #0x280 - stlrb wzr, [x2] - -3: /* The current thread's context has been entirely taken care of. */ - /* Now we want to loop until we successfully switch the thread context. */ - /* Start by saving all the values we care about in callee-save registers. */ - mov x19, x0 /* this */ - mov x20, x1 /* this->state */ - mov x21, x7 /* highest priority thread */ - - /* Set our stack to the idle thread stack. */ - ldr x3, [x20, #0x18] - mov sp, x3 - b 5f - -4: /* We failed to successfully do the context switch, and need to retry. */ - /* Clear the exclusive monitor. */ - clrex - - /* Clear the need's scheduling bool (and dmb ish after, as it's an atomic). */ - /* TODO: Should this be a stlrb? Nintendo does not do one. */ - strb wzr, [x20] - dmb ish - - /* Refresh the highest priority thread. */ - ldr x21, [x20, 16] - -5: /* We're starting to try to do the context switch. */ - /* Check if the highest priority thread if null. */ - /* If it is, we want to branch to a special idle thread loop. */ - cbz x21, 11f - - /* Get the highest priority thread's context, and save it. */ - /* ams::kern::KThread::GetContextForSchedulerLoop() */ - mov x0, x21 - bl _ZN3ams4kern7KThread26GetContextForSchedulerLoopEv - mov x22, x0 - - /* Prepare to try to acquire the context lock. */ - add x1, x22, #0x280 - mov w2, #1 - -6: /* We want to try to lock the highest priority thread's context. */ - /* Check if the lock is already held. */ - ldaxrb w3, [x1] - cbnz w3, 7f - - /* If it's not, try to take it. */ - stxrb w3, w2, [x1] - cbnz w3, 6b - - /* We hold the lock, so we can now switch the thread. */ - b 8f - -7: /* The highest priority thread's context is already locked. */ - /* Check if we need scheduling. If we don't, we can retry directly. */ - ldarb w3, [x20] - cbz w3, 6b - - /* If we do, another core is interfering, and we must start from the top. */ - b 4b - -8: /* It's time to switch the thread. */ - /* Switch to the highest priority thread. */ - mov x0, x19 - mov x1, x21 - - /* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */ - bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE - - /* Check if we need scheduling. If we don't, then we can't complete the switch and should retry. */ - ldarb w1, [x20] - cbnz w1, 10f - - /* Restore the thread context. */ - mov x0, x22 - RESTORE_THREAD_CONTEXT(x0, x1, x2, 9f) - -9: /* We're done restoring the thread context, and can return safely. */ - ret - -10: /* Our switch failed. */ - /* We should unlock the thread context, and then retry. */ - add x1, x22, #0x280 - stlrb wzr, [x1] - b 4b - -11: /* The next thread is nullptr! */ - /* Switch to nullptr. This will actually switch to the idle thread. */ - mov x0, x19 - mov x1, #0 - - /* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */ - bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE - -12: /* We've switched to the idle thread, so we want to loop until we schedule a non-idle thread. */ - /* Check if we need scheduling. */ - ldarb w3, [x20] - cbnz w3, 13f - - /* If we don't, wait for an interrupt and check again. */ - wfi - msr daifclr, #2 - msr daifset, #2 - b 12b - -13: /* We need scheduling again! */ - /* Check whether the interrupt task thread needs to be set runnable. */ - ldrb w3, [x20, #1] - cbz w3, 4b - - /* It does, so do so. We're using the idle thread stack so no register state preserve needed. */ - bl _ZN3ams4kern10KScheduler30SetInterruptTaskThreadRunnableEv - - /* Clear the interrupt task thread as runnable. */ - strb wzr, [x20, #1] - - /* Retry the scheduling loop. */ - b 4b diff --git a/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context_asm.s b/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context_asm.s deleted file mode 100644 index 730712b..0000000 --- a/libraries/libmesosphere/source/arch/arm64/kern_k_thread_context_asm.s +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) 2018-2020 Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -/* ams::kern::arm64::UserModeThreadStarter() */ -.section .text._ZN3ams4kern5arm6421UserModeThreadStarterEv, "ax", %progbits -.global _ZN3ams4kern5arm6421UserModeThreadStarterEv -.type _ZN3ams4kern5arm6421UserModeThreadStarterEv, %function -_ZN3ams4kern5arm6421UserModeThreadStarterEv: - /* NOTE: Stack layout on entry looks like following: */ - /* SP */ - /* | */ - /* v */ - /* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x30) | */ - - /* Clear the disable count for this thread's stack parameters. */ - str wzr, [sp, #(0x120 + 0x18)] - - /* Call ams::kern::arm64::OnThreadStart() */ - bl _ZN3ams4kern5arm6413OnThreadStartEv - - /* Restore thread state from the KExceptionContext on stack */ - ldp x30, x19, [sp, #(8 * 30)] /* x30 = lr, x19 = sp */ - ldp x20, x21, [sp, #(8 * 30 + 16)] /* x20 = pc, x21 = psr */ - ldr x22, [sp, #(8 * 30 + 32)] /* x22 = tpidr */ - - msr sp_el0, x19 - msr elr_el1, x20 - msr spsr_el1, x21 - msr tpidr_el1, x22 - - ldp x0, x1, [sp, #(8 * 0)] - ldp x2, x3, [sp, #(8 * 2)] - ldp x4, x5, [sp, #(8 * 4)] - ldp x6, x7, [sp, #(8 * 6)] - ldp x8, x9, [sp, #(8 * 8)] - ldp x10, x11, [sp, #(8 * 10)] - ldp x12, x13, [sp, #(8 * 12)] - ldp x14, x15, [sp, #(8 * 14)] - ldp x16, x17, [sp, #(8 * 16)] - ldp x18, x19, [sp, #(8 * 18)] - ldp x20, x21, [sp, #(8 * 20)] - ldp x22, x23, [sp, #(8 * 22)] - ldp x24, x25, [sp, #(8 * 24)] - ldp x26, x27, [sp, #(8 * 26)] - ldp x28, x29, [sp, #(8 * 28)] - - /* Increment stack pointer above the KExceptionContext */ - add sp, sp, #0x120 - - /* Return to EL0 */ - eret - -/* ams::kern::arm64::SupervisorModeThreadStarter() */ -.section .text._ZN3ams4kern5arm6427SupervisorModeThreadStarterEv, "ax", %progbits -.global _ZN3ams4kern5arm6427SupervisorModeThreadStarterEv -.type _ZN3ams4kern5arm6427SupervisorModeThreadStarterEv, %function -_ZN3ams4kern5arm6427SupervisorModeThreadStarterEv: - /* NOTE: Stack layout on entry looks like following: */ - /* SP */ - /* | */ - /* v */ - /* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x30) | */ - - /* Load the argument and entrypoint. */ - ldp x0, x1, [sp], #0x10 - - /* Clear the disable count for this thread's stack parameters. */ - str wzr, [sp, #(0x18)] - - /* Mask I bit in DAIF */ - msr daifclr, #2 - br x1 - - /* This should never execute, but Nintendo includes an ERET here. */ - eret - - -/* ams::kern::arm64::KThreadContext::RestoreFpuRegisters64(const KThreadContext &) */ -.section .text._ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_, "ax", %progbits -.global _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_ -.type _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_, %function -_ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_: - /* Load and restore FPCR and FPSR from the context. */ - ldr x1, [x0, #0x70] - msr fpcr, x1 - ldr x1, [x0, #0x78] - msr fpsr, x1 - - /* Restore the FPU registers. */ - ldp q0, q1, [sp, #(16 * 0 + 0x80)] - ldp q2, q3, [sp, #(16 * 2 + 0x80)] - ldp q4, q5, [sp, #(16 * 4 + 0x80)] - ldp q6, q7, [sp, #(16 * 6 + 0x80)] - ldp q8, q9, [sp, #(16 * 8 + 0x80)] - ldp q10, q11, [sp, #(16 * 10 + 0x80)] - ldp q12, q13, [sp, #(16 * 12 + 0x80)] - ldp q14, q15, [sp, #(16 * 14 + 0x80)] - ldp q16, q17, [sp, #(16 * 16 + 0x80)] - ldp q18, q19, [sp, #(16 * 18 + 0x80)] - ldp q20, q21, [sp, #(16 * 20 + 0x80)] - ldp q22, q23, [sp, #(16 * 22 + 0x80)] - ldp q24, q25, [sp, #(16 * 24 + 0x80)] - ldp q26, q27, [sp, #(16 * 26 + 0x80)] - ldp q28, q29, [sp, #(16 * 28 + 0x80)] - ldp q30, q31, [sp, #(16 * 30 + 0x80)] - - ret - -/* ams::kern::arm64::KThreadContext::RestoreFpuRegisters32(const KThreadContext &) */ -.section .text._ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_, "ax", %progbits -.global _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_ -.type _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_, %function -_ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_: - /* Load and restore FPCR and FPSR from the context. */ - ldr x1, [x0, #0x70] - msr fpcr, x1 - ldr x1, [x0, #0x78] - msr fpsr, x1 - - /* Restore the FPU registers. */ - ldp q0, q1, [sp, #(16 * 0 + 0x80)] - ldp q2, q3, [sp, #(16 * 2 + 0x80)] - ldp q4, q5, [sp, #(16 * 4 + 0x80)] - ldp q6, q7, [sp, #(16 * 6 + 0x80)] - ldp q8, q9, [sp, #(16 * 8 + 0x80)] - ldp q10, q11, [sp, #(16 * 10 + 0x80)] - ldp q12, q13, [sp, #(16 * 12 + 0x80)] - ldp q14, q15, [sp, #(16 * 14 + 0x80)] - - ret diff --git a/libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s b/libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s deleted file mode 100644 index 3f7deae..0000000 --- a/libraries/libmesosphere/source/arch/arm64/kern_userspace_memory_access_asm.s +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2018-2020 Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaBegin() */ -.section .text._ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, "ax", %progbits -.global _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv -.type _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, %function -_ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv: -/* NOTE: This is not a real function, and only exists as a label for safety. */ - -/* ================ All Userspace Memory Functions after this line. ================ */ - -/* TODO */ - -/* ================ All Userspace Memory Functions before this line. ================ */ - -/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaEnd() */ -.section .text._ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, "ax", %progbits -.global _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv -.type _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, %function -_ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv: -/* NOTE: This is not a real function, and only exists as a label for safety. */ \ No newline at end of file diff --git a/libraries/libmesosphere/source/board/nintendo/switch/kern_debug_log_impl.cpp b/libraries/libmesosphere/source/board/nintendo/switch/kern_debug_log_impl.cpp index 6623779..8985447 100644 --- a/libraries/libmesosphere/source/board/nintendo/switch/kern_debug_log_impl.cpp +++ b/libraries/libmesosphere/source/board/nintendo/switch/kern_debug_log_impl.cpp @@ -28,6 +28,8 @@ UartRegister_LSR = 5, + UartRegister_IRSA_CSR = 8, + UartRegister_DLL = 0, UartRegister_DLH = 1, }; @@ -71,8 +73,9 @@ /* Disable UART interrupts. */ WriteUartRegister(UartRegister_IER, 0x00); - /* Configure the FIFOO to be enabled and clear receive. */ + /* Configure the FIFO to be enabled and clear receive. */ WriteUartRegister(UartRegister_FCR, 0x03); + WriteUartRegister(UartRegister_IRSA_CSR, 0x02); ReadUartRegister(UartRegister_FCR); return true; diff --git a/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp b/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp index 4d37cb5..358645e 100644 --- a/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp +++ b/libraries/libmesosphere/source/board/nintendo/switch/kern_k_system_control.cpp @@ -260,7 +260,7 @@ void KSystemControl::StopSystem() { if (g_call_smc_on_panic) { /* Display a panic screen via secure monitor. */ - smc::Panic(0xF00); + /* TODO: Enable in release: smc::Panic(0xF00); */ } while (true) { /* ... */ } } diff --git a/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.cpp b/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.cpp index 1aac226..7c49443 100644 --- a/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.cpp +++ b/libraries/libmesosphere/source/board/nintendo/switch/kern_secure_monitor.cpp @@ -116,7 +116,7 @@ void GetConfig(u64 *out, size_t num_qwords, ConfigItem config_item) { SecureMonitorArguments args = { FunctionId_GetConfig, static_cast(config_item) }; CallPrivilegedSecureMonitorFunctionForInit(args); - MESOSPHERE_ABORT_UNLESS((static_cast(args.x[0]) == SmcResult::Success)); + MESOSPHERE_INIT_ABORT_UNLESS((static_cast(args.x[0]) == SmcResult::Success)); for (size_t i = 0; i < num_qwords && i < 7; i++) { out[i] = args.x[1 + i]; } @@ -125,10 +125,10 @@ void GenerateRandomBytes(void *dst, size_t size) { /* Call SmcGenerateRandomBytes() */ SecureMonitorArguments args = { FunctionId_GenerateRandomBytes, size }; - MESOSPHERE_ABORT_UNLESS(size <= sizeof(args) - sizeof(args.x[0])); + MESOSPHERE_INIT_ABORT_UNLESS(size <= sizeof(args) - sizeof(args.x[0])); CallPrivilegedSecureMonitorFunctionForInit(args); - MESOSPHERE_ABORT_UNLESS((static_cast(args.x[0]) == SmcResult::Success)); + MESOSPHERE_INIT_ABORT_UNLESS((static_cast(args.x[0]) == SmcResult::Success)); /* Copy output. */ std::memcpy(dst, &args.x[1], size); diff --git a/libraries/libmesosphere/source/kern_k_scheduler.cpp b/libraries/libmesosphere/source/kern_k_scheduler.cpp index 4bd37ed..50e778c 100644 --- a/libraries/libmesosphere/source/kern_k_scheduler.cpp +++ b/libraries/libmesosphere/source/kern_k_scheduler.cpp @@ -17,6 +17,10 @@ namespace ams::kern { + bool KScheduler::s_scheduler_update_needed; + KScheduler::LockType KScheduler::s_scheduler_lock; + KSchedulerPriorityQueue KScheduler::s_priority_queue; + namespace { class KSchedulerInterruptTask : public KInterruptTask { diff --git a/libraries/libmesosphere/source/kern_kernel.cpp b/libraries/libmesosphere/source/kern_kernel.cpp index 09bc065..d312d27 100644 --- a/libraries/libmesosphere/source/kern_kernel.cpp +++ b/libraries/libmesosphere/source/kern_kernel.cpp @@ -17,18 +17,6 @@ namespace ams::kern { - /* Declare kernel data members in kernel TU. */ - Kernel::State Kernel::s_state = Kernel::State::Invalid; - KThread Kernel::s_main_threads[cpu::NumCores]; - KThread Kernel::s_idle_threads[cpu::NumCores]; - KResourceLimit Kernel::s_system_resource_limit; - KMemoryManager Kernel::s_memory_manager; - KPageTableManager Kernel::s_page_table_manager; - KMemoryBlockSlabManager Kernel::s_app_memory_block_manager; - KMemoryBlockSlabManager Kernel::s_sys_memory_block_manager; - KBlockInfoManager Kernel::s_block_info_manager; - KSupervisorPageTable Kernel::s_supervisor_page_table; - namespace { template diff --git a/libraries/libvapours/include/vapours/util/util_intrusive_list.hpp b/libraries/libvapours/include/vapours/util/util_intrusive_list.hpp index c7d52e7..771aa38 100644 --- a/libraries/libvapours/include/vapours/util/util_intrusive_list.hpp +++ b/libraries/libvapours/include/vapours/util/util_intrusive_list.hpp @@ -405,7 +405,7 @@ return Traits::GetParent(node); } public: - IntrusiveList() : impl() { /* ... */ } + constexpr IntrusiveList() : impl() { /* ... */ } /* Iterator accessors. */ iterator begin() { diff --git a/mesosphere/build_mesosphere.py b/mesosphere/build_mesosphere.py index e5fc15d..eb53e7e 100644 --- a/mesosphere/build_mesosphere.py +++ b/mesosphere/build_mesosphere.py @@ -22,11 +22,17 @@ assert (kernel_end >= len(kernel)) embedded_ini = b'' + try: + with open('ini.bin', 'rb') as f: + embedded_ini = f.read() + except: + pass embedded_ini_offset = align_up(kernel_end, 0x1000) + 0x1000 - embedded_ini_end = embedded_ini_offset + 0 # TODO: Create and embed an INI, eventually. + embedded_ini_end = embedded_ini_offset + len(embedded_ini) # TODO: Create and embed an INI, eventually. kernel_ldr_offset = align_up(embedded_ini_end, 0x1000) + 0x1000 kernel_ldr_end = kernel_ldr_offset + len(kernel_ldr) + mesosphere_end = align_up(kernel_ldr_end, 0x1000) with open('mesosphere.bin', 'wb') as f: f.write(kernel[:kernel_metadata_offset + 4]) @@ -37,7 +43,8 @@ f.seek(embedded_ini_end) f.seek(kernel_ldr_offset) f.write(kernel_ldr) - f.seek(kernel_ldr_end) + f.seek(mesosphere_end) + f.write(b'\x00'*0x1000) return 0 diff --git a/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s b/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s new file mode 100644 index 0000000..68983e5 --- /dev/null +++ b/mesosphere/kernel/source/arch/arm64/kern_exception_handlers_asm.s @@ -0,0 +1,610 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* ams::kern::arm64::EL1IrqExceptionHandler() */ +.section .text._ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv +.type _ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv, %function +_ZN3ams4kern5arm6422EL1IrqExceptionHandlerEv: + /* Save registers that need saving. */ + sub sp, sp, #(8 * 24) + + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x30, [sp, #(8 * 22)] + + mrs x19, sp_el0 + mrs x20, elr_el1 + mrs x21, spsr_el1 + mov w21, w21 + + /* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */ + mrs x18, tpidr_el1 + mov x0, #0 + bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb + + /* Restore registers that we saved. */ + msr sp_el0, x19 + msr elr_el1, x20 + msr spsr_el1, x21 + + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x30, [sp, #(8 * 22)] + + add sp, sp, #(8 * 24) + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL0IrqExceptionHandler() */ +.section .text._ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv +.type _ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv, %function +_ZN3ams4kern5arm6422EL0IrqExceptionHandlerEv: + /* Save registers that need saving. */ + sub sp, sp, #(8 * 36) + + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Invoke KInterruptManager::HandleInterrupt(bool user_mode). */ + mrs x18, tpidr_el1 + mov x0, #1 + bl _ZN3ams4kern5arm6417KInterruptManager15HandleInterruptEb + + /* Restore state from the context. */ + ldp x30, x20, [sp, #(8 * 30)] + ldp x21, x22, [sp, #(8 * 32)] + ldr x23, [sp, #(8 * 34)] + msr sp_el0, x20 + msr elr_el1, x21 + msr spsr_el1, x22 + msr tpidr_el0, x23 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + add sp, sp, #0x120 + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL0SynchronousExceptionHandler() */ +.section .text._ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv +.type _ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv, %function +_ZN3ams4kern5arm6430EL0SynchronousExceptionHandlerEv: + /* Save x16 and x17, so that we can use them as scratch. */ + stp x16, x17, [sp, #-16]! + + /* Get and parse the exception syndrome register. */ + mrs x16, esr_el1 + lsr x17, x16, #0x1a + + /* Is this an aarch32 SVC? */ + cmp x17, #0x11 + b.eq 2f + + /* Is this an aarch64 SVC? */ + cmp x17, #0x15 + b.eq 3f + + /* Is this an FPU error? */ + cmp x17, #0x7 + b.eq 4f + + /* Is this an instruction abort? */ + cmp x17, #0x21 + b.eq 5f + + /* Is this a data abort? */ + cmp x17, #0x25 + b.eq 5f + +1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */ + /* It is also not an SVC or an FPU exception. Handle it generically! */ + + /* Restore x16 and x17. */ + ldp x16, x17, [sp], 16 + + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + + /* Restore state from the context. */ + ldp x30, x20, [sp, #(8 * 30)] + ldp x21, x22, [sp, #(8 * 32)] + ldr x23, [sp, #(8 * 34)] + msr sp_el0, x20 + msr elr_el1, x21 + msr spsr_el1, x22 + msr tpidr_el0, x23 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + add sp, sp, #0x120 + + /* Return from the exception. */ + eret + +2: /* SVC from aarch32. */ + ldp x16, x17, [sp], 16 + b _ZN3ams4kern5arm6412SvcHandler32Ev + +3: /* SVC from aarch64. */ + ldp x16, x17, [sp], 16 + b _ZN3ams4kern5arm6412SvcHandler64Ev + +4: /* FPU exception. */ + ldp x16, x17, [sp], 16 + b _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv + +5: /* Check if there's a TLB conflict that caused the abort. */ + and x17, x16, #0x3F + cmp x17, #0x30 + b.ne 1b + + /* Get the ASID in x17. */ + mrs x17, ttbr0_el1 + and x17, x17, #(0xFFFF << 48) + + /* Check if FAR is valid by examining the FnV bit. */ + tbnz x16, #10, 6f + + /* FAR is valid, so we can invalidate the address it holds. */ + mrs x16, far_el1 + lsr x16, x16, #12 + orr x17, x16, x17 + tlbi vaae1, x17 + b 7f + +6: /* There's a TLB conflict and FAR isn't valid. */ + /* Invalidate the entire TLB. */ + tlbi vmalle1 + +7: /* Return from a TLB conflict. */ + /* Ensure instruction consistency. */ + dsb ish + isb + + /* Restore x16 and x17. */ + ldp x16, x17, [sp], 16 + + /* Return from the exception. */ + eret + + +/* ams::kern::arm64::EL1SynchronousExceptionHandler() */ +.section .text._ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv +.type _ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv, %function +_ZN3ams4kern5arm6430EL1SynchronousExceptionHandlerEv: + /* Nintendo uses the "unused" virtual timer compare value as a scratch register. */ + msr cntv_cval_el0, x0 + + /* Get and parse the exception syndrome register. */ + mrs x0, esr_el1 + lsr x0, x0, #0x1a + + /* Is this an instruction abort? */ + cmp x0, #0x21 + b.eq 5f + + /* Is this a data abort? */ + cmp x0, #0x25 + b.eq 5f + +1: /* The exception is not a data abort or instruction abort caused by a TLB conflict. */ + /* Load the CoreLocalContext into x0. */ + mrs x0, tpidr_el1 + cbz x0, 2f + + /* Load the exception stack top from the context. */ + ldr x0, [x0, #0x28] + + /* Setup the stack for a generic exception handle */ + sub x0, x0, #0x20 + str x1, [x0, #16] + mov x1, sp + str x1, [x0] + mov sp, x0 + ldr x1, [x0, #16] + mrs x0, cntv_cval_el0 + str x0, [sp, #8] + + /* Check again if this is a data abort from EL1. */ + mrs x0, esr_el1 + lsr x1, x0, #0x1a + cmp x1, #0x25 + b.ne 3f + + /* Data abort. Check if it was from trying to access userspace memory. */ + mrs x1, elr_el1 + adr x0, _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv + cmp x1, x0 + b.lo 3f + adr x0, _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv + cmp x1, x0 + b.hs 3f + + /* We aborted trying to access userspace memory. */ + /* All functions that access user memory return a boolean for whether they succeeded. */ + /* With that in mind, we can simply restore the stack pointer and return false directly. */ + ldr x0, [sp] + mov sp, x0 + + /* Return false. */ + mov x0, #0x0 + msr elr_el1, x30 + eret + +2: /* The CoreLocalContext is nullptr. */ + /* Setup the stack for a generic exception handle. */ + /* NOTE: Nintendo does not restore X0 here, and thus saves nullptr. */ + /* This is probably not their intention, so we'll fix it. */ + /* NOTE: Nintendo also does not really save SP correctly, and so we */ + /* will also fix that. */ + mov x0, sp + sub x0, x0, #0x20 + str x1, [x0, #16] + mov x1, sp + str x1, [x0] + mov sp, x0 + mrs x0, cntv_cval_el0 + str x0, [sp, #8] + +3: /* The exception wasn't an triggered by copying memory from userspace. */ + ldr x0, [sp, #8] + ldr x1, [sp, #16] + + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Call ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *) */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + +4: /* HandleException should never return. The best we can do is infinite loop. */ + b 4b + +5: /* Check if there's a TLB conflict that caused the abort. */ + /* NOTE: There is a Nintendo bug in this code that we correct. */ + /* Nintendo compares the low 6 bits of x0 without restoring the value. */ + /* They intend to check the DFSC/IFSC bits of esr_el1, but because they */ + /* shifted esr earlier, the check is invalid and always fails. */ + mrs x0, esr_el1 + and x0, x0, #0x3F + cmp x0, #0x30 + b.ne 1b + + /* Check if FAR is valid by examining the FnV bit. */ + /* NOTE: Nintendo again has a bug here, the same as above. */ + /* They do not refresh the value of x0, and again compare with */ + /* the relevant bit already masked out of x0. */ + mrs x0, esr_el1 + tbnz x0, #10, 6f + + /* FAR is valid, so we can invalidate the address it holds. */ + mrs x0, far_el1 + lsr x0, x0, #12 + tlbi vaae1, x0 + b 7f + +6: /* There's a TLB conflict and FAR isn't valid. */ + /* Invalidate the entire TLB. */ + tlbi vmalle1 + +7: /* Return from a TLB conflict. */ + /* Ensure instruction consistency. */ + dsb ish + isb + + /* Restore x0 from scratch. */ + mrs x0, cntv_cval_el0 + + /* Return from the exception. */ + eret + + +/* ams::kern::arm64::FpuAccessExceptionHandler() */ +.section .text._ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv +.type _ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv, %function +_ZN3ams4kern5arm6425FpuAccessExceptionHandlerEv: + /* Save registers that need saving. */ + sub sp, sp, #(8 * 24) + + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x30, [sp, #(8 * 22)] + + mrs x18, tpidr_el1 + mrs x19, sp_el0 + mrs x20, elr_el1 + mrs x21, spsr_el1 + + /* Invoke the FPU context switch handler. */ + bl _ZN3ams4kern5arm6423FpuContextSwitchHandlerEv + + /* Restore registers that we saved. */ + msr sp_el0, x19 + msr elr_el1, x20 + msr spsr_el1, x21 + + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x30, [sp, #(8 * 22)] + + add sp, sp, #(8 * 24) + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL1SystemErrorHandler() */ +.section .text._ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv +.type _ZN3ams4kern5arm6421EL1SystemErrorHandlerEv, %function +_ZN3ams4kern5arm6421EL1SystemErrorHandlerEv: + /* Nintendo uses the "unused" virtual timer compare value as a scratch register. */ + msr cntv_cval_el0, x0 + + /* Load the exception stack top from the context. */ + ldr x0, [x0, #0x28] + + /* Setup the stack for a generic exception handle */ + sub x0, x0, #0x20 + str x1, [x0, #16] + mov x1, sp + str x1, [x0] + mov sp, x0 + ldr x1, [x0, #16] + mrs x0, cntv_cval_el0 + str x0, [sp, #8] + + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + +1: /* HandleException should never return. The best we can do is infinite loop. */ + b 1b + + /* Return from the exception. */ + eret + +/* ams::kern::arm64::EL0SystemErrorHandler() */ +.section .text._ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, "ax", %progbits +.global _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv +.type _ZN3ams4kern5arm6421EL0SystemErrorHandlerEv, %function +_ZN3ams4kern5arm6421EL0SystemErrorHandlerEv: + /* Create a KExceptionContext to pass to HandleException. */ + sub sp, sp, #0x120 + stp x0, x1, [sp, #(8 * 0)] + stp x2, x3, [sp, #(8 * 2)] + stp x4, x5, [sp, #(8 * 4)] + stp x6, x7, [sp, #(8 * 6)] + stp x8, x9, [sp, #(8 * 8)] + stp x10, x11, [sp, #(8 * 10)] + stp x12, x13, [sp, #(8 * 12)] + stp x14, x15, [sp, #(8 * 14)] + stp x16, x17, [sp, #(8 * 16)] + stp x18, x19, [sp, #(8 * 18)] + stp x20, x21, [sp, #(8 * 20)] + stp x22, x23, [sp, #(8 * 22)] + stp x24, x25, [sp, #(8 * 24)] + stp x26, x27, [sp, #(8 * 26)] + stp x28, x29, [sp, #(8 * 28)] + mrs x20, sp_el0 + mrs x21, elr_el1 + mrs x22, spsr_el1 + mrs x23, tpidr_el0 + mov w22, w22 + stp x30, x20, [sp, #(8 * 30)] + stp x21, x22, [sp, #(8 * 32)] + str x23, [sp, #(8 * 34)] + + /* Invoke ams::kern::arm64::HandleException(ams::kern::arm64::KExceptionContext *). */ + mrs x18, tpidr_el1 + mov x0, sp + bl _ZN3ams4kern5arm6415HandleExceptionEPNS1_17KExceptionContextE + + /* Restore state from the context. */ + ldp x30, x20, [sp, #(8 * 30)] + ldp x21, x22, [sp, #(8 * 32)] + ldr x23, [sp, #(8 * 34)] + msr sp_el0, x20 + msr elr_el1, x21 + msr spsr_el1, x22 + msr tpidr_el0, x23 + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + add sp, sp, #0x120 + + /* Return from the exception. */ + eret + diff --git a/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s b/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s new file mode 100644 index 0000000..ce23648 --- /dev/null +++ b/mesosphere/kernel/source/arch/arm64/kern_k_scheduler_asm.s @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#define SAVE_THREAD_CONTEXT(ctx, tmp0, tmp1, done_label) \ + /* Save the callee save registers + SP and cpacr. */ \ + mov tmp0, sp; \ + mrs tmp1, cpacr_el1; \ + stp x19, x20, [ctx, #(8 * 0)]; \ + stp x21, x22, [ctx, #(8 * 2)]; \ + stp x23, x24, [ctx, #(8 * 4)]; \ + stp x25, x26, [ctx, #(8 * 6)]; \ + stp x27, x28, [ctx, #(8 * 8)]; \ + stp x29, x30, [ctx, #(8 * 10)]; \ + \ + stp tmp0, tmp1, [ctx, #0x60]; \ + \ + /* Check whether the FPU is enabled. */ \ + /* If it isn't, skip saving FPU state. */ \ + and tmp1, tmp1, #0x300000; \ + cbz tmp1, done_label; \ + \ + /* Save fpcr and fpsr. */ \ + mrs tmp0, fpcr; \ + mrs tmp1, fpsr; \ + stp tmp0, tmp1, [ctx, #0x70]; \ + \ + /* Save the FPU registers. */ \ + stp q0, q1, [ctx, #(16 * 0 + 0x80)]; \ + stp q2, q3, [ctx, #(16 * 2 + 0x80)]; \ + stp q4, q5, [ctx, #(16 * 4 + 0x80)]; \ + stp q6, q7, [ctx, #(16 * 6 + 0x80)]; \ + stp q8, q9, [ctx, #(16 * 8 + 0x80)]; \ + stp q10, q11, [ctx, #(16 * 10 + 0x80)]; \ + stp q12, q13, [ctx, #(16 * 12 + 0x80)]; \ + stp q14, q15, [ctx, #(16 * 14 + 0x80)]; \ + stp q16, q17, [ctx, #(16 * 16 + 0x80)]; \ + stp q18, q19, [ctx, #(16 * 18 + 0x80)]; \ + stp q20, q21, [ctx, #(16 * 20 + 0x80)]; \ + stp q22, q23, [ctx, #(16 * 22 + 0x80)]; \ + stp q24, q25, [ctx, #(16 * 24 + 0x80)]; \ + stp q26, q27, [ctx, #(16 * 26 + 0x80)]; \ + stp q28, q29, [ctx, #(16 * 28 + 0x80)]; \ + stp q30, q31, [ctx, #(16 * 30 + 0x80)]; + +#define RESTORE_THREAD_CONTEXT(ctx, tmp0, tmp1, done_label) \ + /* Restore the callee save registers + SP and cpacr. */ \ + ldp tmp0, tmp1, [ctx, #0x60]; \ + mov sp, tmp0; \ + ldp x19, x20, [ctx, #(8 * 0)]; \ + ldp x21, x22, [ctx, #(8 * 2)]; \ + ldp x23, x24, [ctx, #(8 * 4)]; \ + ldp x25, x26, [ctx, #(8 * 6)]; \ + ldp x27, x28, [ctx, #(8 * 8)]; \ + ldp x29, x30, [ctx, #(8 * 10)]; \ + \ + msr cpacr_el1, tmp1; \ + isb; \ + \ + /* Check whether the FPU is enabled. */ \ + /* If it isn't, skip saving FPU state. */ \ + and tmp1, tmp1, #0x300000; \ + cbz tmp1, done_label; \ + \ + /* Save fpcr and fpsr. */ \ + ldp tmp0, tmp1, [ctx, #0x70]; \ + msr fpcr, tmp0; \ + msr fpsr, tmp1; \ + \ + /* Save the FPU registers. */ \ + ldp q0, q1, [ctx, #(16 * 0 + 0x80)]; \ + ldp q2, q3, [ctx, #(16 * 2 + 0x80)]; \ + ldp q4, q5, [ctx, #(16 * 4 + 0x80)]; \ + ldp q6, q7, [ctx, #(16 * 6 + 0x80)]; \ + ldp q8, q9, [ctx, #(16 * 8 + 0x80)]; \ + ldp q10, q11, [ctx, #(16 * 10 + 0x80)]; \ + ldp q12, q13, [ctx, #(16 * 12 + 0x80)]; \ + ldp q14, q15, [ctx, #(16 * 14 + 0x80)]; \ + ldp q16, q17, [ctx, #(16 * 16 + 0x80)]; \ + ldp q18, q19, [ctx, #(16 * 18 + 0x80)]; \ + ldp q20, q21, [ctx, #(16 * 20 + 0x80)]; \ + ldp q22, q23, [ctx, #(16 * 22 + 0x80)]; \ + ldp q24, q25, [ctx, #(16 * 24 + 0x80)]; \ + ldp q26, q27, [ctx, #(16 * 26 + 0x80)]; \ + ldp q28, q29, [ctx, #(16 * 28 + 0x80)]; \ + ldp q30, q31, [ctx, #(16 * 30 + 0x80)]; + + +/* ams::kern::KScheduler::ScheduleImpl() */ +.section .text._ZN3ams4kern10KScheduler12ScheduleImplEv, "ax", %progbits +.global _ZN3ams4kern10KScheduler12ScheduleImplEv +.type _ZN3ams4kern10KScheduler12ScheduleImplEv, %function + +/* Ensure ScheduleImpl is aligned to 0x40 bytes. */ +.balign 0x40 + +_ZN3ams4kern10KScheduler12ScheduleImplEv: + /* Right now, x0 contains (this). We want x1 to point to the scheduling state, */ + /* Current KScheduler layout has state at +0x0. */ + mov x1, x0 + + /* First thing we want to do is check whether the interrupt task thread is runnable. */ + ldrb w3, [x1, #1] + cbz w3, 0f + + /* If it is, we want to call KScheduler::SetInterruptTaskThreadRunnable() to note it runnable. */ + stp x0, x1, [sp, #-16]! + stp x30, xzr, [sp, #-16]! + bl _ZN3ams4kern10KScheduler30SetInterruptTaskThreadRunnableEv + ldp x30, xzr, [sp], 16 + ldp x0, x1, [sp], 16 + + /* Clear the interrupt task thread as runnable. */ + strb wzr, [x1, #1] + +0: /* Interrupt task thread runnable checked. */ + /* Now we want to check if there's any scheduling to do. */ + + /* First, clear the need's scheduling bool (and dmb ish after, as it's an atomic). */ + /* TODO: Should this be a stlrb? Nintendo does not do one. */ + strb wzr, [x1] + dmb ish + + /* Check if the highest priority thread is the same as the current thread. */ + ldr x7, [x1, 16] + ldr x2, [x18] + cmp x7, x2 + b.ne 1f + + /* If they're the same, then we can just return as there's nothing to do. */ + ret + +1: /* The highest priority thread is not the same as the current thread. */ + /* Get a reference to the current thread's stack parameters. */ + add x2, sp, #0x1000 + and x2, x2, #~(0x1000-1) + + /* Check if the thread has terminated. We can do this by checking the DPC flags for DpcFlag_Terminated. */ + ldurb w3, [x2, #-0x20] + tbnz w3, #1, 3f + + /* The current thread hasn't terminated, so we want to save its context. */ + ldur x2, [x2, #-0x10] + SAVE_THREAD_CONTEXT(x2, x4, x5, 2f) + +2: /* We're done saving this thread's context, so we need to unlock it. */ + /* We can just do an atomic write to the relevant KThreadContext member. */ + add x2, x2, #0x280 + stlrb wzr, [x2] + +3: /* The current thread's context has been entirely taken care of. */ + /* Now we want to loop until we successfully switch the thread context. */ + /* Start by saving all the values we care about in callee-save registers. */ + mov x19, x0 /* this */ + mov x20, x1 /* this->state */ + mov x21, x7 /* highest priority thread */ + + /* Set our stack to the idle thread stack. */ + ldr x3, [x20, #0x18] + mov sp, x3 + b 5f + +4: /* We failed to successfully do the context switch, and need to retry. */ + /* Clear the exclusive monitor. */ + clrex + + /* Clear the need's scheduling bool (and dmb ish after, as it's an atomic). */ + /* TODO: Should this be a stlrb? Nintendo does not do one. */ + strb wzr, [x20] + dmb ish + + /* Refresh the highest priority thread. */ + ldr x21, [x20, 16] + +5: /* We're starting to try to do the context switch. */ + /* Check if the highest priority thread if null. */ + /* If it is, we want to branch to a special idle thread loop. */ + cbz x21, 11f + + /* Get the highest priority thread's context, and save it. */ + /* ams::kern::KThread::GetContextForSchedulerLoop() */ + mov x0, x21 + bl _ZN3ams4kern7KThread26GetContextForSchedulerLoopEv + mov x22, x0 + + /* Prepare to try to acquire the context lock. */ + add x1, x22, #0x280 + mov w2, #1 + +6: /* We want to try to lock the highest priority thread's context. */ + /* Check if the lock is already held. */ + ldaxrb w3, [x1] + cbnz w3, 7f + + /* If it's not, try to take it. */ + stxrb w3, w2, [x1] + cbnz w3, 6b + + /* We hold the lock, so we can now switch the thread. */ + b 8f + +7: /* The highest priority thread's context is already locked. */ + /* Check if we need scheduling. If we don't, we can retry directly. */ + ldarb w3, [x20] + cbz w3, 6b + + /* If we do, another core is interfering, and we must start from the top. */ + b 4b + +8: /* It's time to switch the thread. */ + /* Switch to the highest priority thread. */ + mov x0, x19 + mov x1, x21 + + /* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */ + bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE + + /* Check if we need scheduling. If we don't, then we can't complete the switch and should retry. */ + ldarb w1, [x20] + cbnz w1, 10f + + /* Restore the thread context. */ + mov x0, x22 + RESTORE_THREAD_CONTEXT(x0, x1, x2, 9f) + +9: /* We're done restoring the thread context, and can return safely. */ + ret + +10: /* Our switch failed. */ + /* We should unlock the thread context, and then retry. */ + add x1, x22, #0x280 + stlrb wzr, [x1] + b 4b + +11: /* The next thread is nullptr! */ + /* Switch to nullptr. This will actually switch to the idle thread. */ + mov x0, x19 + mov x1, #0 + + /* Call ams::kern::KScheduler::SwitchThread(ams::kern::KThread *) */ + bl _ZN3ams4kern10KScheduler12SwitchThreadEPNS0_7KThreadE + +12: /* We've switched to the idle thread, so we want to loop until we schedule a non-idle thread. */ + /* Check if we need scheduling. */ + ldarb w3, [x20] + cbnz w3, 13f + + /* If we don't, wait for an interrupt and check again. */ + wfi + msr daifclr, #2 + msr daifset, #2 + b 12b + +13: /* We need scheduling again! */ + /* Check whether the interrupt task thread needs to be set runnable. */ + ldrb w3, [x20, #1] + cbz w3, 4b + + /* It does, so do so. We're using the idle thread stack so no register state preserve needed. */ + bl _ZN3ams4kern10KScheduler30SetInterruptTaskThreadRunnableEv + + /* Clear the interrupt task thread as runnable. */ + strb wzr, [x20, #1] + + /* Retry the scheduling loop. */ + b 4b diff --git a/mesosphere/kernel/source/arch/arm64/kern_k_thread_context_asm.s b/mesosphere/kernel/source/arch/arm64/kern_k_thread_context_asm.s new file mode 100644 index 0000000..730712b --- /dev/null +++ b/mesosphere/kernel/source/arch/arm64/kern_k_thread_context_asm.s @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* ams::kern::arm64::UserModeThreadStarter() */ +.section .text._ZN3ams4kern5arm6421UserModeThreadStarterEv, "ax", %progbits +.global _ZN3ams4kern5arm6421UserModeThreadStarterEv +.type _ZN3ams4kern5arm6421UserModeThreadStarterEv, %function +_ZN3ams4kern5arm6421UserModeThreadStarterEv: + /* NOTE: Stack layout on entry looks like following: */ + /* SP */ + /* | */ + /* v */ + /* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x30) | */ + + /* Clear the disable count for this thread's stack parameters. */ + str wzr, [sp, #(0x120 + 0x18)] + + /* Call ams::kern::arm64::OnThreadStart() */ + bl _ZN3ams4kern5arm6413OnThreadStartEv + + /* Restore thread state from the KExceptionContext on stack */ + ldp x30, x19, [sp, #(8 * 30)] /* x30 = lr, x19 = sp */ + ldp x20, x21, [sp, #(8 * 30 + 16)] /* x20 = pc, x21 = psr */ + ldr x22, [sp, #(8 * 30 + 32)] /* x22 = tpidr */ + + msr sp_el0, x19 + msr elr_el1, x20 + msr spsr_el1, x21 + msr tpidr_el1, x22 + + ldp x0, x1, [sp, #(8 * 0)] + ldp x2, x3, [sp, #(8 * 2)] + ldp x4, x5, [sp, #(8 * 4)] + ldp x6, x7, [sp, #(8 * 6)] + ldp x8, x9, [sp, #(8 * 8)] + ldp x10, x11, [sp, #(8 * 10)] + ldp x12, x13, [sp, #(8 * 12)] + ldp x14, x15, [sp, #(8 * 14)] + ldp x16, x17, [sp, #(8 * 16)] + ldp x18, x19, [sp, #(8 * 18)] + ldp x20, x21, [sp, #(8 * 20)] + ldp x22, x23, [sp, #(8 * 22)] + ldp x24, x25, [sp, #(8 * 24)] + ldp x26, x27, [sp, #(8 * 26)] + ldp x28, x29, [sp, #(8 * 28)] + + /* Increment stack pointer above the KExceptionContext */ + add sp, sp, #0x120 + + /* Return to EL0 */ + eret + +/* ams::kern::arm64::SupervisorModeThreadStarter() */ +.section .text._ZN3ams4kern5arm6427SupervisorModeThreadStarterEv, "ax", %progbits +.global _ZN3ams4kern5arm6427SupervisorModeThreadStarterEv +.type _ZN3ams4kern5arm6427SupervisorModeThreadStarterEv, %function +_ZN3ams4kern5arm6427SupervisorModeThreadStarterEv: + /* NOTE: Stack layout on entry looks like following: */ + /* SP */ + /* | */ + /* v */ + /* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x30) | */ + + /* Load the argument and entrypoint. */ + ldp x0, x1, [sp], #0x10 + + /* Clear the disable count for this thread's stack parameters. */ + str wzr, [sp, #(0x18)] + + /* Mask I bit in DAIF */ + msr daifclr, #2 + br x1 + + /* This should never execute, but Nintendo includes an ERET here. */ + eret + + +/* ams::kern::arm64::KThreadContext::RestoreFpuRegisters64(const KThreadContext &) */ +.section .text._ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_, "ax", %progbits +.global _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_ +.type _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_, %function +_ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters64ERKS2_: + /* Load and restore FPCR and FPSR from the context. */ + ldr x1, [x0, #0x70] + msr fpcr, x1 + ldr x1, [x0, #0x78] + msr fpsr, x1 + + /* Restore the FPU registers. */ + ldp q0, q1, [sp, #(16 * 0 + 0x80)] + ldp q2, q3, [sp, #(16 * 2 + 0x80)] + ldp q4, q5, [sp, #(16 * 4 + 0x80)] + ldp q6, q7, [sp, #(16 * 6 + 0x80)] + ldp q8, q9, [sp, #(16 * 8 + 0x80)] + ldp q10, q11, [sp, #(16 * 10 + 0x80)] + ldp q12, q13, [sp, #(16 * 12 + 0x80)] + ldp q14, q15, [sp, #(16 * 14 + 0x80)] + ldp q16, q17, [sp, #(16 * 16 + 0x80)] + ldp q18, q19, [sp, #(16 * 18 + 0x80)] + ldp q20, q21, [sp, #(16 * 20 + 0x80)] + ldp q22, q23, [sp, #(16 * 22 + 0x80)] + ldp q24, q25, [sp, #(16 * 24 + 0x80)] + ldp q26, q27, [sp, #(16 * 26 + 0x80)] + ldp q28, q29, [sp, #(16 * 28 + 0x80)] + ldp q30, q31, [sp, #(16 * 30 + 0x80)] + + ret + +/* ams::kern::arm64::KThreadContext::RestoreFpuRegisters32(const KThreadContext &) */ +.section .text._ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_, "ax", %progbits +.global _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_ +.type _ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_, %function +_ZN3ams4kern5arm6414KThreadContext21RestoreFpuRegisters32ERKS2_: + /* Load and restore FPCR and FPSR from the context. */ + ldr x1, [x0, #0x70] + msr fpcr, x1 + ldr x1, [x0, #0x78] + msr fpsr, x1 + + /* Restore the FPU registers. */ + ldp q0, q1, [sp, #(16 * 0 + 0x80)] + ldp q2, q3, [sp, #(16 * 2 + 0x80)] + ldp q4, q5, [sp, #(16 * 4 + 0x80)] + ldp q6, q7, [sp, #(16 * 6 + 0x80)] + ldp q8, q9, [sp, #(16 * 8 + 0x80)] + ldp q10, q11, [sp, #(16 * 10 + 0x80)] + ldp q12, q13, [sp, #(16 * 12 + 0x80)] + ldp q14, q15, [sp, #(16 * 14 + 0x80)] + + ret diff --git a/mesosphere/kernel/source/arch/arm64/kern_userspace_memory_access_asm.s b/mesosphere/kernel/source/arch/arm64/kern_userspace_memory_access_asm.s new file mode 100644 index 0000000..3f7deae --- /dev/null +++ b/mesosphere/kernel/source/arch/arm64/kern_userspace_memory_access_asm.s @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaBegin() */ +.section .text._ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, "ax", %progbits +.global _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv +.type _ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv, %function +_ZN3ams4kern5arm6438UserspaceMemoryAccessFunctionAreaBeginEv: +/* NOTE: This is not a real function, and only exists as a label for safety. */ + +/* ================ All Userspace Memory Functions after this line. ================ */ + +/* TODO */ + +/* ================ All Userspace Memory Functions before this line. ================ */ + +/* ams::kern::arm64::UserspaceMemoryAccessFunctionAreaEnd() */ +.section .text._ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, "ax", %progbits +.global _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv +.type _ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv, %function +_ZN3ams4kern5arm6436UserspaceMemoryAccessFunctionAreaEndEv: +/* NOTE: This is not a real function, and only exists as a label for safety. */ \ No newline at end of file diff --git a/mesosphere/kernel/source/kern_kernel_instantiations.cpp b/mesosphere/kernel/source/kern_kernel_instantiations.cpp new file mode 100644 index 0000000..6167c27 --- /dev/null +++ b/mesosphere/kernel/source/kern_kernel_instantiations.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2018-2020 Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include + +namespace ams::kern { + + /* Declare kernel data members in kernel TU. */ + Kernel::State Kernel::s_state = Kernel::State::Invalid; + KResourceLimit Kernel::s_system_resource_limit; + KMemoryManager Kernel::s_memory_manager; + KPageTableManager Kernel::s_page_table_manager; + KMemoryBlockSlabManager Kernel::s_app_memory_block_manager; + KMemoryBlockSlabManager Kernel::s_sys_memory_block_manager; + KBlockInfoManager Kernel::s_block_info_manager; + KSupervisorPageTable Kernel::s_supervisor_page_table; + + namespace { + + KThread g_main_threads[cpu::NumCores]; + KThread g_idle_threads[cpu::NumCores]; + } + + KThread &Kernel::GetMainThread(s32 core_id) { return g_main_threads[core_id]; } + KThread &Kernel::GetIdleThread(s32 core_id) { return g_idle_threads[core_id]; } + +} diff --git a/mesosphere/kernel_ldr/source/arch/arm64/start.s b/mesosphere/kernel_ldr/source/arch/arm64/start.s index 2d9c3df..18412f5 100644 --- a/mesosphere/kernel_ldr/source/arch/arm64/start.s +++ b/mesosphere/kernel_ldr/source/arch/arm64/start.s @@ -73,6 +73,7 @@ /* Call ams::kern::init::loader::Main(uintptr_t, ams::kern::init::KernelLayout *, uintptr_t) */ ldp x0, x1, [sp, #0x00] ldr x2, [sp, #0x10] + bl _ZN3ams4kern4init6loader4MainEmPNS1_12KernelLayoutEm str x0, [sp, #0x00] diff --git a/mesosphere/kernel_ldr/source/kern_init_loader.cpp b/mesosphere/kernel_ldr/source/kern_init_loader.cpp index 004259a..226e20d 100644 --- a/mesosphere/kernel_ldr/source/kern_init_loader.cpp +++ b/mesosphere/kernel_ldr/source/kern_init_loader.cpp @@ -218,7 +218,6 @@ } - uintptr_t Main(uintptr_t base_address, KernelLayout *layout, uintptr_t ini_base_address) { /* Relocate the kernel to the correct physical base address. */ /* Base address and layout are passed by reference and modified. */ @@ -235,12 +234,12 @@ const uintptr_t rw_offset = layout->rw_offset; /* UNUSED: const uintptr_t rw_end_offset = layout->rw_end_offset; */ const uintptr_t bss_end_offset = layout->bss_end_offset; - MESOSPHERE_ABORT_UNLESS(util::IsAligned(rx_offset, 0x1000)); - MESOSPHERE_ABORT_UNLESS(util::IsAligned(rx_end_offset, 0x1000)); - MESOSPHERE_ABORT_UNLESS(util::IsAligned(ro_offset, 0x1000)); - MESOSPHERE_ABORT_UNLESS(util::IsAligned(ro_end_offset, 0x1000)); - MESOSPHERE_ABORT_UNLESS(util::IsAligned(rw_offset, 0x1000)); - MESOSPHERE_ABORT_UNLESS(util::IsAligned(bss_end_offset, 0x1000)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rx_offset, 0x1000)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rx_end_offset, 0x1000)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(ro_offset, 0x1000)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(ro_end_offset, 0x1000)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(rw_offset, 0x1000)); + MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(bss_end_offset, 0x1000)); const uintptr_t bss_offset = layout->bss_offset; const uintptr_t ini_load_offset = layout->ini_load_offset; const uintptr_t dynamic_offset = layout->dynamic_offset; diff --git a/stratosphere/ro/source/impl/ro_nro_utils.cpp b/stratosphere/ro/source/impl/ro_nro_utils.cpp index 181512e..90de057 100644 --- a/stratosphere/ro/source/impl/ro_nro_utils.cpp +++ b/stratosphere/ro/source/impl/ro_nro_utils.cpp @@ -38,7 +38,7 @@ if (bss_heap_size > 0) { map::MappedCodeMemory tmp_bss_mcm(process_handle, base_address + nro_heap_size, bss_heap_address, bss_heap_size); R_TRY_CATCH(tmp_bss_mcm.GetResult()) { - R_CATCH(svc::ResultInvalidCurrentMemoryState) { + R_CATCH(svc::ResultInvalidCurrentMemory) { continue; } } R_END_TRY_CATCH;