GitBucket
Newer
/*
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <mesosphere.hpp>
#include "kern_lps_driver.hpp"
#include "kern_k_sleep_manager.hpp"
#include "kern_bpmp_api.hpp"
#include "kern_atomics_registers.hpp"
#include "kern_ictlr_registers.hpp"
#include "kern_sema_registers.hpp"
namespace ams::kern::board::nintendo::nx::lps {
namespace {
constexpr inline int ChannelCount = 12;
constexpr inline TimeSpan ChannelTimeout = TimeSpan::FromSeconds(1);
constinit bool g_lps_init_done = false;
constinit bool g_bpmp_connected = false;
constinit bool g_bpmp_mail_initialized = false;
constinit KSpinLock g_bpmp_mrq_lock;
constinit KVirtualAddress g_evp_address = Null<KVirtualAddress>;
constinit KVirtualAddress g_flow_address = Null<KVirtualAddress>;
constinit KVirtualAddress g_prictlr_address = Null<KVirtualAddress>;
constinit KVirtualAddress g_sema_address = Null<KVirtualAddress>;
constinit KVirtualAddress g_atomics_address = Null<KVirtualAddress>;
constinit KVirtualAddress g_clkrst_address = Null<KVirtualAddress>;
constinit KVirtualAddress g_pmc_address = Null<KVirtualAddress>;
constinit ChannelData g_channel_area[ChannelCount] = {};
constinit u32 g_csite_clk_source = 0;
ALWAYS_INLINE u32 Read(KVirtualAddress address) {
return *GetPointer<volatile u32>(address);
}
ALWAYS_INLINE void Write(KVirtualAddress address, u32 value) {
*GetPointer<volatile u32>(address) = value;
}
void InitializeDeviceVirtualAddresses() {
/* Retrieve randomized mappings. */
g_evp_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_LegacyLpsExceptionVectors);
g_flow_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_LegacyLpsFlowController);
g_prictlr_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_LegacyLpsPrimaryICtlr);
g_sema_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_LegacyLpsSemaphore);
g_atomics_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_LegacyLpsAtomics);
g_clkrst_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_LegacyLpsClkRst);
g_pmc_address = KMemoryLayout::GetDeviceVirtualAddress(KMemoryRegionType_PowerManagementController);
}
/* NOTE: linux "do_cc4_init" */
void ConfigureCc3AndCc4() {
/* Configure CC4/CC3 as enabled with time threshold as 2 microseconds. */
Write(g_flow_address + FLOW_CTLR_CC4_HVC_CONTROL, (0x2 << 3) | 0x1);
/* Configure Retention with threshold 2 microseconds. */
Write(g_flow_address + FLOW_CTLR_CC4_RETENTION_CONTROL, (0x2 << 3));
/* Configure CC3/CC3 retry threshold as 2 microseconds. */
Write(g_flow_address + FLOW_CTLR_CC4_HVC_RETRY, (0x2 << 3));
/* Read the retry register to ensure writes take. */
Read(g_flow_address + FLOW_CTLR_CC4_HVC_RETRY);
}
constexpr bool IsValidMessageDataSize(int size) {
return 0 <= size && size < MessageDataSizeMax;
}
/* NOTE: linux "bpmp_valid_txfer" */
constexpr bool IsTransferValid(const void *ob, int ob_size, void *ib, int ib_size) {
return IsValidMessageDataSize(ob_size) && IsValidMessageDataSize(ib_size) && (ob_size == 0 || ob != nullptr) && (ib_size == 0 || ib != nullptr);
}
/* NOTE: linux "bpmp_ob_channel" */
int BpmpGetOutboundChannel() {
return GetCurrentCoreId();
}
/* NOTE: linux "bpmp_ch_sta" */
u32 BpmpGetChannelState(int channel) {
cpu::DataSynchronizationBarrier();
return Read(g_sema_address + RES_SEMA_SHRD_SMP_STA) & CH_MASK(channel);
}
/* NOTE: linux "bpmp_master_free" */
bool BpmpIsMasterFree(int channel) {
return BpmpGetChannelState(channel) == MA_FREE(channel);
}
/* NOTE: linux "bpmp_master_acked" */
bool BpmpIsMasterAcked(int channel) {
return BpmpGetChannelState(channel) == MA_ACKD(channel);
}
/* NOTE: linux "bpmp_signal_slave" */
void BpmpSignalSlave(int channel) {
Write(g_sema_address + RES_SEMA_SHRD_SMP_CLR, CH_MASK(channel));
cpu::DataSynchronizationBarrier();
}
/* NOTE: linux "bpmp_free_master" */
void BpmpFreeMaster(int channel) {
/* Transition state from ack'd to free. */
Write(g_sema_address + RES_SEMA_SHRD_SMP_CLR, ((MA_ACKD(channel)) ^ (MA_FREE(channel))));
cpu::DataSynchronizationBarrier();
}
/* NOTE: linux "bpmp_ring_doorbell" */
void BpmpRingDoorbell() {
Write(g_prictlr_address + ICTLR_FIR_SET(INT_SHR_SEM_OUTBOX_IBF), FIR_BIT(INT_SHR_SEM_OUTBOX_IBF));
cpu::DataSynchronizationBarrier();
}
/* NOTE: linux "bpmp_wait_master_free" */
int BpmpWaitMasterFree(int channel) {
/* Check if the master is already freed. */
if (BpmpIsMasterFree(channel)) {
return 0;
}
/* Spin-poll for the master to be freed until timeout occurs. */
const auto start_tick = KHardwareTimer::GetTick();
const auto timeout = ams::svc::Tick(ChannelTimeout);
do {
if (BpmpIsMasterFree(channel)) {
return 0;
}
} while ((KHardwareTimer::GetTick() - start_tick) < timeout);
/* The master didn't become free. */
return -1;
}
/* NOTE: linux "bpmp_wait_ack" */
int BpmpWaitAck(int channel) {
/* Check if the master is already ACK'd. */
if (BpmpIsMasterAcked(channel)) {
return 0;
}
/* Spin-poll for the master to be ACK'd until timeout occurs. */
const auto start_tick = KHardwareTimer::GetTick();
const auto timeout = ams::svc::Tick(ChannelTimeout);
do {
if (BpmpIsMasterAcked(channel)) {
return 0;
}
} while ((KHardwareTimer::GetTick() - start_tick) < timeout);
/* The master didn't get ACK'd. */
return -1;
}
/* NOTE: linux "bpmp_write_ch" */
int BpmpWriteChannel(int channel, int mrq, int flags, const void *data, size_t data_size) {
/* Wait to be able to master the mailbox. */
if (int res = BpmpWaitMasterFree(channel); res != 0) {
return res;
}
/* Prepare the message. */
MailboxData *mb = g_channel_area[channel].ob;
mb->code = mrq;
mb->flags = flags;
if (data != nullptr) {
std::memcpy(mb->data, data, data_size);
}
/* Signal to slave that message is available. */
BpmpSignalSlave(channel);
return 0;
}
/* NOTE: linux "__bpmp_read_ch" */
int BpmpReadChannel(int channel, void *data, size_t data_size) {
/* Get the message. */
MailboxData *mb = g_channel_area[channel].ib;
/* Copy any return data. */
if (data != nullptr) {
std::memcpy(data, mb->data, data_size);
}
/* Free the channel. */
BpmpFreeMaster(channel);
/* Return result. */
return mb->code;
}
/* NOTE: linux "tegra_bpmp_send_receive_atomic" or "tegra_bpmp_send_receive". */
int BpmpSendAndReceive(int mrq, const void *ob, int ob_size, void *ib, int ib_size) {
/* Validate that the data transfer is valid. */
if (!IsTransferValid(ob, ob_size, ib, ib_size)) {
return -1;
}
/* Validate that the bpmp is connected. */
if (!g_bpmp_connected) {
return -1;
}
/* Disable interrupts. */
KScopedInterruptDisable di;
/* Acquire exclusive access to send mrqs. */
KScopedSpinLock lk(g_bpmp_mrq_lock);
/* Send the message. */
int channel = BpmpGetOutboundChannel();
if (int res = BpmpWriteChannel(channel, mrq, BPMP_MSG_DO_ACK, ob, ob_size); res != 0) {
return res;
}
/* Send "doorbell" irq to the bpmp firmware. */
BpmpRingDoorbell();
/* Wait for the bpmp firmware to acknowledge our request. */
if (int res = BpmpWaitAck(channel); res != 0) {
return res;
}
/* Read the data the bpmp sent back. */
return BpmpReadChannel(channel, ib, ib_size);
}
/* NOTE: linux "tegra_bpmp_send" */
int BpmpSend(int mrq, const void *ob, int ob_size) {
/* Validate that the data transfer is valid. */
if (!IsTransferValid(ob, ob_size, nullptr, 0)) {
return -1;
}
/* Validate that the bpmp is connected. */
if (!g_bpmp_connected) {
return -1;
}
/* Disable interrupts. */
KScopedInterruptDisable di;
/* Acquire exclusive access to send mrqs. */
KScopedSpinLock lk(g_bpmp_mrq_lock);
/* Send the message. */
int channel = BpmpGetOutboundChannel();
if (int res = BpmpWriteChannel(channel, mrq, 0, ob, ob_size); res != 0) {
return res;
}
/* Send "doorbell" irq to the bpmp firmware. */
BpmpRingDoorbell();
return 0;
}
/* NOTE: modified linux "tegra_bpmp_enable_suspend" */
int BpmpEnableSuspend(int mode, int flags) {
/* Prepare data for bpmp. */
const s32 data[] = { mode, flags };
/* Send the data. */
return BpmpSend(MRQ_ENABLE_SUSPEND, data, sizeof(data));
}
/* NOTE: linux "__bpmp_connect" */
int ConnectToBpmp() {
/* Check if we've already connected. */
if (g_bpmp_connected) {
return 0;
}
/* Verify that the resource semaphore state is set. */
if (Read(g_sema_address + RES_SEMA_SHRD_SMP_STA) == 0) {
return -1;
}
/* Get the channels, which the bpmp firmware has configured in advance. */
{
const KVirtualAddress iram_virt_addr = KMemoryLayout::GetDeviceVirtualAddress (KMemoryRegionType_LegacyLpsIram);
const KPhysicalAddress iram_phys_addr = KMemoryLayout::GetDevicePhysicalAddress(KMemoryRegionType_LegacyLpsIram);
for (auto i = 0; i < ChannelCount; ++i) {
/* Trigger a get command for the desired channel. */
Write(g_atomics_address + ATOMICS_AP0_TRIGGER, TRIGGER_CMD_GET | (i << 16));
/* Retrieve the channel phys-addr-in-iram, and convert it to a kernel address. */
auto *ch = GetPointer<MailboxData>(iram_virt_addr + (Read(g_atomics_address + ATOMICS_AP0_RESULT(i)) - GetInteger(iram_phys_addr)));
/* Verify the channel isn't null. */
/* NOTE: This is an utterly nonsense check, as this would require the bpmp firmware to specify */
/* a phys-to-virt diff as an address. On 1.0.0, which had no ASLR, this was 0x8028C000. */
/* However, Nintendo has the check, and we'll preserve it to be faithful. */
if (ch == nullptr) {
return -1;
}
/* Set the channel in the channel area. */
g_channel_area[i].ib = ch;
g_channel_area[i].ob = ch;
}
}
/* Mark driver as connected to bpmp. */
g_bpmp_connected = true;
return 0;
}
/* NOTE: Modified linux "bpmp_mail_init" */
int InitializeBpmpMail() {
/* Check if we've already initialized. */
if (g_bpmp_mail_initialized) {
return 0;
}
/* Mark function as having been called. */
g_bpmp_mail_initialized = true;
/* Forward declare result/reply variables. */
int res, request = 0, reply = 0;
/* Try to connect to the bpmp. */
if (res = ConnectToBpmp(); res != 0) {
MESOSPHERE_LOG("bpmp: connect error returns %d\n", res);
return res;
}
/* Ensure that we can successfully ping the bpmp. */
request = 1;
if (res = BpmpSendAndReceive(MRQ_PING, std::addressof(request), sizeof(request), std::addressof(reply), sizeof(reply)); res != 0) {
MESOSPHERE_LOG("bpmp: MRQ_PING error returns %d with reply %d\n", res, reply);
return res;
}
/* Configure the PMIC. */
request = 1;
if (res = BpmpSendAndReceive(MRQ_CPU_PMIC_SELECT, std::addressof(request), sizeof(request), std::addressof(reply), sizeof(reply)); res != 0) {
MESOSPHERE_LOG("bpmp: MRQ_CPU_PMIC_SELECT for MAX77621 error returns %d with reply %d\n", res, reply);
return res;
}
return 0;
}
}
void Initialize() {
if (!g_lps_init_done) {
/* Get the addresses of the devices the driver needs. */
InitializeDeviceVirtualAddresses();
/* Configure CC3/CC4. */
ConfigureCc3AndCc4();
/* Initialize ccplex <-> bpmp mail. */
/* NOTE: Nintendo does not check that this call succeeds. */
InitializeBpmpMail();
g_lps_init_done = true;
}
}
Result EnableSuspend(bool enable) {
/* If we're not on core 0, there's nothing to do. */
R_SUCCEED_IF(GetCurrentCoreId() != 0);
/* If we're not enabling suspend, there's nothing to do. */
R_SUCCEED_IF(!enable);
/* Instruct BPMP to enable suspend-to-sc7. */
R_UNLESS(BpmpEnableSuspend(TEGRA_BPMP_PM_SC7, 0) == 0, svc::ResultInvalidState());
return ResultSuccess();
}
void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry) {
/* Verify that we're allowed to perform suspension. */
MESOSPHERE_ABORT_UNLESS(g_lps_init_done);
MESOSPHERE_ABORT_UNLESS(GetCurrentCoreId() == 0);
/* Save the CSITE clock source. */
g_csite_clk_source = Read(g_clkrst_address + CLK_RST_CONTROLLER_CLK_SOURCE_CSITE);
/* Configure CSITE clock source as CLK_M. */
Write(g_clkrst_address + CLK_RST_CONTROLLER_CLK_SOURCE_CSITE, (0x6 << 29));
/* Clear the top bit of PMC_SCRATCH4. */
Write(g_pmc_address + APBDEV_PMC_SCRATCH4, Read(g_pmc_address + APBDEV_PMC_SCRATCH4) & 0x7FFFFFFF);
/* Write 1 to PMC_SCRATCH0. This will cause the bootrom to use the warmboot code-path. */
Write(g_pmc_address + APBDEV_PMC_SCRATCH0, 1);
/* Read PMC_SCRATCH0 to be sure our write takes. */
Read(g_pmc_address + APBDEV_PMC_SCRATCH0);
/* Invoke the sleep hander. */
KSleepManager::CpuSleepHandler(arg, entry);
/* Disable deep power down. */
Write(g_pmc_address + APBDEV_PMC_DPD_ENABLE, 0);
/* Restore the saved CSITE clock source. */
Write(g_clkrst_address + CLK_RST_CONTROLLER_CLK_SOURCE_CSITE, g_csite_clk_source);
/* Read the CSITE clock source to ensure our configuration takes. */
Read(g_clkrst_address + CLK_RST_CONTROLLER_CLK_SOURCE_CSITE);
/* Configure CC3/CC4. */
ConfigureCc3AndCc4();
}
void ResumeBpmpFirmware() {
/* Halt the bpmp. */
Write(g_flow_address + FLOW_CTLR_HALT_COP_EVENTS, (0x2 << 29));
/* Hold the bpmp in reset. */
Write(g_clkrst_address + CLK_RST_CONTROLLER_RST_DEV_L_SET, 0x2);
/* Read the saved bpmp entrypoint, and write it to the relevant exception vector. */
const u32 bpmp_entry = Read(g_pmc_address + APBDEV_PMC_SCRATCH39);
Write(g_evp_address + EVP_COP_RESET_VECTOR, bpmp_entry);
/* Verify that we can read back the address we wrote. */
while (Read(g_evp_address + EVP_COP_RESET_VECTOR) != bpmp_entry) {
/* ... */
}
/* Spin for 40 ticks, to give enough time for the bpmp to be reset. */
const auto start_tick = KHardwareTimer::GetTick();
do {
__asm__ __volatile__("" ::: "memory");
} while ((KHardwareTimer::GetTick() - start_tick) < 40);
/* Take the bpmp out of reset. */
Write(g_clkrst_address + CLK_RST_CONTROLLER_RST_DEV_L_CLR, 0x2);
/* Resume the bpmp. */
Write(g_flow_address + FLOW_CTLR_HALT_COP_EVENTS, (0x0 << 29));
}
}