diff --git a/src/core/hle/kernel/memory.cpp b/src/core/hle/kernel/memory.cpp
index 496d07cb59..7f27e9655c 100644
--- a/src/core/hle/kernel/memory.cpp
+++ b/src/core/hle/kernel/memory.cpp
@@ -8,7 +8,6 @@
 #include <memory>
 #include <utility>
 #include <vector>
-#include "audio_core/audio_core.h"
 #include "common/assert.h"
 #include "common/common_types.h"
 #include "common/logging/log.h"
@@ -24,7 +23,7 @@
 
 namespace Kernel {
 
-static MemoryRegionInfo memory_regions[3];
+MemoryRegionInfo memory_regions[3];
 
 /// Size of the APPLICATION, SYSTEM and BASE memory regions (respectively) for each system
 /// memory configuration type.
@@ -96,9 +95,6 @@ MemoryRegionInfo* GetMemoryRegion(MemoryRegion region) {
     }
 }
 
-std::array<u8, Memory::VRAM_SIZE> vram;
-std::array<u8, Memory::N3DS_EXTRA_RAM_SIZE> n3ds_extra_ram;
-
 void HandleSpecialMapping(VMManager& address_space, const AddressMapping& mapping) {
     using namespace Memory;
 
@@ -143,30 +139,14 @@ void HandleSpecialMapping(VMManager& address_space, const AddressMapping& mappin
         return;
     }
 
-    // TODO(yuriks): Use GetPhysicalPointer when that becomes independent of the virtual
-    // mappings.
-    u8* target_pointer = nullptr;
-    switch (area->paddr_base) {
-    case VRAM_PADDR:
-        target_pointer = vram.data();
-        break;
-    case DSP_RAM_PADDR:
-        target_pointer = AudioCore::GetDspMemory().data();
-        break;
-    case N3DS_EXTRA_RAM_PADDR:
-        target_pointer = n3ds_extra_ram.data();
-        break;
-    default:
-        UNREACHABLE();
-    }
+    u8* target_pointer = Memory::GetPhysicalPointer(area->paddr_base + offset_into_region);
 
     // TODO(yuriks): This flag seems to have some other effect, but it's unknown what
     MemoryState memory_state = mapping.unk_flag ? MemoryState::Static : MemoryState::IO;
 
-    auto vma = address_space
-                   .MapBackingMemory(mapping.address, target_pointer + offset_into_region,
-                                     mapping.size, memory_state)
-                   .Unwrap();
+    auto vma =
+        address_space.MapBackingMemory(mapping.address, target_pointer, mapping.size, memory_state)
+            .Unwrap();
     address_space.Reprotect(vma,
                             mapping.read_only ? VMAPermission::Read : VMAPermission::ReadWrite);
 }
diff --git a/src/core/hle/kernel/memory.h b/src/core/hle/kernel/memory.h
index 08c1a99892..da6bb35633 100644
--- a/src/core/hle/kernel/memory.h
+++ b/src/core/hle/kernel/memory.h
@@ -26,4 +26,6 @@ MemoryRegionInfo* GetMemoryRegion(MemoryRegion region);
 
 void HandleSpecialMapping(VMManager& address_space, const AddressMapping& mapping);
 void MapSharedPages(VMManager& address_space);
+
+extern MemoryRegionInfo memory_regions[3];
 } // namespace Kernel
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index ea46b6ead8..4dcbf22748 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -4,10 +4,12 @@
 
 #include <array>
 #include <cstring>
+#include "audio_core/audio_core.h"
 #include "common/assert.h"
 #include "common/common_types.h"
 #include "common/logging/log.h"
 #include "common/swap.h"
+#include "core/hle/kernel/memory.h"
 #include "core/hle/kernel/process.h"
 #include "core/memory.h"
 #include "core/memory_setup.h"
@@ -16,6 +18,9 @@
 
 namespace Memory {
 
+static std::array<u8, Memory::VRAM_SIZE> vram;
+static std::array<u8, Memory::N3DS_EXTRA_RAM_SIZE> n3ds_extra_ram;
+
 PageTable* current_page_table = nullptr;
 
 std::array<u8*, PAGE_TABLE_NUM_ENTRIES>* GetCurrentPageTablePointers() {
@@ -236,9 +241,63 @@ std::string ReadCString(VAddr vaddr, std::size_t max_length) {
 }
 
 u8* GetPhysicalPointer(PAddr address) {
-    // TODO(Subv): This call should not go through the application's memory mapping.
-    boost::optional<VAddr> vaddr = PhysicalToVirtualAddress(address);
-    return vaddr ? GetPointer(*vaddr) : nullptr;
+    struct MemoryArea {
+        PAddr paddr_base;
+        u32 size;
+    };
+
+    static constexpr MemoryArea memory_areas[] = {
+        {VRAM_PADDR, VRAM_SIZE},
+        {IO_AREA_PADDR, IO_AREA_SIZE},
+        {DSP_RAM_PADDR, DSP_RAM_SIZE},
+        {FCRAM_PADDR, FCRAM_N3DS_SIZE},
+        {N3DS_EXTRA_RAM_PADDR, N3DS_EXTRA_RAM_SIZE},
+    };
+
+    const auto area =
+        std::find_if(std::begin(memory_areas), std::end(memory_areas), [&](const auto& area) {
+            return address >= area.paddr_base && address < area.paddr_base + area.size;
+        });
+
+    if (area == std::end(memory_areas)) {
+        LOG_ERROR(HW_Memory, "unknown GetPhysicalPointer @ 0x%08X", address);
+        return nullptr;
+    }
+
+    if (area->paddr_base == IO_AREA_PADDR) {
+        LOG_ERROR(HW_Memory, "MMIO mappings are not supported yet. phys_addr=0x%08X", address);
+        return nullptr;
+    }
+
+    u32 offset_into_region = address - area->paddr_base;
+
+    u8* target_pointer = nullptr;
+    switch (area->paddr_base) {
+    case VRAM_PADDR:
+        target_pointer = vram.data() + offset_into_region;
+        break;
+    case DSP_RAM_PADDR:
+        target_pointer = AudioCore::GetDspMemory().data() + offset_into_region;
+        break;
+    case FCRAM_PADDR:
+        for (const auto& region : Kernel::memory_regions) {
+            if (offset_into_region >= region.base &&
+                offset_into_region < region.base + region.size) {
+                target_pointer =
+                    region.linear_heap_memory->data() + offset_into_region - region.base;
+                break;
+            }
+        }
+        ASSERT_MSG(target_pointer != nullptr, "Invalid FCRAM address");
+        break;
+    case N3DS_EXTRA_RAM_PADDR:
+        target_pointer = n3ds_extra_ram.data() + offset_into_region;
+        break;
+    default:
+        UNREACHABLE();
+    }
+
+    return target_pointer;
 }
 
 void RasterizerMarkRegionCached(PAddr start, u32 size, int count_delta) {
diff --git a/src/core/memory.h b/src/core/memory.h
index 859a142022..b228a48c2e 100644
--- a/src/core/memory.h
+++ b/src/core/memory.h
@@ -227,8 +227,6 @@ boost::optional<VAddr> PhysicalToVirtualAddress(PAddr addr);
 
 /**
  * Gets a pointer to the memory region beginning at the specified physical address.
- *
- * @note This is currently implemented using PhysicalToVirtualAddress().
  */
 u8* GetPhysicalPointer(PAddr address);