192 lines
7.7 KiB
Diff
Executable File
192 lines
7.7 KiB
Diff
Executable File
From 0e4c1a4392bcf58bac954c023ba9b2da48bc0dbe Mon Sep 17 00:00:00 2001
|
|
From: Raffaele Aquilone <[email protected]>
|
|
Date: Wed, 27 Mar 2024 16:59:34 +0000
|
|
Subject: [PATCH] GPUCORE-41547 Improve handling of page metadata after migration
|
|
|
|
Handling of page metadata after migration in kbase_mmu_migrate_page()
|
|
has been made slightly safer. Only if the page is still in the
|
|
ALLOCATED_MAPPED state is it safe to access and update the
|
|
metadata. There's no problem if the page became NOT_MOVABLE.
|
|
It is an error, instead, if the page has changed to any other
|
|
state.
|
|
|
|
The kbase_set_phy_alloc_page_status() helper must be called
|
|
while keeping the region lock to prevent race conditions where
|
|
the pages of a GPU VA region could be set to NOT_MOVABLE
|
|
while a page migration is in progress.
|
|
|
|
TI2: 1151297 (page migration tests, ps 7)
|
|
TI2: 1151300 (DDK precommit, ps 7)
|
|
|
|
Change-Id: I062431075fcd29fcd13e65acd779ab03315f51d0
|
|
---
|
|
|
|
diff --git a/product/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c b/product/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c
|
|
index a0d9789..5a5a4c3 100644
|
|
--- a/product/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c
|
|
+++ b/product/kernel/drivers/gpu/arm/midgard/csf/mali_kbase_csf_tiler_heap.c
|
|
@@ -351,13 +351,14 @@
|
|
}
|
|
|
|
remove_external_chunk_mappings(kctx, chunk);
|
|
- kbase_gpu_vm_unlock(kctx);
|
|
|
|
/* If page migration is enabled, we don't want to migrate tiler heap pages.
|
|
* This does not change if the constituent pages are already marked as isolated.
|
|
*/
|
|
if (kbase_is_page_migration_enabled())
|
|
- kbase_set_phy_alloc_page_status(chunk->region->gpu_alloc, NOT_MOVABLE);
|
|
+ kbase_set_phy_alloc_page_status(kctx, chunk->region->gpu_alloc, NOT_MOVABLE);
|
|
+
|
|
+ kbase_gpu_vm_unlock(kctx);
|
|
|
|
return chunk;
|
|
|
|
@@ -738,7 +739,7 @@
|
|
KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING);
|
|
|
|
if (kbase_is_page_migration_enabled())
|
|
- kbase_set_phy_alloc_page_status(buf_desc_reg->gpu_alloc, NOT_MOVABLE);
|
|
+ kbase_set_phy_alloc_page_status(kctx, buf_desc_reg->gpu_alloc, NOT_MOVABLE);
|
|
|
|
kbase_gpu_vm_unlock(kctx);
|
|
|
|
diff --git a/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c b/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c
|
|
index 1a0b9fa..76886ea 100644
|
|
--- a/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c
|
|
+++ b/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.c
|
|
@@ -2027,11 +2027,13 @@
|
|
}
|
|
KBASE_EXPORT_TEST_API(kbase_alloc_phy_pages);
|
|
|
|
-void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc,
|
|
+void kbase_set_phy_alloc_page_status(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc,
|
|
enum kbase_page_status status)
|
|
{
|
|
u32 i = 0;
|
|
|
|
+ lockdep_assert_held(&kctx->reg_lock);
|
|
+
|
|
for (; i < alloc->nents; i++) {
|
|
struct tagged_addr phys = alloc->pages[i];
|
|
struct kbase_page_metadata *page_md = kbase_page_private(as_page(phys));
|
|
@@ -3293,7 +3295,8 @@
|
|
if (kbase_is_page_migration_enabled()) {
|
|
kbase_gpu_vm_lock(kctx);
|
|
mutex_lock(&kctx->jit_evict_lock);
|
|
- kbase_set_phy_alloc_page_status(reg->gpu_alloc, ALLOCATED_MAPPED);
|
|
+ kbase_set_phy_alloc_page_status(kctx, reg->gpu_alloc,
|
|
+ ALLOCATED_MAPPED);
|
|
mutex_unlock(&kctx->jit_evict_lock);
|
|
kbase_gpu_vm_unlock(kctx);
|
|
}
|
|
@@ -3443,9 +3446,18 @@
|
|
kbase_mem_evictable_mark_reclaim(reg->gpu_alloc);
|
|
|
|
kbase_gpu_vm_lock(kctx);
|
|
+
|
|
reg->flags |= KBASE_REG_DONT_NEED;
|
|
reg->flags &= ~KBASE_REG_ACTIVE_JIT_ALLOC;
|
|
kbase_mem_shrink_cpu_mapping(kctx, reg, 0, reg->gpu_alloc->nents);
|
|
+
|
|
+ /* Inactive JIT regions should be freed by the shrinker and not impacted
|
|
+ * by page migration. Once freed, they will enter into the page migration
|
|
+ * state machine via the mempools.
|
|
+ */
|
|
+ if (kbase_is_page_migration_enabled())
|
|
+ kbase_set_phy_alloc_page_status(kctx, reg->gpu_alloc, NOT_MOVABLE);
|
|
+
|
|
kbase_gpu_vm_unlock(kctx);
|
|
|
|
/*
|
|
@@ -3461,12 +3473,6 @@
|
|
|
|
list_move(®->jit_node, &kctx->jit_pool_head);
|
|
|
|
- /* Inactive JIT regions should be freed by the shrinker and not impacted
|
|
- * by page migration. Once freed, they will enter into the page migration
|
|
- * state machine via the mempools.
|
|
- */
|
|
- if (kbase_is_page_migration_enabled())
|
|
- kbase_set_phy_alloc_page_status(reg->gpu_alloc, NOT_MOVABLE);
|
|
mutex_unlock(&kctx->jit_evict_lock);
|
|
}
|
|
|
|
diff --git a/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h b/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h
|
|
index 59b6ddc0..e35f47e 100644
|
|
--- a/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h
|
|
+++ b/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem.h
|
|
@@ -530,6 +530,7 @@
|
|
/**
|
|
* kbase_set_phy_alloc_page_status - Set the page migration status of the underlying
|
|
* physical allocation.
|
|
+ * @kctx: Pointer to Kbase context.
|
|
* @alloc: the physical allocation containing the pages whose metadata is going
|
|
* to be modified
|
|
* @status: the status the pages should end up in
|
|
@@ -538,7 +539,7 @@
|
|
* proper states are set. Instead, it is only used when we change the allocation
|
|
* to NOT_MOVABLE or from NOT_MOVABLE to ALLOCATED_MAPPED
|
|
*/
|
|
-void kbase_set_phy_alloc_page_status(struct kbase_mem_phy_alloc *alloc,
|
|
+void kbase_set_phy_alloc_page_status(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc,
|
|
enum kbase_page_status status);
|
|
|
|
static inline void kbase_mem_phy_alloc_gpu_mapped(struct kbase_mem_phy_alloc *alloc)
|
|
diff --git a/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c b/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c
|
|
index 70b7163..925bc1d 100644
|
|
--- a/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c
|
|
+++ b/product/kernel/drivers/gpu/arm/midgard/mali_kbase_mem_linux.c
|
|
@@ -885,7 +885,7 @@
|
|
/* Indicate to page migration that the memory can be reclaimed by the shrinker.
|
|
*/
|
|
if (kbase_is_page_migration_enabled())
|
|
- kbase_set_phy_alloc_page_status(gpu_alloc, NOT_MOVABLE);
|
|
+ kbase_set_phy_alloc_page_status(kctx, gpu_alloc, NOT_MOVABLE);
|
|
|
|
mutex_unlock(&kctx->jit_evict_lock);
|
|
kbase_mem_evictable_mark_reclaim(gpu_alloc);
|
|
@@ -943,7 +943,7 @@
|
|
* from.
|
|
*/
|
|
if (kbase_is_page_migration_enabled())
|
|
- kbase_set_phy_alloc_page_status(gpu_alloc, ALLOCATED_MAPPED);
|
|
+ kbase_set_phy_alloc_page_status(kctx, gpu_alloc, ALLOCATED_MAPPED);
|
|
}
|
|
}
|
|
|
|
diff --git a/product/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c b/product/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c
|
|
index c6f96a81..ab0d85b 100644
|
|
--- a/product/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c
|
|
+++ b/product/kernel/drivers/gpu/arm/midgard/mmu/mali_kbase_mmu.c
|
|
@@ -4003,13 +4003,24 @@
|
|
/* Undertaking metadata transfer, while we are holding the mmu_lock */
|
|
spin_lock(&page_md->migrate_lock);
|
|
if (level == MIDGARD_MMU_BOTTOMLEVEL) {
|
|
- size_t page_array_index = (page_md->data.mapped.vpfn / GPU_PAGES_PER_CPU_PAGE) -
|
|
- page_md->data.mapped.reg->start_pfn;
|
|
+ enum kbase_page_status page_status = PAGE_STATUS_GET(page_md->status);
|
|
|
|
- WARN_ON(PAGE_STATUS_GET(page_md->status) != ALLOCATED_MAPPED);
|
|
+ if (page_status == ALLOCATED_MAPPED) {
|
|
+ /* Replace page in array of pages of the physical allocation. */
|
|
+ size_t page_array_index =
|
|
+ div_u64(page_md->data.mapped.vpfn, GPU_PAGES_PER_CPU_PAGE) -
|
|
+ page_md->data.mapped.reg->start_pfn;
|
|
|
|
- /* Replace page in array of pages of the physical allocation. */
|
|
- page_md->data.mapped.reg->gpu_alloc->pages[page_array_index] = new_phys;
|
|
+ page_md->data.mapped.reg->gpu_alloc->pages[page_array_index] = new_phys;
|
|
+ } else if (page_status == NOT_MOVABLE) {
|
|
+ dev_dbg(kbdev->dev,
|
|
+ "%s: migration completed and page has become NOT_MOVABLE.",
|
|
+ __func__);
|
|
+ } else {
|
|
+ dev_WARN(kbdev->dev,
|
|
+ "%s: migration completed but page has moved to status %d.",
|
|
+ __func__, page_status);
|
|
+ }
|
|
}
|
|
/* Update the new page dma_addr with the transferred metadata from the old_page */
|
|
page_md->dma_addr = new_dma_addr;
|