drm/amdkfd: heavy-weight flush TLB after unmap

Need do a heavy-weight TLB flush to make sure we have no more dirty data
in the cache for the unmapped pages.

Define enum TLB_FLUSH_TYPE, add flush_type parameter to
amdgpu_amdkfd_flush_gpu_tlb_pasid.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Reviewed-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 5f6696a3c778c599f52c23868546442c1687afeb..bfab2f9fdd17910bc0f0ff17f74a42d08e9b808a 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -670,10 +670,10 @@ int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid)
        return 0;
 }
 
-int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid)
+int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid,
+                                     enum TLB_FLUSH_TYPE flush_type)
 {
        struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
-       const uint32_t flush_type = 0;
        bool all_hub = false;
 
        if (adev->family == AMDGPU_FAMILY_AI)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 313ee49b9f17dbc568edc42d494f517578d817bd..f4de50de359b12c2dc54a5364659ede8c6da79da 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -36,6 +36,12 @@
 
 extern uint64_t amdgpu_amdkfd_total_mem_size;
 
+enum TLB_FLUSH_TYPE {
+       TLB_FLUSH_LEGACY = 0,
+       TLB_FLUSH_LIGHTWEIGHT,
+       TLB_FLUSH_HEAVYWEIGHT
+};
+
 struct amdgpu_device;
 
 struct kfd_bo_va_list {
@@ -135,7 +141,8 @@ int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
 void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
 bool amdgpu_amdkfd_have_atomics_support(struct kgd_dev *kgd);
 int amdgpu_amdkfd_flush_gpu_tlb_vmid(struct kgd_dev *kgd, uint16_t vmid);
-int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid);
+int amdgpu_amdkfd_flush_gpu_tlb_pasid(struct kgd_dev *kgd, uint16_t pasid,
+                                     enum TLB_FLUSH_TYPE flush_type);
 
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
 

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 9d4f527bda7c4f1c4fd02027c082b5aa3be640bb..5b6c5669c03de7f6807562df8599365017772027 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1853,7 +1853,7 @@ void kfd_flush_tlb(struct kfd_process_device *pdd)
                                                        pdd->qpd.vmid);
        } else {
                amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
-                                               pdd->process->pasid);
+                                       pdd->process->pasid, TLB_FLUSH_LEGACY);
        }
 }
 

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index a52e4594785f604be1eca3cc03b036e282138d26..f2901e7a993ecf75d3417758407206829328da0d 100644 (file)
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -1127,7 +1127,7 @@ svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
                                break;
                }
                amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
-                                                 p->pasid);
+                                       p->pasid, TLB_FLUSH_HEAVYWEIGHT);
        }
 
        return r;
@@ -1183,7 +1183,7 @@ svm_range_map_to_gpu(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
                p = container_of(prange->svms, struct kfd_process, svms);
                amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
-                                                 p->pasid);
+                                               p->pasid, TLB_FLUSH_LEGACY);
        }
 out:
        prange->mapping.bo_va = NULL;