drm/amdgpu: Add PASID management

Allows assigning a PASID to a VM for identifying VMs involved in page
faults. The global PASID manager is also exported in the KFD
interface so that AMDGPU and KFD can share the PASID space.

PASIDs of different sizes can be requested. On APUs, the PASID size
is deterined by the capabilities of the IOMMU. So KFD must be able
to allocate PASIDs in a smaller range.

Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Acked-by: Alex Deucher <alexander.deucher@amd.com>
Reviewed-by: Oded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
index b9dbbf9cb8b07eea5a367822131e249dd17f1e94..dc7e25cce7412a1488687caaf1111b2d299fb07d 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c
@@ -169,6 +169,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
        .get_vmem_size = get_vmem_size,
        .get_gpu_clock_counter = get_gpu_clock_counter,
        .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+       .alloc_pasid = amdgpu_vm_alloc_pasid,
+       .free_pasid = amdgpu_vm_free_pasid,
        .program_sh_mem_settings = kgd_program_sh_mem_settings,
        .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
        .init_pipeline = kgd_init_pipeline,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
index 309f2419c6d8aaed250f1fd2235cc45c3cdef1d7..c678c69936a0b8370f14ddd62d872043150b1b35 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c
@@ -128,6 +128,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
        .get_vmem_size = get_vmem_size,
        .get_gpu_clock_counter = get_gpu_clock_counter,
        .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
+       .alloc_pasid = amdgpu_vm_alloc_pasid,
+       .free_pasid = amdgpu_vm_free_pasid,
        .program_sh_mem_settings = kgd_program_sh_mem_settings,
        .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
        .init_pipeline = kgd_init_pipeline,

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index e16229000a983e8cc068a70af7ec32074be06c30..79d9ab43d42c0994f3a00ff0ceaf8ab9026df091 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -825,7 +825,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
        }
 
        r = amdgpu_vm_init(adev, &fpriv->vm,
-                          AMDGPU_VM_CONTEXT_GFX);
+                          AMDGPU_VM_CONTEXT_GFX, 0);
        if (r) {
                kfree(fpriv);
                goto out_suspend;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 2196bca7331c262fb6aade12513e1d5cfb7ca990..9b795915cab1d731d9dbfad72191a88948b68118 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -27,11 +27,58 @@
  */
 #include <linux/dma-fence-array.h>
 #include <linux/interval_tree_generic.h>
+#include <linux/idr.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
+/*
+ * PASID manager
+ *
+ * PASIDs are global address space identifiers that can be shared
+ * between the GPU, an IOMMU and the driver. VMs on different devices
+ * may use the same PASID if they share the same address
+ * space. Therefore PASIDs are allocated using a global IDA. VMs are
+ * looked up from the PASID per amdgpu_device.
+ */
+static DEFINE_IDA(amdgpu_vm_pasid_ida);
+
+/**
+ * amdgpu_vm_alloc_pasid - Allocate a PASID
+ * @bits: Maximum width of the PASID in bits, must be at least 1
+ *
+ * Allocates a PASID of the given width while keeping smaller PASIDs
+ * available if possible.
+ *
+ * Returns a positive integer on success. Returns %-EINVAL if bits==0.
+ * Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
+ * memory allocation failure.
+ */
+int amdgpu_vm_alloc_pasid(unsigned int bits)
+{
+       int pasid = -EINVAL;
+
+       for (bits = min(bits, 31U); bits > 0; bits--) {
+               pasid = ida_simple_get(&amdgpu_vm_pasid_ida,
+                                      1U << (bits - 1), 1U << bits,
+                                      GFP_KERNEL);
+               if (pasid != -ENOSPC)
+                       break;
+       }
+
+       return pasid;
+}
+
+/**
+ * amdgpu_vm_free_pasid - Free a PASID
+ * @pasid: PASID to free
+ */
+void amdgpu_vm_free_pasid(unsigned int pasid)
+{
+       ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);
+}
+
 /*
  * GPUVM
  * GPUVM is similar to the legacy gart on older asics, however
@@ -2539,7 +2586,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_
  * Init @vm fields.
  */
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                  int vm_context)
+                  int vm_context, unsigned int pasid)
 {
        const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
                AMDGPU_VM_PTE_COUNT(adev) * 8);
@@ -2620,6 +2667,19 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
                        goto error_free_root;
        }
 
+       if (pasid) {
+               unsigned long flags;
+
+               spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+               r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
+                             GFP_ATOMIC);
+               spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+               if (r < 0)
+                       goto error_free_root;
+
+               vm->pasid = pasid;
+       }
+
        return 0;
 
 error_free_root:
@@ -2673,6 +2733,14 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
        bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
        int i;
 
+       if (vm->pasid) {
+               unsigned long flags;
+
+               spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
+               idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
+               spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
+       }
+
        amd_sched_entity_fini(vm->entity.sched, &vm->entity);
 
        if (!RB_EMPTY_ROOT(&vm->va)) {
@@ -2752,6 +2820,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
        adev->vm_manager.vm_update_mode = 0;
 #endif
 
+       idr_init(&adev->vm_manager.pasid_idr);
+       spin_lock_init(&adev->vm_manager.pasid_lock);
 }
 
 /**
@@ -2765,6 +2835,9 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
 {
        unsigned i, j;
 
+       WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
+       idr_destroy(&adev->vm_manager.pasid_idr);
+
        for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
                struct amdgpu_vm_id_manager *id_mgr =
                        &adev->vm_manager.id_mgr[i];

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 48c58ae4bb3a14270179c8181258ed7f6aa0f960..7873dfa8c0f953bd55b05d9f28241592534a966b 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -25,6 +25,7 @@
 #define __AMDGPU_VM_H__
 
 #include <linux/rbtree.h>
+#include <linux/idr.h>
 
 #include "gpu_scheduler.h"
 #include "amdgpu_sync.h"
@@ -148,8 +149,9 @@ struct amdgpu_vm {
        /* Scheduler entity for page table updates */
        struct amd_sched_entity entity;
 
-       /* client id */
+       /* client id and PASID (TODO: replace client_id with PASID) */
        u64                     client_id;
+       unsigned int            pasid;
        /* dedicated to vm */
        struct amdgpu_vm_id     *reserved_vmid[AMDGPU_MAX_VMHUBS];
 
@@ -220,12 +222,20 @@ struct amdgpu_vm_manager {
         * BIT1[= 0] Compute updated by SDMA [= 1] by CPU
         */
        int                                     vm_update_mode;
+
+       /* PASID to VM mapping, will be used in interrupt context to
+        * look up VM of a page fault
+        */
+       struct idr                              pasid_idr;
+       spinlock_t                              pasid_lock;
 };
 
+int amdgpu_vm_alloc_pasid(unsigned int bits);
+void amdgpu_vm_free_pasid(unsigned int pasid);
 void amdgpu_vm_manager_init(struct amdgpu_device *adev);
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
-                  int vm_context);
+                  int vm_context, unsigned int pasid);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
                         struct list_head *validated,

diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 94277cb734d2f6febbeeef143db081a3c78e2f66..f516fd10e6ba7469856a59d039a8e1d715251a0e 100644 (file)
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -112,6 +112,9 @@ struct tile_config {
  *
  * @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz
  *
+ * @alloc_pasid: Allocate a PASID
+ * @free_pasid: Free a PASID
+ *
  * @program_sh_mem_settings: A function that should initiate the memory
  * properties such as main aperture memory type (cache / non cached) and
  * secondary aperture base address, size and memory type.
@@ -160,6 +163,9 @@ struct kfd2kgd_calls {
 
        uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd);
 
+       int (*alloc_pasid)(unsigned int bits);
+       void (*free_pasid)(unsigned int pasid);
+
        /* Register access functions */
        void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid,
                        uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,