drm/amdgpu: stop reserving the BO in the MMU callback v3

Instead take the callback lock during the final parts of CS.

This should solve the last remaining locking order problems with BO reservations.

v2: rebase, make dummy functions static inline
v3: add one more missing inline and comments

Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 134a049f87bddfddc6f4b716d53bc7ef844e3076..740683474a1c4fc04a0d51afc1827799259e71c5 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -178,6 +178,7 @@ struct amdgpu_cs_parser;
 struct amdgpu_job;
 struct amdgpu_irq_src;
 struct amdgpu_fpriv;
+struct amdgpu_mn;
 
 enum amdgpu_cp_irq {
        AMDGPU_CP_IRQ_GFX_EOP = 0,
@@ -1057,6 +1058,7 @@ struct amdgpu_cs_parser {
        /* buffer objects */
        struct ww_acquire_ctx           ticket;
        struct amdgpu_bo_list           *bo_list;
+       struct amdgpu_mn                *mn;
        struct amdgpu_bo_list_entry     vm_pd;
        struct list_head                validated;
        struct dma_fence                *fence;
@@ -1201,9 +1203,18 @@ void amdgpu_test_moves(struct amdgpu_device *adev);
  * MMU Notifier
  */
 #if defined(CONFIG_MMU_NOTIFIER)
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev);
 int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
 void amdgpu_mn_unregister(struct amdgpu_bo *bo);
+void amdgpu_mn_lock(struct amdgpu_mn *mn);
+void amdgpu_mn_unlock(struct amdgpu_mn *mn);
 #else
+static inline void amdgpu_mn_lock(struct amdgpu_mn *mn) {}
+static inline void amdgpu_mn_unlock(struct amdgpu_mn *mn) {}
+static inline struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+{
+       return NULL;
+}
 static inline int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr)
 {
        return -ENODEV;

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 4877df83b801de151df7afc44bc9af1cc4843008..c2310d4eebc8bc2864f526633c5fd53ee9b9d73a 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -513,8 +513,11 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
        INIT_LIST_HEAD(&p->validated);
 
        p->bo_list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle);
-       if (p->bo_list)
+       if (p->bo_list) {
                amdgpu_bo_list_get_list(p->bo_list, &p->validated);
+               if (p->bo_list->first_userptr != p->bo_list->num_entries)
+                       p->mn = amdgpu_mn_get(p->adev);
+       }
 
        INIT_LIST_HEAD(&duplicates);
        amdgpu_vm_get_pd_bo(&fpriv->vm, &p->validated, &p->vm_pd);
@@ -722,11 +725,7 @@ static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
 {
        unsigned i;
 
-       if (!error)
-               ttm_eu_fence_buffer_objects(&parser->ticket,
-                                           &parser->validated,
-                                           parser->fence);
-       else if (backoff)
+       if (error && backoff)
                ttm_eu_backoff_reservation(&parser->ticket,
                                           &parser->validated);
 
@@ -1127,14 +1126,29 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
        struct amdgpu_ring *ring = p->job->ring;
        struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
        struct amdgpu_job *job;
+       unsigned i;
        int r;
 
+       amdgpu_mn_lock(p->mn);
+       if (p->bo_list) {
+               for (i = p->bo_list->first_userptr;
+                    i < p->bo_list->num_entries; ++i) {
+                       struct amdgpu_bo *bo = p->bo_list->array[i].robj;
+
+                       if (amdgpu_ttm_tt_userptr_needs_pages(bo->tbo.ttm)) {
+                               amdgpu_mn_unlock(p->mn);
+                               return -ERESTARTSYS;
+                       }
+               }
+       }
+
        job = p->job;
        p->job = NULL;
 
        r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
        if (r) {
                amdgpu_job_free(job);
+               amdgpu_mn_unlock(p->mn);
                return r;
        }
 
@@ -1150,6 +1164,10 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 
        trace_amdgpu_cs_ioctl(job);
        amd_sched_entity_push_job(&job->base);
+
+       ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
+       amdgpu_mn_unlock(p->mn);
+
        return 0;
 }
 

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
index 6d216abd0e1d7572377880ca4218c2b6a0e06a21..99edb40b5f998eef76a45a43c8ed433a3e2e711f 100644 (file)
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c
@@ -106,6 +106,25 @@ static void amdgpu_mn_release(struct mmu_notifier *mn,
        schedule_work(&rmn->work);
 }
 
+
+/**
+ * amdgpu_mn_lock - take the write side lock for this mn
+ */
+void amdgpu_mn_lock(struct amdgpu_mn *mn)
+{
+       if (mn)
+               down_write(&mn->lock);
+}
+
+/**
+ * amdgpu_mn_unlock - drop the write side lock for this mn
+ */
+void amdgpu_mn_unlock(struct amdgpu_mn *mn)
+{
+       if (mn)
+               up_write(&mn->lock);
+}
+
 /**
  * amdgpu_mn_invalidate_node - unmap all BOs of a node
  *
@@ -126,20 +145,12 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node,
                if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end))
                        continue;
 
-               r = amdgpu_bo_reserve(bo, true);
-               if (r) {
-                       DRM_ERROR("(%ld) failed to reserve user bo\n", r);
-                       continue;
-               }
-
                r = reservation_object_wait_timeout_rcu(bo->tbo.resv,
                        true, false, MAX_SCHEDULE_TIMEOUT);
                if (r <= 0)
                        DRM_ERROR("(%ld) failed to wait for user bo\n", r);
 
                amdgpu_ttm_tt_mark_user_pages(bo->tbo.ttm);
-
-               amdgpu_bo_unreserve(bo);
        }
 }
 
@@ -223,7 +234,7 @@ static const struct mmu_notifier_ops amdgpu_mn_ops = {
  *
  * Creates a notifier context for current->mm.
  */
-static struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
+struct amdgpu_mn *amdgpu_mn_get(struct amdgpu_device *adev)
 {
        struct mm_struct *mm = current->mm;
        struct amdgpu_mn *rmn;
@@ -368,3 +379,4 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo)
        up_write(&rmn->lock);
        mutex_unlock(&adev->mn_lock);
 }
+