--- /dev/null
+/*
+ * Copyright 2018 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "nouveau_dmem.h"
+#include "nouveau_drv.h"
+#include "nouveau_chan.h"
+#include "nouveau_dma.h"
+#include "nouveau_mem.h"
+#include "nouveau_bo.h"
+
+#include <nvif/class.h>
+#include <nvif/object.h>
+#include <nvif/if500b.h>
+#include <nvif/if900b.h>
+
+#include <linux/sched/mm.h>
+#include <linux/hmm.h>
+
+/*
+ * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
+ * it in vram while in use. We likely want to overhaul memory management for
+ * nouveau to be more page like (not necessarily with system page size but a
+ * bigger page size) at lowest level and have some shim layer on top that would
+ * provide the same functionality as TTM.
+ */
+#define DMEM_CHUNK_SIZE (2UL << 20)
+#define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
+
+struct nouveau_migrate;
+
+typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
+ u64 dst_addr, u64 src_addr);
+
+struct nouveau_dmem_chunk {
+ struct list_head list;
+ struct nouveau_bo *bo;
+ struct nouveau_drm *drm;
+ unsigned long pfn_first;
+ unsigned long callocated;
+ unsigned long bitmap[BITS_TO_LONGS(DMEM_CHUNK_NPAGES)];
+ struct nvif_vma vma;
+ spinlock_t lock;
+};
+
+struct nouveau_dmem_migrate {
+ nouveau_migrate_copy_t copy_func;
+ struct nouveau_channel *chan;
+};
+
+struct nouveau_dmem {
+ struct hmm_devmem *devmem;
+ struct nouveau_dmem_migrate migrate;
+ struct list_head chunk_free;
+ struct list_head chunk_full;
+ struct list_head chunk_empty;
+ struct mutex mutex;
+};
+
+struct nouveau_migrate_hmem {
+ struct scatterlist *sg;
+ struct nouveau_mem mem;
+ unsigned long npages;
+ struct nvif_vma vma;
+};
+
+struct nouveau_dmem_fault {
+ struct nouveau_drm *drm;
+ struct nouveau_fence *fence;
+ struct nouveau_migrate_hmem hmem;
+};
+
+struct nouveau_migrate {
+ struct vm_area_struct *vma;
+ struct nouveau_drm *drm;
+ struct nouveau_fence *fence;
+ unsigned long npages;
+ struct nouveau_migrate_hmem hmem;
+};
+
+static void
+nouveau_migrate_hmem_fini(struct nouveau_drm *drm,
+ struct nouveau_migrate_hmem *hmem)
+{
+ struct nvif_vmm *vmm = &drm->client.vmm.vmm;
+
+ nouveau_mem_fini(&hmem->mem);
+ nvif_vmm_put(vmm, &hmem->vma);
+
+ if (hmem->sg) {
+ dma_unmap_sg_attrs(drm->dev->dev, hmem->sg,
+ hmem->npages, DMA_BIDIRECTIONAL,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ kfree(hmem->sg);
+ hmem->sg = NULL;
+ }
+}
+
+static int
+nouveau_migrate_hmem_init(struct nouveau_drm *drm,
+ struct nouveau_migrate_hmem *hmem,
+ unsigned long npages,
+ const unsigned long *pfns)
+{
+ struct nvif_vmm *vmm = &drm->client.vmm.vmm;
+ unsigned long i;
+ int ret;
+
+ hmem->sg = kzalloc(npages * sizeof(*hmem->sg), GFP_KERNEL);
+ if (hmem->sg == NULL)
+ return -ENOMEM;
+
+ for (i = 0, hmem->npages = 0; hmem->npages < npages; ++i) {
+ struct page *page;
+
+ if (!pfns[i] || pfns[i] == MIGRATE_PFN_ERROR)
+ continue;
+
+ page = migrate_pfn_to_page(pfns[i]);
+ if (page == NULL) {
+ ret = -EINVAL;
+ goto error;
+ }
+
+ sg_set_page(&hmem->sg[hmem->npages], page, PAGE_SIZE, 0);
+ hmem->npages++;
+ }
+ sg_mark_end(&hmem->sg[hmem->npages - 1]);
+
+ i = dma_map_sg_attrs(drm->dev->dev, hmem->sg, hmem->npages,
+ DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC);
+ if (i != hmem->npages) {
+ ret = -ENOMEM;
+ goto error;
+ }
+
+ ret = nouveau_mem_sgl(&hmem->mem, &drm->client,
+ hmem->npages, hmem->sg);
+ if (ret)
+ goto error;
+
+ ret = nvif_vmm_get(vmm, LAZY, false, hmem->mem.mem.page,
+ 0, hmem->mem.mem.size, &hmem->vma);
+ if (ret)
+ goto error;
+
+ ret = nouveau_mem_map(&hmem->mem, vmm, &hmem->vma);
+ if (ret)
+ goto error;
+
+ return 0;
+
+error:
+ nouveau_migrate_hmem_fini(drm, hmem);
+ return ret;
+}
+
+
+static void
+nouveau_dmem_free(struct hmm_devmem *devmem, struct page *page)
+{
+ struct nouveau_dmem_chunk *chunk;
+ struct nouveau_drm *drm;
+ unsigned long idx;
+
+ chunk = (void *)hmm_devmem_page_get_drvdata(page);
+ idx = page_to_pfn(page) - chunk->pfn_first;
+ drm = chunk->drm;
+
+ /*
+ * FIXME:
+ *
+ * This is really a bad example, we need to overhaul nouveau memory
+ * management to be more page focus and allow lighter locking scheme
+ * to be use in the process.
+ */
+ spin_lock(&chunk->lock);
+ clear_bit(idx, chunk->bitmap);
+ WARN_ON(!chunk->callocated);
+ chunk->callocated--;
+ /*
+ * FIXME when chunk->callocated reach 0 we should add the chunk to
+ * a reclaim list so that it can be freed in case of memory pressure.
+ */
+ spin_unlock(&chunk->lock);
+}
+
+static void
+nouveau_dmem_fault_alloc_and_copy(struct vm_area_struct *vma,
+ const unsigned long *src_pfns,
+ unsigned long *dst_pfns,
+ unsigned long start,
+ unsigned long end,
+ void *private)
+{
+ struct nouveau_dmem_fault *fault = private;
+ struct nouveau_drm *drm = fault->drm;
+ unsigned long addr, i, c, npages = 0;
+ nouveau_migrate_copy_t copy;
+ int ret;
+
+
+ /* First allocate new memory */
+ for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
+ struct page *dpage, *spage;
+
+ dst_pfns[i] = 0;
+ spage = migrate_pfn_to_page(src_pfns[i]);
+ if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
+ continue;
+
+ dpage = hmm_vma_alloc_locked_page(vma, addr);
+ if (!dpage) {
+ dst_pfns[i] = MIGRATE_PFN_ERROR;
+ continue;
+ }
+
+ dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
+ MIGRATE_PFN_LOCKED;
+ npages++;
+ }
+
+ /* Create scatter list FIXME: get rid of scatter list */
+ ret = nouveau_migrate_hmem_init(drm, &fault->hmem, npages, dst_pfns);
+ if (ret)
+ goto error;
+
+ /* Copy things over */
+ copy = drm->dmem->migrate.copy_func;
+ for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {
+ struct nouveau_dmem_chunk *chunk;
+ struct page *spage, *dpage;
+ u64 src_addr, dst_addr;
+
+ dpage = migrate_pfn_to_page(dst_pfns[i]);
+ if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
+ continue;
+
+ dst_addr = fault->hmem.vma.addr + (c << PAGE_SHIFT);
+ c++;
+
+ spage = migrate_pfn_to_page(src_pfns[i]);
+ if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
+ dst_pfns[i] = MIGRATE_PFN_ERROR;
+ __free_page(dpage);
+ continue;
+ }
+
+ chunk = (void *)hmm_devmem_page_get_drvdata(spage);
+ src_addr = page_to_pfn(spage) - chunk->pfn_first;
+ src_addr = (src_addr << PAGE_SHIFT) + chunk->vma.addr;
+
+ ret = copy(drm, 1, dst_addr, src_addr);
+ if (ret) {
+ dst_pfns[i] = MIGRATE_PFN_ERROR;
+ __free_page(dpage);
+ continue;
+ }
+ }
+
+ nouveau_fence_new(drm->dmem->migrate.chan, false, &fault->fence);
+
+ return;
+
+error:
+ for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
+ struct page *page;
+
+ if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
+ continue;
+
+ page = migrate_pfn_to_page(dst_pfns[i]);
+ dst_pfns[i] = MIGRATE_PFN_ERROR;
+ if (page == NULL)
+ continue;
+
+ __free_page(page);
+ }
+}
+
+void nouveau_dmem_fault_finalize_and_map(struct vm_area_struct *vma,
+ const unsigned long *src_pfns,
+ const unsigned long *dst_pfns,
+ unsigned long start,
+ unsigned long end,
+ void *private)
+{
+ struct nouveau_dmem_fault *fault = private;
+ struct nouveau_drm *drm = fault->drm;
+
+ if (fault->fence) {
+ nouveau_fence_wait(fault->fence, true, false);
+ nouveau_fence_unref(&fault->fence);
+ } else {
+ /*
+ * FIXME wait for channel to be IDLE before calling finalizing
+ * the hmem object below (nouveau_migrate_hmem_fini()).
+ */
+ }
+ nouveau_migrate_hmem_fini(drm, &fault->hmem);
+}
+
+static const struct migrate_vma_ops nouveau_dmem_fault_migrate_ops = {
+ .alloc_and_copy = nouveau_dmem_fault_alloc_and_copy,
+ .finalize_and_map = nouveau_dmem_fault_finalize_and_map,
+};
+
+static int
+nouveau_dmem_fault(struct hmm_devmem *devmem,
+ struct vm_area_struct *vma,
+ unsigned long addr,
+ const struct page *page,
+ unsigned int flags,
+ pmd_t *pmdp)
+{
+ struct drm_device *drm_dev = dev_get_drvdata(devmem->device);
+ unsigned long src[1] = {0}, dst[1] = {0};
+ struct nouveau_dmem_fault fault = {0};
+ int ret;
+
+
+
+ /*
+ * FIXME what we really want is to find some heuristic to migrate more
+ * than just one page on CPU fault. When such fault happens it is very
+ * likely that more surrounding page will CPU fault too.
+ */
+ fault.drm = nouveau_drm(drm_dev);
+ ret = migrate_vma(&nouveau_dmem_fault_migrate_ops, vma, addr,
+ addr + PAGE_SIZE, src, dst, &fault);
+ if (ret)
+ return VM_FAULT_SIGBUS;
+
+ if (dst[0] == MIGRATE_PFN_ERROR)
+ return VM_FAULT_SIGBUS;
+
+ return 0;
+}
+
+static const struct hmm_devmem_ops
+nouveau_dmem_devmem_ops = {
+ .free = nouveau_dmem_free,
+ .fault = nouveau_dmem_fault,
+};
+
+static int
+nouveau_dmem_chunk_alloc(struct nouveau_drm *drm)
+{
+ struct nvif_vmm *vmm = &drm->client.vmm.vmm;
+ struct nouveau_dmem_chunk *chunk;
+ int ret;
+
+ if (drm->dmem == NULL)
+ return -EINVAL;
+
+ mutex_lock(&drm->dmem->mutex);
+ chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
+ struct nouveau_dmem_chunk,
+ list);
+ if (chunk == NULL) {
+ mutex_unlock(&drm->dmem->mutex);
+ return -ENOMEM;
+ }
+
+ list_del(&chunk->list);
+ mutex_unlock(&drm->dmem->mutex);
+
+ ret = nvif_vmm_get(vmm, LAZY, false, 12, 0,
+ DMEM_CHUNK_SIZE, &chunk->vma);
+ if (ret)
+ goto out;
+
+ ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
+ TTM_PL_FLAG_VRAM, 0, 0, NULL, NULL,
+ &chunk->bo);
+ if (ret)
+ goto out;
+
+ ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
+ if (ret) {
+ nouveau_bo_ref(NULL, &chunk->bo);
+ goto out;
+ }
+
+ ret = nouveau_mem_map(nouveau_mem(&chunk->bo->bo.mem), vmm, &chunk->vma);
+ if (ret) {
+ nouveau_bo_unpin(chunk->bo);
+ nouveau_bo_ref(NULL, &chunk->bo);
+ goto out;
+ }
+
+ bitmap_zero(chunk->bitmap, DMEM_CHUNK_NPAGES);
+ spin_lock_init(&chunk->lock);
+
+out:
+ mutex_lock(&drm->dmem->mutex);
+ if (chunk->bo)
+ list_add(&chunk->list, &drm->dmem->chunk_empty);
+ else
+ list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
+ mutex_unlock(&drm->dmem->mutex);
+
+ return ret;
+}
+
+static struct nouveau_dmem_chunk *
+nouveau_dmem_chunk_first_free_locked(struct nouveau_drm *drm)
+{
+ struct nouveau_dmem_chunk *chunk;
+
+ chunk = list_first_entry_or_null(&drm->dmem->chunk_free,
+ struct nouveau_dmem_chunk,
+ list);
+ if (chunk)
+ return chunk;
+
+ chunk = list_first_entry_or_null(&drm->dmem->chunk_empty,
+ struct nouveau_dmem_chunk,
+ list);
+ if (chunk->bo)
+ return chunk;
+
+ return NULL;
+}
+
+static int
+nouveau_dmem_pages_alloc(struct nouveau_drm *drm,
+ unsigned long npages,
+ unsigned long *pages)
+{
+ struct nouveau_dmem_chunk *chunk;
+ unsigned long c;
+ int ret;
+
+ memset(pages, 0xff, npages * sizeof(*pages));
+
+ mutex_lock(&drm->dmem->mutex);
+ for (c = 0; c < npages;) {
+ unsigned long i;
+
+ chunk = nouveau_dmem_chunk_first_free_locked(drm);
+ if (chunk == NULL) {
+ mutex_unlock(&drm->dmem->mutex);
+ ret = nouveau_dmem_chunk_alloc(drm);
+ if (ret) {
+ if (c)
+ break;
+ return ret;
+ }
+ continue;
+ }
+
+ spin_lock(&chunk->lock);
+ i = find_first_zero_bit(chunk->bitmap, DMEM_CHUNK_NPAGES);
+ while (i < DMEM_CHUNK_NPAGES && c < npages) {
+ pages[c] = chunk->pfn_first + i;
+ set_bit(i, chunk->bitmap);
+ chunk->callocated++;
+ c++;
+
+ i = find_next_zero_bit(chunk->bitmap,
+ DMEM_CHUNK_NPAGES, i);
+ }
+ spin_unlock(&chunk->lock);
+ }
+ mutex_unlock(&drm->dmem->mutex);
+
+ return 0;
+}
+
+static struct page *
+nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
+{
+ unsigned long pfns[1];
+ struct page *page;
+ int ret;
+
+ /* FIXME stop all the miss-match API ... */
+ ret = nouveau_dmem_pages_alloc(drm, 1, pfns);
+ if (ret)
+ return NULL;
+
+ page = pfn_to_page(pfns[0]);
+ get_page(page);
+ lock_page(page);
+ return page;
+}
+
+static void
+nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
+{
+ unlock_page(page);
+ put_page(page);
+}
+
+void
+nouveau_dmem_resume(struct nouveau_drm *drm)
+{
+ struct nouveau_dmem_chunk *chunk;
+ int ret;
+
+ if (drm->dmem == NULL)
+ return;
+
+ mutex_lock(&drm->dmem->mutex);
+ list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
+ ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
+ /* FIXME handle pin failure */
+ WARN_ON(ret);
+ }
+ list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
+ ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
+ /* FIXME handle pin failure */
+ WARN_ON(ret);
+ }
+ list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
+ ret = nouveau_bo_pin(chunk->bo, TTM_PL_FLAG_VRAM, false);
+ /* FIXME handle pin failure */
+ WARN_ON(ret);
+ }
+ mutex_unlock(&drm->dmem->mutex);
+}
+
+void
+nouveau_dmem_suspend(struct nouveau_drm *drm)
+{
+ struct nouveau_dmem_chunk *chunk;
+
+ if (drm->dmem == NULL)
+ return;
+
+ mutex_lock(&drm->dmem->mutex);
+ list_for_each_entry (chunk, &drm->dmem->chunk_free, list) {
+ nouveau_bo_unpin(chunk->bo);
+ }
+ list_for_each_entry (chunk, &drm->dmem->chunk_full, list) {
+ nouveau_bo_unpin(chunk->bo);
+ }
+ list_for_each_entry (chunk, &drm->dmem->chunk_empty, list) {
+ nouveau_bo_unpin(chunk->bo);
+ }
+ mutex_unlock(&drm->dmem->mutex);
+}
+
+void
+nouveau_dmem_fini(struct nouveau_drm *drm)
+{
+ struct nvif_vmm *vmm = &drm->client.vmm.vmm;
+ struct nouveau_dmem_chunk *chunk, *tmp;
+
+ if (drm->dmem == NULL)
+ return;
+
+ mutex_lock(&drm->dmem->mutex);
+
+ WARN_ON(!list_empty(&drm->dmem->chunk_free));
+ WARN_ON(!list_empty(&drm->dmem->chunk_full));
+
+ list_for_each_entry_safe (chunk, tmp, &drm->dmem->chunk_empty, list) {
+ if (chunk->bo) {
+ nouveau_bo_unpin(chunk->bo);
+ nouveau_bo_ref(NULL, &chunk->bo);
+ }
+ nvif_vmm_put(vmm, &chunk->vma);
+ list_del(&chunk->list);
+ kfree(chunk);
+ }
+
+ mutex_unlock(&drm->dmem->mutex);
+}
+
+static int
+nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
+ u64 dst_addr, u64 src_addr)
+{
+ struct nouveau_channel *chan = drm->dmem->migrate.chan;
+ int ret;
+
+ ret = RING_SPACE(chan, 10);
+ if (ret)
+ return ret;
+
+ BEGIN_NVC0(chan, NvSubCopy, 0x0400, 8);
+ OUT_RING (chan, upper_32_bits(src_addr));
+ OUT_RING (chan, lower_32_bits(src_addr));
+ OUT_RING (chan, upper_32_bits(dst_addr));
+ OUT_RING (chan, lower_32_bits(dst_addr));
+ OUT_RING (chan, PAGE_SIZE);
+ OUT_RING (chan, PAGE_SIZE);
+ OUT_RING (chan, PAGE_SIZE);
+ OUT_RING (chan, npages);
+ BEGIN_IMC0(chan, NvSubCopy, 0x0300, 0x0386);
+ return 0;
+}
+
+static int
+nouveau_dmem_migrate_init(struct nouveau_drm *drm)
+{
+ switch (drm->ttm.copy.oclass) {
+ case PASCAL_DMA_COPY_A:
+ case PASCAL_DMA_COPY_B:
+ case VOLTA_DMA_COPY_A:
+ case TURING_DMA_COPY_A:
+ drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
+ drm->dmem->migrate.chan = drm->ttm.chan;
+ return 0;
+ default:
+ break;
+ }
+ return -ENODEV;
+}
+
+void
+nouveau_dmem_init(struct nouveau_drm *drm)
+{
+ struct device *device = drm->dev->dev;
+ unsigned long i, size;
+ int ret;
+
+ /* This only make sense on PASCAL or newer */
+ if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
+ return;
+
+ if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
+ return;
+
+ mutex_init(&drm->dmem->mutex);
+ INIT_LIST_HEAD(&drm->dmem->chunk_free);
+ INIT_LIST_HEAD(&drm->dmem->chunk_full);
+ INIT_LIST_HEAD(&drm->dmem->chunk_empty);
+
+ size = ALIGN(drm->client.device.info.ram_user, DMEM_CHUNK_SIZE);
+
+ /* Initialize migration dma helpers before registering memory */
+ ret = nouveau_dmem_migrate_init(drm);
+ if (ret) {
+ kfree(drm->dmem);
+ drm->dmem = NULL;
+ return;
+ }
+
+ /*
+ * FIXME we need some kind of policy to decide how much VRAM we
+ * want to register with HMM. For now just register everything
+ * and latter if we want to do thing like over commit then we
+ * could revisit this.
+ */
+ drm->dmem->devmem = hmm_devmem_add(&nouveau_dmem_devmem_ops,
+ device, size);
+ if (drm->dmem->devmem == NULL) {
+ kfree(drm->dmem);
+ drm->dmem = NULL;
+ return;
+ }
+
+ for (i = 0; i < (size / DMEM_CHUNK_SIZE); ++i) {
+ struct nouveau_dmem_chunk *chunk;
+ struct page *page;
+ unsigned long j;
+
+ chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
+ if (chunk == NULL) {
+ nouveau_dmem_fini(drm);
+ return;
+ }
+
+ chunk->drm = drm;
+ chunk->pfn_first = drm->dmem->devmem->pfn_first;
+ chunk->pfn_first += (i * DMEM_CHUNK_NPAGES);
+ list_add_tail(&chunk->list, &drm->dmem->chunk_empty);
+
+ page = pfn_to_page(chunk->pfn_first);
+ for (j = 0; j < DMEM_CHUNK_NPAGES; ++j, ++page) {
+ hmm_devmem_page_set_drvdata(page, (long)chunk);
+ }
+ }
+
+ NV_INFO(drm, "DMEM: registered %ldMB of device memory\n", size >> 20);
+}
+
+static void
+nouveau_dmem_migrate_alloc_and_copy(struct vm_area_struct *vma,
+ const unsigned long *src_pfns,
+ unsigned long *dst_pfns,
+ unsigned long start,
+ unsigned long end,
+ void *private)
+{
+ struct nouveau_migrate *migrate = private;
+ struct nouveau_drm *drm = migrate->drm;
+ unsigned long addr, i, c, npages = 0;
+ nouveau_migrate_copy_t copy;
+ int ret;
+
+ /* First allocate new memory */
+ for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, i++) {
+ struct page *dpage, *spage;
+
+ dst_pfns[i] = 0;
+ spage = migrate_pfn_to_page(src_pfns[i]);
+ if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))
+ continue;
+
+ dpage = nouveau_dmem_page_alloc_locked(drm);
+ if (!dpage)
+ continue;
+
+ dst_pfns[i] = migrate_pfn(page_to_pfn(dpage)) |
+ MIGRATE_PFN_LOCKED |
+ MIGRATE_PFN_DEVICE;
+ npages++;
+ }
+
+ if (!npages)
+ return;
+
+ /* Create scatter list FIXME: get rid of scatter list */
+ ret = nouveau_migrate_hmem_init(drm, &migrate->hmem, npages, src_pfns);
+ if (ret)
+ goto error;
+
+ /* Copy things over */
+ copy = drm->dmem->migrate.copy_func;
+ for (addr = start, i = c = 0; addr < end; addr += PAGE_SIZE, i++) {
+ struct nouveau_dmem_chunk *chunk;
+ struct page *spage, *dpage;
+ u64 src_addr, dst_addr;
+
+ dpage = migrate_pfn_to_page(dst_pfns[i]);
+ if (!dpage || dst_pfns[i] == MIGRATE_PFN_ERROR)
+ continue;
+
+ chunk = (void *)hmm_devmem_page_get_drvdata(dpage);
+ dst_addr = page_to_pfn(dpage) - chunk->pfn_first;
+ dst_addr = (dst_addr << PAGE_SHIFT) + chunk->vma.addr;
+
+ spage = migrate_pfn_to_page(src_pfns[i]);
+ if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE)) {
+ nouveau_dmem_page_free_locked(drm, dpage);
+ dst_pfns[i] = 0;
+ continue;
+ }
+
+ src_addr = migrate->hmem.vma.addr + (c << PAGE_SHIFT);
+ c++;
+
+ ret = copy(drm, 1, dst_addr, src_addr);
+ if (ret) {
+ nouveau_dmem_page_free_locked(drm, dpage);
+ dst_pfns[i] = 0;
+ continue;
+ }
+ }
+
+ nouveau_fence_new(drm->dmem->migrate.chan, false, &migrate->fence);
+
+ return;
+
+error:
+ for (addr = start, i = 0; addr < end; addr += PAGE_SIZE, ++i) {
+ struct page *page;
+
+ if (!dst_pfns[i] || dst_pfns[i] == MIGRATE_PFN_ERROR)
+ continue;
+
+ page = migrate_pfn_to_page(dst_pfns[i]);
+ dst_pfns[i] = MIGRATE_PFN_ERROR;
+ if (page == NULL)
+ continue;
+
+ __free_page(page);
+ }
+}
+
+void nouveau_dmem_migrate_finalize_and_map(struct vm_area_struct *vma,
+ const unsigned long *src_pfns,
+ const unsigned long *dst_pfns,
+ unsigned long start,
+ unsigned long end,
+ void *private)
+{
+ struct nouveau_migrate *migrate = private;
+ struct nouveau_drm *drm = migrate->drm;
+
+ if (migrate->fence) {
+ nouveau_fence_wait(migrate->fence, true, false);
+ nouveau_fence_unref(&migrate->fence);
+ } else {
+ /*
+ * FIXME wait for channel to be IDLE before finalizing
+ * the hmem object below (nouveau_migrate_hmem_fini()) ?
+ */
+ }
+ nouveau_migrate_hmem_fini(drm, &migrate->hmem);
+
+ /*
+ * FIXME optimization: update GPU page table to point to newly
+ * migrated memory.
+ */
+}
+
+static const struct migrate_vma_ops nouveau_dmem_migrate_ops = {
+ .alloc_and_copy = nouveau_dmem_migrate_alloc_and_copy,
+ .finalize_and_map = nouveau_dmem_migrate_finalize_and_map,
+};
+
+int
+nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
+ struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end)
+{
+ unsigned long *src_pfns, *dst_pfns, npages;
+ struct nouveau_migrate migrate = {0};
+ unsigned long i, c, max;
+ int ret = 0;
+
+ npages = (end - start) >> PAGE_SHIFT;
+ max = min(SG_MAX_SINGLE_ALLOC, npages);
+ src_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
+ if (src_pfns == NULL)
+ return -ENOMEM;
+ dst_pfns = kzalloc(sizeof(long) * max, GFP_KERNEL);
+ if (dst_pfns == NULL) {
+ kfree(src_pfns);
+ return -ENOMEM;
+ }
+
+ migrate.drm = drm;
+ migrate.vma = vma;
+ migrate.npages = npages;
+ for (i = 0; i < npages; i += c) {
+ unsigned long next;
+
+ c = min(SG_MAX_SINGLE_ALLOC, npages);
+ next = start + (c << PAGE_SHIFT);
+ ret = migrate_vma(&nouveau_dmem_migrate_ops, vma, start,
+ next, src_pfns, dst_pfns, &migrate);
+ if (ret)
+ goto out;
+ start = next;
+ }
+
+out:
+ kfree(dst_pfns);
+ kfree(src_pfns);
+ return ret;
+}
+
+static inline bool
+nouveau_dmem_page(struct nouveau_drm *drm, struct page *page)
+{
+ if (!is_device_private_page(page))
+ return false;
+
+ if (drm->dmem->devmem != page->pgmap->data)
+ return false;
+
+ return true;
+}
+
+void
+nouveau_dmem_convert_pfn(struct nouveau_drm *drm,
+ struct hmm_range *range)
+{
+ unsigned long i, npages;
+
+ npages = (range->end - range->start) >> PAGE_SHIFT;
+ for (i = 0; i < npages; ++i) {
+ struct nouveau_dmem_chunk *chunk;
+ struct page *page;
+ uint64_t addr;
+
+ page = hmm_pfn_to_page(range, range->pfns[i]);
+ if (page == NULL)
+ continue;
+
+ if (!(range->pfns[i] & range->flags[HMM_PFN_DEVICE_PRIVATE])) {
+ continue;
+ }
+
+ if (!nouveau_dmem_page(drm, page)) {
+ WARN(1, "Some unknown device memory !\n");
+ range->pfns[i] = 0;
+ continue;
+ }
+
+ chunk = (void *)hmm_devmem_page_get_drvdata(page);
+ addr = page_to_pfn(page) - chunk->pfn_first;
+ addr = (addr + chunk->bo->bo.mem.start) << PAGE_SHIFT;
+
+ range->pfns[i] &= ((1UL << range->pfn_shift) - 1);
+ range->pfns[i] |= (addr >> PAGE_SHIFT) << range->pfn_shift;
+ }
+}
projects / kernel.git / commitdiff