diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 103 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 16 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 7 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_memory.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_module.h | 40 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_range_manager.c | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 15 |
10 files changed, 99 insertions, 94 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index 03c86628e4ac..8f9fa4188897 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -32,7 +32,6 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_module.h> #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> #include <linux/agp_backend.h> diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 9a03c7834b1e..31e8b3da5563 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -31,7 +31,6 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_module.h> #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> #include <linux/jiffies.h> @@ -43,6 +42,8 @@ #include <linux/atomic.h> #include <linux/dma-resv.h> +#include "ttm_module.h" + static void ttm_bo_global_kobj_release(struct kobject *kobj); /* @@ -71,9 +72,9 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_resource_manager *man; int i, mem_type; - drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n", - bo, bo->mem.num_pages, bo->mem.size >> 10, - bo->mem.size >> 20); + drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n", + bo, bo->mem.num_pages, bo->base.size >> 10, + bo->base.size >> 20); for (i = 0; i < placement->num_placement; i++) { mem_type = placement->placement[i].mem_type; drm_printf(&p, " placement[%d]=0x%08X (%d)\n", @@ -109,40 +110,14 @@ static struct kobj_type ttm_bo_glob_kobj_type = { .default_attrs = ttm_bo_global_attrs }; -static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo, - struct ttm_resource *mem) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_resource_manager *man; - - if (!list_empty(&bo->lru) || bo->pin_count) - return; - - man = ttm_manager_type(bdev, mem->mem_type); - list_add_tail(&bo->lru, &man->lru[bo->priority]); - - if (man->use_tt && bo->ttm && - !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG | - TTM_PAGE_FLAG_SWAPPED))) { - list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]); - } -} - static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; - bool notify = false; - if (!list_empty(&bo->swap)) { - list_del_init(&bo->swap); - notify = true; - } - if (!list_empty(&bo->lru)) { - list_del_init(&bo->lru); - notify = true; - } + list_del_init(&bo->swap); + list_del_init(&bo->lru); - if (notify && bdev->driver->del_from_lru_notify) + if (bdev->driver->del_from_lru_notify) bdev->driver->del_from_lru_notify(bo); } @@ -155,12 +130,30 @@ static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, } void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, + struct ttm_resource *mem, struct ttm_lru_bulk_move *bulk) { + struct ttm_bo_device *bdev = bo->bdev; + struct ttm_resource_manager *man; + dma_resv_assert_held(bo->base.resv); - ttm_bo_del_from_lru(bo); - ttm_bo_add_mem_to_lru(bo, &bo->mem); + if (bo->pin_count) + return; + + man = ttm_manager_type(bdev, mem->mem_type); + list_move_tail(&bo->lru, &man->lru[bo->priority]); + if (man->use_tt && bo->ttm && + !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG | + TTM_PAGE_FLAG_SWAPPED))) { + struct list_head *swap; + + swap = &ttm_bo_glob.swap_lru[bo->priority]; + list_move_tail(&bo->swap, swap); + } + + if (bdev->driver->del_from_lru_notify) + bdev->driver->del_from_lru_notify(bo); if (bulk && !bo->pin_count) { switch (bo->mem.mem_type) { @@ -267,7 +260,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, goto out_err; } - ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; + ctx->bytes_moved += bo->base.size; return 0; out_err: @@ -514,10 +507,9 @@ static void ttm_bo_release(struct kref *kref) * shrinkers, now that they are queued for * destruction. */ - if (bo->pin_count) { + if (WARN_ON(bo->pin_count)) { bo->pin_count = 0; - ttm_bo_del_from_lru(bo); - ttm_bo_add_mem_to_lru(bo, &bo->mem); + ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL); } kref_init(&bo->kref); @@ -859,8 +851,7 @@ static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, mem->placement = place->flags; spin_lock(&ttm_bo_glob.lru_lock); - ttm_bo_del_from_lru(bo); - ttm_bo_add_mem_to_lru(bo, mem); + ttm_bo_move_to_lru_tail(bo, mem, NULL); spin_unlock(&ttm_bo_glob.lru_lock); return 0; @@ -937,9 +928,8 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo, } error: - if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) { + if (bo->mem.mem_type == TTM_PL_SYSTEM && !bo->pin_count) ttm_bo_move_to_lru_tail_unlocked(bo); - } return ret; } @@ -984,8 +974,7 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, memset(&hop, 0, sizeof(hop)); - mem.num_pages = bo->num_pages; - mem.size = mem.num_pages << PAGE_SHIFT; + mem.num_pages = PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT; mem.page_alignment = bo->mem.page_alignment; mem.bus.offset = 0; mem.bus.addr = NULL; @@ -1101,7 +1090,7 @@ EXPORT_SYMBOL(ttm_bo_validate); int ttm_bo_init_reserved(struct ttm_bo_device *bdev, struct ttm_buffer_object *bo, - unsigned long size, + size_t size, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, @@ -1112,9 +1101,8 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, void (*destroy) (struct ttm_buffer_object *)) { struct ttm_mem_global *mem_glob = &ttm_mem_glob; - int ret = 0; - unsigned long num_pages; bool locked; + int ret = 0; ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); if (ret) { @@ -1126,16 +1114,6 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, return -ENOMEM; } - num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (num_pages == 0) { - pr_err("Illegal buffer object size\n"); - if (destroy) - (*destroy)(bo); - else - kfree(bo); - ttm_mem_global_free(mem_glob, acc_size); - return -EINVAL; - } bo->destroy = destroy ? destroy : ttm_bo_default_destroy; kref_init(&bo->kref); @@ -1144,10 +1122,8 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, INIT_LIST_HEAD(&bo->swap); bo->bdev = bdev; bo->type = type; - bo->num_pages = num_pages; - bo->mem.size = num_pages << PAGE_SHIFT; bo->mem.mem_type = TTM_PL_SYSTEM; - bo->mem.num_pages = bo->num_pages; + bo->mem.num_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; bo->mem.mm_node = NULL; bo->mem.page_alignment = page_alignment; bo->mem.bus.offset = 0; @@ -1165,9 +1141,10 @@ int ttm_bo_init_reserved(struct ttm_bo_device *bdev, } if (!ttm_bo_uses_embedded_gem_object(bo)) { /* - * bo.gem is not initialized, so we have to setup the + * bo.base is not initialized, so we have to setup the * struct elements we want use regardless. */ + bo->base.size = size; dma_resv_init(&bo->base._resv); drm_vma_node_reset(&bo->base.vma_node); } @@ -1209,7 +1186,7 @@ EXPORT_SYMBOL(ttm_bo_init_reserved); int ttm_bo_init(struct ttm_bo_device *bdev, struct ttm_buffer_object *bo, - unsigned long size, + size_t size, enum ttm_bo_type type, struct ttm_placement *placement, uint32_t page_alignment, diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 7ccb2295cac1..398d5013fc39 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -310,7 +310,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, kref_init(&fbo->base.kref); fbo->base.destroy = &ttm_transfered_destroy; fbo->base.acc_size = 0; - fbo->base.pin_count = 1; + fbo->base.pin_count = 0; if (bo->type != ttm_bo_type_sg) fbo->base.base.resv = &fbo->base.base._resv; @@ -319,6 +319,8 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, ret = dma_resv_trylock(&fbo->base.base._resv); WARN_ON(!ret); + ttm_bo_move_to_lru_tail_unlocked(&fbo->base); + *new_obj = &fbo->base; return 0; } @@ -429,9 +431,9 @@ int ttm_bo_kmap(struct ttm_buffer_object *bo, map->virtual = NULL; map->bo = bo; - if (num_pages > bo->num_pages) + if (num_pages > bo->mem.num_pages) return -EINVAL; - if (start_page > bo->num_pages) + if ((start_page + num_pages) > bo->mem.num_pages) return -EINVAL; ret = ttm_mem_io_reserve(bo->bdev, &bo->mem); @@ -483,14 +485,14 @@ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) if (mem->bus.is_iomem) { void __iomem *vaddr_iomem; - size_t size = bo->num_pages << PAGE_SHIFT; if (mem->bus.addr) vaddr_iomem = (void __iomem *)mem->bus.addr; else if (mem->bus.caching == ttm_write_combined) - vaddr_iomem = ioremap_wc(mem->bus.offset, size); + vaddr_iomem = ioremap_wc(mem->bus.offset, + bo->base.size); else - vaddr_iomem = ioremap(mem->bus.offset, size); + vaddr_iomem = ioremap(mem->bus.offset, bo->base.size); if (!vaddr_iomem) return -ENOMEM; @@ -515,7 +517,7 @@ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct dma_buf_map *map) * or to make the buffer object look contiguous. */ prot = ttm_io_prot(bo, mem, PAGE_KERNEL); - vaddr = vmap(ttm->pages, bo->num_pages, 0, prot); + vaddr = vmap(ttm->pages, ttm->num_pages, 0, prot); if (!vaddr) return -ENOMEM; diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 2944fa0af493..6dc96cf66744 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -31,7 +31,6 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_module.h> #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> #include <drm/drm_vma_manager.h> @@ -199,7 +198,7 @@ static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, /* Fault should not cross bo boundary. */ page_offset &= ~(fault_page_size - 1); - if (page_offset + fault_page_size > bo->num_pages) + if (page_offset + fault_page_size > bo->mem.num_pages) goto out_fallback; if (bo->mem.bus.is_iomem) @@ -307,7 +306,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, page_last = vma_pages(vma) + vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); - if (unlikely(page_offset >= bo->num_pages)) + if (unlikely(page_offset >= bo->mem.num_pages)) return VM_FAULT_SIGBUS; prot = ttm_io_prot(bo, &bo->mem, prot); @@ -470,7 +469,7 @@ int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, << PAGE_SHIFT); int ret; - if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages) + if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->mem.num_pages) return -EIO; ret = ttm_bo_reserve(bo, true, false, NULL); diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index 8a8f1a6a83a6..9fa36ed59429 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -55,7 +55,7 @@ void ttm_eu_backoff_reservation(struct ww_acquire_ctx *ticket, list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; - ttm_bo_move_to_lru_tail(bo, NULL); + ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL); dma_resv_unlock(bo->base.resv); } spin_unlock(&ttm_bo_glob.lru_lock); @@ -162,7 +162,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket, dma_resv_add_shared_fence(bo->base.resv, fence); else dma_resv_add_excl_fence(bo->base.resv, fence); - ttm_bo_move_to_lru_tail(bo, NULL); + ttm_bo_move_to_lru_tail(bo, &bo->mem, NULL); dma_resv_unlock(bo->base.resv); } spin_unlock(&ttm_bo_glob.lru_lock); diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c index 5ed1fc8f2ace..a3bfbd9cea68 100644 --- a/drivers/gpu/drm/ttm/ttm_memory.c +++ b/drivers/gpu/drm/ttm/ttm_memory.c @@ -29,7 +29,6 @@ #define pr_fmt(fmt) "[TTM] " fmt #include <drm/ttm/ttm_memory.h> -#include <drm/ttm/ttm_module.h> #include <linux/spinlock.h> #include <linux/sched.h> #include <linux/wait.h> @@ -39,6 +38,8 @@ #include <linux/swap.h> #include <drm/ttm/ttm_pool.h> +#include "ttm_module.h" + #define TTM_MEMORY_ALLOC_RETRIES 4 struct ttm_mem_global ttm_mem_glob; diff --git a/drivers/gpu/drm/ttm/ttm_module.c b/drivers/gpu/drm/ttm/ttm_module.c index 6ff40c041d79..c0906437cb1c 100644 --- a/drivers/gpu/drm/ttm/ttm_module.c +++ b/drivers/gpu/drm/ttm/ttm_module.c @@ -32,9 +32,10 @@ #include <linux/module.h> #include <linux/device.h> #include <linux/sched.h> -#include <drm/ttm/ttm_module.h> #include <drm/drm_sysfs.h> +#include "ttm_module.h" + static DECLARE_WAIT_QUEUE_HEAD(exit_q); static atomic_t device_released; diff --git a/drivers/gpu/drm/ttm/ttm_module.h b/drivers/gpu/drm/ttm/ttm_module.h new file mode 100644 index 000000000000..45fa318c1585 --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_module.h @@ -0,0 +1,40 @@ +/************************************************************************** + * + * Copyright 2008-2009 VMware, Inc., Palo Alto, CA., USA + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ +/* + * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> + */ + +#ifndef _TTM_MODULE_H_ +#define _TTM_MODULE_H_ + +#include <linux/kernel.h> +struct kobject; + +#define TTM_PFX "[TTM] " +extern struct kobject *ttm_get_kobj(void); + +#endif /* _TTM_MODULE_H_ */ diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c index e0952444cea9..a39305f742da 100644 --- a/drivers/gpu/drm/ttm/ttm_range_manager.c +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -29,7 +29,6 @@ * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ -#include <drm/ttm/ttm_module.h> #include <drm/ttm/ttm_bo_driver.h> #include <drm/ttm/ttm_placement.h> #include <drm/drm_mm.h> diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index da9eeffe0c6d..7f75a13163f0 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -129,7 +129,7 @@ static void ttm_tt_init_fields(struct ttm_tt *ttm, uint32_t page_flags, enum ttm_caching caching) { - ttm->num_pages = bo->num_pages; + ttm->num_pages = PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT; ttm->caching = ttm_cached; ttm->page_flags = page_flags; ttm->dma_address = NULL; @@ -162,19 +162,6 @@ void ttm_tt_fini(struct ttm_tt *ttm) } EXPORT_SYMBOL(ttm_tt_fini); -int ttm_dma_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, - uint32_t page_flags, enum ttm_caching caching) -{ - ttm_tt_init_fields(ttm, bo, page_flags, caching); - - if (ttm_dma_tt_alloc_page_directory(ttm)) { - pr_err("Failed allocating page table\n"); - return -ENOMEM; - } - return 0; -} -EXPORT_SYMBOL(ttm_dma_tt_init); - int ttm_sg_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, uint32_t page_flags, enum ttm_caching caching) { |