summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c')
-rw-r--r--drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c208
1 files changed, 77 insertions, 131 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 798985b94685..371bc099fda2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -63,61 +63,13 @@
#define AMDGPU_TTM_VRAM_MAX_DW_READ (size_t)128
-
-/**
- * amdgpu_init_mem_type - Initialize a memory manager for a specific type of
- * memory request.
- *
- * @bdev: The TTM BO device object (contains a reference to amdgpu_device)
- * @type: The type of memory requested
- * @man: The memory type manager for each domain
- *
- * This is called by ttm_bo_init_mm() when a buffer object is being
- * initialized.
- */
-static int amdgpu_init_mem_type(struct ttm_bo_device *bdev, uint32_t type,
- struct ttm_mem_type_manager *man)
+static int amdgpu_ttm_init_on_chip(struct amdgpu_device *adev,
+ unsigned int type,
+ uint64_t size)
{
- struct amdgpu_device *adev;
-
- adev = amdgpu_ttm_adev(bdev);
-
- switch (type) {
- case TTM_PL_SYSTEM:
- /* System memory */
- man->flags = TTM_MEMTYPE_FLAG_MAPPABLE;
- man->available_caching = TTM_PL_MASK_CACHING;
- man->default_caching = TTM_PL_FLAG_CACHED;
- break;
- case TTM_PL_TT:
- /* GTT memory */
- man->func = &amdgpu_gtt_mgr_func;
- man->available_caching = TTM_PL_MASK_CACHING;
- man->default_caching = TTM_PL_FLAG_CACHED;
- man->flags = TTM_MEMTYPE_FLAG_MAPPABLE | TTM_MEMTYPE_FLAG_CMA;
- break;
- case TTM_PL_VRAM:
- /* "On-card" video ram */
- man->func = &amdgpu_vram_mgr_func;
- man->flags = TTM_MEMTYPE_FLAG_FIXED |
- TTM_MEMTYPE_FLAG_MAPPABLE;
- man->available_caching = TTM_PL_FLAG_UNCACHED | TTM_PL_FLAG_WC;
- man->default_caching = TTM_PL_FLAG_WC;
- break;
- case AMDGPU_PL_GDS:
- case AMDGPU_PL_GWS:
- case AMDGPU_PL_OA:
- /* On-chip GDS memory*/
- man->func = &ttm_bo_manager_func;
- man->flags = TTM_MEMTYPE_FLAG_FIXED | TTM_MEMTYPE_FLAG_CMA;
- man->available_caching = TTM_PL_FLAG_UNCACHED;
- man->default_caching = TTM_PL_FLAG_UNCACHED;
- break;
- default:
- DRM_ERROR("Unsupported memory type %u\n", (unsigned)type);
- return -EINVAL;
- }
- return 0;
+ return ttm_range_man_init(&adev->mman.bdev, type,
+ TTM_PL_FLAG_UNCACHED, TTM_PL_FLAG_UNCACHED,
+ false, size >> PAGE_SHIFT);
}
/**
@@ -231,9 +183,9 @@ static int amdgpu_verify_access(struct ttm_buffer_object *bo, struct file *filp)
* Assign the memory from new_mem to the memory of the buffer object bo.
*/
static void amdgpu_move_null(struct ttm_buffer_object *bo,
- struct ttm_mem_reg *new_mem)
+ struct ttm_resource *new_mem)
{
- struct ttm_mem_reg *old_mem = &bo->mem;
+ struct ttm_resource *old_mem = &bo->mem;
BUG_ON(old_mem->mm_node != NULL);
*old_mem = *new_mem;
@@ -250,7 +202,7 @@ static void amdgpu_move_null(struct ttm_buffer_object *bo,
*/
static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
struct drm_mm_node *mm_node,
- struct ttm_mem_reg *mem)
+ struct ttm_resource *mem)
{
uint64_t addr = 0;
@@ -270,7 +222,7 @@ static uint64_t amdgpu_mm_node_addr(struct ttm_buffer_object *bo,
* @offset: The offset that drm_mm_node is used for finding.
*
*/
-static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
+static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_resource *mem,
uint64_t *offset)
{
struct drm_mm_node *mm_node = mem->mm_node;
@@ -298,7 +250,7 @@ static struct drm_mm_node *amdgpu_find_mm_node(struct ttm_mem_reg *mem,
* the physical address for local memory.
*/
static int amdgpu_ttm_map_buffer(struct ttm_buffer_object *bo,
- struct ttm_mem_reg *mem,
+ struct ttm_resource *mem,
struct drm_mm_node *mm_node,
unsigned num_pages, uint64_t offset,
unsigned window, struct amdgpu_ring *ring,
@@ -430,12 +382,22 @@ int amdgpu_ttm_copy_mem_to_mem(struct amdgpu_device *adev,
}
src_offset = src->offset;
- src_mm = amdgpu_find_mm_node(src->mem, &src_offset);
- src_node_size = (src_mm->size << PAGE_SHIFT) - src_offset;
+ if (src->mem->mm_node) {
+ src_mm = amdgpu_find_mm_node(src->mem, &src_offset);
+ src_node_size = (src_mm->size << PAGE_SHIFT) - src_offset;
+ } else {
+ src_mm = NULL;
+ src_node_size = ULLONG_MAX;
+ }
dst_offset = dst->offset;
- dst_mm = amdgpu_find_mm_node(dst->mem, &dst_offset);
- dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst_offset;
+ if (dst->mem->mm_node) {
+ dst_mm = amdgpu_find_mm_node(dst->mem, &dst_offset);
+ dst_node_size = (dst_mm->size << PAGE_SHIFT) - dst_offset;
+ } else {
+ dst_mm = NULL;
+ dst_node_size = ULLONG_MAX;
+ }
mutex_lock(&adev->mman.gtt_window_lock);
@@ -512,8 +474,8 @@ error:
*/
static int amdgpu_move_blit(struct ttm_buffer_object *bo,
bool evict, bool no_wait_gpu,
- struct ttm_mem_reg *new_mem,
- struct ttm_mem_reg *old_mem)
+ struct ttm_resource *new_mem,
+ struct ttm_resource *old_mem)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
@@ -572,10 +534,10 @@ error:
*/
static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
struct ttm_operation_ctx *ctx,
- struct ttm_mem_reg *new_mem)
+ struct ttm_resource *new_mem)
{
- struct ttm_mem_reg *old_mem = &bo->mem;
- struct ttm_mem_reg tmp_mem;
+ struct ttm_resource *old_mem = &bo->mem;
+ struct ttm_resource tmp_mem;
struct ttm_place placements;
struct ttm_placement placement;
int r;
@@ -617,7 +579,7 @@ static int amdgpu_move_vram_ram(struct ttm_buffer_object *bo, bool evict,
/* move BO (in tmp_mem) to new_mem */
r = ttm_bo_move_ttm(bo, ctx, new_mem);
out_cleanup:
- ttm_bo_mem_put(bo, &tmp_mem);
+ ttm_resource_free(bo, &tmp_mem);
return r;
}
@@ -628,10 +590,10 @@ out_cleanup:
*/
static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
struct ttm_operation_ctx *ctx,
- struct ttm_mem_reg *new_mem)
+ struct ttm_resource *new_mem)
{
- struct ttm_mem_reg *old_mem = &bo->mem;
- struct ttm_mem_reg tmp_mem;
+ struct ttm_resource *old_mem = &bo->mem;
+ struct ttm_resource tmp_mem;
struct ttm_placement placement;
struct ttm_place placements;
int r;
@@ -664,7 +626,7 @@ static int amdgpu_move_ram_vram(struct ttm_buffer_object *bo, bool evict,
goto out_cleanup;
}
out_cleanup:
- ttm_bo_mem_put(bo, &tmp_mem);
+ ttm_resource_free(bo, &tmp_mem);
return r;
}
@@ -674,7 +636,7 @@ out_cleanup:
* Called by amdgpu_bo_move()
*/
static bool amdgpu_mem_visible(struct amdgpu_device *adev,
- struct ttm_mem_reg *mem)
+ struct ttm_resource *mem)
{
struct drm_mm_node *nodes = mem->mm_node;
@@ -684,7 +646,7 @@ static bool amdgpu_mem_visible(struct amdgpu_device *adev,
if (mem->mem_type != TTM_PL_VRAM)
return false;
- /* ttm_mem_reg_ioremap only supports contiguous memory */
+ /* ttm_resource_ioremap only supports contiguous memory */
if (nodes->size != mem->num_pages)
return false;
@@ -699,11 +661,11 @@ static bool amdgpu_mem_visible(struct amdgpu_device *adev,
*/
static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
struct ttm_operation_ctx *ctx,
- struct ttm_mem_reg *new_mem)
+ struct ttm_resource *new_mem)
{
struct amdgpu_device *adev;
struct amdgpu_bo *abo;
- struct ttm_mem_reg *old_mem = &bo->mem;
+ struct ttm_resource *old_mem = &bo->mem;
int r;
/* Can't move a pinned BO */
@@ -785,19 +747,12 @@ memcpy:
*
* Called by ttm_mem_io_reserve() ultimately via ttm_bo_vm_fault()
*/
-static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
+static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *mem)
{
- struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type];
struct amdgpu_device *adev = amdgpu_ttm_adev(bdev);
struct drm_mm_node *mm_node = mem->mm_node;
+ size_t bus_size = (size_t)mem->num_pages << PAGE_SHIFT;
- mem->bus.addr = NULL;
- mem->bus.offset = 0;
- mem->bus.size = mem->num_pages << PAGE_SHIFT;
- mem->bus.base = 0;
- mem->bus.is_iomem = false;
- if (!(man->flags & TTM_MEMTYPE_FLAG_MAPPABLE))
- return -EINVAL;
switch (mem->mem_type) {
case TTM_PL_SYSTEM:
/* system memory */
@@ -807,11 +762,11 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_
case TTM_PL_VRAM:
mem->bus.offset = mem->start << PAGE_SHIFT;
/* check if it's visible */
- if ((mem->bus.offset + mem->bus.size) > adev->gmc.visible_vram_size)
+ if ((mem->bus.offset + bus_size) > adev->gmc.visible_vram_size)
return -EINVAL;
/* Only physically contiguous buffers apply. In a contiguous
* buffer, size of the first mm_node would match the number of
- * pages in ttm_mem_reg.
+ * pages in ttm_resource.
*/
if (adev->mman.aper_base_kaddr &&
(mm_node->size == mem->num_pages))
@@ -827,10 +782,6 @@ static int amdgpu_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_
return 0;
}
-static void amdgpu_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
-{
-}
-
static unsigned long amdgpu_ttm_io_mem_pfn(struct ttm_buffer_object *bo,
unsigned long page_offset)
{
@@ -1044,7 +995,6 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
struct amdgpu_ttm_tt *gtt = (void *)ttm;
- unsigned nents;
int r;
int write = !(gtt->userflags & AMDGPU_GEM_USERPTR_READONLY);
@@ -1059,9 +1009,8 @@ static int amdgpu_ttm_tt_pin_userptr(struct ttm_tt *ttm)
goto release_sg;
/* Map SG to device */
- r = -ENOMEM;
- nents = dma_map_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
- if (nents == 0)
+ r = dma_map_sgtable(adev->dev, ttm->sg, direction, 0);
+ if (r)
goto release_sg;
/* convert SG to linear array of pages and dma addresses */
@@ -1092,8 +1041,7 @@ static void amdgpu_ttm_tt_unpin_userptr(struct ttm_tt *ttm)
return;
/* unmap the pages mapped to the device */
- dma_unmap_sg(adev->dev, ttm->sg->sgl, ttm->sg->nents, direction);
-
+ dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);
sg_free_table(ttm->sg);
#if IS_ENABLED(CONFIG_DRM_AMDGPU_USERPTR)
@@ -1163,7 +1111,7 @@ gart_bind_fail:
* This handles binding GTT memory to the device address space.
*/
static int amdgpu_ttm_backend_bind(struct ttm_tt *ttm,
- struct ttm_mem_reg *bo_mem)
+ struct ttm_resource *bo_mem)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
struct amdgpu_ttm_tt *gtt = (void*)ttm;
@@ -1214,7 +1162,7 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev);
struct ttm_operation_ctx ctx = { false, false };
struct amdgpu_ttm_tt *gtt = (void*)bo->ttm;
- struct ttm_mem_reg tmp;
+ struct ttm_resource tmp;
struct ttm_placement placement;
struct ttm_place placements;
uint64_t addr, flags;
@@ -1251,11 +1199,11 @@ int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo)
gtt->offset = (u64)tmp.start << PAGE_SHIFT;
r = amdgpu_ttm_gart_bind(adev, bo, flags);
if (unlikely(r)) {
- ttm_bo_mem_put(bo, &tmp);
+ ttm_resource_free(bo, &tmp);
return r;
}
- ttm_bo_mem_put(bo, &bo->mem);
+ ttm_resource_free(bo, &bo->mem);
bo->mem = tmp;
}
@@ -1289,7 +1237,7 @@ int amdgpu_ttm_recover_gart(struct ttm_buffer_object *tbo)
* Called by ttm_tt_unbind() on behalf of ttm_bo_move_ttm() and
* ttm_tt_destroy().
*/
-static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
+static void amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
{
struct amdgpu_device *adev = amdgpu_ttm_adev(ttm->bdev);
struct amdgpu_ttm_tt *gtt = (void *)ttm;
@@ -1300,14 +1248,13 @@ static int amdgpu_ttm_backend_unbind(struct ttm_tt *ttm)
amdgpu_ttm_tt_unpin_userptr(ttm);
if (gtt->offset == AMDGPU_BO_INVALID_OFFSET)
- return 0;
+ return;
/* unbind shouldn't be done for GDS/GWS/OA in ttm_bo_clean_mm */
r = amdgpu_gart_unbind(adev, gtt->offset, ttm->num_pages);
if (r)
DRM_ERROR("failed to unbind %lu pages at 0x%08llX\n",
gtt->ttm.ttm.num_pages, gtt->offset);
- return r;
}
static void amdgpu_ttm_backend_destroy(struct ttm_tt *ttm)
@@ -1455,21 +1402,26 @@ static void amdgpu_ttm_tt_unpopulate(struct ttm_tt *ttm)
* amdgpu_ttm_tt_set_userptr - Initialize userptr GTT ttm_tt for the current
* task
*
- * @ttm: The ttm_tt object to bind this userptr object to
+ * @bo: The ttm_buffer_object to bind this userptr to
* @addr: The address in the current tasks VM space to use
* @flags: Requirements of userptr object.
*
* Called by amdgpu_gem_userptr_ioctl() to bind userptr pages
* to current task
*/
-int amdgpu_ttm_tt_set_userptr(struct ttm_tt *ttm, uint64_t addr,
- uint32_t flags)
+int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo,
+ uint64_t addr, uint32_t flags)
{
- struct amdgpu_ttm_tt *gtt = (void *)ttm;
+ struct amdgpu_ttm_tt *gtt;
- if (gtt == NULL)
- return -EINVAL;
+ if (!bo->ttm) {
+ /* TODO: We want a separate TTM object type for userptrs */
+ bo->ttm = amdgpu_ttm_tt_create(bo, 0);
+ if (bo->ttm == NULL)
+ return -ENOMEM;
+ }
+ gtt = (void*)bo->ttm;
gtt->userptr = addr;
gtt->userflags = flags;
@@ -1555,7 +1507,7 @@ bool amdgpu_ttm_tt_is_readonly(struct ttm_tt *ttm)
*
* Figure out the flags to use for a VM PDE (Page Directory Entry).
*/
-uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
+uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_resource *mem)
{
uint64_t flags = 0;
@@ -1581,7 +1533,7 @@ uint64_t amdgpu_ttm_tt_pde_flags(struct ttm_tt *ttm, struct ttm_mem_reg *mem)
* Figure out the flags to use for a VM PTE (Page Table Entry).
*/
uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
- struct ttm_mem_reg *mem)
+ struct ttm_resource *mem)
{
uint64_t flags = amdgpu_ttm_tt_pde_flags(ttm, mem);
@@ -1739,7 +1691,6 @@ static struct ttm_bo_driver amdgpu_bo_driver = {
.ttm_tt_create = &amdgpu_ttm_tt_create,
.ttm_tt_populate = &amdgpu_ttm_tt_populate,
.ttm_tt_unpopulate = &amdgpu_ttm_tt_unpopulate,
- .init_mem_type = &amdgpu_init_mem_type,
.eviction_valuable = amdgpu_ttm_bo_eviction_valuable,
.evict_flags = &amdgpu_evict_flags,
.move = &amdgpu_bo_move,
@@ -1748,7 +1699,6 @@ static struct ttm_bo_driver amdgpu_bo_driver = {
.release_notify = &amdgpu_bo_release_notify,
.fault_reserve_notify = &amdgpu_bo_fault_reserve_notify,
.io_mem_reserve = &amdgpu_ttm_io_mem_reserve,
- .io_mem_free = &amdgpu_ttm_io_mem_free,
.io_mem_pfn = amdgpu_ttm_io_mem_pfn,
.access_memory = &amdgpu_ttm_access_memory,
.del_from_lru_notify = &amdgpu_vm_del_from_lru_notify
@@ -1934,8 +1884,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
adev->mman.bdev.no_retry = true;
/* Initialize VRAM pool with all of VRAM divided into pages */
- r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_VRAM,
- adev->gmc.real_vram_size >> PAGE_SHIFT);
+ r = amdgpu_vram_mgr_init(adev);
if (r) {
DRM_ERROR("Failed initializing VRAM heap.\n");
return r;
@@ -2009,7 +1958,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
gtt_size = (uint64_t)amdgpu_gtt_size << 20;
/* Initialize GTT memory pool */
- r = ttm_bo_init_mm(&adev->mman.bdev, TTM_PL_TT, gtt_size >> PAGE_SHIFT);
+ r = amdgpu_gtt_mgr_init(adev, gtt_size);
if (r) {
DRM_ERROR("Failed initializing GTT heap.\n");
return r;
@@ -2018,22 +1967,19 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
(unsigned)(gtt_size / (1024 * 1024)));
/* Initialize various on-chip memory pools */
- r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GDS,
- adev->gds.gds_size);
+ r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GDS, adev->gds.gds_size);
if (r) {
DRM_ERROR("Failed initializing GDS heap.\n");
return r;
}
- r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_GWS,
- adev->gds.gws_size);
+ r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_GWS, adev->gds.gws_size);
if (r) {
DRM_ERROR("Failed initializing gws heap.\n");
return r;
}
- r = ttm_bo_init_mm(&adev->mman.bdev, AMDGPU_PL_OA,
- adev->gds.oa_size);
+ r = amdgpu_ttm_init_on_chip(adev, AMDGPU_PL_OA, adev->gds.oa_size);
if (r) {
DRM_ERROR("Failed initializing oa heap.\n");
return r;
@@ -2073,11 +2019,11 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
iounmap(adev->mman.aper_base_kaddr);
adev->mman.aper_base_kaddr = NULL;
- ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_VRAM);
- ttm_bo_clean_mm(&adev->mman.bdev, TTM_PL_TT);
- ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GDS);
- ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_GWS);
- ttm_bo_clean_mm(&adev->mman.bdev, AMDGPU_PL_OA);
+ amdgpu_vram_mgr_fini(adev);
+ amdgpu_gtt_mgr_fini(adev);
+ ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GDS);
+ ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_GWS);
+ ttm_range_man_fini(&adev->mman.bdev, AMDGPU_PL_OA);
ttm_bo_device_release(&adev->mman.bdev);
adev->mman.initialized = false;
DRM_INFO("amdgpu: ttm finalized\n");
@@ -2094,7 +2040,7 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
*/
void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
{
- struct ttm_mem_type_manager *man = &adev->mman.bdev.man[TTM_PL_VRAM];
+ struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
uint64_t size;
int r;
@@ -2316,7 +2262,7 @@ static int amdgpu_mm_dump_table(struct seq_file *m, void *data)
unsigned ttm_pl = (uintptr_t)node->info_ent->data;
struct drm_device *dev = node->minor->dev;
struct amdgpu_device *adev = drm_to_adev(dev);
- struct ttm_mem_type_manager *man = &adev->mman.bdev.man[ttm_pl];
+ struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, ttm_pl);
struct drm_printer p = drm_seq_file_printer(m);
man->func->debug(man, &p);