#include <linux/iosys-map.h>
#include <linux/io-mapping.h>
#include <linux/scatterlist.h>
#include <drm/ttm/ttm_bo.h>
#include <drm/ttm/ttm_placement.h>
#include <drm/ttm/ttm_resource.h>
void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk)
{
memset(bulk, 0, sizeof(*bulk));
}
EXPORT_SYMBOL(ttm_lru_bulk_move_init);
void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk)
{
unsigned i, j;
for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) {
for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j];
struct ttm_resource_manager *man;
if (!pos->first)
continue;
lockdep_assert_held(&pos->first->bo->bdev->lru_lock);
dma_resv_assert_held(pos->first->bo->base.resv);
dma_resv_assert_held(pos->last->bo->base.resv);
man = ttm_manager_type(pos->first->bo->bdev, i);
list_bulk_move_tail(&man->lru[j], &pos->first->lru,
&pos->last->lru);
}
}
}
EXPORT_SYMBOL(ttm_lru_bulk_move_tail);
static struct ttm_lru_bulk_move_pos *
ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res)
{
return &bulk->pos[res->mem_type][res->bo->priority];
}
static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos,
struct ttm_resource *res)
{
if (pos->last != res) {
if (pos->first == res)
pos->first = list_next_entry(res, lru);
list_move(&res->lru, &pos->last->lru);
pos->last = res;
}
}
static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk,
struct ttm_resource *res)
{
struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
if (!pos->first) {
pos->first = res;
pos->last = res;
} else {
ttm_lru_bulk_move_pos_tail(pos, res);
}
}
static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk,
struct ttm_resource *res)
{
struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res);
if (unlikely(WARN_ON(!pos->first || !pos->last) ||
(pos->first == res && pos->last == res))) {
pos->first = NULL;
pos->last = NULL;
} else if (pos->first == res) {
pos->first = list_next_entry(res, lru);
} else if (pos->last == res) {
pos->last = list_prev_entry(res, lru);
} else {
list_move(&res->lru, &pos->last->lru);
}
}
void ttm_resource_add_bulk_move(struct ttm_resource *res,
struct ttm_buffer_object *bo)
{
if (bo->bulk_move && !bo->pin_count)
ttm_lru_bulk_move_add(bo->bulk_move, res);
}
void ttm_resource_del_bulk_move(struct ttm_resource *res,
struct ttm_buffer_object *bo)
{
if (bo->bulk_move && !bo->pin_count)
ttm_lru_bulk_move_del(bo->bulk_move, res);
}
void ttm_resource_move_to_lru_tail(struct ttm_resource *res)
{
struct ttm_buffer_object *bo = res->bo;
struct ttm_device *bdev = bo->bdev;
lockdep_assert_held(&bo->bdev->lru_lock);
if (bo->pin_count) {
list_move_tail(&res->lru, &bdev->pinned);
} else if (bo->bulk_move) {
struct ttm_lru_bulk_move_pos *pos =
ttm_lru_bulk_move_pos(bo->bulk_move, res);
ttm_lru_bulk_move_pos_tail(pos, res);
} else {
struct ttm_resource_manager *man;
man = ttm_manager_type(bdev, res->mem_type);
list_move_tail(&res->lru, &man->lru[bo->priority]);
}
}
void ttm_resource_init(struct ttm_buffer_object *bo,
const struct ttm_place *place,
struct ttm_resource *res)
{
struct ttm_resource_manager *man;
res->start = 0;
res->size = bo->base.size;
res->mem_type = place->mem_type;
res->placement = place->flags;
res->bus.addr = NULL;
res->bus.offset = 0;
res->bus.is_iomem = false;
res->bus.caching = ttm_cached;
res->bo = bo;
man = ttm_manager_type(bo->bdev, place->mem_type);
spin_lock(&bo->bdev->lru_lock);
if (bo->pin_count)
list_add_tail(&res->lru, &bo->bdev->pinned);
else
list_add_tail(&res->lru, &man->lru[bo->priority]);
man->usage += res->size;
spin_unlock(&bo->bdev->lru_lock);
}
EXPORT_SYMBOL(ttm_resource_init);
void ttm_resource_fini(struct ttm_resource_manager *man,
struct ttm_resource *res)
{
struct ttm_device *bdev = man->bdev;
spin_lock(&bdev->lru_lock);
list_del_init(&res->lru);
man->usage -= res->size;
spin_unlock(&bdev->lru_lock);
}
EXPORT_SYMBOL(ttm_resource_fini);
int ttm_resource_alloc(struct ttm_buffer_object *bo,
const struct ttm_place *place,
struct ttm_resource **res_ptr)
{
struct ttm_resource_manager *man =
ttm_manager_type(bo->bdev, place->mem_type);
int ret;
ret = man->func->alloc(man, bo, place, res_ptr);
if (ret)
return ret;
spin_lock(&bo->bdev->lru_lock);
ttm_resource_add_bulk_move(*res_ptr, bo);
spin_unlock(&bo->bdev->lru_lock);
return 0;
}
void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
{
struct ttm_resource_manager *man;
if (!*res)
return;
spin_lock(&bo->bdev->lru_lock);
ttm_resource_del_bulk_move(*res, bo);
spin_unlock(&bo->bdev->lru_lock);
man = ttm_manager_type(bo->bdev, (*res)->mem_type);
man->func->free(man, *res);
*res = NULL;
}
EXPORT_SYMBOL(ttm_resource_free);
bool ttm_resource_intersects(struct ttm_device *bdev,
struct ttm_resource *res,
const struct ttm_place *place,
size_t size)
{
struct ttm_resource_manager *man;
if (!res)
return false;
man = ttm_manager_type(bdev, res->mem_type);
if (!place || !man->func->intersects)
return true;
return man->func->intersects(man, res, place, size);
}
bool ttm_resource_compatible(struct ttm_device *bdev,
struct ttm_resource *res,
const struct ttm_place *place,
size_t size)
{
struct ttm_resource_manager *man;
if (!res || !place)
return false;
man = ttm_manager_type(bdev, res->mem_type);
if (!man->func->compatible)
return true;
return man->func->compatible(man, res, place, size);
}
static bool ttm_resource_places_compat(struct ttm_resource *res,
const struct ttm_place *places,
unsigned num_placement)
{
struct ttm_buffer_object *bo = res->bo;
struct ttm_device *bdev = bo->bdev;
unsigned i;
if (res->placement & TTM_PL_FLAG_TEMPORARY)
return false;
for (i = 0; i < num_placement; i++) {
const struct ttm_place *heap = &places[i];
if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
continue;
if ((res->mem_type == heap->mem_type) &&
(!(heap->flags & TTM_PL_FLAG_CONTIGUOUS) ||
(res->placement & TTM_PL_FLAG_CONTIGUOUS)))
return true;
}
return false;
}
bool ttm_resource_compat(struct ttm_resource *res,
struct ttm_placement *placement)
{
if (ttm_resource_places_compat(res, placement->placement,
placement->num_placement))
return true;
if ((placement->busy_placement != placement->placement ||
placement->num_busy_placement > placement->num_placement) &&
ttm_resource_places_compat(res, placement->busy_placement,
placement->num_busy_placement))
return true;
return false;
}
void ttm_resource_set_bo(struct ttm_resource *res,
struct ttm_buffer_object *bo)
{
spin_lock(&bo->bdev->lru_lock);
res->bo = bo;
spin_unlock(&bo->bdev->lru_lock);
}
void ttm_resource_manager_init(struct ttm_resource_manager *man,
struct ttm_device *bdev,
uint64_t size)
{
unsigned i;
spin_lock_init(&man->move_lock);
man->bdev = bdev;
man->size = size;
man->usage = 0;
for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i)
INIT_LIST_HEAD(&man->lru[i]);
man->move = NULL;
}
EXPORT_SYMBOL(ttm_resource_manager_init);
int ttm_resource_manager_evict_all(struct ttm_device *bdev,
struct ttm_resource_manager *man)
{
struct ttm_operation_ctx ctx = {
.interruptible = false,
.no_wait_gpu = false,
.force_alloc = true
};
struct dma_fence *fence;
int ret;
unsigned i;
spin_lock(&bdev->lru_lock);
for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
while (!list_empty(&man->lru[i])) {
spin_unlock(&bdev->lru_lock);
ret = ttm_mem_evict_first(bdev, man, NULL, &ctx,
NULL);
if (ret)
return ret;
spin_lock(&bdev->lru_lock);
}
}
spin_unlock(&bdev->lru_lock);
spin_lock(&man->move_lock);
fence = dma_fence_get(man->move);
spin_unlock(&man->move_lock);
if (fence) {
ret = dma_fence_wait(fence, false);
dma_fence_put(fence);
if (ret)
return ret;
}
return 0;
}
EXPORT_SYMBOL(ttm_resource_manager_evict_all);
uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man)
{
uint64_t usage;
spin_lock(&man->bdev->lru_lock);
usage = man->usage;
spin_unlock(&man->bdev->lru_lock);
return usage;
}
EXPORT_SYMBOL(ttm_resource_manager_usage);
void ttm_resource_manager_debug(struct ttm_resource_manager *man,
struct drm_printer *p)
{
drm_printf(p, " use_type: %d\n", man->use_type);
drm_printf(p, " use_tt: %d\n", man->use_tt);
drm_printf(p, " size: %llu\n", man->size);
drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man));
if (man->func->debug)
man->func->debug(man, p);
}
EXPORT_SYMBOL(ttm_resource_manager_debug);
struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_manager *man,
struct ttm_resource_cursor *cursor)
{
struct ttm_resource *res;
lockdep_assert_held(&man->bdev->lru_lock);
for (cursor->priority = 0; cursor->priority < TTM_MAX_BO_PRIORITY;
++cursor->priority)
list_for_each_entry(res, &man->lru[cursor->priority], lru)
return res;
return NULL;
}
struct ttm_resource *
ttm_resource_manager_next(struct ttm_resource_manager *man,
struct ttm_resource_cursor *cursor,
struct ttm_resource *res)
{
lockdep_assert_held(&man->bdev->lru_lock);
list_for_each_entry_continue(res, &man->lru[cursor->priority], lru)
return res;
for (++cursor->priority; cursor->priority < TTM_MAX_BO_PRIORITY;
++cursor->priority)
list_for_each_entry(res, &man->lru[cursor->priority], lru)
return res;
return NULL;
}
static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter,
struct iosys_map *dmap,
pgoff_t i)
{
struct ttm_kmap_iter_iomap *iter_io =
container_of(iter, typeof(*iter_io), base);
void __iomem *addr;
retry:
while (i >= iter_io->cache.end) {
iter_io->cache.sg = iter_io->cache.sg ?
sg_next(iter_io->cache.sg) : iter_io->st->sgl;
iter_io->cache.i = iter_io->cache.end;
iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >>
PAGE_SHIFT;
iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) -
iter_io->start;
}
if (i < iter_io->cache.i) {
iter_io->cache.end = 0;
iter_io->cache.sg = NULL;
goto retry;
}
addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs +
(((resource_size_t)i - iter_io->cache.i)
<< PAGE_SHIFT));
iosys_map_set_vaddr_iomem(dmap, addr);
}
static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter,
struct iosys_map *map)
{
io_mapping_unmap_local(map->vaddr_iomem);
}
static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = {
.map_local = ttm_kmap_iter_iomap_map_local,
.unmap_local = ttm_kmap_iter_iomap_unmap_local,
.maps_tt = false,
};
struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
struct io_mapping *iomap,
struct sg_table *st,
resource_size_t start)
{
iter_io->base.ops = &ttm_kmap_iter_io_ops;
iter_io->iomap = iomap;
iter_io->st = st;
iter_io->start = start;
memset(&iter_io->cache, 0, sizeof(iter_io->cache));
return &iter_io->base;
}
EXPORT_SYMBOL(ttm_kmap_iter_iomap_init);
static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter,
struct iosys_map *dmap,
pgoff_t i)
{
struct ttm_kmap_iter_linear_io *iter_io =
container_of(iter, typeof(*iter_io), base);
*dmap = iter_io->dmap;
iosys_map_incr(dmap, i * PAGE_SIZE);
}
static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = {
.map_local = ttm_kmap_iter_linear_io_map_local,
.maps_tt = false,
};
struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
struct ttm_device *bdev,
struct ttm_resource *mem)
{
int ret;
ret = ttm_mem_io_reserve(bdev, mem);
if (ret)
goto out_err;
if (!mem->bus.is_iomem) {
ret = -EINVAL;
goto out_io_free;
}
if (mem->bus.addr) {
iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr);
iter_io->needs_unmap = false;
} else {
iter_io->needs_unmap = true;
memset(&iter_io->dmap, 0, sizeof(iter_io->dmap));
if (mem->bus.caching == ttm_write_combined)
iosys_map_set_vaddr_iomem(&iter_io->dmap,
ioremap_wc(mem->bus.offset,
mem->size));
else if (mem->bus.caching == ttm_cached)
iosys_map_set_vaddr(&iter_io->dmap,
memremap(mem->bus.offset, mem->size,
MEMREMAP_WB |
MEMREMAP_WT |
MEMREMAP_WC));
if (iosys_map_is_null(&iter_io->dmap))
iosys_map_set_vaddr_iomem(&iter_io->dmap,
ioremap(mem->bus.offset,
mem->size));
if (iosys_map_is_null(&iter_io->dmap)) {
ret = -ENOMEM;
goto out_io_free;
}
}
iter_io->base.ops = &ttm_kmap_iter_linear_io_ops;
return &iter_io->base;
out_io_free:
ttm_mem_io_free(bdev, mem);
out_err:
return ERR_PTR(ret);
}
void
ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
struct ttm_device *bdev,
struct ttm_resource *mem)
{
if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) {
if (iter_io->dmap.is_iomem)
iounmap(iter_io->dmap.vaddr_iomem);
else
memunmap(iter_io->dmap.vaddr);
}
ttm_mem_io_free(bdev, mem);
}
#if defined(CONFIG_DEBUG_FS)
static int ttm_resource_manager_show(struct seq_file *m, void *unused)
{
struct ttm_resource_manager *man =
(struct ttm_resource_manager *)m->private;
struct drm_printer p = drm_seq_file_printer(m);
ttm_resource_manager_debug(man, &p);
return 0;
}
DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager);
#endif
void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
struct dentry * parent,
const char *name)
{
#if defined(CONFIG_DEBUG_FS)
debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops);
#endif
}
EXPORT_SYMBOL