summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/gpu/msm/kgsl.c602
-rw-r--r--drivers/gpu/msm/kgsl.h37
-rw-r--r--drivers/gpu/msm/kgsl_compat.c12
-rw-r--r--drivers/gpu/msm/kgsl_debugfs.c81
-rw-r--r--drivers/gpu/msm/kgsl_ioctl.c12
-rw-r--r--drivers/gpu/msm/kgsl_iommu.c165
-rw-r--r--drivers/gpu/msm/kgsl_mmu.c93
-rw-r--r--drivers/gpu/msm/kgsl_mmu.h6
-rw-r--r--drivers/gpu/msm/kgsl_sharedmem.h34
-rw-r--r--drivers/gpu/msm/kgsl_snapshot.c7
-rw-r--r--drivers/gpu/msm/kgsl_trace.h94
-rw-r--r--include/uapi/linux/msm_kgsl.h99
12 files changed, 1154 insertions, 88 deletions
diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c
index c203ac7bfe8c..24005a1fda72 100644
--- a/drivers/gpu/msm/kgsl.c
+++ b/drivers/gpu/msm/kgsl.c
@@ -78,6 +78,16 @@ struct kgsl_dma_buf_meta {
struct sg_table *table;
};
+static inline struct kgsl_pagetable *_get_memdesc_pagetable(
+ struct kgsl_pagetable *pt, struct kgsl_mem_entry *entry)
+{
+ /* if a secured buffer, map it to secure global pagetable */
+ if (kgsl_memdesc_is_secured(&entry->memdesc))
+ return pt->mmu->securepagetable;
+
+ return pt;
+}
+
static void kgsl_mem_entry_detach_process(struct kgsl_mem_entry *entry);
static const struct file_operations kgsl_fops;
@@ -445,14 +455,17 @@ kgsl_mem_entry_attach_process(struct kgsl_mem_entry *entry,
/* map the memory after unlocking if gpuaddr has been assigned */
if (entry->memdesc.gpuaddr) {
- /* if a secured buffer map it to secure global pagetable */
+ pagetable = process->pagetable;
if (kgsl_memdesc_is_secured(&entry->memdesc))
- pagetable = process->pagetable->mmu->securepagetable;
- else
- pagetable = process->pagetable;
+ pagetable = pagetable->mmu->securepagetable;
entry->memdesc.pagetable = pagetable;
- ret = kgsl_mmu_map(pagetable, &entry->memdesc);
+
+ if (entry->memdesc.flags & KGSL_MEMFLAGS_SPARSE_VIRT)
+ ret = kgsl_mmu_sparse_dummy_map(pagetable,
+ &entry->memdesc, 0, entry->memdesc.size);
+ else if (entry->memdesc.gpuaddr)
+ ret = kgsl_mmu_map(pagetable, &entry->memdesc);
if (ret)
kgsl_mem_entry_detach_process(entry);
}
@@ -1270,6 +1283,24 @@ kgsl_sharedmem_find(struct kgsl_process_private *private, uint64_t gpuaddr)
}
EXPORT_SYMBOL(kgsl_sharedmem_find);
+struct kgsl_mem_entry * __must_check
+kgsl_sharedmem_find_id_flags(struct kgsl_process_private *process,
+ unsigned int id, uint64_t flags)
+{
+ int count = 0;
+ struct kgsl_mem_entry *entry;
+
+ spin_lock(&process->mem_lock);
+ entry = idr_find(&process->mem_idr, id);
+ if (entry)
+ if (!entry->pending_free &&
+ (flags & entry->memdesc.flags) == flags)
+ count = kgsl_mem_entry_get(entry);
+ spin_unlock(&process->mem_lock);
+
+ return (count == 0) ? NULL : entry;
+}
+
/**
* kgsl_sharedmem_find_id() - find a memory entry by id
* @process: the owning process
@@ -1283,19 +1314,7 @@ EXPORT_SYMBOL(kgsl_sharedmem_find);
struct kgsl_mem_entry * __must_check
kgsl_sharedmem_find_id(struct kgsl_process_private *process, unsigned int id)
{
- int result;
- struct kgsl_mem_entry *entry;
-
- drain_workqueue(kgsl_driver.mem_workqueue);
-
- spin_lock(&process->mem_lock);
- entry = idr_find(&process->mem_idr, id);
- result = kgsl_mem_entry_get(entry);
- spin_unlock(&process->mem_lock);
-
- if (result == 0)
- return NULL;
- return entry;
+ return kgsl_sharedmem_find_id_flags(process, id, 0);
}
/**
@@ -3121,6 +3140,546 @@ long kgsl_ioctl_gpumem_get_info(struct kgsl_device_private *dev_priv,
return result;
}
+static inline int _sparse_alloc_param_sanity_check(uint64_t size,
+ uint64_t pagesize)
+{
+ if (size == 0 || pagesize == 0)
+ return -EINVAL;
+
+ if (pagesize != PAGE_SIZE && pagesize != SZ_64K)
+ return -EINVAL;
+
+ if (pagesize > size || !IS_ALIGNED(size, pagesize))
+ return -EINVAL;
+
+ return 0;
+}
+
+long kgsl_ioctl_sparse_phys_alloc(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data)
+{
+ struct kgsl_process_private *process = dev_priv->process_priv;
+ struct kgsl_sparse_phys_alloc *param = data;
+ struct kgsl_mem_entry *entry;
+ int ret;
+ int id;
+
+ ret = _sparse_alloc_param_sanity_check(param->size, param->pagesize);
+ if (ret)
+ return ret;
+
+ entry = kgsl_mem_entry_create();
+ if (entry == NULL)
+ return -ENOMEM;
+
+ ret = kgsl_process_private_get(process);
+ if (!ret) {
+ ret = -EBADF;
+ goto err_free_entry;
+ }
+
+ idr_preload(GFP_KERNEL);
+ spin_lock(&process->mem_lock);
+ /* Allocate the ID but don't attach the pointer just yet */
+ id = idr_alloc(&process->mem_idr, NULL, 1, 0, GFP_NOWAIT);
+ spin_unlock(&process->mem_lock);
+ idr_preload_end();
+
+ if (id < 0) {
+ ret = id;
+ goto err_put_proc_priv;
+ }
+
+ entry->id = id;
+ entry->priv = process;
+
+ entry->memdesc.flags = KGSL_MEMFLAGS_SPARSE_PHYS;
+ kgsl_memdesc_set_align(&entry->memdesc, ilog2(param->pagesize));
+
+ ret = kgsl_allocate_user(dev_priv->device, &entry->memdesc,
+ process->pagetable, param->size, entry->memdesc.flags);
+ if (ret)
+ goto err_remove_idr;
+
+ /* Sanity check to verify we got correct pagesize */
+ if (param->pagesize != PAGE_SIZE && entry->memdesc.sgt != NULL) {
+ struct scatterlist *s;
+ int i;
+
+ for_each_sg(entry->memdesc.sgt->sgl, s,
+ entry->memdesc.sgt->nents, i) {
+ if (!IS_ALIGNED(s->length, param->pagesize))
+ goto err_invalid_pages;
+ }
+ }
+
+ param->id = entry->id;
+ param->flags = entry->memdesc.flags;
+
+ trace_sparse_phys_alloc(entry->id, param->size, param->pagesize);
+ kgsl_mem_entry_commit_process(entry);
+
+ return 0;
+
+err_invalid_pages:
+ kgsl_sharedmem_free(&entry->memdesc);
+err_remove_idr:
+ spin_lock(&process->mem_lock);
+ idr_remove(&process->mem_idr, entry->id);
+ spin_unlock(&process->mem_lock);
+err_put_proc_priv:
+ kgsl_process_private_put(process);
+err_free_entry:
+ kfree(entry);
+
+ return ret;
+}
+
+long kgsl_ioctl_sparse_phys_free(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data)
+{
+ struct kgsl_process_private *process = dev_priv->process_priv;
+ struct kgsl_sparse_phys_free *param = data;
+ struct kgsl_mem_entry *entry;
+
+ entry = kgsl_sharedmem_find_id_flags(process, param->id,
+ KGSL_MEMFLAGS_SPARSE_PHYS);
+ if (entry == NULL)
+ return -EINVAL;
+
+ if (entry->memdesc.cur_bindings != 0) {
+ kgsl_mem_entry_put(entry);
+ return -EINVAL;
+ }
+
+ trace_sparse_phys_free(entry->id);
+
+ /* One put for find_id(), one put for the kgsl_mem_entry_create() */
+ kgsl_mem_entry_put(entry);
+ kgsl_mem_entry_put(entry);
+
+ return 0;
+}
+
+long kgsl_ioctl_sparse_virt_alloc(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data)
+{
+ struct kgsl_sparse_virt_alloc *param = data;
+ struct kgsl_mem_entry *entry;
+ int ret;
+
+ ret = _sparse_alloc_param_sanity_check(param->size, param->pagesize);
+ if (ret)
+ return ret;
+
+ entry = kgsl_mem_entry_create();
+ if (entry == NULL)
+ return -ENOMEM;
+
+ entry->memdesc.flags = KGSL_MEMFLAGS_SPARSE_VIRT;
+ entry->memdesc.size = param->size;
+ entry->memdesc.cur_bindings = 0;
+ kgsl_memdesc_set_align(&entry->memdesc, ilog2(param->pagesize));
+
+ spin_lock_init(&entry->bind_lock);
+ entry->bind_tree = RB_ROOT;
+
+ ret = kgsl_mem_entry_attach_process(entry, dev_priv);
+ if (ret) {
+ kfree(entry);
+ return ret;
+ }
+
+ param->id = entry->id;
+ param->gpuaddr = entry->memdesc.gpuaddr;
+ param->flags = entry->memdesc.flags;
+
+ trace_sparse_virt_alloc(entry->id, param->size, param->pagesize);
+ kgsl_mem_entry_commit_process(entry);
+
+ return 0;
+}
+
+long kgsl_ioctl_sparse_virt_free(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data)
+{
+ struct kgsl_process_private *process = dev_priv->process_priv;
+ struct kgsl_sparse_virt_free *param = data;
+ struct kgsl_mem_entry *entry = NULL;
+
+ entry = kgsl_sharedmem_find_id_flags(process, param->id,
+ KGSL_MEMFLAGS_SPARSE_VIRT);
+ if (entry == NULL)
+ return -EINVAL;
+
+ if (entry->bind_tree.rb_node != NULL) {
+ kgsl_mem_entry_put(entry);
+ return -EINVAL;
+ }
+
+ trace_sparse_virt_free(entry->id);
+
+ /* One put for find_id(), one put for the kgsl_mem_entry_create() */
+ kgsl_mem_entry_put(entry);
+ kgsl_mem_entry_put(entry);
+
+ return 0;
+}
+
+static int _sparse_add_to_bind_tree(struct kgsl_mem_entry *entry,
+ uint64_t v_offset,
+ struct kgsl_memdesc *memdesc,
+ uint64_t p_offset,
+ uint64_t size,
+ uint64_t flags)
+{
+ struct sparse_bind_object *new;
+ struct rb_node **node, *parent = NULL;
+
+ new = kzalloc(sizeof(*new), GFP_KERNEL);
+ if (new == NULL)
+ return -ENOMEM;
+
+ new->v_off = v_offset;
+ new->p_off = p_offset;
+ new->p_memdesc = memdesc;
+ new->size = size;
+ new->flags = flags;
+
+ node = &entry->bind_tree.rb_node;
+
+ while (*node != NULL) {
+ struct sparse_bind_object *this;
+
+ parent = *node;
+ this = rb_entry(parent, struct sparse_bind_object, node);
+
+ if (new->v_off < this->v_off)
+ node = &parent->rb_left;
+ else if (new->v_off > this->v_off)
+ node = &parent->rb_right;
+ }
+
+ rb_link_node(&new->node, parent, node);
+ rb_insert_color(&new->node, &entry->bind_tree);
+
+ return 0;
+}
+
+static int _sparse_rm_from_bind_tree(struct kgsl_mem_entry *entry,
+ struct sparse_bind_object *obj,
+ uint64_t v_offset, uint64_t size)
+{
+ spin_lock(&entry->bind_lock);
+ if (v_offset == obj->v_off && size >= obj->size) {
+ /*
+ * We are all encompassing, remove the entry and free
+ * things up
+ */
+ rb_erase(&obj->node, &entry->bind_tree);
+ kfree(obj);
+ } else if (v_offset == obj->v_off) {
+ /*
+ * We are the front of the node, adjust the front of
+ * the node
+ */
+ obj->v_off += size;
+ obj->p_off += size;
+ obj->size -= size;
+ } else if ((v_offset + size) == (obj->v_off + obj->size)) {
+ /*
+ * We are at the end of the obj, adjust the beginning
+ * points
+ */
+ obj->size -= size;
+ } else {
+ /*
+ * We are in the middle of a node, split it up and
+ * create a new mini node. Adjust this node's bounds
+ * and add the new node to the list.
+ */
+ uint64_t tmp_size = obj->size;
+ int ret;
+
+ obj->size = v_offset - obj->v_off;
+
+ spin_unlock(&entry->bind_lock);
+ ret = _sparse_add_to_bind_tree(entry, v_offset + size,
+ obj->p_memdesc,
+ obj->p_off + (v_offset - obj->v_off) + size,
+ tmp_size - (v_offset - obj->v_off) - size,
+ obj->flags);
+
+ return ret;
+ }
+
+ spin_unlock(&entry->bind_lock);
+
+ return 0;
+}
+
+static struct sparse_bind_object *_find_containing_bind_obj(
+ struct kgsl_mem_entry *entry,
+ uint64_t offset, uint64_t size)
+{
+ struct sparse_bind_object *obj = NULL;
+ struct rb_node *node = entry->bind_tree.rb_node;
+
+ spin_lock(&entry->bind_lock);
+
+ while (node != NULL) {
+ obj = rb_entry(node, struct sparse_bind_object, node);
+
+ if (offset == obj->v_off) {
+ break;
+ } else if (offset < obj->v_off) {
+ if (offset + size > obj->v_off)
+ break;
+ node = node->rb_left;
+ obj = NULL;
+ } else if (offset > obj->v_off) {
+ if (offset < obj->v_off + obj->size)
+ break;
+ node = node->rb_right;
+ obj = NULL;
+ }
+ }
+
+ spin_unlock(&entry->bind_lock);
+
+ return obj;
+}
+
+static int _sparse_unbind(struct kgsl_mem_entry *entry,
+ struct sparse_bind_object *bind_obj,
+ uint64_t offset, uint64_t size)
+{
+ struct kgsl_memdesc *memdesc = bind_obj->p_memdesc;
+ struct kgsl_pagetable *pt = memdesc->pagetable;
+ int ret;
+
+ if (memdesc->cur_bindings < (size / PAGE_SIZE))
+ return -EINVAL;
+
+ memdesc->cur_bindings -= size / PAGE_SIZE;
+
+ ret = kgsl_mmu_unmap_offset(pt, memdesc,
+ entry->memdesc.gpuaddr, offset, size);
+ if (ret)
+ return ret;
+
+ ret = kgsl_mmu_sparse_dummy_map(pt, &entry->memdesc, offset, size);
+ if (ret)
+ return ret;
+
+ ret = _sparse_rm_from_bind_tree(entry, bind_obj, offset, size);
+ if (ret == 0) {
+ atomic_long_sub(size, &kgsl_driver.stats.mapped);
+ trace_sparse_unbind(entry->id, offset, size);
+ }
+
+ return ret;
+}
+
+static long sparse_unbind_range(struct kgsl_sparse_binding_object *obj,
+ struct kgsl_mem_entry *virt_entry)
+{
+ struct sparse_bind_object *bind_obj;
+ int ret = 0;
+ uint64_t size = obj->size;
+ uint64_t tmp_size = obj->size;
+ uint64_t offset = obj->virtoffset;
+
+ while (size > 0 && ret == 0) {
+ tmp_size = size;
+ bind_obj = _find_containing_bind_obj(virt_entry, offset, size);
+ if (bind_obj == NULL)
+ return 0;
+
+ if (bind_obj->v_off > offset) {
+ tmp_size = size - bind_obj->v_off - offset;
+ if (tmp_size > bind_obj->size)
+ tmp_size = bind_obj->size;
+ offset = bind_obj->v_off;
+ } else if (bind_obj->v_off < offset) {
+ uint64_t diff = offset - bind_obj->v_off;
+
+ if (diff + size > bind_obj->size)
+ tmp_size = bind_obj->size - diff;
+ } else {
+ if (tmp_size > bind_obj->size)
+ tmp_size = bind_obj->size;
+ }
+
+ ret = _sparse_unbind(virt_entry, bind_obj, offset, tmp_size);
+ if (ret == 0) {
+ offset += tmp_size;
+ size -= tmp_size;
+ }
+ }
+
+ return ret;
+}
+
+static inline bool _is_phys_bindable(struct kgsl_mem_entry *phys_entry,
+ uint64_t offset, uint64_t size, uint64_t flags)
+{
+ struct kgsl_memdesc *memdesc = &phys_entry->memdesc;
+
+ if (!IS_ALIGNED(offset | size, kgsl_memdesc_get_pagesize(memdesc)))
+ return false;
+
+ if (!(flags & KGSL_SPARSE_BIND_MULTIPLE_TO_PHYS) &&
+ offset + size > memdesc->size)
+ return false;
+
+ return true;
+}
+
+static int _sparse_bind(struct kgsl_process_private *process,
+ struct kgsl_mem_entry *virt_entry, uint64_t v_offset,
+ struct kgsl_mem_entry *phys_entry, uint64_t p_offset,
+ uint64_t size, uint64_t flags)
+{
+ int ret;
+ struct kgsl_pagetable *pagetable;
+ struct kgsl_memdesc *memdesc = &phys_entry->memdesc;
+
+ /* map the memory after unlocking if gpuaddr has been assigned */
+ if (memdesc->gpuaddr)
+ return -EINVAL;
+
+ if (memdesc->useraddr != 0)
+ return -EINVAL;
+
+ pagetable = memdesc->pagetable;
+
+ /* Clear out any mappings */
+ ret = kgsl_mmu_unmap_offset(pagetable, &virt_entry->memdesc,
+ virt_entry->memdesc.gpuaddr, v_offset, size);
+ if (ret)
+ return ret;
+
+ ret = kgsl_mmu_map_offset(pagetable, virt_entry->memdesc.gpuaddr,
+ v_offset, memdesc, p_offset, size, flags);
+ if (ret) {
+ /* Try to clean up, but not the end of the world */
+ kgsl_mmu_sparse_dummy_map(pagetable, &virt_entry->memdesc,
+ v_offset, size);
+ return ret;
+ }
+
+ ret = _sparse_add_to_bind_tree(virt_entry, v_offset, memdesc,
+ p_offset, size, flags);
+ if (ret == 0)
+ memdesc->cur_bindings += size / PAGE_SIZE;
+
+ return ret;
+}
+
+static long sparse_bind_range(struct kgsl_process_private *private,
+ struct kgsl_sparse_binding_object *obj,
+ struct kgsl_mem_entry *virt_entry)
+{
+ struct kgsl_mem_entry *phys_entry;
+ int ret;
+
+ phys_entry = kgsl_sharedmem_find_id_flags(private, obj->id,
+ KGSL_MEMFLAGS_SPARSE_PHYS);
+ if (phys_entry == NULL)
+ return -EINVAL;
+
+ if (!_is_phys_bindable(phys_entry, obj->physoffset, obj->size,
+ obj->flags)) {
+ kgsl_mem_entry_put(phys_entry);
+ return -EINVAL;
+ }
+
+ if (kgsl_memdesc_get_align(&virt_entry->memdesc) !=
+ kgsl_memdesc_get_align(&phys_entry->memdesc)) {
+ kgsl_mem_entry_put(phys_entry);
+ return -EINVAL;
+ }
+
+ ret = sparse_unbind_range(obj, virt_entry);
+ if (ret) {
+ kgsl_mem_entry_put(phys_entry);
+ return -EINVAL;
+ }
+
+ ret = _sparse_bind(private, virt_entry, obj->virtoffset,
+ phys_entry, obj->physoffset, obj->size,
+ obj->flags & KGSL_SPARSE_BIND_MULTIPLE_TO_PHYS);
+ if (ret == 0) {
+ KGSL_STATS_ADD(obj->size, &kgsl_driver.stats.mapped,
+ &kgsl_driver.stats.mapped_max);
+
+ trace_sparse_bind(virt_entry->id, obj->virtoffset,
+ phys_entry->id, obj->physoffset,
+ obj->size, obj->flags);
+ }
+
+ kgsl_mem_entry_put(phys_entry);
+
+ return ret;
+}
+
+long kgsl_ioctl_sparse_bind(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data)
+{
+ struct kgsl_process_private *private = dev_priv->process_priv;
+ struct kgsl_sparse_bind *param = data;
+ struct kgsl_sparse_binding_object obj;
+ struct kgsl_mem_entry *virt_entry;
+ int pg_sz;
+ void __user *ptr;
+ int ret = 0;
+ int i = 0;
+
+ ptr = (void __user *) (uintptr_t) param->list;
+
+ if (param->size > sizeof(struct kgsl_sparse_binding_object) ||
+ param->count == 0 || ptr == NULL)
+ return -EINVAL;
+
+ virt_entry = kgsl_sharedmem_find_id_flags(private, param->id,
+ KGSL_MEMFLAGS_SPARSE_VIRT);
+ if (virt_entry == NULL)
+ return -EINVAL;
+
+ pg_sz = kgsl_memdesc_get_pagesize(&virt_entry->memdesc);
+
+ for (i = 0; i < param->count; i++) {
+ memset(&obj, 0, sizeof(obj));
+ ret = _copy_from_user(&obj, ptr, sizeof(obj), param->size);
+ if (ret)
+ break;
+
+ /* Sanity check initial range */
+ if (obj.size == 0 ||
+ obj.virtoffset + obj.size > virt_entry->memdesc.size ||
+ !(IS_ALIGNED(obj.virtoffset | obj.size, pg_sz))) {
+ ret = -EINVAL;
+ break;
+ }
+
+ if (obj.flags & KGSL_SPARSE_BIND)
+ ret = sparse_bind_range(private, &obj, virt_entry);
+ else if (obj.flags & KGSL_SPARSE_UNBIND)
+ ret = sparse_unbind_range(&obj, virt_entry);
+ else
+ ret = -EINVAL;
+ if (ret)
+ break;
+
+ ptr += sizeof(obj);
+ }
+
+ kgsl_mem_entry_put(virt_entry);
+
+ return ret;
+}
+
long kgsl_ioctl_gpuobj_info(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data)
{
@@ -3356,6 +3915,13 @@ get_mmap_entry(struct kgsl_process_private *private,
goto err_put;
}
+ if (entry->memdesc.flags & KGSL_MEMFLAGS_SPARSE_PHYS) {
+ if (len != entry->memdesc.size) {
+ ret = -EINVAL;
+ goto err_put;
+ }
+ }
+
if (entry->memdesc.useraddr != 0) {
ret = -EBUSY;
goto err_put;
diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h
index ee7149e1fd41..25f5de6ce645 100644
--- a/drivers/gpu/msm/kgsl.h
+++ b/drivers/gpu/msm/kgsl.h
@@ -184,6 +184,7 @@ struct kgsl_memdesc_ops {
* @attrs: dma attributes for this memory
* @pages: An array of pointers to allocated pages
* @page_count: Total number of pages allocated
+ * @cur_bindings: Number of sparse pages actively bound
*/
struct kgsl_memdesc {
struct kgsl_pagetable *pagetable;
@@ -202,6 +203,7 @@ struct kgsl_memdesc {
struct dma_attrs attrs;
struct page **pages;
unsigned int page_count;
+ unsigned int cur_bindings;
};
/*
@@ -235,6 +237,8 @@ struct kgsl_memdesc {
* @dev_priv: back pointer to the device file that created this entry.
* @metadata: String containing user specified metadata for the entry
* @work: Work struct used to schedule a kgsl_mem_entry_put in atomic contexts
+ * @bind_lock: Lock for sparse memory bindings
+ * @bind_tree: RB Tree for sparse memory bindings
*/
struct kgsl_mem_entry {
struct kref refcount;
@@ -246,6 +250,8 @@ struct kgsl_mem_entry {
int pending_free;
char metadata[KGSL_GPUOBJ_ALLOC_METADATA_MAX + 1];
struct work_struct work;
+ spinlock_t bind_lock;
+ struct rb_root bind_tree;
};
struct kgsl_device_private;
@@ -315,6 +321,24 @@ struct kgsl_protected_registers {
int range;
};
+/**
+ * struct sparse_bind_object - Bind metadata
+ * @node: Node for the rb tree
+ * @p_memdesc: Physical memdesc bound to
+ * @v_off: Offset of bind in the virtual entry
+ * @p_off: Offset of bind in the physical memdesc
+ * @size: Size of the bind
+ * @flags: Flags for the bind
+ */
+struct sparse_bind_object {
+ struct rb_node node;
+ struct kgsl_memdesc *p_memdesc;
+ uint64_t v_off;
+ uint64_t p_off;
+ uint64_t size;
+ uint64_t flags;
+};
+
long kgsl_ioctl_device_getproperty(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
long kgsl_ioctl_device_setproperty(struct kgsl_device_private *dev_priv,
@@ -377,6 +401,19 @@ long kgsl_ioctl_gpu_command(struct kgsl_device_private *dev_priv,
long kgsl_ioctl_gpuobj_set_info(struct kgsl_device_private *dev_priv,
unsigned int cmd, void *data);
+long kgsl_ioctl_sparse_phys_alloc(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data);
+long kgsl_ioctl_sparse_phys_free(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data);
+long kgsl_ioctl_sparse_virt_alloc(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data);
+long kgsl_ioctl_sparse_virt_free(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data);
+long kgsl_ioctl_sparse_bind(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data);
+long kgsl_ioctl_sparse_unbind(struct kgsl_device_private *dev_priv,
+ unsigned int cmd, void *data);
+
void kgsl_mem_entry_destroy(struct kref *kref);
struct kgsl_mem_entry * __must_check
diff --git a/drivers/gpu/msm/kgsl_compat.c b/drivers/gpu/msm/kgsl_compat.c
index 248c78b7e5c4..028a9566fa14 100644
--- a/drivers/gpu/msm/kgsl_compat.c
+++ b/drivers/gpu/msm/kgsl_compat.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2015, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -372,6 +372,16 @@ static const struct kgsl_ioctl kgsl_compat_ioctl_funcs[] = {
kgsl_ioctl_gpu_command),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_SET_INFO,
kgsl_ioctl_gpuobj_set_info),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_ALLOC,
+ kgsl_ioctl_sparse_phys_alloc),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_FREE,
+ kgsl_ioctl_sparse_phys_free),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_ALLOC,
+ kgsl_ioctl_sparse_virt_alloc),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_FREE,
+ kgsl_ioctl_sparse_virt_free),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_BIND,
+ kgsl_ioctl_sparse_bind),
};
long kgsl_compat_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
diff --git a/drivers/gpu/msm/kgsl_debugfs.c b/drivers/gpu/msm/kgsl_debugfs.c
index 93ac790f3a55..df9eb9ebd779 100644
--- a/drivers/gpu/msm/kgsl_debugfs.c
+++ b/drivers/gpu/msm/kgsl_debugfs.c
@@ -129,10 +129,13 @@ static int print_mem_entry(int id, void *ptr, void *data)
{
struct seq_file *s = data;
struct kgsl_mem_entry *entry = ptr;
- char flags[9];
+ char flags[10];
char usage[16];
struct kgsl_memdesc *m = &entry->memdesc;
+ if (m->flags & KGSL_MEMFLAGS_SPARSE_VIRT)
+ return 0;
+
flags[0] = kgsl_memdesc_is_global(m) ? 'g' : '-';
flags[1] = '-';
flags[2] = !(m->flags & KGSL_MEMFLAGS_GPUREADONLY) ? 'w' : '-';
@@ -141,7 +144,8 @@ static int print_mem_entry(int id, void *ptr, void *data)
flags[5] = kgsl_memdesc_use_cpu_map(m) ? 'p' : '-';
flags[6] = (m->useraddr) ? 'Y' : 'N';
flags[7] = kgsl_memdesc_is_secured(m) ? 's' : '-';
- flags[8] = '\0';
+ flags[8] = m->flags & KGSL_MEMFLAGS_SPARSE_PHYS ? 'P' : '-';
+ flags[9] = '\0';
kgsl_get_memory_usage(usage, sizeof(usage), m->flags);
@@ -211,6 +215,70 @@ static const struct file_operations process_mem_fops = {
.release = process_mem_release,
};
+static int print_sparse_mem_entry(int id, void *ptr, void *data)
+{
+ struct seq_file *s = data;
+ struct kgsl_mem_entry *entry = ptr;
+ struct kgsl_memdesc *m = &entry->memdesc;
+ struct rb_node *node;
+
+ if (!(m->flags & KGSL_MEMFLAGS_SPARSE_VIRT))
+ return 0;
+
+ node = rb_first(&entry->bind_tree);
+
+ while (node != NULL) {
+ struct sparse_bind_object *obj = rb_entry(node,
+ struct sparse_bind_object, node);
+ seq_printf(s, "%5d %16llx %16llx %16llx %16llx\n",
+ entry->id, entry->memdesc.gpuaddr,
+ obj->v_off, obj->size, obj->p_off);
+ node = rb_next(node);
+ }
+
+ seq_putc(s, '\n');
+
+ return 0;
+}
+
+static int process_sparse_mem_print(struct seq_file *s, void *unused)
+{
+ struct kgsl_process_private *private = s->private;
+
+ seq_printf(s, "%5s %16s %16s %16s %16s\n",
+ "v_id", "gpuaddr", "v_offset", "v_size", "p_offset");
+
+ spin_lock(&private->mem_lock);
+ idr_for_each(&private->mem_idr, print_sparse_mem_entry, s);
+ spin_unlock(&private->mem_lock);
+
+ return 0;
+}
+
+static int process_sparse_mem_open(struct inode *inode, struct file *file)
+{
+ int ret;
+ pid_t pid = (pid_t) (unsigned long) inode->i_private;
+ struct kgsl_process_private *private = NULL;
+
+ private = kgsl_process_private_find(pid);
+
+ if (!private)
+ return -ENODEV;
+
+ ret = single_open(file, process_sparse_mem_print, private);
+ if (ret)
+ kgsl_process_private_put(private);
+
+ return ret;
+}
+
+static const struct file_operations process_sparse_mem_fops = {
+ .open = process_sparse_mem_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = process_mem_release,
+};
/**
* kgsl_process_init_debugfs() - Initialize debugfs for a process
@@ -251,6 +319,15 @@ void kgsl_process_init_debugfs(struct kgsl_process_private *private)
if (IS_ERR_OR_NULL(dentry))
WARN((dentry == NULL),
"Unable to create 'mem' file for %s\n", name);
+
+ dentry = debugfs_create_file("sparse_mem", 0444, private->debug_root,
+ (void *) ((unsigned long) private->pid),
+ &process_sparse_mem_fops);
+
+ if (IS_ERR_OR_NULL(dentry))
+ WARN((dentry == NULL),
+ "Unable to create 'sparse_mem' file for %s\n", name);
+
}
void kgsl_core_debugfs_init(void)
diff --git a/drivers/gpu/msm/kgsl_ioctl.c b/drivers/gpu/msm/kgsl_ioctl.c
index 0802e94f56ad..894e6a4a146b 100644
--- a/drivers/gpu/msm/kgsl_ioctl.c
+++ b/drivers/gpu/msm/kgsl_ioctl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2015, The Linux Foundation. All rights reserved.
+/* Copyright (c) 2008-2016, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
@@ -90,6 +90,16 @@ static const struct kgsl_ioctl kgsl_ioctl_funcs[] = {
kgsl_ioctl_gpu_command),
KGSL_IOCTL_FUNC(IOCTL_KGSL_GPUOBJ_SET_INFO,
kgsl_ioctl_gpuobj_set_info),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_ALLOC,
+ kgsl_ioctl_sparse_phys_alloc),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_PHYS_FREE,
+ kgsl_ioctl_sparse_phys_free),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_ALLOC,
+ kgsl_ioctl_sparse_virt_alloc),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_VIRT_FREE,
+ kgsl_ioctl_sparse_virt_free),
+ KGSL_IOCTL_FUNC(IOCTL_KGSL_SPARSE_BIND,
+ kgsl_ioctl_sparse_bind),
};
long kgsl_ioctl_copy_in(unsigned int kernel_cmd, unsigned int user_cmd,
diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c
index b467ef81d257..166bb68e64a1 100644
--- a/drivers/gpu/msm/kgsl_iommu.c
+++ b/drivers/gpu/msm/kgsl_iommu.c
@@ -323,8 +323,8 @@ static int _iommu_map_sync_pc(struct kgsl_pagetable *pt,
_unlock_if_secure_mmu(memdesc, pt->mmu);
if (ret) {
- KGSL_CORE_ERR("map err: %p, 0x%016llX, 0x%llx, 0x%x, %d\n",
- iommu_pt->domain, gpuaddr, size, flags, ret);
+ KGSL_CORE_ERR("map err: 0x%016llX, 0x%llx, 0x%x, %d\n",
+ gpuaddr, size, flags, ret);
return -ENODEV;
}
@@ -351,8 +351,8 @@ static int _iommu_unmap_sync_pc(struct kgsl_pagetable *pt,
_unlock_if_secure_mmu(memdesc, pt->mmu);
if (unmapped != size) {
- KGSL_CORE_ERR("unmap err: %p, 0x%016llx, 0x%llx, %zd\n",
- iommu_pt->domain, addr, size, unmapped);
+ KGSL_CORE_ERR("unmap err: 0x%016llx, 0x%llx, %zd\n",
+ addr, size, unmapped);
return -ENODEV;
}
@@ -421,8 +421,9 @@ static int _iommu_map_sg_offset_sync_pc(struct kgsl_pagetable *pt,
if (size != 0) {
/* Cleanup on error */
_iommu_unmap_sync_pc(pt, memdesc, addr, mapped);
- KGSL_CORE_ERR("map err: %p, 0x%016llX, %d, %x, %zd\n",
- iommu_pt->domain, addr, nents, flags, mapped);
+ KGSL_CORE_ERR(
+ "map sg offset err: 0x%016llX, %d, %x, %zd\n",
+ addr, nents, flags, mapped);
return -ENODEV;
}
@@ -451,8 +452,8 @@ static int _iommu_map_sg_sync_pc(struct kgsl_pagetable *pt,
_unlock_if_secure_mmu(memdesc, pt->mmu);
if (mapped == 0) {
- KGSL_CORE_ERR("map err: %p, 0x%016llX, %d, %x, %zd\n",
- iommu_pt->domain, addr, nents, flags, mapped);
+ KGSL_CORE_ERR("map sg err: 0x%016llX, %d, %x, %zd\n",
+ addr, nents, flags, mapped);
return -ENODEV;
}
@@ -467,6 +468,13 @@ static int _iommu_map_sg_sync_pc(struct kgsl_pagetable *pt,
static struct page *kgsl_guard_page;
static struct kgsl_memdesc kgsl_secure_guard_page_memdesc;
+/*
+ * The dummy page is a placeholder/extra page to be used for sparse mappings.
+ * This page will be mapped to all virtual sparse bindings that are not
+ * physically backed.
+ */
+static struct page *kgsl_dummy_page;
+
/* These functions help find the nearest allocated memory entries on either side
* of a faulting address. If we know the nearby allocations memory we can
* get a better determination of what we think should have been located in the
@@ -1309,6 +1317,11 @@ static void kgsl_iommu_close(struct kgsl_mmu *mmu)
kgsl_guard_page = NULL;
}
+ if (kgsl_dummy_page != NULL) {
+ __free_page(kgsl_dummy_page);
+ kgsl_dummy_page = NULL;
+ }
+
kgsl_iommu_remove_global(mmu, &iommu->setstate);
kgsl_sharedmem_free(&iommu->setstate);
kgsl_cleanup_qdss_desc(mmu);
@@ -1523,6 +1536,8 @@ kgsl_iommu_unmap_offset(struct kgsl_pagetable *pt,
struct kgsl_memdesc *memdesc, uint64_t addr,
uint64_t offset, uint64_t size)
{
+ if (size == 0 || (size + offset) > kgsl_memdesc_footprint(memdesc))
+ return -EINVAL;
/*
* All GPU addresses as assigned are page aligned, but some
* functions perturb the gpuaddr with an offset, so apply the
@@ -1530,9 +1545,8 @@ kgsl_iommu_unmap_offset(struct kgsl_pagetable *pt,
*/
addr = PAGE_ALIGN(addr);
-
- if (size == 0 || addr == 0)
- return 0;
+ if (addr == 0)
+ return -EINVAL;
return _iommu_unmap_sync_pc(pt, memdesc, addr + offset, size);
}
@@ -1540,13 +1554,11 @@ kgsl_iommu_unmap_offset(struct kgsl_pagetable *pt,
static int
kgsl_iommu_unmap(struct kgsl_pagetable *pt, struct kgsl_memdesc *memdesc)
{
- uint64_t size = memdesc->size;
-
- if (kgsl_memdesc_has_guard_page(memdesc))
- size += kgsl_memdesc_guard_page_size(pt->mmu, memdesc);
+ if (memdesc->size == 0 || memdesc->gpuaddr == 0)
+ return -EINVAL;
return kgsl_iommu_unmap_offset(pt, memdesc, memdesc->gpuaddr, 0,
- size);
+ kgsl_memdesc_footprint(memdesc));
}
/**
@@ -1593,7 +1605,7 @@ static int _iommu_map_guard_page(struct kgsl_pagetable *pt,
} else {
if (kgsl_guard_page == NULL) {
kgsl_guard_page = alloc_page(GFP_KERNEL | __GFP_ZERO |
- __GFP_HIGHMEM);
+ __GFP_NORETRY | __GFP_HIGHMEM);
if (kgsl_guard_page == NULL)
return -ENOMEM;
}
@@ -1602,7 +1614,7 @@ static int _iommu_map_guard_page(struct kgsl_pagetable *pt,
}
return _iommu_map_sync_pc(pt, memdesc, gpuaddr, physaddr,
- kgsl_memdesc_guard_page_size(pt->mmu, memdesc),
+ kgsl_memdesc_guard_page_size(memdesc),
protflags & ~IOMMU_WRITE);
}
@@ -1658,6 +1670,100 @@ done:
return ret;
}
+static int kgsl_iommu_sparse_dummy_map(struct kgsl_pagetable *pt,
+ struct kgsl_memdesc *memdesc, uint64_t offset, uint64_t size)
+{
+ int ret = 0, i;
+ struct page **pages = NULL;
+ struct sg_table sgt;
+ int count = size >> PAGE_SHIFT;
+
+ /* verify the offset is within our range */
+ if (size + offset > memdesc->size)
+ return -EINVAL;
+
+ if (kgsl_dummy_page == NULL) {
+ kgsl_dummy_page = alloc_page(GFP_KERNEL | __GFP_ZERO |
+ __GFP_HIGHMEM);
+ if (kgsl_dummy_page == NULL)
+ return -ENOMEM;
+ }
+
+ pages = kcalloc(count, sizeof(struct page *), GFP_KERNEL);
+ if (pages == NULL)
+ return -ENOMEM;
+
+ for (i = 0; i < count; i++)
+ pages[i] = kgsl_dummy_page;
+
+ ret = sg_alloc_table_from_pages(&sgt, pages, count,
+ 0, size, GFP_KERNEL);
+ if (ret == 0) {
+ ret = _iommu_map_sg_sync_pc(pt, memdesc->gpuaddr + offset,
+ memdesc, sgt.sgl, sgt.nents,
+ IOMMU_READ | IOMMU_NOEXEC);
+ sg_free_table(&sgt);
+ }
+
+ kfree(pages);
+
+ return ret;
+}
+
+static int _map_to_one_page(struct kgsl_pagetable *pt, uint64_t addr,
+ struct kgsl_memdesc *memdesc, uint64_t physoffset,
+ uint64_t size, unsigned int map_flags)
+{
+ int ret = 0, i;
+ int pg_sz = kgsl_memdesc_get_pagesize(memdesc);
+ int count = size >> PAGE_SHIFT;
+ struct page *page = NULL;
+ struct page **pages = NULL;
+ struct sg_page_iter sg_iter;
+ struct sg_table sgt;
+
+ /* Find our physaddr offset addr */
+ if (memdesc->pages != NULL)
+ page = memdesc->pages[physoffset >> PAGE_SHIFT];
+ else {
+ for_each_sg_page(memdesc->sgt->sgl, &sg_iter,
+ memdesc->sgt->nents, physoffset >> PAGE_SHIFT) {
+ page = sg_page_iter_page(&sg_iter);
+ break;
+ }
+ }
+
+ if (page == NULL)
+ return -EINVAL;
+
+ pages = kcalloc(count, sizeof(struct page *), GFP_KERNEL);
+ if (pages == NULL)
+ return -ENOMEM;
+
+ for (i = 0; i < count; i++) {
+ if (pg_sz != PAGE_SIZE) {
+ struct page *tmp_page = page;
+ int j;
+
+ for (j = 0; j < 16; j++, tmp_page += PAGE_SIZE)
+ pages[i++] = tmp_page;
+ } else
+ pages[i] = page;
+ }
+
+ ret = sg_alloc_table_from_pages(&sgt, pages, count,
+ 0, size, GFP_KERNEL);
+ if (ret == 0) {
+ ret = _iommu_map_sg_sync_pc(pt, addr, memdesc, sgt.sgl,
+ sgt.nents, map_flags);
+ sg_free_table(&sgt);
+ }
+
+ kfree(pages);
+
+ return ret;
+}
+
static int kgsl_iommu_map_offset(struct kgsl_pagetable *pt,
uint64_t virtaddr, uint64_t virtoffset,
struct kgsl_memdesc *memdesc, uint64_t physoffset,
@@ -1668,13 +1774,17 @@ static int kgsl_iommu_map_offset(struct kgsl_pagetable *pt,
int ret;
struct sg_table *sgt = NULL;
- pg_sz = (1 << kgsl_memdesc_get_align(memdesc));
+ pg_sz = kgsl_memdesc_get_pagesize(memdesc);
if (!IS_ALIGNED(virtaddr | virtoffset | physoffset | size, pg_sz))
return -EINVAL;
if (size == 0)
return -EINVAL;
+ if (!(feature_flag & KGSL_SPARSE_BIND_MULTIPLE_TO_PHYS) &&
+ size + physoffset > kgsl_memdesc_footprint(memdesc))
+ return -EINVAL;
+
/*
* For paged memory allocated through kgsl, memdesc->pages is not NULL.
* Allocate sgt here just for its map operation. Contiguous memory
@@ -1688,9 +1798,13 @@ static int kgsl_iommu_map_offset(struct kgsl_pagetable *pt,
if (IS_ERR(sgt))
return PTR_ERR(sgt);
- ret = _iommu_map_sg_offset_sync_pc(pt, virtaddr + virtoffset,
- memdesc, sgt->sgl, sgt->nents,
- physoffset, size, protflags);
+ if (feature_flag & KGSL_SPARSE_BIND_MULTIPLE_TO_PHYS)
+ ret = _map_to_one_page(pt, virtaddr + virtoffset,
+ memdesc, physoffset, size, protflags);
+ else
+ ret = _iommu_map_sg_offset_sync_pc(pt, virtaddr + virtoffset,
+ memdesc, sgt->sgl, sgt->nents,
+ physoffset, size, protflags);
if (memdesc->pages != NULL)
kgsl_free_sgt(sgt);
@@ -2152,8 +2266,7 @@ static int kgsl_iommu_get_gpuaddr(struct kgsl_pagetable *pagetable,
{
struct kgsl_iommu_pt *pt = pagetable->priv;
int ret = 0;
- uint64_t addr, start, end;
- uint64_t size = memdesc->size;
+ uint64_t addr, start, end, size;
unsigned int align;
BUG_ON(kgsl_memdesc_use_cpu_map(memdesc));
@@ -2162,8 +2275,7 @@ static int kgsl_iommu_get_gpuaddr(struct kgsl_pagetable *pagetable,
pagetable->name != KGSL_MMU_SECURE_PT)
return -EINVAL;
- if (kgsl_memdesc_has_guard_page(memdesc))
- size += kgsl_memdesc_guard_page_size(pagetable->mmu, memdesc);
+ size = kgsl_memdesc_footprint(memdesc);
align = 1 << kgsl_memdesc_get_align(memdesc);
@@ -2445,4 +2557,5 @@ static struct kgsl_mmu_pt_ops iommu_pt_ops = {
.addr_in_range = kgsl_iommu_addr_in_range,
.mmu_map_offset = kgsl_iommu_map_offset,
.mmu_unmap_offset = kgsl_iommu_unmap_offset,
+ .mmu_sparse_dummy_map = kgsl_iommu_sparse_dummy_map,
};
diff --git a/drivers/gpu/msm/kgsl_mmu.c b/drivers/gpu/msm/kgsl_mmu.c
index 8b0d93fda32c..10f6b8049d36 100644
--- a/drivers/gpu/msm/kgsl_mmu.c
+++ b/drivers/gpu/msm/kgsl_mmu.c
@@ -386,29 +386,24 @@ int
kgsl_mmu_map(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc)
{
- int ret = 0;
int size;
if (!memdesc->gpuaddr)
return -EINVAL;
- /* Only global mappings should be mapped multiple times */
- if (!kgsl_memdesc_is_global(memdesc) &&
- (KGSL_MEMDESC_MAPPED & memdesc->priv))
- return -EINVAL;
size = kgsl_memdesc_footprint(memdesc);
- if (PT_OP_VALID(pagetable, mmu_map))
- ret = pagetable->pt_ops->mmu_map(pagetable, memdesc);
-
- if (ret)
- return ret;
+ if (PT_OP_VALID(pagetable, mmu_map)) {
+ int ret;
- atomic_inc(&pagetable->stats.entries);
- KGSL_STATS_ADD(size, &pagetable->stats.mapped,
- &pagetable->stats.max_mapped);
+ ret = pagetable->pt_ops->mmu_map(pagetable, memdesc);
+ if (ret)
+ return ret;
- memdesc->priv |= KGSL_MEMDESC_MAPPED;
+ atomic_inc(&pagetable->stats.entries);
+ KGSL_STATS_ADD(size, &pagetable->stats.mapped,
+ &pagetable->stats.max_mapped);
+ }
return 0;
}
@@ -455,22 +450,22 @@ int
kgsl_mmu_unmap(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc)
{
- uint64_t size;
-
- if (memdesc->size == 0 || memdesc->gpuaddr == 0 ||
- !(KGSL_MEMDESC_MAPPED & memdesc->priv))
+ if (memdesc->size == 0)
return -EINVAL;
- size = kgsl_memdesc_footprint(memdesc);
+ if (PT_OP_VALID(pagetable, mmu_unmap)) {
+ int ret;
+ uint64_t size;
- if (PT_OP_VALID(pagetable, mmu_unmap))
- pagetable->pt_ops->mmu_unmap(pagetable, memdesc);
+ size = kgsl_memdesc_footprint(memdesc);
- atomic_dec(&pagetable->stats.entries);
- atomic_long_sub(size, &pagetable->stats.mapped);
+ ret = pagetable->pt_ops->mmu_unmap(pagetable, memdesc);
+ if (ret)
+ return ret;
- if (!kgsl_memdesc_is_global(memdesc))
- memdesc->priv &= ~KGSL_MEMDESC_MAPPED;
+ atomic_dec(&pagetable->stats.entries);
+ atomic_long_sub(size, &pagetable->stats.mapped);
+ }
return 0;
}
@@ -481,11 +476,20 @@ int kgsl_mmu_map_offset(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc, uint64_t physoffset,
uint64_t size, uint64_t flags)
{
- if (PT_OP_VALID(pagetable, mmu_map_offset))
- return pagetable->pt_ops->mmu_map_offset(pagetable, virtaddr,
+ if (PT_OP_VALID(pagetable, mmu_map_offset)) {
+ int ret;
+
+ ret = pagetable->pt_ops->mmu_map_offset(pagetable, virtaddr,
virtoffset, memdesc, physoffset, size, flags);
+ if (ret)
+ return ret;
+
+ atomic_inc(&pagetable->stats.entries);
+ KGSL_STATS_ADD(size, &pagetable->stats.mapped,
+ &pagetable->stats.max_mapped);
+ }
- return -EINVAL;
+ return 0;
}
EXPORT_SYMBOL(kgsl_mmu_map_offset);
@@ -493,14 +497,41 @@ int kgsl_mmu_unmap_offset(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *memdesc, uint64_t addr, uint64_t offset,
uint64_t size)
{
- if (PT_OP_VALID(pagetable, mmu_unmap_offset))
- return pagetable->pt_ops->mmu_unmap_offset(pagetable, memdesc,
+ if (PT_OP_VALID(pagetable, mmu_unmap_offset)) {
+ int ret;
+
+ ret = pagetable->pt_ops->mmu_unmap_offset(pagetable, memdesc,
addr, offset, size);
+ if (ret)
+ return ret;
+
+ atomic_dec(&pagetable->stats.entries);
+ atomic_long_sub(size, &pagetable->stats.mapped);
+ }
- return -EINVAL;
+ return 0;
}
EXPORT_SYMBOL(kgsl_mmu_unmap_offset);
+int kgsl_mmu_sparse_dummy_map(struct kgsl_pagetable *pagetable,
+ struct kgsl_memdesc *memdesc, uint64_t offset, uint64_t size)
+{
+ if (PT_OP_VALID(pagetable, mmu_sparse_dummy_map)) {
+ int ret;
+
+ ret = pagetable->pt_ops->mmu_sparse_dummy_map(pagetable,
+ memdesc, offset, size);
+ if (ret)
+ return ret;
+
+ atomic_dec(&pagetable->stats.entries);
+ atomic_long_sub(size, &pagetable->stats.mapped);
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(kgsl_mmu_sparse_dummy_map);
+
void kgsl_mmu_remove_global(struct kgsl_device *device,
struct kgsl_memdesc *memdesc)
{
diff --git a/drivers/gpu/msm/kgsl_mmu.h b/drivers/gpu/msm/kgsl_mmu.h
index 588777af353f..53645cc1741c 100644
--- a/drivers/gpu/msm/kgsl_mmu.h
+++ b/drivers/gpu/msm/kgsl_mmu.h
@@ -106,6 +106,9 @@ struct kgsl_mmu_pt_ops {
int (*mmu_unmap_offset)(struct kgsl_pagetable *pt,
struct kgsl_memdesc *memdesc, uint64_t addr,
uint64_t offset, uint64_t size);
+ int (*mmu_sparse_dummy_map)(struct kgsl_pagetable *pt,
+ struct kgsl_memdesc *memdesc, uint64_t offset,
+ uint64_t size);
};
/*
@@ -230,6 +233,9 @@ int kgsl_mmu_unmap_offset(struct kgsl_pagetable *pagetable,
struct kgsl_memdesc *kgsl_mmu_get_qdss_global_entry(struct kgsl_device *device);
+int kgsl_mmu_sparse_dummy_map(struct kgsl_pagetable *pagetable,
+ struct kgsl_memdesc *memdesc, uint64_t offset, uint64_t size);
+
/*
* Static inline functions of MMU that simply call the SMMU specific
* function using a function pointer. These functions can be thought
diff --git a/drivers/gpu/msm/kgsl_sharedmem.h b/drivers/gpu/msm/kgsl_sharedmem.h
index c05aaecb5284..565ae4c39fdd 100644
--- a/drivers/gpu/msm/kgsl_sharedmem.h
+++ b/drivers/gpu/msm/kgsl_sharedmem.h
@@ -92,6 +92,18 @@ kgsl_memdesc_get_align(const struct kgsl_memdesc *memdesc)
}
/*
+ * kgsl_memdesc_get_pagesize - Get pagesize based on alignment
+ * @memdesc - the memdesc
+ *
+ * Returns the pagesize based on memdesc alignment
+ */
+static inline int
+kgsl_memdesc_get_pagesize(const struct kgsl_memdesc *memdesc)
+{
+ return (1 << kgsl_memdesc_get_align(memdesc));
+}
+
+/*
* kgsl_memdesc_get_cachemode - Get cache mode of a memdesc
* @memdesc: the memdesc
*
@@ -211,12 +223,19 @@ kgsl_memdesc_has_guard_page(const struct kgsl_memdesc *memdesc)
*
* Returns guard page size
*/
-static inline int
-kgsl_memdesc_guard_page_size(const struct kgsl_mmu *mmu,
- const struct kgsl_memdesc *memdesc)
+static inline uint64_t
+kgsl_memdesc_guard_page_size(const struct kgsl_memdesc *memdesc)
{
- return kgsl_memdesc_is_secured(memdesc) ? mmu->secure_align_mask + 1 :
- PAGE_SIZE;
+ if (!kgsl_memdesc_has_guard_page(memdesc))
+ return 0;
+
+ if (kgsl_memdesc_is_secured(memdesc)) {
+ if (memdesc->pagetable != NULL &&
+ memdesc->pagetable->mmu != NULL)
+ return memdesc->pagetable->mmu->secure_align_mask + 1;
+ }
+
+ return PAGE_SIZE;
}
/*
@@ -241,10 +260,7 @@ kgsl_memdesc_use_cpu_map(const struct kgsl_memdesc *memdesc)
static inline uint64_t
kgsl_memdesc_footprint(const struct kgsl_memdesc *memdesc)
{
- uint64_t size = memdesc->size;
- if (kgsl_memdesc_has_guard_page(memdesc))
- size += SZ_4K;
- return size;
+ return memdesc->size + kgsl_memdesc_guard_page_size(memdesc);
}
/*
diff --git a/drivers/gpu/msm/kgsl_snapshot.c b/drivers/gpu/msm/kgsl_snapshot.c
index 69ae2e3fd2d7..f9d3ede718ab 100644
--- a/drivers/gpu/msm/kgsl_snapshot.c
+++ b/drivers/gpu/msm/kgsl_snapshot.c
@@ -313,6 +313,13 @@ int kgsl_snapshot_get_object(struct kgsl_snapshot *snapshot,
goto err_put;
}
+ /* Do not save sparse memory */
+ if (entry->memdesc.flags & KGSL_MEMFLAGS_SPARSE_VIRT ||
+ entry->memdesc.flags & KGSL_MEMFLAGS_SPARSE_PHYS) {
+ ret = 0;
+ goto err_put;
+ }
+
/*
* size indicates the number of bytes in the region to save. This might
* not always be the entire size of the region because some buffers are
diff --git a/drivers/gpu/msm/kgsl_trace.h b/drivers/gpu/msm/kgsl_trace.h
index 8988dc12839f..bac09175cf12 100644
--- a/drivers/gpu/msm/kgsl_trace.h
+++ b/drivers/gpu/msm/kgsl_trace.h
@@ -1102,6 +1102,100 @@ TRACE_EVENT(kgsl_msg,
)
);
+DECLARE_EVENT_CLASS(sparse_alloc_template,
+ TP_PROTO(unsigned int id, uint64_t size, unsigned int pagesize),
+ TP_ARGS(id, size, pagesize),
+ TP_STRUCT__entry(
+ __field(unsigned int, id)
+ __field(uint64_t, size)
+ __field(unsigned int, pagesize)
+ ),
+ TP_fast_assign(
+ __entry->id = id;
+ __entry->size = size;
+ __entry->pagesize = pagesize;
+ ),
+ TP_printk("id=%d size=0x%llX pagesize=0x%X",
+ __entry->id, __entry->size, __entry->pagesize)
+);
+
+DEFINE_EVENT(sparse_alloc_template, sparse_phys_alloc,
+ TP_PROTO(unsigned int id, uint64_t size, unsigned int pagesize),
+ TP_ARGS(id, size, pagesize)
+);
+
+DEFINE_EVENT(sparse_alloc_template, sparse_virt_alloc,
+ TP_PROTO(unsigned int id, uint64_t size, unsigned int pagesize),
+ TP_ARGS(id, size, pagesize)
+);
+
+DECLARE_EVENT_CLASS(sparse_free_template,
+ TP_PROTO(unsigned int id),
+ TP_ARGS(id),
+ TP_STRUCT__entry(
+ __field(unsigned int, id)
+ ),
+ TP_fast_assign(
+ __entry->id = id;
+ ),
+ TP_printk("id=%d", __entry->id)
+);
+
+DEFINE_EVENT(sparse_free_template, sparse_phys_free,
+ TP_PROTO(unsigned int id),
+ TP_ARGS(id)
+);
+
+DEFINE_EVENT(sparse_free_template, sparse_virt_free,
+ TP_PROTO(unsigned int id),
+ TP_ARGS(id)
+);
+
+TRACE_EVENT(sparse_bind,
+ TP_PROTO(unsigned int v_id, uint64_t v_off,
+ unsigned int p_id, uint64_t p_off,
+ uint64_t size, uint64_t flags),
+ TP_ARGS(v_id, v_off, p_id, p_off, size, flags),
+ TP_STRUCT__entry(
+ __field(unsigned int, v_id)
+ __field(uint64_t, v_off)
+ __field(unsigned int, p_id)
+ __field(uint64_t, p_off)
+ __field(uint64_t, size)
+ __field(uint64_t, flags)
+ ),
+ TP_fast_assign(
+ __entry->v_id = v_id;
+ __entry->v_off = v_off;
+ __entry->p_id = p_id;
+ __entry->p_off = p_off;
+ __entry->size = size;
+ __entry->flags = flags;
+ ),
+ TP_printk(
+ "v_id=%d v_off=0x%llX p_id=%d p_off=0x%llX size=0x%llX flags=0x%llX",
+ __entry->v_id, __entry->v_off,
+ __entry->p_id, __entry->p_off,
+ __entry->size, __entry->flags)
+);
+
+TRACE_EVENT(sparse_unbind,
+ TP_PROTO(unsigned int v_id, uint64_t v_off, uint64_t size),
+ TP_ARGS(v_id, v_off, size),
+ TP_STRUCT__entry(
+ __field(unsigned int, v_id)
+ __field(uint64_t, v_off)
+ __field(uint64_t, size)
+ ),
+ TP_fast_assign(
+ __entry->v_id = v_id;
+ __entry->v_off = v_off;
+ __entry->size = size;
+ ),
+ TP_printk("v_id=%d v_off=0x%llX size=0x%llX",
+ __entry->v_id, __entry->v_off, __entry->size)
+);
+
#endif /* _KGSL_TRACE_H */
diff --git a/include/uapi/linux/msm_kgsl.h b/include/uapi/linux/msm_kgsl.h
index a80278954a77..aac11dbe5984 100644
--- a/include/uapi/linux/msm_kgsl.h
+++ b/include/uapi/linux/msm_kgsl.h
@@ -121,6 +121,11 @@
#define KGSL_MEMFLAGS_GPUWRITEONLY 0x02000000U
#define KGSL_MEMFLAGS_FORCE_32BIT 0x100000000ULL
+/* Flag for binding all the virt range to single phys data */
+#define KGSL_SPARSE_BIND_MULTIPLE_TO_PHYS 0x400000000ULL
+#define KGSL_SPARSE_BIND 0x1ULL
+#define KGSL_SPARSE_UNBIND 0x2ULL
+
/* Memory caching hints */
#define KGSL_CACHEMODE_MASK 0x0C000000U
#define KGSL_CACHEMODE_SHIFT 26
@@ -131,6 +136,8 @@
#define KGSL_CACHEMODE_WRITEBACK 3
#define KGSL_MEMFLAGS_USE_CPU_MAP 0x10000000ULL
+#define KGSL_MEMFLAGS_SPARSE_PHYS 0x20000000ULL
+#define KGSL_MEMFLAGS_SPARSE_VIRT 0x40000000ULL
/* Memory types for which allocations are made */
#define KGSL_MEMTYPE_MASK 0x0000FF00
@@ -1457,4 +1464,96 @@ struct kgsl_gpuobj_set_info {
#define IOCTL_KGSL_GPUOBJ_SET_INFO \
_IOW(KGSL_IOC_TYPE, 0x4C, struct kgsl_gpuobj_set_info)
+/**
+ * struct kgsl_sparse_phys_alloc - Argument for IOCTL_KGSL_SPARSE_PHYS_ALLOC
+ * @size: Size in bytes to back
+ * @pagesize: Pagesize alignment required
+ * @flags: Flags for this allocation
+ * @id: Returned ID for this allocation
+ */
+struct kgsl_sparse_phys_alloc {
+ uint64_t size;
+ uint64_t pagesize;
+ uint64_t flags;
+ unsigned int id;
+};
+
+#define IOCTL_KGSL_SPARSE_PHYS_ALLOC \
+ _IOWR(KGSL_IOC_TYPE, 0x50, struct kgsl_sparse_phys_alloc)
+
+/**
+ * struct kgsl_sparse_phys_free - Argument for IOCTL_KGSL_SPARSE_PHYS_FREE
+ * @id: ID to free
+ */
+struct kgsl_sparse_phys_free {
+ unsigned int id;
+};
+
+#define IOCTL_KGSL_SPARSE_PHYS_FREE \
+ _IOW(KGSL_IOC_TYPE, 0x51, struct kgsl_sparse_phys_free)
+
+/**
+ * struct kgsl_sparse_virt_alloc - Argument for IOCTL_KGSL_SPARSE_VIRT_ALLOC
+ * @size: Size in bytes to reserve
+ * @pagesize: Pagesize alignment required
+ * @flags: Flags for this allocation
+ * @id: Returned ID for this allocation
+ * @gpuaddr: Returned GPU address for this allocation
+ */
+struct kgsl_sparse_virt_alloc {
+ uint64_t size;
+ uint64_t pagesize;
+ uint64_t flags;
+ uint64_t gpuaddr;
+ unsigned int id;
+};
+
+#define IOCTL_KGSL_SPARSE_VIRT_ALLOC \
+ _IOWR(KGSL_IOC_TYPE, 0x52, struct kgsl_sparse_virt_alloc)
+
+/**
+ * struct kgsl_sparse_virt_free - Argument for IOCTL_KGSL_SPARSE_VIRT_FREE
+ * @id: ID to free
+ */
+struct kgsl_sparse_virt_free {
+ unsigned int id;
+};
+
+#define IOCTL_KGSL_SPARSE_VIRT_FREE \
+ _IOW(KGSL_IOC_TYPE, 0x53, struct kgsl_sparse_virt_free)
+
+/**
+ * struct kgsl_sparse_binding_object - Argument for kgsl_sparse_bind
+ * @virtoffset: Offset into the virtual ID
+ * @physoffset: Offset into the physical ID (bind only)
+ * @size: Size in bytes to reserve
+ * @flags: Flags for this kgsl_sparse_binding_object
+ * @id: Physical ID to bind (bind only)
+ */
+struct kgsl_sparse_binding_object {
+ uint64_t virtoffset;
+ uint64_t physoffset;
+ uint64_t size;
+ uint64_t flags;
+ unsigned int id;
+};
+
+/**
+ * struct kgsl_sparse_bind - Argument for IOCTL_KGSL_SPARSE_BIND
+ * @list: List of kgsl_sparse_bind_objects to bind/unbind
+ * @id: Virtual ID to bind/unbind
+ * @size: Size of kgsl_sparse_bind_object
+ * @count: Number of elements in list
+ *
+ */
+struct kgsl_sparse_bind {
+ uint64_t __user list;
+ unsigned int id;
+ unsigned int size;
+ unsigned int count;
+};
+
+#define IOCTL_KGSL_SPARSE_BIND \
+ _IOW(KGSL_IOC_TYPE, 0x54, struct kgsl_sparse_bind)
+
#endif /* _UAPI_MSM_KGSL_H */
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-23 10:23:03 +0000