diff options
| -rw-r--r-- | drivers/gpu/msm/kgsl.c | 16 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl.h | 7 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl_debugfs.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl_iommu.c | 50 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl_pool.c | 25 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl_pool.h | 1 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl_sharedmem.c | 131 | ||||
| -rw-r--r-- | drivers/gpu/msm/kgsl_sharedmem.h | 45 |
8 files changed, 186 insertions, 91 deletions
diff --git a/drivers/gpu/msm/kgsl.c b/drivers/gpu/msm/kgsl.c index f77dbb7f20af..7dcb4df9607d 100644 --- a/drivers/gpu/msm/kgsl.c +++ b/drivers/gpu/msm/kgsl.c @@ -3663,19 +3663,15 @@ static int kgsl_mmap(struct file *file, struct vm_area_struct *vma) if (cache == KGSL_CACHEMODE_WRITEBACK || cache == KGSL_CACHEMODE_WRITETHROUGH) { - struct scatterlist *s; int i; unsigned long addr = vma->vm_start; + struct kgsl_memdesc *m = &entry->memdesc; - for_each_sg(entry->memdesc.sgt->sgl, s, - entry->memdesc.sgt->nents, i) { - int j; - for (j = 0; j < (s->length >> PAGE_SHIFT); j++) { - struct page *page = sg_page(s); - page = nth_page(page, j); - vm_insert_page(vma, addr, page); - addr += PAGE_SIZE; - } + for (i = 0; i < m->page_count; i++) { + struct page *page = m->pages[i]; + + vm_insert_page(vma, addr, page); + addr += PAGE_SIZE; } } diff --git a/drivers/gpu/msm/kgsl.h b/drivers/gpu/msm/kgsl.h index c172021c8944..90dc550a29b2 100644 --- a/drivers/gpu/msm/kgsl.h +++ b/drivers/gpu/msm/kgsl.h @@ -179,8 +179,9 @@ struct kgsl_memdesc_ops { * @ops: Function hooks for the memdesc memory type * @flags: Flags set from userspace * @dev: Pointer to the struct device that owns this memory - * @memmap: bitmap of pages for mmapsize - * @memmap_len: Number of bits for memmap + * @attrs: dma attributes for this memory + * @pages: An array of pointers to allocated pages + * @page_count: Total number of pages allocated */ struct kgsl_memdesc { struct kgsl_pagetable *pagetable; @@ -197,6 +198,8 @@ struct kgsl_memdesc { uint64_t flags; struct device *dev; struct dma_attrs attrs; + struct page **pages; + unsigned int page_count; }; /* diff --git a/drivers/gpu/msm/kgsl_debugfs.c b/drivers/gpu/msm/kgsl_debugfs.c index 766cd811588c..93ac790f3a55 100644 --- a/drivers/gpu/msm/kgsl_debugfs.c +++ b/drivers/gpu/msm/kgsl_debugfs.c @@ -150,7 +150,7 @@ static int print_mem_entry(int id, void *ptr, void *data) (unsigned long *) m->useraddr, m->size, entry->id, flags, memtype_str(kgsl_memdesc_usermem_type(m)), - usage, m->sgt->nents, m->mapsize); + usage, (m->sgt ? m->sgt->nents : 0), m->mapsize); if (entry->metadata[0] != 0) seq_printf(s, " %s", entry->metadata); diff --git a/drivers/gpu/msm/kgsl_iommu.c b/drivers/gpu/msm/kgsl_iommu.c index 865cd9d8f498..b467ef81d257 100644 --- a/drivers/gpu/msm/kgsl_iommu.c +++ b/drivers/gpu/msm/kgsl_iommu.c @@ -1627,16 +1627,34 @@ kgsl_iommu_map(struct kgsl_pagetable *pt, uint64_t addr = memdesc->gpuaddr; uint64_t size = memdesc->size; unsigned int flags = _get_protection_flags(memdesc); + struct sg_table *sgt = NULL; - ret = _iommu_map_sg_sync_pc(pt, addr, memdesc, memdesc->sgt->sgl, - memdesc->sgt->nents, flags); + /* + * For paged memory allocated through kgsl, memdesc->pages is not NULL. + * Allocate sgt here just for its map operation. Contiguous memory + * already has its sgt, so no need to allocate it here. + */ + if (memdesc->pages != NULL) + sgt = kgsl_alloc_sgt_from_pages(memdesc); + else + sgt = memdesc->sgt; + + if (IS_ERR(sgt)) + return PTR_ERR(sgt); + + ret = _iommu_map_sg_sync_pc(pt, addr, memdesc, sgt->sgl, + sgt->nents, flags); if (ret) - return ret; + goto done; ret = _iommu_map_guard_page(pt, memdesc, addr + size, flags); if (ret) _iommu_unmap_sync_pc(pt, memdesc, addr, size); +done: + if (memdesc->pages != NULL) + kgsl_free_sgt(sgt); + return ret; } @@ -1647,6 +1665,8 @@ static int kgsl_iommu_map_offset(struct kgsl_pagetable *pt, { int pg_sz; unsigned int protflags = _get_protection_flags(memdesc); + int ret; + struct sg_table *sgt = NULL; pg_sz = (1 << kgsl_memdesc_get_align(memdesc)); if (!IS_ALIGNED(virtaddr | virtoffset | physoffset | size, pg_sz)) @@ -1655,9 +1675,27 @@ static int kgsl_iommu_map_offset(struct kgsl_pagetable *pt, if (size == 0) return -EINVAL; - return _iommu_map_sg_offset_sync_pc(pt, virtaddr + virtoffset, - memdesc, memdesc->sgt->sgl, memdesc->sgt->nents, - physoffset, size, protflags); + /* + * For paged memory allocated through kgsl, memdesc->pages is not NULL. + * Allocate sgt here just for its map operation. Contiguous memory + * already has its sgt, so no need to allocate it here. + */ + if (memdesc->pages != NULL) + sgt = kgsl_alloc_sgt_from_pages(memdesc); + else + sgt = memdesc->sgt; + + if (IS_ERR(sgt)) + return PTR_ERR(sgt); + + ret = _iommu_map_sg_offset_sync_pc(pt, virtaddr + virtoffset, + memdesc, sgt->sgl, sgt->nents, + physoffset, size, protflags); + + if (memdesc->pages != NULL) + kgsl_free_sgt(sgt); + + return ret; } /* This function must be called with context bank attached */ diff --git a/drivers/gpu/msm/kgsl_pool.c b/drivers/gpu/msm/kgsl_pool.c index 7fb3b37ac191..7967b19779db 100644 --- a/drivers/gpu/msm/kgsl_pool.c +++ b/drivers/gpu/msm/kgsl_pool.c @@ -263,6 +263,31 @@ void kgsl_pool_free_sgt(struct sg_table *sgt) } } +/** + * kgsl_pool_free_pages() - Free pages in the pages array + * @pages: pointer of the pages array + * + * Free the pages by collapsing any physical adjacent pages. + * Pages are added back to the pool, if pool has sufficient space + * otherwise they are given back to system. + */ +void kgsl_pool_free_pages(struct page **pages, unsigned int pcount) +{ + int i; + + if (pages == NULL || pcount == 0) + return; + + for (i = 0; i < pcount;) { + /* + * Free each page or compound page group individually. + */ + struct page *p = pages[i]; + + i += 1 << compound_order(p); + kgsl_pool_free_page(p); + } +} static int kgsl_pool_idx_lookup(unsigned int order) { int i; diff --git a/drivers/gpu/msm/kgsl_pool.h b/drivers/gpu/msm/kgsl_pool.h index f2cdda19140b..efbfa96f1498 100644 --- a/drivers/gpu/msm/kgsl_pool.h +++ b/drivers/gpu/msm/kgsl_pool.h @@ -34,6 +34,7 @@ kgsl_gfp_mask(unsigned int page_order) } void kgsl_pool_free_sgt(struct sg_table *sgt); +void kgsl_pool_free_pages(struct page **pages, unsigned int page_count); void kgsl_init_page_pools(void); void kgsl_exit_page_pools(void); int kgsl_pool_alloc_page(int *page_size, struct page **pages, diff --git a/drivers/gpu/msm/kgsl_sharedmem.c b/drivers/gpu/msm/kgsl_sharedmem.c index 50dcd39fac58..e144d6481905 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.c +++ b/drivers/gpu/msm/kgsl_sharedmem.c @@ -358,8 +358,7 @@ static int kgsl_page_alloc_vmfault(struct kgsl_memdesc *memdesc, struct vm_area_struct *vma, struct vm_fault *vmf) { - int i, pgoff; - struct scatterlist *s = memdesc->sgt->sgl; + int pgoff; unsigned int offset; offset = ((unsigned long) vmf->virtual_address - vma->vm_start); @@ -369,30 +368,15 @@ static int kgsl_page_alloc_vmfault(struct kgsl_memdesc *memdesc, pgoff = offset >> PAGE_SHIFT; - /* - * The sglist might be comprised of mixed blocks of memory depending - * on how many 64K pages were allocated. This means we have to do math - * to find the actual 4K page to map in user space - */ - - for (i = 0; i < memdesc->sgt->nents; i++) { - int npages = s->length >> PAGE_SHIFT; + if (pgoff < memdesc->page_count) { + struct page *page = memdesc->pages[pgoff]; - if (pgoff < npages) { - struct page *page = sg_page(s); + get_page(page); + vmf->page = page; - page = nth_page(page, pgoff); + memdesc->mapsize += PAGE_SIZE; - get_page(page); - vmf->page = page; - - memdesc->mapsize += PAGE_SIZE; - - return 0; - } - - pgoff -= npages; - s = sg_next(s); + return 0; } return VM_FAULT_SIGBUS; @@ -455,9 +439,15 @@ static void kgsl_page_alloc_free(struct kgsl_memdesc *memdesc) for_each_sg_page(memdesc->sgt->sgl, &sg_iter, memdesc->sgt->nents, 0) ClearPagePrivate(sg_page_iter_page(&sg_iter)); + } - kgsl_pool_free_sgt(memdesc->sgt); + /* Free pages using the pages array for non secure paged memory */ + if (memdesc->pages != NULL) + kgsl_pool_free_pages(memdesc->pages, memdesc->page_count); + else + kgsl_pool_free_sgt(memdesc->sgt); + } /* @@ -477,31 +467,10 @@ static int kgsl_page_alloc_map_kernel(struct kgsl_memdesc *memdesc) return -ENOMEM; mutex_lock(&kernel_map_global_lock); - if (!memdesc->hostptr) { + if ((!memdesc->hostptr) && (memdesc->pages != NULL)) { pgprot_t page_prot = pgprot_writecombine(PAGE_KERNEL); - struct page **pages = NULL; - struct scatterlist *sg; - int npages = PAGE_ALIGN(memdesc->size) >> PAGE_SHIFT; - int sglen = memdesc->sgt->nents; - int i, count = 0; - - /* create a list of pages to call vmap */ - pages = kgsl_malloc(npages * sizeof(struct page *)); - if (pages == NULL) { - ret = -ENOMEM; - goto done; - } - - for_each_sg(memdesc->sgt->sgl, sg, sglen, i) { - struct page *page = sg_page(sg); - int j; - - for (j = 0; j < sg->length >> PAGE_SHIFT; j++) - pages[count++] = page++; - } - - memdesc->hostptr = vmap(pages, count, + memdesc->hostptr = vmap(memdesc->pages, memdesc->page_count, VM_IOREMAP, page_prot); if (memdesc->hostptr) KGSL_STATS_ADD(memdesc->size, @@ -509,11 +478,10 @@ static int kgsl_page_alloc_map_kernel(struct kgsl_memdesc *memdesc) &kgsl_driver.stats.vmalloc_max); else ret = -ENOMEM; - kgsl_free(pages); } if (memdesc->hostptr) memdesc->hostptr_count++; -done: + mutex_unlock(&kernel_map_global_lock); return ret; @@ -681,7 +649,6 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, unsigned int j, page_size, len_alloc; unsigned int pcount = 0; size_t len; - struct page **pages = NULL; unsigned int align; size = PAGE_ALIGN(size); @@ -712,18 +679,17 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, memdesc->pagetable = pagetable; memdesc->ops = &kgsl_page_alloc_ops; - memdesc->sgt = kmalloc(sizeof(struct sg_table), GFP_KERNEL); - if (memdesc->sgt == NULL) - return -ENOMEM; - /* - * Allocate space to store the list of pages to send to vmap. This is an - * array of pointers so we can track 1024 pages per page of allocation + * Allocate space to store the list of pages. This is an array of + * pointers so we can track 1024 pages per page of allocation. + * Keep this array around for non global non secure buffers that + * are allocated by kgsl. This helps with improving the vm fault + * routine by finding the faulted page in constant time. */ - pages = kgsl_malloc(len_alloc * sizeof(struct page *)); + memdesc->pages = kgsl_malloc(len_alloc * sizeof(struct page *)); - if (pages == NULL) { + if (memdesc->pages == NULL) { ret = -ENOMEM; goto done; } @@ -734,9 +700,9 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, int page_count; page_count = kgsl_pool_alloc_page(&page_size, - pages + pcount, len_alloc - pcount, + memdesc->pages + pcount, + len_alloc - pcount, &align); - if (page_count <= 0) { if (page_count == -EAGAIN) continue; @@ -760,16 +726,12 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, pcount += page_count; len -= page_size; memdesc->size += page_size; + memdesc->page_count += page_count; /* Get the needed page size for the next iteration */ page_size = get_page_size(len, align); } - ret = sg_alloc_table_from_pages(memdesc->sgt, pages, pcount, 0, - memdesc->size, GFP_KERNEL); - if (ret) - goto done; - /* Call to the hypervisor to lock any secure buffer allocations */ if (memdesc->flags & KGSL_MEMFLAGS_SECURE) { unsigned int i; @@ -778,10 +740,27 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, int source_vm = VMID_HLOS; int dest_vm = VMID_CP_PIXEL; + memdesc->sgt = kmalloc(sizeof(struct sg_table), GFP_KERNEL); + if (memdesc->sgt == NULL) { + ret = -ENOMEM; + goto done; + } + + ret = sg_alloc_table_from_pages(memdesc->sgt, memdesc->pages, + memdesc->page_count, 0, memdesc->size, GFP_KERNEL); + if (ret) { + kfree(memdesc->sgt); + goto done; + } + ret = hyp_assign_table(memdesc->sgt, &source_vm, 1, &dest_vm, &dest_perms, 1); - if (ret) + if (ret) { + sg_free_table(memdesc->sgt); + kfree(memdesc->sgt); + memdesc->sgt = NULL; goto done; + } /* Set private bit for each sg to indicate that its secured */ for_each_sg(memdesc->sgt->sgl, sg, memdesc->sgt->nents, i) @@ -793,6 +772,14 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, KGSL_STATS_ADD(memdesc->size, &kgsl_driver.stats.secure, &kgsl_driver.stats.secure_max); + /* + * We don't need the array for secure buffers because they are + * not mapped to CPU + */ + kgsl_free(memdesc->pages); + memdesc->pages = NULL; + memdesc->page_count = 0; + /* Don't map and zero the locked secure buffer */ goto done; } @@ -802,19 +789,18 @@ kgsl_sharedmem_page_alloc_user(struct kgsl_memdesc *memdesc, done: if (ret) { - if (pages) { + if (memdesc->pages) { unsigned int count = 1; for (j = 0; j < pcount; j += count) { - count = 1 << compound_order(pages[j]); - kgsl_pool_free_page(pages[j]); + count = 1 << compound_order(memdesc->pages[j]); + kgsl_pool_free_page(memdesc->pages[j]); } } - kfree(memdesc->sgt); + kgsl_free(memdesc->pages); memset(memdesc, 0, sizeof(*memdesc)); } - kgsl_free(pages); return ret; } @@ -837,6 +823,9 @@ void kgsl_sharedmem_free(struct kgsl_memdesc *memdesc) kfree(memdesc->sgt); } + if (memdesc->pages) + kgsl_free(memdesc->pages); + memset(memdesc, 0, sizeof(*memdesc)); } EXPORT_SYMBOL(kgsl_sharedmem_free); diff --git a/drivers/gpu/msm/kgsl_sharedmem.h b/drivers/gpu/msm/kgsl_sharedmem.h index 5093ebd6e51a..52c2b7ddd183 100644 --- a/drivers/gpu/msm/kgsl_sharedmem.h +++ b/drivers/gpu/msm/kgsl_sharedmem.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2002,2007-2015, The Linux Foundation. All rights reserved. +/* Copyright (c) 2002,2007-2016, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -293,4 +293,47 @@ static inline void kgsl_free_global(struct kgsl_device *device, void kgsl_sharedmem_set_noretry(bool val); bool kgsl_sharedmem_get_noretry(void); +/** + * kgsl_alloc_sgt_from_pages() - Allocate a sg table + * + * @memdesc: memory descriptor of the allocation + * + * Allocate and return pointer to a sg table + */ +static inline struct sg_table *kgsl_alloc_sgt_from_pages( + struct kgsl_memdesc *m) +{ + int ret; + struct sg_table *sgt; + + sgt = kmalloc(sizeof(struct sg_table), GFP_KERNEL); + if (sgt == NULL) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table_from_pages(sgt, m->pages, m->page_count, 0, + m->size, GFP_KERNEL); + if (ret) { + kfree(sgt); + return ERR_PTR(ret); + } + + return sgt; +} + +/** + * kgsl_free_sgt() - Free a sg table structure + * + * @sgt: sg table pointer to be freed + * + * Free the sg table allocated using sgt and free the + * sgt structure itself + */ +static inline void kgsl_free_sgt(struct sg_table *sgt) +{ + if (sgt != NULL) { + sg_free_table(sgt); + kfree(sgt); + } +} + #endif /* __KGSL_SHAREDMEM_H */ |