Lubancat2/kernel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

BACKPORT: dma-buf: heaps: add dma-heaps

Browse Source 
Change-Id: Idf952c4f923a570f97da47e0d8d490d6427dd5d3
Signed-off-by: Jianqun Xu <jay.xu@rock-chips.com>
This commit is contained in:
Jianqun Xu
2022-09-13 15:09:24 +08:00
committed by Tao Huang
parent 43eba68e77
commit a414b9416b
13 changed files with 2672 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
drivers/dma-buf/Kconfig
View File

@ -35,9 +35,21 @@ config SW_SYNC
WARNING: improper use of this can result in deadlocking kernel
drivers from userspace. Intended for test and debug only.
menuconfig DMABUF_HEAPS
bool "DMA-BUF Userland Memory Heaps"
select DMA_SHARED_BUFFER
help
Choose this option to enable the DMA-BUF userland memory heaps.
This options creates per heap chardevs in /dev/dma_heap/ which
allows userspace to allocate dma-bufs that can be shared
between drivers.
config DMABUF_PAGE_POOL
bool "DMA-BUF page-pool library"
depends on !DMABUF_HEAPS
help
Choose this option to enable the DMA-BUF page-pool library.
source "drivers/dma-buf/heaps/Kconfig"
endmenu

2
drivers/dma-buf/Makefile
View File

@ -1,4 +1,6 @@
obj-y := dma-buf.o dma-fence.o dma-fence-array.o reservation.o seqno-fence.o
obj-$(CONFIG_DMABUF_HEAPS) += dma-heap.o
obj-$(CONFIG_DMABUF_HEAPS) += heaps/
obj-$(CONFIG_DMABUF_CACHE) += dma-buf-cache.o
obj-$(CONFIG_DMABUF_PAGE_POOL) += dmabuf_page_pool.o
obj-$(CONFIG_SYNC_FILE) += sync_file.o

494
drivers/dma-buf/dma-heap.c Normal file
View File

@ -0,0 +1,494 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Framework for userspace DMA-BUF allocations
*
* Copyright (C) 2011 Google, Inc.
* Copyright (C) 2019 Linaro Ltd.
*/
#include <linux/cdev.h>
#include <linux/debugfs.h>
#include <linux/device.h>
#include <linux/dma-buf.h>
#include <linux/err.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/nospec.h>
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/dma-heap.h>
#include <uapi/linux/dma-heap.h>
#define DEVNAME "dma_heap"
#define NUM_HEAP_MINORS 128
/**
* struct dma_heap - represents a dmabuf heap in the system
* @name: used for debugging/device-node name
* @ops: ops struct for this heap
* @heap_devt heap device node
* @list list head connecting to list of heaps
* @heap_cdev heap char device
* @heap_dev heap device struct
*
* Represents a heap of memory from which buffers can be made.
*/
struct dma_heap {
const char *name;
const struct dma_heap_ops *ops;
void *priv;
dev_t heap_devt;
struct list_head list;
struct cdev heap_cdev;
struct kref refcount;
struct device *heap_dev;
};
static LIST_HEAD(heap_list);
static DEFINE_MUTEX(heap_list_lock);
static dev_t dma_heap_devt;
static struct class *dma_heap_class;
static DEFINE_MUTEX(heap_devnode_lock);
static DECLARE_BITMAP(heap_devnode_nums, NUM_HEAP_MINORS);
struct dma_heap *dma_heap_find(const char *name)
{
struct dma_heap *h;
mutex_lock(&heap_list_lock);
list_for_each_entry(h, &heap_list, list) {
if (!strcmp(h->name, name)) {
kref_get(&h->refcount);
mutex_unlock(&heap_list_lock);
return h;
}
}
mutex_unlock(&heap_list_lock);
return NULL;
}
EXPORT_SYMBOL_GPL(dma_heap_find);
void dma_heap_buffer_free(struct dma_buf *dmabuf)
{
dma_buf_put(dmabuf);
}
EXPORT_SYMBOL_GPL(dma_heap_buffer_free);
struct dma_buf *dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
unsigned int fd_flags,
unsigned int heap_flags)
{
if (fd_flags & ~DMA_HEAP_VALID_FD_FLAGS)
return ERR_PTR(-EINVAL);
if (heap_flags & ~DMA_HEAP_VALID_HEAP_FLAGS)
return ERR_PTR(-EINVAL);
/*
* Allocations from all heaps have to begin
* and end on page boundaries.
*/
len = PAGE_ALIGN(len);
if (!len)
return ERR_PTR(-EINVAL);
return heap->ops->allocate(heap, len, fd_flags, heap_flags);
}
EXPORT_SYMBOL_GPL(dma_heap_buffer_alloc);
int dma_heap_bufferfd_alloc(struct dma_heap *heap, size_t len,
unsigned int fd_flags,
unsigned int heap_flags)
{
struct dma_buf *dmabuf;
int fd;
dmabuf = dma_heap_buffer_alloc(heap, len, fd_flags, heap_flags);
if (IS_ERR(dmabuf))
return PTR_ERR(dmabuf);
fd = dma_buf_fd(dmabuf, fd_flags);
if (fd < 0) {
dma_buf_put(dmabuf);
/* just return, as put will call release and that will free */
}
return fd;
}
EXPORT_SYMBOL_GPL(dma_heap_bufferfd_alloc);
static int dma_heap_open(struct inode *inode, struct file *file)
{
struct dma_heap *heap;
mutex_lock(&heap_devnode_lock);
heap = container_of(inode->i_cdev, struct dma_heap, heap_cdev);
if (!heap) {
pr_err("dma_heap: minor %d unknown.\n", iminor(inode));
mutex_unlock(&heap_devnode_lock);
return -ENODEV;
}
mutex_unlock(&heap_devnode_lock);
/* instance data as context */
file->private_data = heap;
nonseekable_open(inode, file);
return 0;
}
static long dma_heap_ioctl_allocate(struct file *file, void *data)
{
struct dma_heap_allocation_data *heap_allocation = data;
struct dma_heap *heap = file->private_data;
int fd;
if (heap_allocation->fd)
return -EINVAL;
fd = dma_heap_bufferfd_alloc(heap, heap_allocation->len,
heap_allocation->fd_flags,
heap_allocation->heap_flags);
if (fd < 0)
return fd;
heap_allocation->fd = fd;
return 0;
}
static int dma_heap_ioctl_get_phys(struct file *file, void *data)
{
#if IS_ENABLED(CONFIG_NO_GKI)
struct dma_heap *heap = file->private_data;
struct dma_heap_phys_data *phys = data;
if (heap->ops->get_phys)
return heap->ops->get_phys(heap, phys);
#endif
return -EINVAL;
}
static unsigned int dma_heap_ioctl_cmds[] = {
DMA_HEAP_IOCTL_ALLOC,
DMA_HEAP_IOCTL_GET_PHYS,
};
static long dma_heap_ioctl(struct file *file, unsigned int ucmd,
unsigned long arg)
{
char stack_kdata[128];
char *kdata = stack_kdata;
unsigned int kcmd;
unsigned int in_size, out_size, drv_size, ksize;
int nr = _IOC_NR(ucmd);
int ret = 0;
if (nr >= ARRAY_SIZE(dma_heap_ioctl_cmds))
return -EINVAL;
nr = array_index_nospec(nr, ARRAY_SIZE(dma_heap_ioctl_cmds));
/* Get the kernel ioctl cmd that matches */
kcmd = dma_heap_ioctl_cmds[nr];
/* Figure out the delta between user cmd size and kernel cmd size */
drv_size = _IOC_SIZE(kcmd);
out_size = _IOC_SIZE(ucmd);
in_size = out_size;
if ((ucmd & kcmd & IOC_IN) == 0)
in_size = 0;
if ((ucmd & kcmd & IOC_OUT) == 0)
out_size = 0;
ksize = max(max(in_size, out_size), drv_size);
/* If necessary, allocate buffer for ioctl argument */
if (ksize > sizeof(stack_kdata)) {
kdata = kmalloc(ksize, GFP_KERNEL);
if (!kdata)
return -ENOMEM;
}
if (copy_from_user(kdata, (void __user *)arg, in_size) != 0) {
ret = -EFAULT;
goto err;
}
/* zero out any difference between the kernel/user structure size */
if (ksize > in_size)
memset(kdata + in_size, 0, ksize - in_size);
switch (kcmd) {
case DMA_HEAP_IOCTL_ALLOC:
ret = dma_heap_ioctl_allocate(file, kdata);
break;
case DMA_HEAP_IOCTL_GET_PHYS:
ret = dma_heap_ioctl_get_phys(file, kdata);
break;
default:
ret = -ENOTTY;
goto err;
}
if (copy_to_user((void __user *)arg, kdata, out_size) != 0)
ret = -EFAULT;
err:
if (kdata != stack_kdata)
kfree(kdata);
return ret;
}
static const struct file_operations dma_heap_fops = {
.owner = THIS_MODULE,
.open = dma_heap_open,
.unlocked_ioctl = dma_heap_ioctl,
#ifdef CONFIG_COMPAT
.compat_ioctl = dma_heap_ioctl,
#endif
};
/**
* dma_heap_get_drvdata() - get per-subdriver data for the heap
* @heap: DMA-Heap to retrieve private data for
*
* Returns:
* The per-subdriver data for the heap.
*/
void *dma_heap_get_drvdata(struct dma_heap *heap)
{
return heap->priv;
}
EXPORT_SYMBOL_GPL(dma_heap_get_drvdata);
static void dma_heap_release(struct kref *ref)
{
struct dma_heap *heap = container_of(ref, struct dma_heap, refcount);
int minor = MINOR(heap->heap_devt);
/* Note, we already holding the heap_list_lock here */
list_del(&heap->list);
device_destroy(dma_heap_class, heap->heap_devt);
cdev_del(&heap->heap_cdev);
mutex_lock(&heap_devnode_lock);
clear_bit(minor, heap_devnode_nums);
mutex_unlock(&heap_devnode_lock);
kfree(heap);
}
void dma_heap_put(struct dma_heap *h)
{
/*
* Take the heap_list_lock now to avoid racing with code
* scanning the list and then taking a kref.
*/
mutex_lock(&heap_list_lock);
kref_put(&h->refcount, dma_heap_release);
mutex_unlock(&heap_list_lock);
}
EXPORT_SYMBOL_GPL(dma_heap_put);
/**
* dma_heap_get_dev() - get device struct for the heap
* @heap: DMA-Heap to retrieve device struct from
*
* Returns:
* The device struct for the heap.
*/
struct device *dma_heap_get_dev(struct dma_heap *heap)
{
return heap->heap_dev;
}
EXPORT_SYMBOL_GPL(dma_heap_get_dev);
/**
* dma_heap_get_name() - get heap name
* @heap: DMA-Heap to retrieve private data for
*
* Returns:
* The char* for the heap name.
*/
const char *dma_heap_get_name(struct dma_heap *heap)
{
return heap->name;
}
EXPORT_SYMBOL_GPL(dma_heap_get_name);
struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info)
{
struct dma_heap *heap, *err_ret;
unsigned int minor;
int ret;
if (!exp_info->name || !strcmp(exp_info->name, "")) {
pr_err("dma_heap: Cannot add heap without a name\n");
return ERR_PTR(-EINVAL);
}
if (!exp_info->ops || !exp_info->ops->allocate) {
pr_err("dma_heap: Cannot add heap with invalid ops struct\n");
return ERR_PTR(-EINVAL);
}
/* check the name is unique */
heap = dma_heap_find(exp_info->name);
if (heap) {
pr_err("dma_heap: Already registered heap named %s\n",
exp_info->name);
dma_heap_put(heap);
return ERR_PTR(-EINVAL);
}
heap = kzalloc(sizeof(*heap), GFP_KERNEL);
if (!heap)
return ERR_PTR(-ENOMEM);
kref_init(&heap->refcount);
heap->name = exp_info->name;
heap->ops = exp_info->ops;
heap->priv = exp_info->priv;
/* Part 1: Find a free minor number */
mutex_lock(&heap_devnode_lock);
minor = find_next_zero_bit(heap_devnode_nums, NUM_HEAP_MINORS, 0);
if (minor == NUM_HEAP_MINORS) {
mutex_unlock(&heap_devnode_lock);
pr_err("dma_heap: Unable to get minor number for heap\n");
err_ret = ERR_PTR(-ENFILE);
goto err0;
}
set_bit(minor, heap_devnode_nums);
mutex_unlock(&heap_devnode_lock);
/* Create device */
heap->heap_devt = MKDEV(MAJOR(dma_heap_devt), minor);
cdev_init(&heap->heap_cdev, &dma_heap_fops);
ret = cdev_add(&heap->heap_cdev, heap->heap_devt, 1);
if (ret < 0) {
pr_err("dma_heap: Unable to add char device\n");
err_ret = ERR_PTR(ret);
goto err1;
}
heap->heap_dev = device_create(dma_heap_class,
NULL,
heap->heap_devt,
NULL,
heap->name);
if (IS_ERR(heap->heap_dev)) {
pr_err("dma_heap: Unable to create device\n");
err_ret = ERR_CAST(heap->heap_dev);
goto err2;
}
/* Make sure it doesn't disappear on us */
heap->heap_dev = get_device(heap->heap_dev);
/* Add heap to the list */
mutex_lock(&heap_list_lock);
list_add(&heap->list, &heap_list);
mutex_unlock(&heap_list_lock);
return heap;
err2:
cdev_del(&heap->heap_cdev);
err1:
mutex_lock(&heap_devnode_lock);
clear_bit(minor, heap_devnode_nums);
mutex_unlock(&heap_devnode_lock);
err0:
kfree(heap);
return err_ret;
}
EXPORT_SYMBOL_GPL(dma_heap_add);
static char *dma_heap_devnode(struct device *dev, umode_t *mode)
{
return kasprintf(GFP_KERNEL, "dma_heap/%s", dev_name(dev));
}
static ssize_t total_pools_kb_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf)
{
struct dma_heap *heap;
u64 total_pool_size = 0;
mutex_lock(&heap_list_lock);
list_for_each_entry(heap, &heap_list, list) {
if (heap->ops->get_pool_size)
total_pool_size += heap->ops->get_pool_size(heap);
}
mutex_unlock(&heap_list_lock);
return sysfs_emit(buf, "%llu\n", total_pool_size / 1024);
}
static struct kobj_attribute total_pools_kb_attr =
__ATTR_RO(total_pools_kb);
static struct attribute *dma_heap_sysfs_attrs[] = {
&total_pools_kb_attr.attr,
NULL,
};
ATTRIBUTE_GROUPS(dma_heap_sysfs);
static struct kobject *dma_heap_kobject;
static int dma_heap_sysfs_setup(void)
{
int ret;
dma_heap_kobject = kobject_create_and_add("dma_heap", kernel_kobj);
if (!dma_heap_kobject)
return -ENOMEM;
ret = sysfs_create_groups(dma_heap_kobject, dma_heap_sysfs_groups);
if (ret) {
kobject_put(dma_heap_kobject);
return ret;
}
return 0;
}
static void dma_heap_sysfs_teardown(void)
{
kobject_put(dma_heap_kobject);
}
static int dma_heap_init(void)
{
int ret;
ret = dma_heap_sysfs_setup();
if (ret)
return ret;
ret = alloc_chrdev_region(&dma_heap_devt, 0, NUM_HEAP_MINORS, DEVNAME);
if (ret)
goto err_chrdev;
dma_heap_class = class_create(THIS_MODULE, DEVNAME);
if (IS_ERR(dma_heap_class)) {
ret = PTR_ERR(dma_heap_class);
goto err_class;
}
dma_heap_class->devnode = dma_heap_devnode;
return 0;
err_class:
unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS);
err_chrdev:
dma_heap_sysfs_teardown();
return ret;
}
subsys_initcall(dma_heap_init);

24
drivers/dma-buf/heaps/Kconfig Normal file
View File

@ -0,0 +1,24 @@
menuconfig DMABUF_HEAPS_DEFERRED_FREE
bool "DMA-BUF heaps deferred-free library"
help
Choose this option to enable the DMA-BUF heaps deferred-free library.
menuconfig DMABUF_HEAPS_PAGE_POOL
bool "DMA-BUF heaps page-pool library"
help
Choose this option to enable the DMA-BUF heaps page-pool library.
config DMABUF_HEAPS_SYSTEM
tristate "DMA-BUF System Heap"
depends on DMABUF_HEAPS && DMABUF_HEAPS_DEFERRED_FREE && DMABUF_HEAPS_PAGE_POOL
help
Choose this option to enable the system dmabuf heap. The system heap
is backed by pages from the buddy allocator. If in doubt, say Y.
config DMABUF_HEAPS_CMA
tristate "DMA-BUF CMA Heap"
depends on DMABUF_HEAPS && DMA_CMA
help
Choose this option to enable dma-buf CMA heap. This heap is backed
by the Contiguous Memory Allocator (CMA). If your system has these
regions, you should say Y here.

5
drivers/dma-buf/heaps/Makefile Normal file
View File

@ -0,0 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_DMABUF_HEAPS_DEFERRED_FREE) += deferred-free-helper.o
obj-$(CONFIG_DMABUF_HEAPS_PAGE_POOL) += page_pool.o
obj-$(CONFIG_DMABUF_HEAPS_SYSTEM) += rk_system_heap.o
obj-$(CONFIG_DMABUF_HEAPS_CMA) += cma_heap.o

604
drivers/dma-buf/heaps/cma_heap.c Normal file
View File

@ -0,0 +1,604 @@
// SPDX-License-Identifier: GPL-2.0
/*
* DMABUF CMA heap exporter
*
* Copyright (C) 2012, 2019, 2020 Linaro Ltd.
* Author: <benjamin.gaignard@linaro.org> for ST-Ericsson.
*
* Also utilizing parts of Andrew Davis' SRAM heap:
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
* Andrew F. Davis <afd@ti.com>
*/
#include <linux/cma.h>
#include <linux/dma-buf.h>
#include <linux/dma-heap.h>
#include <linux/dma-contiguous.h>
#include <linux/err.h>
#include <linux/highmem.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <uapi/linux/dma-heap.h>
struct cma_heap {
struct dma_heap *heap;
struct cma *cma;
};
struct cma_heap_buffer {
struct cma_heap *heap;
struct list_head attachments;
struct mutex lock;
unsigned long len;
struct page *cma_pages;
struct page **pages;
pgoff_t pagecount;
int vmap_cnt;
void *vaddr;
bool uncached;
};
struct dma_heap_attachment {
struct device *dev;
struct sg_table table;
struct list_head list;
bool mapped;
bool uncached;
};
static int cma_heap_attach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attachment)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a;
int ret;
a = kzalloc(sizeof(*a), GFP_KERNEL);
if (!a)
return -ENOMEM;
ret = sg_alloc_table_from_pages(&a->table, buffer->pages,
buffer->pagecount, 0,
buffer->pagecount << PAGE_SHIFT,
GFP_KERNEL);
if (ret) {
kfree(a);
return ret;
}
a->dev = attachment->dev;
INIT_LIST_HEAD(&a->list);
a->mapped = false;
a->uncached = buffer->uncached;
attachment->priv = a;
mutex_lock(&buffer->lock);
list_add(&a->list, &buffer->attachments);
mutex_unlock(&buffer->lock);
return 0;
}
static void cma_heap_detach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attachment)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a = attachment->priv;
mutex_lock(&buffer->lock);
list_del(&a->list);
mutex_unlock(&buffer->lock);
sg_free_table(&a->table);
kfree(a);
}
static struct sg_table *cma_heap_map_dma_buf(struct dma_buf_attachment *attachment,
enum dma_data_direction direction)
{
struct dma_heap_attachment *a = attachment->priv;
struct sg_table *table = &a->table;
int attrs = attachment->dma_map_attrs;
int ret;
if (a->uncached)
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
ret = dma_map_sgtable(attachment->dev, table, direction, attrs);
if (ret)
return ERR_PTR(-ENOMEM);
a->mapped = true;
return table;
}
static void cma_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
struct sg_table *table,
enum dma_data_direction direction)
{
struct dma_heap_attachment *a = attachment->priv;
int attrs = attachment->dma_map_attrs;
a->mapped = false;
if (a->uncached)
attrs |= DMA_ATTR_SKIP_CPU_SYNC;
dma_unmap_sgtable(attachment->dev, table, direction, attrs);
}
static int
cma_heap_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf,
enum dma_data_direction direction,
unsigned int offset,
unsigned int len)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a;
if (buffer->vmap_cnt)
invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
if (buffer->uncached)
return 0;
mutex_lock(&buffer->lock);
list_for_each_entry(a, &buffer->attachments, list) {
if (!a->mapped)
continue;
dma_sync_sgtable_for_cpu(a->dev, &a->table, direction);
}
if (list_empty(&buffer->attachments)) {
phys_addr_t phys = page_to_phys(buffer->cma_pages);
dma_sync_single_for_cpu(dma_heap_get_dev(buffer->heap->heap),
phys + offset,
len,
direction);
}
mutex_unlock(&buffer->lock);
return 0;
}
static int
cma_heap_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf,
enum dma_data_direction direction,
unsigned int offset,
unsigned int len)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a;
if (buffer->vmap_cnt)
flush_kernel_vmap_range(buffer->vaddr, buffer->len);
if (buffer->uncached)
return 0;
mutex_lock(&buffer->lock);
list_for_each_entry(a, &buffer->attachments, list) {
if (!a->mapped)
continue;
dma_sync_sgtable_for_device(a->dev, &a->table, direction);
}
if (list_empty(&buffer->attachments)) {
phys_addr_t phys = page_to_phys(buffer->cma_pages);
dma_sync_single_for_device(dma_heap_get_dev(buffer->heap->heap),
phys + offset,
len,
direction);
}
mutex_unlock(&buffer->lock);
return 0;
}
static int cma_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction dir)
{
return cma_heap_dma_buf_begin_cpu_access_partial(dmabuf, dir, 0,
dmabuf->size);
}
static int cma_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction dir)
{
return cma_heap_dma_buf_end_cpu_access_partial(dmabuf, dir, 0,
dmabuf->size);
}
static vm_fault_t cma_heap_vm_fault(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct cma_heap_buffer *buffer = vma->vm_private_data;
if (vmf->pgoff > buffer->pagecount)
return VM_FAULT_SIGBUS;
vmf->page = buffer->pages[vmf->pgoff];
get_page(vmf->page);
return 0;
}
static const struct vm_operations_struct dma_heap_vm_ops = {
.fault = cma_heap_vm_fault,
};
static int cma_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0)
return -EINVAL;
if (buffer->uncached)
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
vma->vm_ops = &dma_heap_vm_ops;
vma->vm_private_data = buffer;
return 0;
}
static void *cma_heap_do_vmap(struct cma_heap_buffer *buffer)
{
void *vaddr;
pgprot_t pgprot = PAGE_KERNEL;
if (buffer->uncached)
pgprot = pgprot_writecombine(PAGE_KERNEL);
vaddr = vmap(buffer->pages, buffer->pagecount, VM_MAP, pgprot);
if (!vaddr)
return ERR_PTR(-ENOMEM);
return vaddr;
}
static void *cma_heap_vmap(struct dma_buf *dmabuf)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
void *vaddr;
mutex_lock(&buffer->lock);
if (buffer->vmap_cnt) {
buffer->vmap_cnt++;
vaddr = buffer->vaddr;
goto out;
}
vaddr = cma_heap_do_vmap(buffer);
if (IS_ERR(vaddr))
goto out;
buffer->vaddr = vaddr;
buffer->vmap_cnt++;
out:
mutex_unlock(&buffer->lock);
return vaddr;
}
static void cma_heap_vunmap(struct dma_buf *dmabuf, void *vaddr)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
mutex_lock(&buffer->lock);
if (!--buffer->vmap_cnt) {
vunmap(buffer->vaddr);
buffer->vaddr = NULL;
}
mutex_unlock(&buffer->lock);
}
static void cma_heap_dma_buf_release(struct dma_buf *dmabuf)
{
struct cma_heap_buffer *buffer = dmabuf->priv;
struct cma_heap *cma_heap = buffer->heap;
if (buffer->vmap_cnt > 0) {
WARN(1, "%s: buffer still mapped in the kernel\n", __func__);
vunmap(buffer->vaddr);
}
/* free page list */
kfree(buffer->pages);
/* release memory */
cma_release(cma_heap->cma, buffer->cma_pages, buffer->pagecount);
kfree(buffer);
}
static const struct dma_buf_ops cma_heap_buf_ops = {
.attach = cma_heap_attach,
.detach = cma_heap_detach,
.map_dma_buf = cma_heap_map_dma_buf,
.unmap_dma_buf = cma_heap_unmap_dma_buf,
.begin_cpu_access = cma_heap_dma_buf_begin_cpu_access,
.end_cpu_access = cma_heap_dma_buf_end_cpu_access,
.begin_cpu_access_partial = cma_heap_dma_buf_begin_cpu_access_partial,
.end_cpu_access_partial = cma_heap_dma_buf_end_cpu_access_partial,
.mmap = cma_heap_mmap,
.vmap = cma_heap_vmap,
.vunmap = cma_heap_vunmap,
.release = cma_heap_dma_buf_release,
};
static struct dma_buf *cma_heap_do_allocate(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags, bool uncached)
{
struct cma_heap *cma_heap = dma_heap_get_drvdata(heap);
struct cma_heap_buffer *buffer;
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
size_t size = PAGE_ALIGN(len);
pgoff_t pagecount = size >> PAGE_SHIFT;
unsigned long align = get_order(size);
struct page *cma_pages;
struct dma_buf *dmabuf;
int ret = -ENOMEM;
pgoff_t pg;
dma_addr_t dma;
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
if (!buffer)
return ERR_PTR(-ENOMEM);
buffer->uncached = uncached;
INIT_LIST_HEAD(&buffer->attachments);
mutex_init(&buffer->lock);
buffer->len = size;
if (align > CONFIG_CMA_ALIGNMENT)
align = CONFIG_CMA_ALIGNMENT;
cma_pages = cma_alloc(cma_heap->cma, pagecount, align, GFP_KERNEL);
if (!cma_pages)
goto free_buffer;
/* Clear the cma pages */
if (PageHighMem(cma_pages)) {
unsigned long nr_clear_pages = pagecount;
struct page *page = cma_pages;
while (nr_clear_pages > 0) {
void *vaddr = kmap_atomic(page);
memset(vaddr, 0, PAGE_SIZE);
kunmap_atomic(vaddr);
/*
* Avoid wasting time zeroing memory if the process
* has been killed by by SIGKILL
*/
if (fatal_signal_pending(current))
goto free_cma;
page++;
nr_clear_pages--;
}
} else {
memset(page_address(cma_pages), 0, size);
}
buffer->pages = kmalloc_array(pagecount, sizeof(*buffer->pages), GFP_KERNEL);
if (!buffer->pages) {
ret = -ENOMEM;
goto free_cma;
}
for (pg = 0; pg < pagecount; pg++)
buffer->pages[pg] = &cma_pages[pg];
buffer->cma_pages = cma_pages;
buffer->heap = cma_heap;
buffer->pagecount = pagecount;
/* create the dmabuf */
exp_info.exp_name = dma_heap_get_name(heap);
exp_info.ops = &cma_heap_buf_ops;
exp_info.size = buffer->len;
exp_info.flags = fd_flags;
exp_info.priv = buffer;
dmabuf = dma_buf_export(&exp_info);
if (IS_ERR(dmabuf)) {
ret = PTR_ERR(dmabuf);
goto free_pages;
}
if (buffer->uncached) {
dma = dma_map_page(dma_heap_get_dev(heap), buffer->cma_pages, 0,
buffer->pagecount * PAGE_SIZE, DMA_FROM_DEVICE);
dma_unmap_page(dma_heap_get_dev(heap), dma,
buffer->pagecount * PAGE_SIZE, DMA_FROM_DEVICE);
}
return dmabuf;
free_pages:
kfree(buffer->pages);
free_cma:
cma_release(cma_heap->cma, cma_pages, pagecount);
free_buffer:
kfree(buffer);
return ERR_PTR(ret);
}
static struct dma_buf *cma_heap_allocate(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags)
{
return cma_heap_do_allocate(heap, len, fd_flags, heap_flags, false);
}
#if IS_ENABLED(CONFIG_NO_GKI)
static int cma_heap_get_phys(struct dma_heap *heap,
struct dma_heap_phys_data *phys)
{
struct cma_heap *cma_heap = dma_heap_get_drvdata(heap);
struct cma_heap_buffer *buffer;
struct dma_buf *dmabuf;
phys->paddr = (__u64)-1;
if (IS_ERR_OR_NULL(phys))
return -EINVAL;
dmabuf = dma_buf_get(phys->fd);
if (IS_ERR_OR_NULL(dmabuf))
return -EBADFD;
buffer = dmabuf->priv;
if (IS_ERR_OR_NULL(buffer))
goto err;
if (buffer->heap != cma_heap)
goto err;
phys->paddr = page_to_phys(buffer->cma_pages);
err:
dma_buf_put(dmabuf);
return (phys->paddr == (__u64)-1) ? -EINVAL : 0;
}
#endif
static const struct dma_heap_ops cma_heap_ops = {
.allocate = cma_heap_allocate,
#if IS_ENABLED(CONFIG_NO_GKI)
.get_phys = cma_heap_get_phys,
#endif
};
static struct dma_buf *cma_uncached_heap_allocate(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags)
{
return cma_heap_do_allocate(heap, len, fd_flags, heap_flags, true);
}
static struct dma_buf *cma_uncached_heap_not_initialized(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags)
{
pr_info("heap %s not initialized\n", dma_heap_get_name(heap));
return ERR_PTR(-EBUSY);
}
static struct dma_heap_ops cma_uncached_heap_ops = {
.allocate = cma_uncached_heap_not_initialized,
};
static int set_heap_dev_dma(struct device *heap_dev)
{
int err = 0;
if (!heap_dev)
return -EINVAL;
dma_coerce_mask_and_coherent(heap_dev, DMA_BIT_MASK(64));
if (!heap_dev->dma_parms) {
heap_dev->dma_parms = devm_kzalloc(heap_dev,
sizeof(*heap_dev->dma_parms),
GFP_KERNEL);
if (!heap_dev->dma_parms)
return -ENOMEM;
err = dma_set_max_seg_size(heap_dev, (unsigned int)DMA_BIT_MASK(64));
if (err) {
devm_kfree(heap_dev, heap_dev->dma_parms);
dev_err(heap_dev, "Failed to set DMA segment size, err:%d\n", err);
return err;
}
}
return 0;
}
static int __add_cma_heap(struct cma *cma, void *data)
{
struct cma_heap *cma_heap, *cma_uncached_heap;
struct dma_heap_export_info exp_info;
int ret;
cma_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL);
if (!cma_heap)
return -ENOMEM;
cma_heap->cma = cma;
exp_info.name = cma_get_name(cma);
exp_info.ops = &cma_heap_ops;
exp_info.priv = cma_heap;
cma_heap->heap = dma_heap_add(&exp_info);
if (IS_ERR(cma_heap->heap)) {
ret = PTR_ERR(cma_heap->heap);
goto free_cma_heap;
}
cma_uncached_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL);
if (!cma_uncached_heap) {
ret = -ENOMEM;
goto put_cma_heap;
}
cma_uncached_heap->cma = cma;
exp_info.name = "cma-uncached";
exp_info.ops = &cma_uncached_heap_ops;
exp_info.priv = cma_uncached_heap;
cma_uncached_heap->heap = dma_heap_add(&exp_info);
if (IS_ERR(cma_uncached_heap->heap)) {
ret = PTR_ERR(cma_uncached_heap->heap);
goto free_uncached_cma_heap;
}
ret = set_heap_dev_dma(dma_heap_get_dev(cma_uncached_heap->heap));
if (ret)
goto put_uncached_cma_heap;
mb(); /* make sure we only set allocate after dma_mask is set */
cma_uncached_heap_ops.allocate = cma_uncached_heap_allocate;
return 0;
put_uncached_cma_heap:
dma_heap_put(cma_uncached_heap->heap);
free_uncached_cma_heap:
kfree(cma_uncached_heap);
put_cma_heap:
dma_heap_put(cma_heap->heap);
free_cma_heap:
kfree(cma_heap);
return ret;
}
static int add_default_cma_heap(void)
{
struct cma *default_cma = dev_get_cma_area(NULL);
int ret = 0;
if (default_cma)
ret = __add_cma_heap(default_cma, NULL);
return ret;
}
module_init(add_default_cma_heap);
MODULE_DESCRIPTION("DMA-BUF CMA Heap");
MODULE_LICENSE("GPL v2");

142
drivers/dma-buf/heaps/deferred-free-helper.c Normal file
View File

@ -0,0 +1,142 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Deferred dmabuf freeing helper
*
* Copyright (C) 2020 Linaro, Ltd.
*
* Based on the ION page pool code
* Copyright (C) 2011 Google, Inc.
*/
#include <linux/module.h>
#include <linux/freezer.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <uapi/linux/sched/types.h>
#include "deferred-free-helper.h"
static LIST_HEAD(free_list);
static size_t list_nr_pages;
wait_queue_head_t freelist_waitqueue;
struct task_struct *freelist_task;
static DEFINE_SPINLOCK(free_list_lock);
void deferred_free(struct deferred_freelist_item *item,
void (*free)(struct deferred_freelist_item*,
enum df_reason),
size_t nr_pages)
{
unsigned long flags;
INIT_LIST_HEAD(&item->list);
item->nr_pages = nr_pages;
item->free = free;
spin_lock_irqsave(&free_list_lock, flags);
list_add(&item->list, &free_list);
list_nr_pages += nr_pages;
spin_unlock_irqrestore(&free_list_lock, flags);
wake_up(&freelist_waitqueue);
}
EXPORT_SYMBOL_GPL(deferred_free);
static size_t free_one_item(enum df_reason reason)
{
unsigned long flags;
size_t nr_pages;
struct deferred_freelist_item *item;
spin_lock_irqsave(&free_list_lock, flags);
if (list_empty(&free_list)) {
spin_unlock_irqrestore(&free_list_lock, flags);
return 0;
}
item = list_first_entry(&free_list, struct deferred_freelist_item, list);
list_del(&item->list);
nr_pages = item->nr_pages;
list_nr_pages -= nr_pages;
spin_unlock_irqrestore(&free_list_lock, flags);
item->free(item, reason);
return nr_pages;
}
unsigned long get_freelist_nr_pages(void)
{
unsigned long nr_pages;
unsigned long flags;
spin_lock_irqsave(&free_list_lock, flags);
nr_pages = list_nr_pages;
spin_unlock_irqrestore(&free_list_lock, flags);
return nr_pages;
}
EXPORT_SYMBOL_GPL(get_freelist_nr_pages);
static unsigned long freelist_shrink_count(struct shrinker *shrinker,
struct shrink_control *sc)
{
return get_freelist_nr_pages();
}
static unsigned long freelist_shrink_scan(struct shrinker *shrinker,
struct shrink_control *sc)
{
unsigned long total_freed = 0;
if (sc->nr_to_scan == 0)
return 0;
while (total_freed < sc->nr_to_scan) {
size_t pages_freed = free_one_item(DF_UNDER_PRESSURE);
if (!pages_freed)
break;
total_freed += pages_freed;
}
return total_freed;
}
static struct shrinker freelist_shrinker = {
.count_objects = freelist_shrink_count,
.scan_objects = freelist_shrink_scan,
.seeks = DEFAULT_SEEKS,
.batch = 0,
};
static int deferred_free_thread(void *data)
{
while (true) {
wait_event_freezable(freelist_waitqueue,
get_freelist_nr_pages() > 0);
free_one_item(DF_NORMAL);
}
return 0;
}
static int deferred_freelist_init(void)
{
struct sched_param param = { .sched_priority = 0 };
list_nr_pages = 0;
init_waitqueue_head(&freelist_waitqueue);
freelist_task = kthread_run(deferred_free_thread, NULL,
"%s", "dmabuf-deferred-free-worker");
if (IS_ERR(freelist_task)) {
pr_err("Creating thread for deferred free failed\n");
return -1;
}
sched_setscheduler(freelist_task, SCHED_IDLE, &param);
return register_shrinker(&freelist_shrinker);
}
module_init(deferred_freelist_init);
MODULE_LICENSE("GPL");

57
drivers/dma-buf/heaps/deferred-free-helper.h Normal file
View File

@ -0,0 +1,57 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef DEFERRED_FREE_HELPER_H
#define DEFERRED_FREE_HELPER_H
/**
* df_reason - enum for reason why item was freed
*
* This provides a reason for why the free function was called
* on the item. This is useful when deferred_free is used in
* combination with a pagepool, so under pressure the page can
* be immediately freed.
*
* DF_NORMAL: Normal deferred free
*
* DF_UNDER_PRESSURE: Free was called because the system
* is under memory pressure. Usually
* from a shrinker. Avoid allocating
* memory in the free call, as it may
* fail.
*/
enum df_reason {
DF_NORMAL,
DF_UNDER_PRESSURE,
};
/**
* deferred_freelist_item - item structure for deferred freelist
*
* This is to be added to the structure for whatever you want to
* defer freeing on.
*
* @nr_pages: number of pages used by item to be freed
* @free: function pointer to be called when freeing the item
* @list: list entry for the deferred list
*/
struct deferred_freelist_item {
size_t nr_pages;
void (*free)(struct deferred_freelist_item *i,
enum df_reason reason);
struct list_head list;
};
/**
* deferred_free - call to add item to the deferred free list
*
* @item: Pointer to deferred_freelist_item field of a structure
* @free: Function pointer to the free call
* @nr_pages: number of pages to be freed
*/
void deferred_free(struct deferred_freelist_item *item,
void (*free)(struct deferred_freelist_item *i,
enum df_reason reason),
size_t nr_pages);
unsigned long get_freelist_nr_pages(void);
#endif

248
drivers/dma-buf/heaps/page_pool.c Normal file
View File

@ -0,0 +1,248 @@
// SPDX-License-Identifier: GPL-2.0
/*
* DMA BUF page pool system
*
* Copyright (C) 2020 Linaro Ltd.
*
* Based on the ION page pool code
* Copyright (C) 2011 Google, Inc.
*/
#include <linux/module.h>
#include <linux/freezer.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/sched/signal.h>
#include "page_pool.h"
static LIST_HEAD(pool_list);
static DEFINE_MUTEX(pool_list_lock);
static inline
struct page *dmabuf_page_pool_alloc_pages(struct dmabuf_page_pool *pool)
{
if (fatal_signal_pending(current))
return NULL;
return alloc_pages(pool->gfp_mask, pool->order);
}
static inline void dmabuf_page_pool_free_pages(struct dmabuf_page_pool *pool,
struct page *page)
{
__free_pages(page, pool->order);
}
static void dmabuf_page_pool_add(struct dmabuf_page_pool *pool, struct page *page)
{
int index;
if (PageHighMem(page))
index = POOL_HIGHPAGE;
else
index = POOL_LOWPAGE;
mutex_lock(&pool->mutex);
list_add_tail(&page->lru, &pool->items[index]);
pool->count[index]++;
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
1 << pool->order);
mutex_unlock(&pool->mutex);
}
static struct page *dmabuf_page_pool_remove(struct dmabuf_page_pool *pool, int index)
{
struct page *page;
mutex_lock(&pool->mutex);
page = list_first_entry_or_null(&pool->items[index], struct page, lru);
if (page) {
pool->count[index]--;
list_del(&page->lru);
mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE,
-(1 << pool->order));
}
mutex_unlock(&pool->mutex);
return page;
}
static struct page *dmabuf_page_pool_fetch(struct dmabuf_page_pool *pool)
{
struct page *page = NULL;
page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
if (!page)
page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
return page;
}
struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool)
{
struct page *page = NULL;
if (WARN_ON(!pool))
return NULL;
page = dmabuf_page_pool_fetch(pool);
if (!page)
page = dmabuf_page_pool_alloc_pages(pool);
return page;
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_alloc);
void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page)
{
if (WARN_ON(pool->order != compound_order(page)))
return;
dmabuf_page_pool_add(pool, page);
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_free);
static int dmabuf_page_pool_total(struct dmabuf_page_pool *pool, bool high)
{
int count = pool->count[POOL_LOWPAGE];
if (high)
count += pool->count[POOL_HIGHPAGE];
return count << pool->order;
}
struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask, unsigned int order)
{
struct dmabuf_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
int i;
if (!pool)
return NULL;
for (i = 0; i < POOL_TYPE_SIZE; i++) {
pool->count[i] = 0;
INIT_LIST_HEAD(&pool->items[i]);
}
pool->gfp_mask = gfp_mask | __GFP_COMP;
pool->order = order;
mutex_init(&pool->mutex);
mutex_lock(&pool_list_lock);
list_add(&pool->list, &pool_list);
mutex_unlock(&pool_list_lock);
return pool;
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_create);
void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool)
{
struct page *page;
int i;
/* Remove us from the pool list */
mutex_lock(&pool_list_lock);
list_del(&pool->list);
mutex_unlock(&pool_list_lock);
/* Free any remaining pages in the pool */
for (i = 0; i < POOL_TYPE_SIZE; i++) {
while ((page = dmabuf_page_pool_remove(pool, i)))
dmabuf_page_pool_free_pages(pool, page);
}
kfree(pool);
}
EXPORT_SYMBOL_GPL(dmabuf_page_pool_destroy);
static int dmabuf_page_pool_do_shrink(struct dmabuf_page_pool *pool, gfp_t gfp_mask,
int nr_to_scan)
{
int freed = 0;
bool high;
if (current_is_kswapd())
high = true;
else
high = !!(gfp_mask & __GFP_HIGHMEM);
if (nr_to_scan == 0)
return dmabuf_page_pool_total(pool, high);
while (freed < nr_to_scan) {
struct page *page;
/* Try to free low pages first */
page = dmabuf_page_pool_remove(pool, POOL_LOWPAGE);
if (!page)
page = dmabuf_page_pool_remove(pool, POOL_HIGHPAGE);
if (!page)
break;
dmabuf_page_pool_free_pages(pool, page);
freed += (1 << pool->order);
}
return freed;
}
static int dmabuf_page_pool_shrink(gfp_t gfp_mask, int nr_to_scan)
{
struct dmabuf_page_pool *pool;
int nr_total = 0;
int nr_freed;
int only_scan = 0;
if (!nr_to_scan)
only_scan = 1;
mutex_lock(&pool_list_lock);
list_for_each_entry(pool, &pool_list, list) {
if (only_scan) {
nr_total += dmabuf_page_pool_do_shrink(pool,
gfp_mask,
nr_to_scan);
} else {
nr_freed = dmabuf_page_pool_do_shrink(pool,
gfp_mask,
nr_to_scan);
nr_to_scan -= nr_freed;
nr_total += nr_freed;
if (nr_to_scan <= 0)
break;
}
}
mutex_unlock(&pool_list_lock);
return nr_total;
}
static unsigned long dmabuf_page_pool_shrink_count(struct shrinker *shrinker,
struct shrink_control *sc)
{
return dmabuf_page_pool_shrink(sc->gfp_mask, 0);
}
static unsigned long dmabuf_page_pool_shrink_scan(struct shrinker *shrinker,
struct shrink_control *sc)
{
if (sc->nr_to_scan == 0)
return 0;
return dmabuf_page_pool_shrink(sc->gfp_mask, sc->nr_to_scan);
}
struct shrinker pool_shrinker = {
.count_objects = dmabuf_page_pool_shrink_count,
.scan_objects = dmabuf_page_pool_shrink_scan,
.seeks = DEFAULT_SEEKS,
.batch = 0,
};
static int dmabuf_page_pool_init_shrinker(void)
{
return register_shrinker(&pool_shrinker);
}
module_init(dmabuf_page_pool_init_shrinker);
MODULE_LICENSE("GPL");

55
drivers/dma-buf/heaps/page_pool.h Normal file
View File

@ -0,0 +1,55 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* DMA BUF PagePool implementation
* Based on earlier ION code by Google
*
* Copyright (C) 2011 Google, Inc.
* Copyright (C) 2020 Linaro Ltd.
*/
#ifndef _DMABUF_PAGE_POOL_H
#define _DMABUF_PAGE_POOL_H
#include <linux/device.h>
#include <linux/kref.h>
#include <linux/mm_types.h>
#include <linux/mutex.h>
#include <linux/shrinker.h>
#include <linux/types.h>
/* page types we track in the pool */
enum {
POOL_LOWPAGE, /* Clean lowmem pages */
POOL_HIGHPAGE, /* Clean highmem pages */
POOL_TYPE_SIZE,
};
/**
* struct dmabuf_page_pool - pagepool struct
* @count[]: array of number of pages of that type in the pool
* @items[]: array of list of pages of the specific type
* @mutex: lock protecting this struct and especially the count
* item list
* @gfp_mask: gfp_mask to use from alloc
* @order: order of pages in the pool
* @list: list node for list of pools
*
* Allows you to keep a pool of pre allocated pages to use
*/
struct dmabuf_page_pool {
int count[POOL_TYPE_SIZE];
struct list_head items[POOL_TYPE_SIZE];
struct mutex mutex;
gfp_t gfp_mask;
unsigned int order;
struct list_head list;
};
struct dmabuf_page_pool *dmabuf_page_pool_create(gfp_t gfp_mask,
unsigned int order);
void dmabuf_page_pool_destroy(struct dmabuf_page_pool *pool);
struct page *dmabuf_page_pool_alloc(struct dmabuf_page_pool *pool);
void dmabuf_page_pool_free(struct dmabuf_page_pool *pool, struct page *page);
#endif /* _DMABUF_PAGE_POOL_H */

840
drivers/dma-buf/heaps/rk_system_heap.c Normal file
View File

@ -0,0 +1,840 @@
// SPDX-License-Identifier: GPL-2.0
/*
* DMABUF System heap exporter for Rockchip
*
* Copyright (C) 2011 Google, Inc.
* Copyright (C) 2019, 2020 Linaro Ltd.
* Copyright (c) 2021, 2022 Rockchip Electronics Co. Ltd.
*
* Portions based off of Andrew Davis' SRAM heap:
* Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/
* Andrew F. Davis <afd@ti.com>
*/
#include <linux/dma-buf.h>
#include <linux/dma-mapping.h>
#include <linux/dma-heap.h>
#include <linux/err.h>
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/swiotlb.h>
#include <linux/vmalloc.h>
#include <linux/rockchip/rockchip_sip.h>
#include "page_pool.h"
#include "deferred-free-helper.h"
static struct dma_heap *sys_heap;
static struct dma_heap *sys_dma32_heap;
static struct dma_heap *sys_uncached_heap;
static struct dma_heap *sys_uncached_dma32_heap;
/* Default setting */
static u32 bank_bit_first = 12;
static u32 bank_bit_mask = 0x7;
struct system_heap_buffer {
struct dma_heap *heap;
struct list_head attachments;
struct mutex lock;
unsigned long len;
struct sg_table sg_table;
int vmap_cnt;
void *vaddr;
struct deferred_freelist_item deferred_free;
struct dmabuf_page_pool **pools;
bool uncached;
};
struct dma_heap_attachment {
struct device *dev;
struct sg_table *table;
struct list_head list;
bool mapped;
bool uncached;
};
#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP)
#define MID_ORDER_GFP (LOW_ORDER_GFP | __GFP_NOWARN)
#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \
| __GFP_NORETRY) & ~__GFP_RECLAIM) \
| __GFP_COMP)
static gfp_t order_flags[] = {HIGH_ORDER_GFP, MID_ORDER_GFP, LOW_ORDER_GFP};
/*
* The selection of the orders used for allocation (1MB, 64K, 4K) is designed
* to match with the sizes often found in IOMMUs. Using order 4 pages instead
* of order 0 pages can significantly improve the performance of many IOMMUs
* by reducing TLB pressure and time spent updating page tables.
*/
static unsigned int orders[] = {8, 4, 0};
#define NUM_ORDERS ARRAY_SIZE(orders)
struct dmabuf_page_pool *pools[NUM_ORDERS];
struct dmabuf_page_pool *dma32_pools[NUM_ORDERS];
static struct sg_table *dup_sg_table(struct sg_table *table)
{
struct sg_table *new_table;
int ret, i;
struct scatterlist *sg, *new_sg;
new_table = kzalloc(sizeof(*new_table), GFP_KERNEL);
if (!new_table)
return ERR_PTR(-ENOMEM);
ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL);
if (ret) {
kfree(new_table);
return ERR_PTR(-ENOMEM);
}
new_sg = new_table->sgl;
for_each_sgtable_sg(table, sg, i) {
sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset);
new_sg = sg_next(new_sg);
}
return new_table;
}
static int system_heap_attach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attachment)
{
struct system_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a;
struct sg_table *table;
a = kzalloc(sizeof(*a), GFP_KERNEL);
if (!a)
return -ENOMEM;
table = dup_sg_table(&buffer->sg_table);
if (IS_ERR(table)) {
kfree(a);
return -ENOMEM;
}
a->table = table;
a->dev = attachment->dev;
INIT_LIST_HEAD(&a->list);
a->mapped = false;
a->uncached = buffer->uncached;
attachment->priv = a;
mutex_lock(&buffer->lock);
list_add(&a->list, &buffer->attachments);
mutex_unlock(&buffer->lock);
return 0;
}
static void system_heap_detach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attachment)
{
struct system_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a = attachment->priv;
mutex_lock(&buffer->lock);
list_del(&a->list);
mutex_unlock(&buffer->lock);
sg_free_table(a->table);
kfree(a->table);
kfree(a);
}
static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment,
enum dma_data_direction direction)
{
struct dma_heap_attachment *a = attachment->priv;
struct sg_table *table = a->table;
int attr = attachment->dma_map_attrs;
int ret;
if (a->uncached)
attr |= DMA_ATTR_SKIP_CPU_SYNC;
ret = dma_map_sgtable(attachment->dev, table, direction, attr);
if (ret)
return ERR_PTR(ret);
a->mapped = true;
return table;
}
static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment,
struct sg_table *table,
enum dma_data_direction direction)
{
struct dma_heap_attachment *a = attachment->priv;
int attr = attachment->dma_map_attrs;
if (a->uncached)
attr |= DMA_ATTR_SKIP_CPU_SYNC;
a->mapped = false;
dma_unmap_sgtable(attachment->dev, table, direction, attr);
}
static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
{
struct system_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a;
mutex_lock(&buffer->lock);
if (buffer->vmap_cnt)
invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
if (!buffer->uncached) {
list_for_each_entry(a, &buffer->attachments, list) {
if (!a->mapped)
continue;
dma_sync_sgtable_for_cpu(a->dev, a->table, direction);
}
}
mutex_unlock(&buffer->lock);
return 0;
}
static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
{
struct system_heap_buffer *buffer = dmabuf->priv;
struct dma_heap_attachment *a;
mutex_lock(&buffer->lock);
if (buffer->vmap_cnt)
flush_kernel_vmap_range(buffer->vaddr, buffer->len);
if (!buffer->uncached) {
list_for_each_entry(a, &buffer->attachments, list) {
if (!a->mapped)
continue;
dma_sync_sgtable_for_device(a->dev, a->table, direction);
}
}
mutex_unlock(&buffer->lock);
return 0;
}
static int system_heap_sgl_sync_range(struct device *dev,
struct sg_table *sgt,
unsigned int offset,
unsigned int length,
enum dma_data_direction dir,
bool for_cpu)
{
struct scatterlist *sg;
unsigned int len = 0;
dma_addr_t sg_dma_addr;
int i;
for_each_sgtable_sg(sgt, sg, i) {
unsigned int sg_offset, sg_left, size = 0;
sg_dma_addr = sg_phys(sg);
len += sg->length;
if (len <= offset)
continue;
sg_left = len - offset;
sg_offset = sg->length - sg_left;
size = (length < sg_left) ? length : sg_left;
if (for_cpu)
dma_sync_single_range_for_cpu(dev, sg_dma_addr,
sg_offset, size, dir);
else
dma_sync_single_range_for_device(dev, sg_dma_addr,
sg_offset, size, dir);
offset += size;
length -= size;
if (length == 0)
break;
}
return 0;
}
static int
system_heap_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf,
enum dma_data_direction direction,
unsigned int offset,
unsigned int len)
{
struct system_heap_buffer *buffer = dmabuf->priv;
struct dma_heap *heap = buffer->heap;
struct sg_table *table = &buffer->sg_table;
int ret;
if (direction == DMA_TO_DEVICE)
return 0;
mutex_lock(&buffer->lock);
if (buffer->vmap_cnt)
invalidate_kernel_vmap_range(buffer->vaddr, buffer->len);
if (buffer->uncached) {
mutex_unlock(&buffer->lock);
return 0;
}
ret = system_heap_sgl_sync_range(dma_heap_get_dev(heap), table,
offset, len, direction, true);
mutex_unlock(&buffer->lock);
return ret;
}
static int
system_heap_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf,
enum dma_data_direction direction,
unsigned int offset,
unsigned int len)
{
struct system_heap_buffer *buffer = dmabuf->priv;
struct dma_heap *heap = buffer->heap;
struct sg_table *table = &buffer->sg_table;
int ret;
mutex_lock(&buffer->lock);
if (buffer->vmap_cnt)
flush_kernel_vmap_range(buffer->vaddr, buffer->len);
if (buffer->uncached) {
mutex_unlock(&buffer->lock);
return 0;
}
ret = system_heap_sgl_sync_range(dma_heap_get_dev(heap), table,
offset, len, direction, false);
mutex_unlock(&buffer->lock);
return ret;
}
static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
{
struct system_heap_buffer *buffer = dmabuf->priv;
struct sg_table *table = &buffer->sg_table;
unsigned long addr = vma->vm_start;
struct sg_page_iter piter;
int ret;
if (buffer->uncached)
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
for_each_sgtable_page(table, &piter, vma->vm_pgoff) {
struct page *page = sg_page_iter_page(&piter);
ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE,
vma->vm_page_prot);
if (ret)
return ret;
addr += PAGE_SIZE;
if (addr >= vma->vm_end)
return 0;
}
return 0;
}
static void *system_heap_do_vmap(struct system_heap_buffer *buffer)
{
struct sg_table *table = &buffer->sg_table;
int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
struct page **pages = vmalloc(sizeof(struct page *) * npages);
struct page **tmp = pages;
struct sg_page_iter piter;
pgprot_t pgprot = PAGE_KERNEL;
void *vaddr;
if (!pages)
return ERR_PTR(-ENOMEM);
if (buffer->uncached)
pgprot = pgprot_writecombine(PAGE_KERNEL);
for_each_sgtable_page(table, &piter, 0) {
WARN_ON(tmp - pages >= npages);
*tmp++ = sg_page_iter_page(&piter);
}
vaddr = vmap(pages, npages, VM_MAP, pgprot);
vfree(pages);
if (!vaddr)
return ERR_PTR(-ENOMEM);
return vaddr;
}
static void *system_heap_vmap(struct dma_buf *dmabuf)
{
struct system_heap_buffer *buffer = dmabuf->priv;
void *vaddr;
mutex_lock(&buffer->lock);
if (buffer->vmap_cnt) {
buffer->vmap_cnt++;
vaddr = buffer->vaddr;
goto out;
}
vaddr = system_heap_do_vmap(buffer);
if (IS_ERR(vaddr))
goto out;
buffer->vaddr = vaddr;
buffer->vmap_cnt++;
out:
mutex_unlock(&buffer->lock);
return vaddr;
}
static void system_heap_vunmap(struct dma_buf *dmabuf, void *vaddr)
{
struct system_heap_buffer *buffer = dmabuf->priv;
mutex_lock(&buffer->lock);
if (!--buffer->vmap_cnt) {
vunmap(buffer->vaddr);
buffer->vaddr = NULL;
}
mutex_unlock(&buffer->lock);
}
static int system_heap_zero_buffer(struct system_heap_buffer *buffer)
{
struct sg_table *sgt = &buffer->sg_table;
struct sg_page_iter piter;
struct page *p;
void *vaddr;
int ret = 0;
for_each_sgtable_page(sgt, &piter, 0) {
p = sg_page_iter_page(&piter);
vaddr = kmap_atomic(p);
memset(vaddr, 0, PAGE_SIZE);
kunmap_atomic(vaddr);
}
return ret;
}
static void system_heap_buf_free(struct deferred_freelist_item *item,
enum df_reason reason)
{
struct system_heap_buffer *buffer;
struct sg_table *table;
struct scatterlist *sg;
int i, j;
buffer = container_of(item, struct system_heap_buffer, deferred_free);
/* Zero the buffer pages before adding back to the pool */
if (reason == DF_NORMAL)
if (system_heap_zero_buffer(buffer))
reason = DF_UNDER_PRESSURE; // On failure, just free
table = &buffer->sg_table;
for_each_sgtable_sg(table, sg, i) {
struct page *page = sg_page(sg);
if (reason == DF_UNDER_PRESSURE) {
__free_pages(page, compound_order(page));
} else {
for (j = 0; j < NUM_ORDERS; j++) {
if (compound_order(page) == orders[j])
break;
}
dmabuf_page_pool_free(buffer->pools[j], page);
}
}
sg_free_table(table);
kfree(buffer);
}
static void system_heap_dma_buf_release(struct dma_buf *dmabuf)
{
struct system_heap_buffer *buffer = dmabuf->priv;
int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE;
deferred_free(&buffer->deferred_free, system_heap_buf_free, npages);
}
static const struct dma_buf_ops system_heap_buf_ops = {
.attach = system_heap_attach,
.detach = system_heap_detach,
.map_dma_buf = system_heap_map_dma_buf,
.unmap_dma_buf = system_heap_unmap_dma_buf,
.begin_cpu_access = system_heap_dma_buf_begin_cpu_access,
.end_cpu_access = system_heap_dma_buf_end_cpu_access,
.begin_cpu_access_partial = system_heap_dma_buf_begin_cpu_access_partial,
.end_cpu_access_partial = system_heap_dma_buf_end_cpu_access_partial,
.mmap = system_heap_mmap,
.vmap = system_heap_vmap,
.vunmap = system_heap_vunmap,
.release = system_heap_dma_buf_release,
};
static struct page *system_heap_alloc_largest_available(struct dma_heap *heap,
struct dmabuf_page_pool **pool,
unsigned long size,
unsigned int max_order)
{
struct page *page;
int i;
for (i = 0; i < NUM_ORDERS; i++) {
if (size < (PAGE_SIZE << orders[i]))
continue;
if (max_order < orders[i])
continue;
page = dmabuf_page_pool_alloc(pool[i]);
if (!page)
continue;
return page;
}
return NULL;
}
static struct dma_buf *system_heap_do_allocate(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags,
bool uncached)
{
struct system_heap_buffer *buffer;
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
unsigned long size_remaining = len;
unsigned int max_order = orders[0];
struct dma_buf *dmabuf;
struct sg_table *table;
struct scatterlist *sg;
struct list_head pages;
struct page *page, *tmp_page;
int i, ret = -ENOMEM;
struct list_head lists[8];
unsigned int block_index[8] = {0};
unsigned int block_1M = 0;
unsigned int block_64K = 0;
unsigned int maximum;
int j;
buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
if (!buffer)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&buffer->attachments);
mutex_init(&buffer->lock);
buffer->heap = heap;
buffer->len = len;
buffer->uncached = uncached;
buffer->pools = strstr(dma_heap_get_name(heap), "dma32") ? dma32_pools : pools;
INIT_LIST_HEAD(&pages);
for (i = 0; i < 8; i++)
INIT_LIST_HEAD(&lists[i]);
i = 0;
while (size_remaining > 0) {
/*
* Avoid trying to allocate memory if the process
* has been killed by SIGKILL
*/
if (fatal_signal_pending(current))
goto free_buffer;
page = system_heap_alloc_largest_available(heap, buffer->pools,
size_remaining,
max_order);
if (!page)
goto free_buffer;
size_remaining -= page_size(page);
max_order = compound_order(page);
if (max_order) {
if (max_order == 8)
block_1M++;
if (max_order == 4)
block_64K++;
list_add_tail(&page->lru, &pages);
} else {
dma_addr_t phys = page_to_phys(page);
unsigned int bit_index = ((phys >> bank_bit_first) & bank_bit_mask) & 0x7;
list_add_tail(&page->lru, &lists[bit_index]);
block_index[bit_index]++;
}
i++;
}
table = &buffer->sg_table;
if (sg_alloc_table(table, i, GFP_KERNEL))
goto free_buffer;
maximum = block_index[0];
for (i = 1; i < 8; i++)
maximum = max(maximum, block_index[i]);
sg = table->sgl;
list_for_each_entry_safe(page, tmp_page, &pages, lru) {
sg_set_page(sg, page, page_size(page), 0);
sg = sg_next(sg);
list_del(&page->lru);
}
for (i = 0; i < maximum; i++) {
for (j = 0; j < 8; j++) {
if (!list_empty(&lists[j])) {
page = list_first_entry(&lists[j], struct page, lru);
sg_set_page(sg, page, PAGE_SIZE, 0);
sg = sg_next(sg);
list_del(&page->lru);
}
}
}
/* create the dmabuf */
exp_info.exp_name = dma_heap_get_name(heap);
exp_info.ops = &system_heap_buf_ops;
exp_info.size = buffer->len;
exp_info.flags = fd_flags;
exp_info.priv = buffer;
dmabuf = dma_buf_export(&exp_info);
if (IS_ERR(dmabuf)) {
ret = PTR_ERR(dmabuf);
goto free_pages;
}
/*
* For uncached buffers, we need to initially flush cpu cache, since
* the __GFP_ZERO on the allocation means the zeroing was done by the
* cpu and thus it is likely cached. Map (and implicitly flush) and
* unmap it now so we don't get corruption later on.
*/
if (buffer->uncached) {
dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0);
}
return dmabuf;
free_pages:
for_each_sgtable_sg(table, sg, i) {
struct page *p = sg_page(sg);
__free_pages(p, compound_order(p));
}
sg_free_table(table);
free_buffer:
list_for_each_entry_safe(page, tmp_page, &pages, lru)
__free_pages(page, compound_order(page));
for (i = 0; i < 8; i++) {
list_for_each_entry_safe(page, tmp_page, &lists[i], lru)
__free_pages(page, compound_order(page));
}
kfree(buffer);
return ERR_PTR(ret);
}
static struct dma_buf *system_heap_allocate(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags)
{
return system_heap_do_allocate(heap, len, fd_flags, heap_flags, false);
}
static long system_get_pool_size(struct dma_heap *heap)
{
int i;
long num_pages = 0;
struct dmabuf_page_pool **pool;
pool = strstr(dma_heap_get_name(heap), "dma32") ? dma32_pools : pools;
for (i = 0; i < NUM_ORDERS; i++, pool++) {
num_pages += ((*pool)->count[POOL_LOWPAGE] +
(*pool)->count[POOL_HIGHPAGE]) << (*pool)->order;
}
return num_pages << PAGE_SHIFT;
}
static const struct dma_heap_ops system_heap_ops = {
.allocate = system_heap_allocate,
.get_pool_size = system_get_pool_size,
};
static struct dma_buf *system_uncached_heap_allocate(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags)
{
return system_heap_do_allocate(heap, len, fd_flags, heap_flags, true);
}
/* Dummy function to be used until we can call coerce_mask_and_coherent */
static struct dma_buf *system_uncached_heap_not_initialized(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags)
{
return ERR_PTR(-EBUSY);
}
static struct dma_heap_ops system_uncached_heap_ops = {
/* After system_heap_create is complete, we will swap this */
.allocate = system_uncached_heap_not_initialized,
};
static int set_heap_dev_dma(struct device *heap_dev)
{
int err = 0;
if (!heap_dev)
return -EINVAL;
dma_coerce_mask_and_coherent(heap_dev, DMA_BIT_MASK(64));
if (!heap_dev->dma_parms) {
heap_dev->dma_parms = devm_kzalloc(heap_dev,
sizeof(*heap_dev->dma_parms),
GFP_KERNEL);
if (!heap_dev->dma_parms)
return -ENOMEM;
err = dma_set_max_seg_size(heap_dev, (unsigned int)DMA_BIT_MASK(64));
if (err) {
devm_kfree(heap_dev, heap_dev->dma_parms);
dev_err(heap_dev, "Failed to set DMA segment size, err:%d\n", err);
return err;
}
}
return 0;
}
static int system_heap_create(void)
{
struct dma_heap_export_info exp_info;
int i, err = 0;
struct dram_addrmap_info *ddr_map_info;
/*
* Since swiotlb has memory size limitation, this will calculate
* the maximum size locally.
*
* Once swiotlb_max_segment() return not '0', means that the totalram size
* is larger than 4GiB and swiotlb is not force mode, in this case, system
* heap should limit largest allocation.
*
* FIX: fix the orders[] as a workaround.
*/
if (swiotlb_max_segment()) {
unsigned int max_size = (1 << IO_TLB_SHIFT) * IO_TLB_SEGSIZE;
int max_order = MAX_ORDER;
int i;
max_size = max_t(unsigned int, max_size, PAGE_SIZE) >> PAGE_SHIFT;
max_order = min(max_order, ilog2(max_size));
for (i = 0; i < NUM_ORDERS; i++) {
if (max_order < orders[i])
orders[i] = max_order;
pr_info("system_heap: orders[%d] = %u\n", i, orders[i]);
}
}
for (i = 0; i < NUM_ORDERS; i++) {
pools[i] = dmabuf_page_pool_create(order_flags[i], orders[i]);
if (!pools[i]) {
int j;
pr_err("%s: page pool creation failed!\n", __func__);
for (j = 0; j < i; j++)
dmabuf_page_pool_destroy(pools[j]);
return -ENOMEM;
}
}
for (i = 0; i < NUM_ORDERS; i++) {
dma32_pools[i] = dmabuf_page_pool_create(order_flags[i] | GFP_DMA32, orders[i]);
if (!dma32_pools[i]) {
int j;
pr_err("%s: page dma32 pool creation failed!\n", __func__);
for (j = 0; j < i; j++)
dmabuf_page_pool_destroy(dma32_pools[j]);
goto err_dma32_pool;
}
}
exp_info.name = "system";
exp_info.ops = &system_heap_ops;
exp_info.priv = NULL;
sys_heap = dma_heap_add(&exp_info);
if (IS_ERR(sys_heap))
return PTR_ERR(sys_heap);
exp_info.name = "system-dma32";
exp_info.ops = &system_heap_ops;
exp_info.priv = NULL;
sys_dma32_heap = dma_heap_add(&exp_info);
if (IS_ERR(sys_dma32_heap))
return PTR_ERR(sys_dma32_heap);
exp_info.name = "system-uncached";
exp_info.ops = &system_uncached_heap_ops;
exp_info.priv = NULL;
sys_uncached_heap = dma_heap_add(&exp_info);
if (IS_ERR(sys_uncached_heap))
return PTR_ERR(sys_uncached_heap);
err = set_heap_dev_dma(dma_heap_get_dev(sys_uncached_heap));
if (err)
return err;
exp_info.name = "system-uncached-dma32";
exp_info.ops = &system_uncached_heap_ops;
exp_info.priv = NULL;
sys_uncached_dma32_heap = dma_heap_add(&exp_info);
if (IS_ERR(sys_uncached_dma32_heap))
return PTR_ERR(sys_uncached_dma32_heap);
err = set_heap_dev_dma(dma_heap_get_dev(sys_uncached_dma32_heap));
if (err)
return err;
dma_coerce_mask_and_coherent(dma_heap_get_dev(sys_uncached_dma32_heap), DMA_BIT_MASK(32));
mb(); /* make sure we only set allocate after dma_mask is set */
system_uncached_heap_ops.allocate = system_uncached_heap_allocate;
ddr_map_info = sip_smc_get_dram_map();
if (ddr_map_info) {
bank_bit_first = ddr_map_info->bank_bit_first;
bank_bit_mask = ddr_map_info->bank_bit_mask;
}
return 0;
err_dma32_pool:
for (i = 0; i < NUM_ORDERS; i++)
dmabuf_page_pool_destroy(pools[i]);
return -ENOMEM;
}
module_init(system_heap_create);
MODULE_LICENSE("GPL v2");

128
include/linux/dma-heap.h Normal file
View File

@ -0,0 +1,128 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
* DMABUF Heaps Allocation Infrastructure
*
* Copyright (C) 2011 Google, Inc.
* Copyright (C) 2019 Linaro Ltd.
*/
#ifndef _DMA_HEAPS_H
#define _DMA_HEAPS_H
#include <linux/cdev.h>
#include <linux/types.h>
#include <uapi/linux/dma-heap.h>
struct dma_heap;
/**
* struct dma_heap_ops - ops to operate on a given heap
* @allocate: allocate dmabuf and return struct dma_buf ptr
* @get_pool_size: if heap maintains memory pools, get pool size in bytes
*
* allocate returns dmabuf on success, ERR_PTR(-errno) on error.
*/
struct dma_heap_ops {
struct dma_buf *(*allocate)(struct dma_heap *heap,
unsigned long len,
unsigned long fd_flags,
unsigned long heap_flags);
long (*get_pool_size)(struct dma_heap *heap);
#if IS_ENABLED(CONFIG_NO_GKI)
int (*get_phys)(struct dma_heap *heap, struct dma_heap_phys_data *phys);
#endif
};
/**
* struct dma_heap_export_info - information needed to export a new dmabuf heap
* @name: used for debugging/device-node name
* @ops: ops struct for this heap
* @priv: heap exporter private data
*
* Information needed to export a new dmabuf heap.
*/
struct dma_heap_export_info {
const char *name;
const struct dma_heap_ops *ops;
void *priv;
};
/**
* dma_heap_get_drvdata() - get per-heap driver data
* @heap: DMA-Heap to retrieve private data for
*
* Returns:
* The per-heap data for the heap.
*/
void *dma_heap_get_drvdata(struct dma_heap *heap);
/**
* dma_heap_get_dev() - get device struct for the heap
* @heap: DMA-Heap to retrieve device struct from
*
* Returns:
* The device struct for the heap.
*/
struct device *dma_heap_get_dev(struct dma_heap *heap);
/**
* dma_heap_get_name() - get heap name
* @heap: DMA-Heap to retrieve private data for
*
* Returns:
* The char* for the heap name.
*/
const char *dma_heap_get_name(struct dma_heap *heap);
/**
* dma_heap_add - adds a heap to dmabuf heaps
* @exp_info: information needed to register this heap
*/
struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info);
/**
* dma_heap_put - drops a reference to a dmabuf heaps, potentially freeing it
* @heap: heap pointer
*/
void dma_heap_put(struct dma_heap *heap);
/**
* dma_heap_find - Returns the registered dma_heap with the specified name
* @name: Name of the heap to find
*
* NOTE: dma_heaps returned from this function MUST be released
* using dma_heap_put() when the user is done.
*/
struct dma_heap *dma_heap_find(const char *name);
/**
* dma_heap_buffer_alloc - Allocate dma-buf from a dma_heap
* @heap: dma_heap to allocate from
* @len: size to allocate
* @fd_flags: flags to set on returned dma-buf fd
* @heap_flags: flags to pass to the dma heap
*
* This is for internal dma-buf allocations only.
*/
struct dma_buf *dma_heap_buffer_alloc(struct dma_heap *heap, size_t len,
unsigned int fd_flags,
unsigned int heap_flags);
/** dma_heap_buffer_free - Free dma_buf allocated by dma_heap_buffer_alloc
* @dma_buf: dma_buf to free
*
* This is really only a simple wrapper to dma_buf_put()
*/
void dma_heap_buffer_free(struct dma_buf *);
/**
* dma_heap_bufferfd_alloc - Allocate dma-buf fd from a dma_heap
* @heap: dma_heap to allocate from
* @len: size to allocate
* @fd_flags: flags to set on returned dma-buf fd
* @heap_flags: flags to pass to the dma heap
*/
int dma_heap_bufferfd_alloc(struct dma_heap *heap, size_t len,
unsigned int fd_flags,
unsigned int heap_flags);
#endif /* _DMA_HEAPS_H */

61
include/uapi/linux/dma-heap.h Normal file
View File

@ -0,0 +1,61 @@
/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
/*
* DMABUF Heaps Userspace API
*
* Copyright (C) 2011 Google, Inc.
* Copyright (C) 2019 Linaro Ltd.
*/
#ifndef _UAPI_LINUX_DMABUF_POOL_H
#define _UAPI_LINUX_DMABUF_POOL_H
#include <linux/ioctl.h>
#include <linux/types.h>
/**
* DOC: DMABUF Heaps Userspace API
*/
/* Valid FD_FLAGS are O_CLOEXEC, O_RDONLY, O_WRONLY, O_RDWR */
#define DMA_HEAP_VALID_FD_FLAGS (O_CLOEXEC | O_ACCMODE)
/* Currently no heap flags */
#define DMA_HEAP_VALID_HEAP_FLAGS (0)
/**
* struct dma_heap_allocation_data - metadata passed from userspace for
* allocations
* @len: size of the allocation
* @fd: will be populated with a fd which provides the
* handle to the allocated dma-buf
* @fd_flags: file descriptor flags used when allocating
* @heap_flags: flags passed to heap
*
* Provided by userspace as an argument to the ioctl
*/
struct dma_heap_allocation_data {
__u64 len;
__u32 fd;
__u32 fd_flags;
__u64 heap_flags;
};
struct dma_heap_phys_data {
__u64 paddr;
__u32 fd;
};
#define DMA_HEAP_IOC_MAGIC 'H'
/**
* DOC: DMA_HEAP_IOCTL_ALLOC - allocate memory from pool
*
* Takes a dma_heap_allocation_data struct and returns it with the fd field
* populated with the dmabuf handle of the allocation.
*/
#define DMA_HEAP_IOCTL_ALLOC _IOWR(DMA_HEAP_IOC_MAGIC, 0x0,\
struct dma_heap_allocation_data)
#define DMA_HEAP_IOCTL_GET_PHYS _IOWR(DMA_HEAP_IOC_MAGIC, 0x1, \
struct dma_heap_phys_data)
#endif /* _UAPI_LINUX_DMABUF_POOL_H */