From: JiangShui Yang yangjiangshui@h-partners.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I94L2V CVE: NA

----------------------------------------------------------------------

The UACCE supports the UACCE_MODE_NOIOMMU mode, device can register to uacce subsystem in UACCE_MODE_NOIOMMU mode, and then the user can directly access the device in the user space.

In addition, the UACCE provides an interface for allocating the DMA memory. Users can call the mmap() in user space to allocate the DMA memory, read and write the DMA memory in the user space and send the DMA address to the hardware for IO task.

Signed-off-by: JiangShui Yang yangjiangshui@h-partners.com

--- drivers/misc/uacce/Kconfig | 3 +- drivers/misc/uacce/uacce.c | 450 +++++++++++++++++++++++++++++++- include/linux/uacce.h | 31 ++- include/uapi/misc/uacce/uacce.h | 53 +++- 4 files changed, 516 insertions(+), 21 deletions(-)

diff --git a/drivers/misc/uacce/Kconfig b/drivers/misc/uacce/Kconfig index 5e39b6083b0f..f0d02be5a057 100644 --- a/drivers/misc/uacce/Kconfig +++ b/drivers/misc/uacce/Kconfig @@ -1,6 +1,7 @@ -config UACCE +menuconfig UACCE tristate "Accelerator Framework for User Land" depends on IOMMU_API + select ANON_INODES help UACCE provides interface for the user process to access the hardware without interaction with the kernel space in data path. diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c index bdc2e6fda782..2f07a12e93db 100644 --- a/drivers/misc/uacce/uacce.c +++ b/drivers/misc/uacce/uacce.c @@ -1,19 +1,26 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include <linux/compat.h> #include <linux/dma-mapping.h> +#include <linux/file.h> #include <linux/iommu.h> #include <linux/module.h> #include <linux/poll.h> #include <linux/slab.h> #include <linux/uacce.h> +#include <linux/wait.h>

static dev_t uacce_devt; static DEFINE_XARRAY_ALLOC(uacce_xa); +static const struct file_operations uacce_fops;

static const struct class uacce_class = { .name = UACCE_NAME, };

+static struct uacce_qfile_region noiommu_ss_default_qfr = { + .type = UACCE_QFRT_SS, +}; + /* * If the parent driver or the device disappears, the queue state is invalid and * ops are not usable anymore. @@ -56,6 +63,141 @@ static int uacce_put_queue(struct uacce_queue *q) return 0; }

+static long uacce_cmd_share_qfr(struct uacce_queue *src, int fd) +{ + struct device *dev = &src->uacce->dev; + struct file *filep = fget(fd); + struct uacce_queue *tgt; + int ret = -EINVAL; + + if (!filep) { + dev_err(dev, "filep is NULL!\n"); + return ret; + } + + if (filep->f_op != &uacce_fops) { + dev_err(dev, "file ops mismatch!\n"); + goto out_with_fd; + } + + tgt = filep->private_data; + if (!tgt) { + dev_err(dev, "target queue is not exist!\n"); + goto out_with_fd; + } + + mutex_lock(&src->mutex); + if (tgt->state == UACCE_Q_ZOMBIE || src->state == UACCE_Q_ZOMBIE) { + dev_err(dev, "target or source queue is zombie!\n"); + goto out_with_fd; + } + + if (!src->qfrs[UACCE_QFRT_SS] || tgt->qfrs[UACCE_QFRT_SS]) { + dev_err(dev, "src q's SS not exists or target q's SS exists!\n"); + goto out_with_fd; + } + + /* In No-IOMMU mode, taget queue uses default SS qfr */ + tgt->qfrs[UACCE_QFRT_SS] = &noiommu_ss_default_qfr; + + ret = 0; + +out_with_fd: + mutex_unlock(&src->mutex); + fput(filep); + + return ret; +} + +static long uacce_get_ss_dma(struct uacce_queue *q, void __user *arg) +{ + struct uacce_device *uacce = q->uacce; + struct uacce_dma_slice *slice; + unsigned long slice_idx = 0; + unsigned long dma, size; + unsigned int max_idx; + long ret = -EFAULT; + + if (q->state == UACCE_Q_ZOMBIE) { + dev_err(&uacce->dev, "queue is zombie!\n"); + ret = -EINVAL; + goto param_check; + } + + if (!q->qfrs[UACCE_QFRT_SS]) { + dev_err(&uacce->dev, "no ss dma region!\n"); + ret = -EINVAL; + goto param_check; + } + + slice = q->qfrs[UACCE_QFRT_SS]->dma_list; + if (copy_from_user(&slice_idx, arg, sizeof(unsigned long))) { + dev_err(&uacce->dev, "copy_from_user fail!\n"); + goto param_check; + } + + if (slice[0].total_num - 1 < slice_idx) { + dev_err(&uacce->dev, "no ss slice idx %lu err, total %u!\n", + slice_idx, slice[0].total_num); + ret = -EINVAL; + goto param_check; + } + + dma = slice[slice_idx].dma; + size = slice[slice_idx].size; + if (!size) { + max_idx = slice[0].total_num - 1; + dev_err(&uacce->dev, "%luth ss region[size = %lu] no exist, range[[0](size = %llu) -> [%u](size = %llu)]\n", + slice_idx, size, slice[0].size, max_idx, slice[max_idx].size); + ret = -ENODEV; + goto param_check; + } + dma = dma | ((size >> UACCE_GRAN_SHIFT) & UACCE_GRAN_NUM_MASK); + if (copy_to_user(arg, &dma, sizeof(unsigned long))) { + dev_err(&uacce->dev, "copy_to_user fail!\n"); + goto param_check; + } + + ret = (long)(slice[0].total_num - 1 - slice_idx); + +param_check: + return ret; +} + +static void uacce_free_dma_buffers(struct uacce_queue *q) +{ + struct uacce_qfile_region *qfr = q->qfrs[UACCE_QFRT_SS]; + struct device *pdev = q->uacce->parent; + int i = 0; + + if (!qfr->dma_list) + return; + + while (i < qfr->dma_list[0].total_num) { + WARN_ON(!qfr->dma_list[i].size || !qfr->dma_list[i].dma); + dev_dbg(pdev, "free dma qfr (index = %d)\n", i); + dma_free_coherent(pdev, qfr->dma_list[i].size, + qfr->dma_list[i].kaddr, + qfr->dma_list[i].dma); + i++; + } + kfree(qfr->dma_list); + qfr->dma_list = NULL; +} + +/** + * uacce_wake_up - Wake up the process who is waiting this queue + * @q: the accelerator queue to wake up + */ +void uacce_wake_up(struct uacce_queue *q) +{ + if (unlikely(!q)) + return; + + wake_up_interruptible(&q->wait); +} +EXPORT_SYMBOL_GPL(uacce_wake_up); + static long uacce_fops_unl_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { @@ -82,6 +224,12 @@ static long uacce_fops_unl_ioctl(struct file *filep, case UACCE_CMD_PUT_Q: ret = uacce_put_queue(q); break; + case UACCE_CMD_SHARE_SVAS: + ret = uacce_cmd_share_qfr(q, (int)arg); + break; + case UACCE_CMD_GET_SS_DMA: + ret = uacce_get_ss_dma(q, (void __user *)(uintptr_t)arg); + break; default: if (uacce->ops->ioctl) ret = uacce->ops->ioctl(q, cmd, arg); @@ -97,7 +245,7 @@ static long uacce_fops_unl_ioctl(struct file *filep, static long uacce_fops_compat_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) { - arg = (unsigned long)compat_ptr(arg); + arg = (unsigned long)(uintptr_t)compat_ptr(arg);

return uacce_fops_unl_ioctl(filep, cmd, arg); } @@ -160,6 +308,7 @@ static int uacce_fops_open(struct inode *inode, struct file *filep) goto out_with_mem;

q->uacce = uacce; + q->filep = filep;

if (uacce->ops->get_queue) { ret = uacce->ops->get_queue(uacce, q->pasid, q); @@ -189,10 +338,16 @@ static int uacce_fops_release(struct inode *inode, struct file *filep) { struct uacce_queue *q = filep->private_data; struct uacce_device *uacce = q->uacce; + struct uacce_qfile_region *ss;

mutex_lock(&uacce->mutex); uacce_put_queue(q); uacce_unbind_queue(q); + ss = q->qfrs[UACCE_QFRT_SS]; + if (ss && ss != &noiommu_ss_default_qfr) { + uacce_free_dma_buffers(q); + kfree(ss); + } list_del(&q->list); mutex_unlock(&uacce->mutex); kfree(q); @@ -203,10 +358,33 @@ static int uacce_fops_release(struct inode *inode, struct file *filep) static void uacce_vma_close(struct vm_area_struct *vma) { struct uacce_queue *q = vma->vm_private_data; + struct uacce_qfile_region *qfr = NULL; + struct uacce_device *uacce = q->uacce; + struct device *dev = &q->uacce->dev; + + if (vma->vm_pgoff >= UACCE_MAX_REGION) + return;

- if (vma->vm_pgoff < UACCE_MAX_REGION) { - struct uacce_qfile_region *qfr = q->qfrs[vma->vm_pgoff]; + qfr = q->qfrs[vma->vm_pgoff]; + if (!qfr) { + dev_err(dev, "qfr NULL, type %lu!\n", vma->vm_pgoff); + return; + }

+ if (qfr->type == UACCE_QFRT_SS && + atomic_read(&current->active_mm->mm_users) > 0) { + /* + * uacce_vma_close() and uacce_remove() may be executed concurrently. + * To avoid accessing the same address at the same time, takes the uacce->mutex. + */ + mutex_lock(&uacce->mutex); + if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue) + uacce->ops->stop_queue(q); + uacce_free_dma_buffers(q); + q->qfrs[vma->vm_pgoff] = NULL; + mutex_unlock(&uacce->mutex); + kfree(qfr); + } else if (qfr->type != UACCE_QFRT_SS) { mutex_lock(&q->mutex); q->qfrs[vma->vm_pgoff] = NULL; mutex_unlock(&q->mutex); @@ -218,6 +396,212 @@ static const struct vm_operations_struct uacce_vm_ops = { .close = uacce_vma_close, };

+static int get_sort_base(struct uacce_dma_slice *list, int low, int high, + struct uacce_dma_slice *tmp) +{ + tmp->kaddr = list[low].kaddr; + tmp->size = list[low].size; + tmp->dma = list[low].dma; + + if (low > high) + return -EINVAL; + else if (low == high) + return 0; + + while (low < high) { + while (low < high && list[high].dma > tmp->dma) + high--; + list[low].kaddr = list[high].kaddr; + list[low].dma = list[high].dma; + list[low].size = list[high].size; + while (low < high && list[low].dma < tmp->dma) + low++; + list[high].kaddr = list[low].kaddr; + list[high].dma = list[low].dma; + list[high].size = list[low].size; + } + list[low].kaddr = tmp->kaddr; + list[low].dma = tmp->dma; + list[low].size = tmp->size; + + return low; +} + +static int uacce_sort_dma_buffers(struct uacce_dma_slice *list, int low, + int high, struct uacce_dma_slice *tmp) +{ + int *idx_list; + int top = 0; + int pilot; + + idx_list = kcalloc(list[0].total_num, sizeof(int), + GFP_KERNEL | __GFP_ZERO); + if (!idx_list) + return -ENOMEM; + + pilot = get_sort_base(list, low, high, tmp); + if (pilot <= 0) { + if (pilot) + pr_err("fail to sort base!\n"); + kfree(idx_list); + return pilot; + } + + if (pilot > low + 1) { + idx_list[top++] = low; + idx_list[top++] = pilot - 1; + } + if (pilot < high - 1) { + idx_list[top++] = pilot + 1; + idx_list[top++] = high; + } + while (top > 0) { + high = idx_list[--top]; + low = idx_list[--top]; + pilot = get_sort_base(list, low, high, tmp); + if (pilot > low + 1) { + idx_list[top++] = low; + idx_list[top++] = pilot - 1; + } + if (pilot < high - 1) { + idx_list[top++] = pilot + 1; + idx_list[top++] = high; + } + } + + kfree(idx_list); + return 0; +} + +static int uacce_alloc_dma_buffers(struct uacce_queue *q, + struct vm_area_struct *vma) +{ + struct uacce_qfile_region *qfr = q->qfrs[UACCE_QFRT_SS]; + unsigned long size = vma->vm_end - vma->vm_start; + unsigned long max_size = PAGE_SIZE << (MAX_ORDER - 1); + struct device *pdev = q->uacce->parent; + struct uacce_device *uacce = q->uacce; + unsigned long start = vma->vm_start; + struct uacce_dma_slice *slice; + unsigned long ss_num; + int ret, i; + + /* + * When IOMMU closed, set maximum slice size is 128M, default is 4M + * when IOMMU opened, set maxinum slice size base on actual size + */ + if (uacce->flags & UACCE_DEV_IOMMU) + max_size = size; + else if (max_size > UACCE_GRAN_NUM_MASK << UACCE_GRAN_SHIFT) + max_size = (UACCE_GRAN_NUM_MASK + 1) << (UACCE_GRAN_SHIFT - 1); + + ss_num = size / max_size + (size % max_size ? 1 : 0); + slice = kcalloc(ss_num + 1, sizeof(*slice), GFP_KERNEL | __GFP_ZERO); + if (!slice) + return -ENOMEM; + + qfr->dma_list = slice; + for (i = 0; i < ss_num; i++) { + if (start + max_size > vma->vm_end) + size = vma->vm_end - start; + else + size = max_size; + dev_dbg(pdev, "allocate dma %ld pages\n", + (size + PAGE_SIZE - 1) >> PAGE_SHIFT); + slice[i].kaddr = dma_alloc_coherent(pdev, (size + + PAGE_SIZE - 1) & PAGE_MASK, + &slice[i].dma, GFP_KERNEL); + if (!slice[i].kaddr) { + dev_err(pdev, "get dma slice(sz = %lu, dma index = %d) fail!\n", + size, i); + slice[0].total_num = i; + ret = -ENOMEM; + goto free_buffer; + } + slice[i].size = (size + PAGE_SIZE - 1) & PAGE_MASK; + slice[i].total_num = ss_num; + start += size; + } + + ret = uacce_sort_dma_buffers(slice, 0, slice[0].total_num - 1, + &slice[ss_num]); + if (ret) { + dev_err(pdev, "failed to sort dma buffers.\n"); + goto free_buffer; + } + + return 0; +free_buffer: + uacce_free_dma_buffers(q); + + return ret; +} + +static int uacce_mmap_dma_buffers(struct uacce_queue *q, + struct vm_area_struct *vma) +{ + struct uacce_qfile_region *qfr = q->qfrs[UACCE_QFRT_SS]; + struct uacce_dma_slice *slice = qfr->dma_list; + struct device *pdev = q->uacce->parent; + unsigned long vm_pgoff; + int ret = 0; + int i = 0; + + /* + * dma_mmap_coherent() requires vm_pgoff as 0 + * restore vm_pfoff to initial value for mmap() + */ + vm_pgoff = vma->vm_pgoff; + vma->vm_pgoff = 0; + while (i < slice[0].total_num && slice[i].size) { + vma->vm_end = vma->vm_start + slice[i].size; + ret = dma_mmap_coherent(pdev, vma, slice[i].kaddr, + slice[i].dma, + slice[i].size); + if (ret) { + dev_err(pdev, "dma mmap fail(dma index = %d, size = %llu)!\n", + i, slice[i].size); + goto DMA_MMAP_FAIL; + } + + i++; + vma->vm_start = vma->vm_end; + } + + /* System unmap_region will clean the results, we need do nothing */ +DMA_MMAP_FAIL: + vma->vm_pgoff = vm_pgoff; + vma->vm_start = qfr->iova; + vma->vm_end = vma->vm_start + (qfr->nr_pages << PAGE_SHIFT); + + return ret; +} + +static int uacce_create_region(struct uacce_queue *q, + struct vm_area_struct *vma, + struct uacce_qfile_region *qfr) +{ + int ret; + + qfr->iova = vma->vm_start; + qfr->nr_pages = vma_pages(vma); + + /* allocate memory */ + ret = uacce_alloc_dma_buffers(q, vma); + if (ret) + return ret; + + ret = uacce_mmap_dma_buffers(q, vma); + if (ret) + goto err_with_pages; + + return ret; + +err_with_pages: + uacce_free_dma_buffers(q); + return ret; +} + static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) { struct uacce_queue *q = filep->private_data; @@ -231,6 +615,9 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) else return -EINVAL;

+ if (q->qfrs[type]) + return -EEXIST; + qfr = kzalloc(sizeof(*qfr), GFP_KERNEL); if (!qfr) return -ENOMEM; @@ -246,10 +633,7 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) goto out_with_lock; }

- if (q->qfrs[type]) { - ret = -EEXIST; - goto out_with_lock; - } + q->qfrs[type] = qfr;

switch (type) { case UACCE_QFRT_MMIO: @@ -264,12 +648,17 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) goto out_with_lock; break;

+ case UACCE_QFRT_SS: + ret = uacce_create_region(q, vma, qfr); + if (ret) + goto out_with_lock; + break; + default: ret = -EINVAL; goto out_with_lock; }

- q->qfrs[type] = qfr; mutex_unlock(&q->mutex);

return ret; @@ -277,6 +666,7 @@ static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma) out_with_lock: mutex_unlock(&q->mutex); kfree(qfr); + q->qfrs[type] = NULL; return ret; }

@@ -407,24 +797,54 @@ static ssize_t isolate_strategy_store(struct device *dev, struct device_attribut return count; }

+static ssize_t node_id_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct uacce_device *uacce = to_uacce_device(dev); + int node_id = -1; + +#ifdef CONFIG_NUMA + node_id = uacce->parent->numa_node; +#endif + return sysfs_emit(buf, "%d\n", node_id); +} + +static ssize_t numa_distance_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + int distance = 0; + +#ifdef CONFIG_NUMA + struct uacce_device *uacce = to_uacce_device(dev); + + distance = node_distance(uacce->parent->numa_node, + cpu_to_node(smp_processor_id())); +#endif + return sysfs_emit(buf, "%d\n", distance); +} + static DEVICE_ATTR_RO(api); static DEVICE_ATTR_RO(flags); +static DEVICE_ATTR_RO(node_id); static DEVICE_ATTR_RO(available_instances); static DEVICE_ATTR_RO(algorithms); static DEVICE_ATTR_RO(region_mmio_size); static DEVICE_ATTR_RO(region_dus_size); static DEVICE_ATTR_RO(isolate); static DEVICE_ATTR_RW(isolate_strategy); +static DEVICE_ATTR_RO(numa_distance);

static struct attribute *uacce_dev_attrs[] = { &dev_attr_api.attr, &dev_attr_flags.attr, + &dev_attr_node_id.attr, &dev_attr_available_instances.attr, &dev_attr_algorithms.attr, &dev_attr_region_mmio_size.attr, &dev_attr_region_dus_size.attr, &dev_attr_isolate.attr, &dev_attr_isolate_strategy.attr, + &dev_attr_numa_distance.attr, NULL, };

@@ -510,14 +930,18 @@ static void uacce_disable_sva(struct uacce_device *uacce) struct uacce_device *uacce_alloc(struct device *parent, struct uacce_interface *interface) { - unsigned int flags = interface->flags; struct uacce_device *uacce; + unsigned int flags; int ret;

+ if (!parent || !interface) + return ERR_PTR(-EINVAL); + uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL); if (!uacce) return ERR_PTR(-ENOMEM);

+ flags = interface->flags; flags = uacce_enable_sva(parent, flags);

uacce->parent = parent; @@ -537,7 +961,10 @@ struct uacce_device *uacce_alloc(struct device *parent, uacce->dev.groups = uacce_dev_groups; uacce->dev.parent = uacce->parent; uacce->dev.release = uacce_release; - dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id); + dev_set_name(&uacce->dev, "%s-%u", interface->name, uacce->dev_id); + + if (flags & UACCE_DEV_NOIOMMU) + dev_warn(&uacce->dev, "register to noiommu mode, it's not safe for kernel\n");

return uacce;

@@ -589,6 +1016,7 @@ void uacce_remove(struct uacce_device *uacce) mutex_lock(&uacce->mutex); /* ensure no open queue remains */ list_for_each_entry_safe(q, next_q, &uacce->queues, list) { + struct uacce_qfile_region *ss = q->qfrs[UACCE_QFRT_SS]; /* * Taking q->mutex ensures that fops do not use the defunct * uacce->ops after the queue is disabled. @@ -603,6 +1031,8 @@ void uacce_remove(struct uacce_device *uacce) * access the mmaped area while parent device is already removed */ unmap_mapping_range(q->mapping, 0, 0, 1); + if (ss && ss != &noiommu_ss_default_qfr) + uacce_free_dma_buffers(q); }

/* disable sva now since no opened queues */ diff --git a/include/linux/uacce.h b/include/linux/uacce.h index e290c0269944..40e0713aac0e 100644 --- a/include/linux/uacce.h +++ b/include/linux/uacce.h @@ -3,22 +3,40 @@ #define _LINUX_UACCE_H

#include <linux/cdev.h> +#include <linux/device.h> +#include <linux/fs.h> +#include <linux/list.h> +#include <linux/iommu.h> #include <uapi/misc/uacce/uacce.h>

#define UACCE_NAME "uacce" -#define UACCE_MAX_REGION 2 +#define UACCE_MAX_REGION 3 #define UACCE_MAX_NAME_SIZE 64 #define UACCE_MAX_ERR_THRESHOLD 65535

struct uacce_queue; struct uacce_device;

+struct uacce_dma_slice { + void *kaddr; /* kernel address for ss */ + dma_addr_t dma; /* dma address, if created by dma api */ + u64 size; /* Size of this dma slice */ + u32 total_num; /* Total slices in this dma list */ +}; + /** * struct uacce_qfile_region - structure of queue file region * @type: type of the region */ struct uacce_qfile_region { enum uacce_qfrt type; + unsigned long iova; /* iova share between user and device space */ + unsigned long nr_pages; + int prot; + unsigned int flags; + struct list_head qs; /* qs sharing the same region, for ss */ + void *kaddr; /* kernel address for dko */ + struct uacce_dma_slice *dma_list; };

/** @@ -29,11 +47,14 @@ struct uacce_qfile_region { * @start_queue: make the queue start work after get_queue * @stop_queue: make the queue stop work before put_queue * @is_q_updated: check whether the task is finished + * @mask_notify: mask the task irq of queue * @mmap: mmap addresses of queue to user space * @ioctl: ioctl for user space users of the queue * @get_isolate_state: get the device state after set the isolate strategy * @isolate_err_threshold_write: stored the isolate error threshold to the device * @isolate_err_threshold_read: read the isolate error threshold value from the device + * @reset: reset the WD device + * @reset_queue: reset the queue */ struct uacce_ops { int (*get_available_instances)(struct uacce_device *uacce); @@ -65,6 +86,7 @@ struct uacce_interface { };

enum uacce_dev_state { + UACCE_DEV_ERR = -1, UACCE_DEV_NORMAL, UACCE_DEV_ISOLATE, }; @@ -90,11 +112,14 @@ enum uacce_q_state { */ struct uacce_queue { struct uacce_device *uacce; + u32 flags; + atomic_t status; void *priv; wait_queue_head_t wait; struct list_head list; struct uacce_qfile_region *qfrs[UACCE_MAX_REGION]; struct mutex mutex; + struct file *filep; enum uacce_q_state state; u32 pasid; struct iommu_sva *handle; @@ -120,6 +145,7 @@ struct uacce_queue { struct uacce_device { const char *algs; const char *api_ver; + int status; const struct uacce_ops *ops; unsigned long qf_pg_num[UACCE_MAX_REGION]; struct device *parent; @@ -139,7 +165,7 @@ struct uacce_device *uacce_alloc(struct device *parent, struct uacce_interface *interface); int uacce_register(struct uacce_device *uacce); void uacce_remove(struct uacce_device *uacce); - +void uacce_wake_up(struct uacce_queue *q); #else /* CONFIG_UACCE */

static inline @@ -156,6 +182,7 @@ static inline int uacce_register(struct uacce_device *uacce)

static inline void uacce_remove(struct uacce_device *uacce) {}

+static inline void uacce_wake_up(struct uacce_queue *q) {} #endif /* CONFIG_UACCE */

#endif /* _LINUX_UACCE_H */ diff --git a/include/uapi/misc/uacce/uacce.h b/include/uapi/misc/uacce/uacce.h index cc7185678f47..788c6ec6f095 100644 --- a/include/uapi/misc/uacce/uacce.h +++ b/include/uapi/misc/uacce/uacce.h @@ -1,10 +1,12 @@ -/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ +/* SPDX-License-Identifier: GPL-2.0-or-later WITH Linux-syscall-note */ +/* Copyright (c) 2018-2019 HiSilicon Limited. */ #ifndef _UAPIUUACCE_H #define _UAPIUUACCE_H

#include <linux/types.h> #include <linux/ioctl.h>

+#define UACCE_CLASS_NAME "uacce" /* * UACCE_CMD_START_Q: Start queue */ @@ -17,22 +19,57 @@ */ #define UACCE_CMD_PUT_Q _IO('W', 1)

-/* - * UACCE Device flags: - * UACCE_DEV_SVA: Shared Virtual Addresses - * Support PASID - * Support device page faults (PCI PRI or SMMU Stall) +#define UACCE_CMD_SHARE_SVAS _IO('W', 2) + +#define UACCE_CMD_GET_SS_DMA _IOR('W', 3, unsigned long) + + +/** + * UACCE Device Attributes: + * + * NOIOMMU: the device has no IOMMU support + * can do ssva, but no map to the dev + * IOMMU: the device has IOMMU support and enable __IOMMU_DOMAIN_PAGING + * PASID: the device has IOMMU which support PASID setting + * can do ssva, mapped to dev per process + * FAULT_FROM_DEV: the device has IOMMU which can do page fault request + * no need for ssva, should be used with PASID + * KMAP_DUS: map the Device user-shared space to kernel + * DRVMAP_DUS: Driver self-maintain its DUS + * SVA: full function device + * SHARE_DOMAIN: no PASID, can do ssva only for one process and the kernel */ #define UACCE_DEV_SVA BIT(0) +#define UACCE_DEV_NOIOMMU BIT(1) +#define UACCE_DEV_IOMMU BIT(7) + + +/* uacce mode of the driver */ +#define UACCE_MODE_NOUACCE 0 /* don't use uacce */ +#define UACCE_MODE_SVA 1 /* use uacce sva mode */ +#define UACCE_MODE_NOIOMMU 2 /* use uacce noiommu mode */ + +#define UACCE_API_VER_NOIOMMU_SUBFIX "_noiommu" + +#define UACCE_QFR_NA ((unsigned long)-1)

/** * enum uacce_qfrt: queue file region type * @UACCE_QFRT_MMIO: device mmio region * @UACCE_QFRT_DUS: device user share region + * @UACCE_QFRT_SS: static share memory(no-sva) */ enum uacce_qfrt { - UACCE_QFRT_MMIO = 0, - UACCE_QFRT_DUS = 1, + UACCE_QFRT_MMIO = 0, /* device mmio region */ + UACCE_QFRT_DUS, /* device user share */ + UACCE_QFRT_SS, /* static share memory */ + UACCE_QFRT_MAX, }; +#define UACCE_QFRT_INVALID UACCE_QFRT_MAX + +/* Pass DMA SS region slice size by granularity 64KB */ +#define UACCE_GRAN_SIZE 0x10000ull +#define UACCE_GRAN_SHIFT 16 +#define UACCE_GRAN_NUM_MASK 0xfffull

#endif

From: JiangShui Yang yangjiangshui@h-partners.com

driver inclusion category: feature bugzilla: https://gitee.com/openeuler/kernel/issues/I94L2V CVE: NA

----------------------------------------------------------------------

Support the HiSilicon Kunpeng accelerators sec/zip/hpre register to UACCE subsystem in UACCE_MODE_NOIOMMU mode so that users can directly perform encryption, decryption, compression and decompression in user space.

Signed-off-by: JiangShui Yang yangjiangshui@h-partners.com Signed-off-by: Weili Qian qianweili@huawei.com --- drivers/crypto/hisilicon/hpre/hpre_main.c | 1 - drivers/crypto/hisilicon/qm.c | 191 ++++++++++++++-------- drivers/crypto/hisilicon/sec2/sec_main.c | 11 +- drivers/crypto/hisilicon/zip/zip_main.c | 11 +- include/linux/hisi_acc_qm.h | 5 +- 5 files changed, 136 insertions(+), 83 deletions(-)

diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index d93aa6630a57..e970dd629234 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -10,7 +10,6 @@ #include <linux/pci.h> #include <linux/pm_runtime.h> #include <linux/topology.h> -#include <linux/uacce.h> #include "hpre.h"

#define HPRE_QM_ABNML_INT_MASK 0x100004 diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 543074742db5..d9caff12cd4d 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -850,6 +850,33 @@ static void qm_cq_head_update(struct hisi_qp *qp) } }

+static void qm_poll_user_event_cb(struct hisi_qp *qp) +{ + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + struct uacce_queue *q = qp->uacce_q; + bool updated = 0; + + /* + * If multi thread poll one queue, each thread will produce + * one event, so we query one cqe and break out of the loop. + * If only one thread poll one queue, we need query all cqe + * to ensure that we poll a cleaned queue next time. + */ + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { + dma_rmb(); + qm_cq_head_update(qp); + cqe = qp->cqe + qp->qp_status.cq_head; + updated = 1; + if (!wq_has_single_sleeper(&q->wait)) + break; + } + + if (updated) { + atomic_inc(&qp->qp_status.complete_task); + qp->event_cb(qp); + } +} + static void qm_poll_req_cb(struct hisi_qp *qp) { struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; @@ -887,7 +914,7 @@ static void qm_work_process(struct work_struct *work) continue;

if (qp->event_cb) { - qp->event_cb(qp); + qm_poll_user_event_cb(qp); continue; }

@@ -1074,6 +1101,7 @@ static void qm_init_qp_status(struct hisi_qp *qp) qp_status->cq_head = 0; qp_status->cqc_phase = true; atomic_set(&qp_status->used, 0); + atomic_set(&qp_status->complete_task, 0); }

static void qm_init_prefetch(struct hisi_qm *qm) @@ -2230,7 +2258,7 @@ static void hisi_qm_cache_wb(struct hisi_qm *qm)

static void qm_qp_event_notifier(struct hisi_qp *qp) { - wake_up_interruptible(&qp->uacce_q->wait); + uacce_wake_up(qp->uacce_q); }

/* This function returns free number of qp in qm. */ @@ -2376,18 +2404,8 @@ static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) static int hisi_qm_is_q_updated(struct uacce_queue *q) { struct hisi_qp *qp = q->priv; - struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; - int updated = 0; - - while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { - /* make sure to read data from memory */ - dma_rmb(); - qm_cq_head_update(qp); - cqe = qp->cqe + qp->qp_status.cq_head; - updated = 1; - }

- return updated; + return atomic_add_unless(&qp->qp_status.complete_task, -1, 0); }

static void qm_set_sqctype(struct uacce_queue *q, u16 type) @@ -2418,7 +2436,7 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, qm_set_sqctype(q, qp_ctx.qc_type); qp_ctx.id = qp->qp_id;

- if (copy_to_user((void __user *)arg, &qp_ctx, + if (copy_to_user((void __user *)(uintptr_t)arg, &qp_ctx, sizeof(struct hisi_qp_ctx))) return -EFAULT;

@@ -2442,6 +2460,66 @@ static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd, return -EINVAL; }

+static void qm_uacce_api_ver_init(struct hisi_qm *qm) +{ + struct uacce_device *uacce = qm->uacce; + + if (uacce->flags & UACCE_DEV_IOMMU) { + qm->use_sva = uacce->flags & UACCE_DEV_SVA ? true : false; + + if (qm->ver == QM_HW_V1) + uacce->api_ver = HISI_QM_API_VER_BASE; + else if (qm->ver == QM_HW_V2) + uacce->api_ver = HISI_QM_API_VER2_BASE; + else + uacce->api_ver = HISI_QM_API_VER3_BASE; + } else { + qm->use_sva = false; + + if (qm->ver == QM_HW_V1) + uacce->api_ver = HISI_QM_API_VER_BASE + UACCE_API_VER_NOIOMMU_SUBFIX; + else if (qm->ver == QM_HW_V2) + uacce->api_ver = HISI_QM_API_VER2_BASE + UACCE_API_VER_NOIOMMU_SUBFIX; + else + uacce->api_ver = HISI_QM_API_VER3_BASE + UACCE_API_VER_NOIOMMU_SUBFIX; + } +} + +static void qm_uacce_base_init(struct hisi_qm *qm) +{ + unsigned long dus_page_nr, mmio_page_nr; + struct uacce_device *uacce = qm->uacce; + struct pci_dev *pdev = qm->pdev; + u16 sq_depth, cq_depth; + + qm_uacce_api_ver_init(qm); + + if (qm->ver == QM_HW_V1) + mmio_page_nr = QM_DOORBELL_PAGE_NR; + else if (qm->ver == QM_HW_V2 || + !test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) + mmio_page_nr = QM_DOORBELL_PAGE_NR + + QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; + else + mmio_page_nr = QM_QP_DB_INTERVAL / PAGE_SIZE; + + uacce->is_vf = pdev->is_virtfn; + uacce->priv = qm; + uacce->parent = &pdev->dev; + qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); + + /* Add one more page for device or qp status */ + dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + + sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >> + PAGE_SHIFT; + + uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; + uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; +} + /** * qm_hw_err_isolate() - Try to set the isolation status of the uacce device * according to user's configuration of error threshold. @@ -2567,7 +2645,7 @@ static void qm_remove_uacce(struct hisi_qm *qm) { struct uacce_device *uacce = qm->uacce;

- if (qm->use_sva) { + if (uacce) { qm_hw_err_destroy(qm); uacce_remove(uacce); qm->uacce = NULL; @@ -2576,15 +2654,9 @@ static void qm_remove_uacce(struct hisi_qm *qm)

static int qm_alloc_uacce(struct hisi_qm *qm) { + struct uacce_interface interface = {0}; struct pci_dev *pdev = qm->pdev; struct uacce_device *uacce; - unsigned long mmio_page_nr; - unsigned long dus_page_nr; - u16 sq_depth, cq_depth; - struct uacce_interface interface = { - .flags = UACCE_DEV_SVA, - .ops = &uacce_qm_ops, - }; int ret;

ret = strscpy(interface.name, dev_driver_string(&pdev->dev), @@ -2592,53 +2664,41 @@ static int qm_alloc_uacce(struct hisi_qm *qm) if (ret < 0) return -ENAMETOOLONG;

- uacce = uacce_alloc(&pdev->dev, &interface); - if (IS_ERR(uacce)) - return PTR_ERR(uacce); + interface.flags = qm->use_iommu ? UACCE_DEV_IOMMU : UACCE_DEV_NOIOMMU; + if (qm->mode == UACCE_MODE_SVA) { + if (!qm->use_iommu) { + pci_err(pdev, "iommu not support sva!\n"); + return -EINVAL; + }

- if (uacce->flags & UACCE_DEV_SVA) { - qm->use_sva = true; - } else { - /* only consider sva case */ - qm_remove_uacce(qm); - return -EINVAL; + interface.flags |= UACCE_DEV_SVA; }

- uacce->is_vf = pdev->is_virtfn; - uacce->priv = qm; - - if (qm->ver == QM_HW_V1) - uacce->api_ver = HISI_QM_API_VER_BASE; - else if (qm->ver == QM_HW_V2) - uacce->api_ver = HISI_QM_API_VER2_BASE; - else - uacce->api_ver = HISI_QM_API_VER3_BASE; - - if (qm->ver == QM_HW_V1) - mmio_page_nr = QM_DOORBELL_PAGE_NR; - else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) - mmio_page_nr = QM_DOORBELL_PAGE_NR + - QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE; - else - mmio_page_nr = qm->db_interval / PAGE_SIZE; - - qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); - - /* Add one more page for device or qp status */ - dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + - sizeof(struct qm_cqe) * cq_depth + PAGE_SIZE) >> - PAGE_SHIFT; - - uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr; - uacce->qf_pg_num[UACCE_QFRT_DUS] = dus_page_nr; - + interface.ops = &uacce_qm_ops; + uacce = uacce_alloc(&pdev->dev, &interface); + if (IS_ERR(uacce)) { + pci_err(pdev, "fail to alloc uacce device\n!"); + return PTR_ERR(uacce); + } qm->uacce = uacce; + + qm_uacce_base_init(qm); INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs); mutex_init(&qm->isolate_data.isolate_lock);

return 0; }

+int qm_register_uacce(struct hisi_qm *qm) +{ + if (!qm->uacce) + return 0; + + dev_info(&qm->pdev->dev, "qm register to uacce\n"); + return uacce_register(qm->uacce); +} +EXPORT_SYMBOL_GPL(qm_register_uacce); + /** * qm_frozen() - Try to froze QM to cut continuous queue request. If * there is user on the QM, return failure without doing anything. @@ -2744,7 +2804,7 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, struct hisi_qp *qp; int ret = -ENOMEM;

- qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), + qm->poll_data[id].qp_finish_id = kcalloc(qm->eq_depth, sizeof(u16), GFP_KERNEL); if (!qm->poll_data[id].qp_finish_id) return -ENOMEM; @@ -2909,12 +2969,9 @@ void hisi_qm_uninit(struct hisi_qm *qm) hisi_qm_set_state(qm, QM_NOT_READY); up_write(&qm->qps_lock);

+ qm_remove_uacce(qm); qm_irqs_unregister(qm); hisi_qm_pci_uninit(qm); - if (qm->use_sva) { - uacce_remove(qm->uacce); - qm->uacce = NULL; - } } EXPORT_SYMBOL_GPL(hisi_qm_uninit);

@@ -4114,7 +4171,7 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) return ret; }

- if (qm->use_sva) { + if (qm->uacce) { ret = qm_hw_err_isolate(qm); if (ret) pci_err(pdev, "failed to isolate hw err!\n"); @@ -4440,7 +4497,7 @@ static int qm_controller_reset(struct hisi_qm *qm) qm_reset_bit_clear(qm);

/* if resetting fails, isolate the device */ - if (qm->use_sva) + if (qm->uacce) qm->isolate_data.is_isolate = true; return ret; } @@ -5402,7 +5459,7 @@ int hisi_qm_init(struct hisi_qm *qm) } }

- if (qm->mode == UACCE_MODE_SVA) { + if (qm->mode != UACCE_MODE_NOUACCE) { ret = qm_alloc_uacce(qm); if (ret < 0) dev_warn(dev, "fail to alloc uacce (%d)\n", ret); diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 377c58bef466..d7047200ee7a 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -13,7 +13,6 @@ #include <linux/pm_runtime.h> #include <linux/seq_file.h> #include <linux/topology.h> -#include <linux/uacce.h>

#include "sec.h"

@@ -1232,12 +1231,10 @@ static int sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_qm_del_list; }

- if (qm->uacce) { - ret = uacce_register(qm->uacce); - if (ret) { - pci_err(pdev, "failed to register uacce (%d)!\n", ret); - goto err_alg_unregister; - } + ret = qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_alg_unregister; }

if (qm->fun_type == QM_HW_PF && vfs_num) { diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index c065fd867161..6239dc47d8a0 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -11,7 +11,6 @@ #include <linux/pm_runtime.h> #include <linux/seq_file.h> #include <linux/topology.h> -#include <linux/uacce.h> #include "zip.h"

#define PCI_DEVICE_ID_HUAWEI_ZIP_PF 0xa250 @@ -1318,12 +1317,10 @@ static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_qm_del_list; }

- if (qm->uacce) { - ret = uacce_register(qm->uacce); - if (ret) { - pci_err(pdev, "failed to register uacce (%d)!\n", ret); - goto err_qm_alg_unregister; - } + ret = qm_register_uacce(qm); + if (ret) { + pci_err(pdev, "failed to register uacce (%d)!\n", ret); + goto err_qm_alg_unregister; }

if (qm->fun_type == QM_HW_PF && vfs_num > 0) { diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 773e0f1c67dc..f6383689974f 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -8,6 +8,7 @@ #include <linux/iopoll.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/uacce.h>

#define QM_QNUM_V1 4096 #define QM_QNUM_V2 1024 @@ -405,6 +406,7 @@ struct hisi_qp_status { u16 cq_head; bool cqc_phase; atomic_t flags; + atomic_t complete_task; };

struct hisi_qp_ops { @@ -495,7 +497,7 @@ static inline int mode_set(const char *val, const struct kernel_param *kp) return -EINVAL;

ret = kstrtou32(val, 10, &n); - if (ret != 0 || (n != UACCE_MODE_SVA && + if (ret != 0 || (n != UACCE_MODE_NOIOMMU && n != UACCE_MODE_SVA && n != UACCE_MODE_NOUACCE)) return -EINVAL;

@@ -527,6 +529,7 @@ static inline void hisi_qm_del_list(struct hisi_qm *qm, struct hisi_qm_list *qm_ mutex_unlock(&qm_list->lock); }

+int qm_register_uacce(struct hisi_qm *qm); int hisi_qm_init(struct hisi_qm *qm); void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm);