diff options
Diffstat (limited to 'drivers/misc')
| -rw-r--r-- | drivers/misc/cxl/fault.c | 2 | ||||
| -rw-r--r-- | drivers/misc/ocxl/Makefile | 3 | ||||
| -rw-r--r-- | drivers/misc/ocxl/afu_irq.c | 102 | ||||
| -rw-r--r-- | drivers/misc/ocxl/config.c | 13 | ||||
| -rw-r--r-- | drivers/misc/ocxl/context.c | 31 | ||||
| -rw-r--r-- | drivers/misc/ocxl/core.c | 574 | ||||
| -rw-r--r-- | drivers/misc/ocxl/file.c | 182 | ||||
| -rw-r--r-- | drivers/misc/ocxl/link.c | 42 | ||||
| -rw-r--r-- | drivers/misc/ocxl/mmio.c | 234 | ||||
| -rw-r--r-- | drivers/misc/ocxl/ocxl_internal.h | 94 | ||||
| -rw-r--r-- | drivers/misc/ocxl/pci.c | 565 | ||||
| -rw-r--r-- | drivers/misc/ocxl/sysfs.c | 54 | ||||
| -rw-r--r-- | drivers/misc/ocxl/trace.h | 12 |
13 files changed, 1150 insertions, 758 deletions
diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index dc7b34174f85..a4d17a5a9763 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -168,7 +168,7 @@ int cxl_handle_mm_fault(struct mm_struct *mm, u64 dsisr, u64 dar) if (dsisr & CXL_PSL_DSISR_An_S) access |= _PAGE_WRITE; - if (!mm && (REGION_ID(dar) != USER_REGION_ID)) + if (!mm && (get_region_id(dar) != USER_REGION_ID)) access |= _PAGE_PRIVILEGED; if (dsisr & DSISR_NOHPTE) diff --git a/drivers/misc/ocxl/Makefile b/drivers/misc/ocxl/Makefile index 5229dcda8297..d07d1bb8e8d4 100644 --- a/drivers/misc/ocxl/Makefile +++ b/drivers/misc/ocxl/Makefile @@ -1,8 +1,9 @@ # SPDX-License-Identifier: GPL-2.0+ ccflags-$(CONFIG_PPC_WERROR) += -Werror -ocxl-y += main.o pci.o config.o file.o pasid.o +ocxl-y += main.o pci.o config.o file.o pasid.o mmio.o ocxl-y += link.o context.o afu_irq.o sysfs.o trace.o +ocxl-y += core.o obj-$(CONFIG_OCXL) += ocxl.o # For tracepoints to include our trace.h from tracepoint infrastructure: diff --git a/drivers/misc/ocxl/afu_irq.c b/drivers/misc/ocxl/afu_irq.c index 11ab996657a2..70f8f1c3929d 100644 --- a/drivers/misc/ocxl/afu_irq.c +++ b/drivers/misc/ocxl/afu_irq.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ // Copyright 2017 IBM Corp. #include <linux/interrupt.h> -#include <linux/eventfd.h> +#include <asm/pnv-ocxl.h> #include "ocxl_internal.h" #include "trace.h" @@ -11,27 +11,59 @@ struct afu_irq { unsigned int virq; char *name; u64 trigger_page; - struct eventfd_ctx *ev_ctx; + irqreturn_t (*handler)(void *private); + void (*free_private)(void *private); + void *private; }; -static int irq_offset_to_id(struct ocxl_context *ctx, u64 offset) +int ocxl_irq_offset_to_id(struct ocxl_context *ctx, u64 offset) { return (offset - ctx->afu->irq_base_offset) >> PAGE_SHIFT; } -static u64 irq_id_to_offset(struct ocxl_context *ctx, int id) +u64 ocxl_irq_id_to_offset(struct ocxl_context *ctx, int irq_id) { - return ctx->afu->irq_base_offset + (id << PAGE_SHIFT); + return ctx->afu->irq_base_offset + (irq_id << PAGE_SHIFT); } +int ocxl_irq_set_handler(struct ocxl_context *ctx, int irq_id, + irqreturn_t (*handler)(void *private), + void (*free_private)(void *private), + void *private) +{ + struct afu_irq *irq; + int rc; + + mutex_lock(&ctx->irq_lock); + irq = idr_find(&ctx->irq_idr, irq_id); + if (!irq) { + rc = -EINVAL; + goto unlock; + } + + irq->handler = handler; + irq->private = private; + irq->free_private = free_private; + + rc = 0; + // Fall through to unlock + +unlock: + mutex_unlock(&ctx->irq_lock); + return rc; +} +EXPORT_SYMBOL_GPL(ocxl_irq_set_handler); + static irqreturn_t afu_irq_handler(int virq, void *data) { struct afu_irq *irq = (struct afu_irq *) data; trace_ocxl_afu_irq_receive(virq); - if (irq->ev_ctx) - eventfd_signal(irq->ev_ctx, 1); - return IRQ_HANDLED; + + if (irq->handler) + return irq->handler(irq->private); + + return IRQ_HANDLED; // Just drop it on the ground } static int setup_afu_irq(struct ocxl_context *ctx, struct afu_irq *irq) @@ -69,7 +101,7 @@ static void release_afu_irq(struct afu_irq *irq) kfree(irq->name); } -int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset) +int ocxl_afu_irq_alloc(struct ocxl_context *ctx, int *irq_id) { struct afu_irq *irq; int rc; @@ -101,11 +133,11 @@ int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset) if (rc) goto err_alloc; - *irq_offset = irq_id_to_offset(ctx, irq->id); - - trace_ocxl_afu_irq_alloc(ctx->pasid, irq->id, irq->virq, irq->hw_irq, - *irq_offset); + trace_ocxl_afu_irq_alloc(ctx->pasid, irq->id, irq->virq, irq->hw_irq); mutex_unlock(&ctx->irq_lock); + + *irq_id = irq->id; + return 0; err_alloc: @@ -117,29 +149,29 @@ err_unlock: kfree(irq); return rc; } +EXPORT_SYMBOL_GPL(ocxl_afu_irq_alloc); static void afu_irq_free(struct afu_irq *irq, struct ocxl_context *ctx) { trace_ocxl_afu_irq_free(ctx->pasid, irq->id); if (ctx->mapping) unmap_mapping_range(ctx->mapping, - irq_id_to_offset(ctx, irq->id), + ocxl_irq_id_to_offset(ctx, irq->id), 1 << PAGE_SHIFT, 1); release_afu_irq(irq); - if (irq->ev_ctx) - eventfd_ctx_put(irq->ev_ctx); + if (irq->free_private) + irq->free_private(irq->private); ocxl_link_free_irq(ctx->afu->fn->link, irq->hw_irq); kfree(irq); } -int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset) +int ocxl_afu_irq_free(struct ocxl_context *ctx, int irq_id) { struct afu_irq *irq; - int id = irq_offset_to_id(ctx, irq_offset); mutex_lock(&ctx->irq_lock); - irq = idr_find(&ctx->irq_idr, id); + irq = idr_find(&ctx->irq_idr, irq_id); if (!irq) { mutex_unlock(&ctx->irq_lock); return -EINVAL; @@ -149,6 +181,7 @@ int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset) mutex_unlock(&ctx->irq_lock); return 0; } +EXPORT_SYMBOL_GPL(ocxl_afu_irq_free); void ocxl_afu_irq_free_all(struct ocxl_context *ctx) { @@ -161,41 +194,16 @@ void ocxl_afu_irq_free_all(struct ocxl_context *ctx) mutex_unlock(&ctx->irq_lock); } -int ocxl_afu_irq_set_fd(struct ocxl_context *ctx, u64 irq_offset, int eventfd) -{ - struct afu_irq *irq; - struct eventfd_ctx *ev_ctx; - int rc = 0, id = irq_offset_to_id(ctx, irq_offset); - - mutex_lock(&ctx->irq_lock); - irq = idr_find(&ctx->irq_idr, id); - if (!irq) { - rc = -EINVAL; - goto unlock; - } - - ev_ctx = eventfd_ctx_fdget(eventfd); - if (IS_ERR(ev_ctx)) { - rc = -EINVAL; - goto unlock; - } - - irq->ev_ctx = ev_ctx; -unlock: - mutex_unlock(&ctx->irq_lock); - return rc; -} - -u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, u64 irq_offset) +u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, int irq_id) { struct afu_irq *irq; - int id = irq_offset_to_id(ctx, irq_offset); u64 addr = 0; mutex_lock(&ctx->irq_lock); - irq = idr_find(&ctx->irq_idr, id); + irq = idr_find(&ctx->irq_idr, irq_id); if (irq) addr = irq->trigger_page; mutex_unlock(&ctx->irq_lock); return addr; } +EXPORT_SYMBOL_GPL(ocxl_afu_irq_get_addr); diff --git a/drivers/misc/ocxl/config.c b/drivers/misc/ocxl/config.c index 8f2c5d8bd2ee..5e65acb8e134 100644 --- a/drivers/misc/ocxl/config.c +++ b/drivers/misc/ocxl/config.c @@ -2,8 +2,8 @@ // Copyright 2017 IBM Corp. #include <linux/pci.h> #include <asm/pnv-ocxl.h> -#include <misc/ocxl.h> #include <misc/ocxl-config.h> +#include "ocxl_internal.h" #define EXTRACT_BIT(val, bit) (!!(val & BIT(bit))) #define EXTRACT_BITS(val, s, e) ((val & GENMASK(e, s)) >> s) @@ -68,7 +68,7 @@ static int find_dvsec_afu_ctrl(struct pci_dev *dev, u8 afu_idx) return 0; } -static int read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn) +static void read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn) { u16 val; int pos; @@ -89,7 +89,6 @@ static int read_pasid(struct pci_dev *dev, struct ocxl_fn_config *fn) out: dev_dbg(&dev->dev, "PASID capability:\n"); dev_dbg(&dev->dev, " Max PASID log = %d\n", fn->max_pasid_log); - return 0; } static int read_dvsec_tl(struct pci_dev *dev, struct ocxl_fn_config *fn) @@ -205,11 +204,7 @@ int ocxl_config_read_function(struct pci_dev *dev, struct ocxl_fn_config *fn) { int rc; - rc = read_pasid(dev, fn); - if (rc) { - dev_err(&dev->dev, "Invalid PASID configuration: %d\n", rc); - return -ENODEV; - } + read_pasid(dev, fn); rc = read_dvsec_tl(dev, fn); if (rc) { @@ -304,7 +299,6 @@ int ocxl_config_check_afu_index(struct pci_dev *dev, } return 1; } -EXPORT_SYMBOL_GPL(ocxl_config_check_afu_index); static int read_afu_name(struct pci_dev *dev, struct ocxl_fn_config *fn, struct ocxl_afu_config *afu) @@ -540,7 +534,6 @@ int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count) { return pnv_ocxl_get_pasid_count(dev, count); } -EXPORT_SYMBOL_GPL(ocxl_config_get_pasid_info); void ocxl_config_set_afu_pasid(struct pci_dev *dev, int pos, int pasid_base, u32 pasid_count_log) diff --git a/drivers/misc/ocxl/context.c b/drivers/misc/ocxl/context.c index c10a940e3b38..bab9c9364184 100644 --- a/drivers/misc/ocxl/context.c +++ b/drivers/misc/ocxl/context.c @@ -4,15 +4,17 @@ #include "trace.h" #include "ocxl_internal.h" -struct ocxl_context *ocxl_context_alloc(void) -{ - return kzalloc(sizeof(struct ocxl_context), GFP_KERNEL); -} - -int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu, +int ocxl_context_alloc(struct ocxl_context **context, struct ocxl_afu *afu, struct address_space *mapping) { int pasid; + struct ocxl_context *ctx; + + *context = kzalloc(sizeof(struct ocxl_context), GFP_KERNEL); + if (!*context) + return -ENOMEM; + + ctx = *context; ctx->afu = afu; mutex_lock(&afu->contexts_lock); @@ -43,6 +45,7 @@ int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu, ocxl_afu_get(afu); return 0; } +EXPORT_SYMBOL_GPL(ocxl_context_alloc); /* * Callback for when a translation fault triggers an error @@ -63,7 +66,7 @@ static void xsl_fault_error(void *data, u64 addr, u64 dsisr) wake_up_all(&ctx->events_wq); } -int ocxl_context_attach(struct ocxl_context *ctx, u64 amr) +int ocxl_context_attach(struct ocxl_context *ctx, u64 amr, struct mm_struct *mm) { int rc; @@ -75,7 +78,7 @@ int ocxl_context_attach(struct ocxl_context *ctx, u64 amr) } rc = ocxl_link_add_pe(ctx->afu->fn->link, ctx->pasid, - current->mm->context.id, ctx->tidr, amr, current->mm, + mm->context.id, ctx->tidr, amr, mm, xsl_fault_error, ctx); if (rc) goto out; @@ -85,13 +88,15 @@ out: mutex_unlock(&ctx->status_mutex); return rc; } +EXPORT_SYMBOL_GPL(ocxl_context_attach); static vm_fault_t map_afu_irq(struct vm_area_struct *vma, unsigned long address, u64 offset, struct ocxl_context *ctx) { u64 trigger_addr; + int irq_id = ocxl_irq_offset_to_id(ctx, offset); - trigger_addr = ocxl_afu_irq_get_addr(ctx, offset); + trigger_addr = ocxl_afu_irq_get_addr(ctx, irq_id); if (!trigger_addr) return VM_FAULT_SIGBUS; @@ -151,12 +156,14 @@ static const struct vm_operations_struct ocxl_vmops = { static int check_mmap_afu_irq(struct ocxl_context *ctx, struct vm_area_struct *vma) { + int irq_id = ocxl_irq_offset_to_id(ctx, vma->vm_pgoff << PAGE_SHIFT); + /* only one page */ if (vma_pages(vma) != 1) return -EINVAL; /* check offset validty */ - if (!ocxl_afu_irq_get_addr(ctx, vma->vm_pgoff << PAGE_SHIFT)) + if (!ocxl_afu_irq_get_addr(ctx, irq_id)) return -EINVAL; /* @@ -238,11 +245,12 @@ int ocxl_context_detach(struct ocxl_context *ctx) } rc = ocxl_link_remove_pe(ctx->afu->fn->link, ctx->pasid); if (rc) { - dev_warn(&ctx->afu->dev, + dev_warn(&dev->dev, "Couldn't remove PE entry cleanly: %d\n", rc); } return 0; } +EXPORT_SYMBOL_GPL(ocxl_context_detach); void ocxl_context_detach_all(struct ocxl_afu *afu) { @@ -280,3 +288,4 @@ void ocxl_context_free(struct ocxl_context *ctx) ocxl_afu_put(ctx->afu); kfree(ctx); } +EXPORT_SYMBOL_GPL(ocxl_context_free); diff --git a/drivers/misc/ocxl/core.c b/drivers/misc/ocxl/core.c new file mode 100644 index 000000000000..b7a09b21ab36 --- /dev/null +++ b/drivers/misc/ocxl/core.c @@ -0,0 +1,574 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2019 IBM Corp. +#include <linux/idr.h> +#include "ocxl_internal.h" + +static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn) +{ + return (get_device(&fn->dev) == NULL) ? NULL : fn; +} + +static void ocxl_fn_put(struct ocxl_fn *fn) +{ + put_device(&fn->dev); +} + +static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn) +{ + struct ocxl_afu *afu; + + afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL); + if (!afu) + return NULL; + + kref_init(&afu->kref); + mutex_init(&afu->contexts_lock); + mutex_init(&afu->afu_control_lock); + idr_init(&afu->contexts_idr); + afu->fn = fn; + ocxl_fn_get(fn); + return afu; +} + +static void free_afu(struct kref *kref) +{ + struct ocxl_afu *afu = container_of(kref, struct ocxl_afu, kref); + + idr_destroy(&afu->contexts_idr); + ocxl_fn_put(afu->fn); + kfree(afu); +} + +void ocxl_afu_get(struct ocxl_afu *afu) +{ + kref_get(&afu->kref); +} +EXPORT_SYMBOL_GPL(ocxl_afu_get); + +void ocxl_afu_put(struct ocxl_afu *afu) +{ + kref_put(&afu->kref, free_afu); +} +EXPORT_SYMBOL_GPL(ocxl_afu_put); + +static int assign_afu_actag(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int actag_count, actag_offset; + struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); + + /* + * if there were not enough actags for the function, each afu + * reduces its count as well + */ + actag_count = afu->config.actag_supported * + fn->actag_enabled / fn->actag_supported; + actag_offset = ocxl_actag_afu_alloc(fn, actag_count); + if (actag_offset < 0) { + dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n", + actag_count, actag_offset); + return actag_offset; + } + afu->actag_base = fn->actag_base + actag_offset; + afu->actag_enabled = actag_count; + + ocxl_config_set_afu_actag(pci_dev, afu->config.dvsec_afu_control_pos, + afu->actag_base, afu->actag_enabled); + dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n", + afu->actag_base, afu->actag_enabled); + return 0; +} + +static void reclaim_afu_actag(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int start_offset, size; + + start_offset = afu->actag_base - fn->actag_base; + size = afu->actag_enabled; + ocxl_actag_afu_free(afu->fn, start_offset, size); +} + +static int assign_afu_pasid(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int pasid_count, pasid_offset; + struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); + + /* + * We only support the case where the function configuration + * requested enough PASIDs to cover all AFUs. + */ + pasid_count = 1 << afu->config.pasid_supported_log; + pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count); + if (pasid_offset < 0) { + dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n", + pasid_count, pasid_offset); + return pasid_offset; + } + afu->pasid_base = fn->pasid_base + pasid_offset; + afu->pasid_count = 0; + afu->pasid_max = pasid_count; + + ocxl_config_set_afu_pasid(pci_dev, afu->config.dvsec_afu_control_pos, + afu->pasid_base, + afu->config.pasid_supported_log); + dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n", + afu->pasid_base, pasid_count); + return 0; +} + +static void reclaim_afu_pasid(struct ocxl_afu *afu) +{ + struct ocxl_fn *fn = afu->fn; + int start_offset, size; + + start_offset = afu->pasid_base - fn->pasid_base; + size = 1 << afu->config.pasid_supported_log; + ocxl_pasid_afu_free(afu->fn, start_offset, size); +} + +static int reserve_fn_bar(struct ocxl_fn *fn, int bar) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + int rc, idx; + + if (bar != 0 && bar != 2 && bar != 4) + return -EINVAL; + + idx = bar >> 1; + if (fn->bar_used[idx]++ == 0) { + rc = pci_request_region(dev, bar, "ocxl"); + if (rc) + return rc; + } + return 0; +} + +static void release_fn_bar(struct ocxl_fn *fn, int bar) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + int idx; + + if (bar != 0 && bar != 2 && bar != 4) + return; + + idx = bar >> 1; + if (--fn->bar_used[idx] == 0) + pci_release_region(dev, bar); + WARN_ON(fn->bar_used[idx] < 0); +} + +static int map_mmio_areas(struct ocxl_afu *afu) +{ + int rc; + struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent); + + rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar); + if (rc) + return rc; + + rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar); + if (rc) { + release_fn_bar(afu->fn, afu->config.global_mmio_bar); + return rc; + } + + afu->global_mmio_start = + pci_resource_start(pci_dev, afu->config.global_mmio_bar) + + afu->config.global_mmio_offset; + afu->pp_mmio_start = + pci_resource_start(pci_dev, afu->config.pp_mmio_bar) + + afu->config.pp_mmio_offset; + + afu->global_mmio_ptr = ioremap(afu->global_mmio_start, + afu->config.global_mmio_size); + if (!afu->global_mmio_ptr) { + release_fn_bar(afu->fn, afu->config.pp_mmio_bar); + release_fn_bar(afu->fn, afu->config.global_mmio_bar); + dev_err(&pci_dev->dev, "Error mapping global mmio area\n"); + return -ENOMEM; + } + + /* + * Leave an empty page between the per-process mmio area and + * the AFU interrupt mappings + */ + afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE; + return 0; +} + +static void unmap_mmio_areas(struct ocxl_afu *afu) +{ + if (afu->global_mmio_ptr) { + iounmap(afu->global_mmio_ptr); + afu->global_mmio_ptr = NULL; + } + afu->global_mmio_start = 0; + afu->pp_mmio_start = 0; + release_fn_bar(afu->fn, afu->config.pp_mmio_bar); + release_fn_bar(afu->fn, afu->config.global_mmio_bar); +} + +static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev) +{ + int rc; + + rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx); + if (rc) + return rc; + + rc = assign_afu_actag(afu); + if (rc) + return rc; + + rc = assign_afu_pasid(afu); + if (rc) + goto err_free_actag; + + rc = map_mmio_areas(afu); + if (rc) + goto err_free_pasid; + + return 0; + +err_free_pasid: + reclaim_afu_pasid(afu); +err_free_actag: + reclaim_afu_actag(afu); + return rc; +} + +static void deconfigure_afu(struct ocxl_afu *afu) +{ + unmap_mmio_areas(afu); + reclaim_afu_pasid(afu); + reclaim_afu_actag(afu); +} + +static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu) +{ + ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1); + + return 0; +} + +static void deactivate_afu(struct ocxl_afu *afu) +{ + struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent); + + ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0); +} + +static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx) +{ + int rc; + struct ocxl_afu *afu; + + afu = alloc_afu(fn); + if (!afu) + return -ENOMEM; + + rc = configure_afu(afu, afu_idx, dev); + if (rc) { + ocxl_afu_put(afu); + return rc; + } + + rc = activate_afu(dev, afu); + if (rc) { + deconfigure_afu(afu); + ocxl_afu_put(afu); + return rc; + } + + list_add_tail(&afu->list, &fn->afu_list); + + return 0; +} + +static void remove_afu(struct ocxl_afu *afu) +{ + list_del(&afu->list); + ocxl_context_detach_all(afu); + deactivate_afu(afu); + deconfigure_afu(afu); + ocxl_afu_put(afu); // matches the implicit get in alloc_afu +} + +static struct ocxl_fn *alloc_function(void) +{ + struct ocxl_fn *fn; + + fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL); + if (!fn) + return NULL; + + INIT_LIST_HEAD(&fn->afu_list); + INIT_LIST_HEAD(&fn->pasid_list); + INIT_LIST_HEAD(&fn->actag_list); + + return fn; +} + +static void free_function(struct ocxl_fn *fn) +{ + WARN_ON(!list_empty(&fn->afu_list)); + WARN_ON(!list_empty(&fn->pasid_list)); + kfree(fn); +} + +static void free_function_dev(struct device *dev) +{ + struct ocxl_fn *fn = container_of(dev, struct ocxl_fn, dev); + + free_function(fn); +} + +static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev) +{ + int rc; + + fn->dev.parent = &dev->dev; + fn->dev.release = free_function_dev; + rc = dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev)); + if (rc) + return rc; + return 0; +} + +static int assign_function_actag(struct ocxl_fn *fn) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + u16 base, enabled, supported; + int rc; + + rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported); + if (rc) + return rc; + + fn->actag_base = base; + fn->actag_enabled = enabled; + fn->actag_supported = supported; + + ocxl_config_set_actag(dev, fn->config.dvsec_function_pos, + fn->actag_base, fn->actag_enabled); + dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n", + fn->actag_base, fn->actag_enabled); + return 0; +} + +static int set_function_pasid(struct ocxl_fn *fn) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + int rc, desired_count, max_count; + + /* A function may not require any PASID */ + if (fn->config.max_pasid_log < 0) + return 0; + + rc = ocxl_config_get_pasid_info(dev, &max_count); + if (rc) + return rc; + + desired_count = 1 << fn->config.max_pasid_log; + + if (desired_count > max_count) { + dev_err(&fn->dev, + "Function requires more PASIDs than is available (%d vs. %d)\n", + desired_count, max_count); + return -ENOSPC; + } + + fn->pasid_base = 0; + return 0; +} + +static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev) +{ + int rc; + + rc = pci_enable_device(dev); + if (rc) { + dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc); + return rc; + } + + /* + * Once it has been confirmed to work on our hardware, we + * should reset the function, to force the adapter to restart + * from scratch. + * A function reset would also reset all its AFUs. + * + * Some hints for implementation: + * + * - there's not status bit to know when the reset is done. We + * should try reading the config space to know when it's + * done. + * - probably something like: + * Reset + * wait 100ms + * issue config read + * allow device up to 1 sec to return success on config + * read before declaring it broken + * + * Some shared logic on the card (CFG, TLX) won't be reset, so + * there's no guarantee that it will be enough. + */ + rc = ocxl_config_read_function(dev, &fn->config); + if (rc) + return rc; + + rc = set_function_device(fn, dev); + if (rc) + return rc; + + rc = assign_function_actag(fn); + if (rc) + return rc; + + rc = set_function_pasid(fn); + if (rc) + return rc; + + rc = ocxl_link_setup(dev, 0, &fn->link); + if (rc) + return rc; + + rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos); + if (rc) { + ocxl_link_release(dev, fn->link); + return rc; + } + return 0; +} + +static void deconfigure_function(struct ocxl_fn *fn) +{ + struct pci_dev *dev = to_pci_dev(fn->dev.parent); + + ocxl_link_release(dev, fn->link); + pci_disable_device(dev); +} + +static struct ocxl_fn *init_function(struct pci_dev *dev) +{ + struct ocxl_fn *fn; + int rc; + + fn = alloc_function(); + if (!fn) + return ERR_PTR(-ENOMEM); + + rc = configure_function(fn, dev); + if (rc) { + free_function(fn); + return ERR_PTR(rc); + } + + rc = device_register(&fn->dev); + if (rc) { + deconfigure_function(fn); + put_device(&fn->dev); + return ERR_PTR(rc); + } + return fn; +} + +// Device detection & initialisation + +struct ocxl_fn *ocxl_function_open(struct pci_dev *dev) +{ + int rc, afu_count = 0; + u8 afu; + struct ocxl_fn *fn; + + if (!radix_enabled()) { + dev_err(&dev->dev, "Unsupported memory model (hash)\n"); + return ERR_PTR(-ENODEV); + } + + fn = init_function(dev); + if (IS_ERR(fn)) { + dev_err(&dev->dev, "function init failed: %li\n", + PTR_ERR(fn)); + return fn; + } + + for (afu = 0; afu <= fn->config.max_afu_index; afu++) { + rc = ocxl_config_check_afu_index(dev, &fn->config, afu); + if (rc > 0) { + rc = init_afu(dev, fn, afu); + if (rc) { + dev_err(&dev->dev, + "Can't initialize AFU index %d\n", afu); + continue; + } + afu_count++; + } + } + dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count); + return fn; +} +EXPORT_SYMBOL_GPL(ocxl_function_open); + +struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn) +{ + return &fn->afu_list; +} +EXPORT_SYMBOL_GPL(ocxl_function_afu_list); + +struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx) +{ + struct ocxl_afu *afu; + + list_for_each_entry(afu, &fn->afu_list, list) { + if (afu->config.idx == afu_idx) + return afu; + } + + return NULL; +} +EXPORT_SYMBOL_GPL(ocxl_function_fetch_afu); + +const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn) +{ + return &fn->config; +} +EXPORT_SYMBOL_GPL(ocxl_function_config); + +void ocxl_function_close(struct ocxl_fn *fn) +{ + struct ocxl_afu *afu, *tmp; + + list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) { + remove_afu(afu); + } + + deconfigure_function(fn); + device_unregister(&fn->dev); +} +EXPORT_SYMBOL_GPL(ocxl_function_close); + +// AFU Metadata + +struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu) +{ + return &afu->config; +} +EXPORT_SYMBOL_GPL(ocxl_afu_config); + +void ocxl_afu_set_private(struct ocxl_afu *afu, void *private) +{ + afu->private = private; +} +EXPORT_SYMBOL_GPL(ocxl_afu_set_private); + +void *ocxl_afu_get_private(struct ocxl_afu *afu) +{ + if (afu) + return afu->private; + + return NULL; +} +EXPORT_SYMBOL_GPL(ocxl_afu_get_private); diff --git a/drivers/misc/ocxl/file.c b/drivers/misc/ocxl/file.c index e6a607488f8a..2870c25da166 100644 --- a/drivers/misc/ocxl/file.c +++ b/drivers/misc/ocxl/file.c @@ -3,6 +3,7 @@ #include <linux/fs.h> #include <linux/poll.h> #include <linux/sched/signal.h> +#include <linux/eventfd.h> #include <linux/uaccess.h> #include <uapi/misc/ocxl.h> #include <asm/reg.h> @@ -17,70 +18,56 @@ static struct class *ocxl_class; static struct mutex minors_idr_lock; static struct idr minors_idr; -static struct ocxl_afu *find_and_get_afu(dev_t devno) +static struct ocxl_file_info *find_file_info(dev_t devno) { - struct ocxl_afu *afu; - int afu_minor; + struct ocxl_file_info *info; - afu_minor = MINOR(devno); /* * We don't declare an RCU critical section here, as our AFU * is protected by a reference counter on the device. By the time the - * minor number of a device is removed from the idr, the ref count of + * info reference is removed from the idr, the ref count of * the device is already at 0, so no user API will access that AFU and * this function can't return it. */ - afu = idr_find(&minors_idr, afu_minor); - if (afu) - ocxl_afu_get(afu); - return afu; + info = idr_find(&minors_idr, MINOR(devno)); + return info; } -static int allocate_afu_minor(struct ocxl_afu *afu) +static int allocate_minor(struct ocxl_file_info *info) { int minor; mutex_lock(&minors_idr_lock); - minor = idr_alloc(&minors_idr, afu, 0, OCXL_NUM_MINORS, GFP_KERNEL); + minor = idr_alloc(&minors_idr, info, 0, OCXL_NUM_MINORS, GFP_KERNEL); mutex_unlock(&minors_idr_lock); return minor; } -static void free_afu_minor(struct ocxl_afu *afu) +static void free_minor(struct ocxl_file_info *info) { mutex_lock(&minors_idr_lock); - idr_remove(&minors_idr, MINOR(afu->dev.devt)); + idr_remove(&minors_idr, MINOR(info->dev.devt)); mutex_unlock(&minors_idr_lock); } static int afu_open(struct inode *inode, struct file *file) { - struct ocxl_afu *afu; + struct ocxl_file_info *info; struct ocxl_context *ctx; int rc; pr_debug("%s for device %x\n", __func__, inode->i_rdev); - afu = find_and_get_afu(inode->i_rdev); - if (!afu) + info = find_file_info(inode->i_rdev); + if (!info) return -ENODEV; - ctx = ocxl_context_alloc(); - if (!ctx) { - rc = -ENOMEM; - goto put_afu; - } - - rc = ocxl_context_init(ctx, afu, inode->i_mapping); + rc = ocxl_context_alloc(&ctx, info->afu, inode->i_mapping); if (rc) - goto put_afu; + return rc; + file->private_data = ctx; - ocxl_afu_put(afu); return 0; - -put_afu: - ocxl_afu_put(afu); - return rc; } static long afu_ioctl_attach(struct ocxl_context *ctx, @@ -100,7 +87,7 @@ static long afu_ioctl_attach(struct ocxl_context *ctx, return -EINVAL; amr = arg.amr & mfspr(SPRN_UAMOR); - rc = ocxl_context_attach(ctx, amr); + rc = ocxl_context_attach(ctx, amr, current->mm); return rc; } @@ -151,10 +138,9 @@ static long afu_ioctl_enable_p9_wait(struct ocxl_context *ctx, mutex_unlock(&ctx->status_mutex); if (status == ATTACHED) { - int rc; - struct link *link = ctx->afu->fn->link; + int rc = ocxl_link_update_pe(ctx->afu->fn->link, + ctx->pasid, ctx->tidr); - rc = ocxl_link_update_pe(link, ctx->pasid, ctx->tidr); if (rc) return rc; } @@ -198,18 +184,40 @@ static long afu_ioctl_get_features(struct ocxl_context *ctx, x == OCXL_IOCTL_GET_FEATURES ? "GET_FEATURES" : \ "UNKNOWN") +static irqreturn_t irq_handler(void *private) +{ + struct eventfd_ctx *ev_ctx = private; + + eventfd_signal(ev_ctx, 1); + return IRQ_HANDLED; +} + +static void irq_free(void *private) +{ + struct eventfd_ctx *ev_ctx = private; + + eventfd_ctx_put(ev_ctx); +} + static long afu_ioctl(struct file *file, unsigned int cmd, unsigned long args) { struct ocxl_context *ctx = file->private_data; struct ocxl_ioctl_irq_fd irq_fd; + struct eventfd_ctx *ev_ctx; + int irq_id; u64 irq_offset; long rc; + bool closed; pr_debug("%s for context %d, command %s\n", __func__, ctx->pasid, CMD_STR(cmd)); - if (ctx->status == CLOSED) + mutex_lock(&ctx->status_mutex); + closed = (ctx->status == CLOSED); + mutex_unlock(&ctx->status_mutex); + + if (closed) return -EIO; switch (cmd) { @@ -219,12 +227,13 @@ static long afu_ioctl(struct file *file, unsigned int cmd, break; case OCXL_IOCTL_IRQ_ALLOC: - rc = ocxl_afu_irq_alloc(ctx, &irq_offset); + rc = ocxl_afu_irq_alloc(ctx, &irq_id); if (!rc) { + irq_offset = ocxl_irq_id_to_offset(ctx, irq_id); rc = copy_to_user((u64 __user *) args, &irq_offset, sizeof(irq_offset)); if (rc) { - ocxl_afu_irq_free(ctx, irq_offset); + ocxl_afu_irq_free(ctx, irq_id); return -EFAULT; } } @@ -235,7 +244,8 @@ static long afu_ioctl(struct file *file, unsigned int cmd, sizeof(irq_offset)); if (rc) return -EFAULT; - rc = ocxl_afu_irq_free(ctx, irq_offset); + irq_id = ocxl_irq_offset_to_id(ctx, irq_offset); + rc = ocxl_afu_irq_free(ctx, irq_id); break; case OCXL_IOCTL_IRQ_SET_FD: @@ -245,8 +255,11 @@ static long afu_ioctl(struct file *file, unsigned int cmd, return -EFAULT; if (irq_fd.reserved) return -EINVAL; - rc = ocxl_afu_irq_set_fd(ctx, irq_fd.irq_offset, - irq_fd.eventfd); + irq_id = ocxl_irq_offset_to_id(ctx, irq_fd.irq_offset); + ev_ctx = eventfd_ctx_fdget(irq_fd.eventfd); + if (IS_ERR(ev_ctx)) + return PTR_ERR(ev_ctx); + rc = ocxl_irq_set_handler(ctx, irq_id, irq_handler, irq_free, ev_ctx); break; case OCXL_IOCTL_GET_METADATA: @@ -469,39 +482,102 @@ static const struct file_operations ocxl_afu_fops = { .release = afu_release, }; -int ocxl_create_cdev(struct ocxl_afu *afu) +// Free the info struct +static void info_release(struct device *dev) +{ + struct ocxl_file_info *info = container_of(dev, struct ocxl_file_info, dev); + + free_minor(info); + ocxl_afu_put(info->afu); + kfree(info); +} + +static int ocxl_file_make_visible(struct ocxl_file_info *info) { int rc; - cdev_init(&afu->cdev, &ocxl_afu_fops); - rc = cdev_add(&afu->cdev, afu->dev.devt, 1); + cdev_init(&info->cdev, &ocxl_afu_fops); + rc = cdev_add(&info->cdev, info->dev.devt, 1); if (rc) { - dev_err(&afu->dev, "Unable to add afu char device: %d\n", rc); + dev_err(&info->dev, "Unable to add afu char device: %d\n", rc); return rc; } + return 0; } -void ocxl_destroy_cdev(struct ocxl_afu *afu) +static void ocxl_file_make_invisible(struct ocxl_file_info *info) { - cdev_del(&afu->cdev); + cdev_del(&info->cdev); } -int ocxl_register_afu(struct ocxl_afu *afu) +int ocxl_file_register_afu(struct ocxl_afu *afu) { int minor; + int rc; + struct ocxl_file_info *info; + struct ocxl_fn *fn = afu->fn; + struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent); - minor = allocate_afu_minor(afu); - if (minor < 0) + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (info == NULL) + return -ENOMEM; + + minor = allocate_minor(info); + if (minor < 0) { + kfree(info); return minor; - afu->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); - afu->dev.class = ocxl_class; - return device_register(&afu->dev); + } + + info->dev.parent = &fn->dev; + info->dev.devt = MKDEV(MAJOR(ocxl_dev), minor); + info->dev.class = ocxl_class; + info->dev.release = info_release; + + info->afu = afu; + ocxl_afu_get(afu); + + rc = dev_set_name(&info->dev, "%s.%s.%hhu", + afu->config.name, dev_name(&pci_dev->dev), afu->config.idx); + if (rc) + goto err_put; + + rc = device_register(&info->dev); + if (rc) + goto err_put; + + rc = ocxl_sysfs_register_afu(info); + if (rc) + goto err_unregister; + + rc = ocxl_file_make_visible(info); + if (rc) + goto err_unregister; + + ocxl_afu_set_private(afu, info); + + return 0; + +err_unregister: + ocxl_sysfs_unregister_afu(info); // safe to call even if register failed + device_unregister(&info->dev); +err_put: + ocxl_afu_put(afu); + free_minor(info); + kfree(info); + return rc; } -void ocxl_unregister_afu(struct ocxl_afu *afu) +void ocxl_file_unregister_afu(struct ocxl_afu *afu) { - free_afu_minor(afu); + struct ocxl_file_info *info = ocxl_afu_get_private(afu); + + if (!info) + return; + + ocxl_file_make_invisible(info); + ocxl_sysfs_unregister_afu(info); + device_unregister(&info->dev); } static char *ocxl_devnode(struct device *dev, umode_t *mode) diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c index d50b861d7e57..cce5b0d64505 100644 --- a/drivers/misc/ocxl/link.c +++ b/drivers/misc/ocxl/link.c @@ -76,7 +76,7 @@ struct spa { * limited number of opencapi slots on a system and lookup is only * done when the device is probed */ -struct link { +struct ocxl_link { struct list_head list; struct kref ref; int domain; @@ -163,7 +163,7 @@ static void xsl_fault_handler_bh(struct work_struct *fault_work) if (fault->dsisr & SPA_XSL_S) access |= _PAGE_WRITE; - if (REGION_ID(fault->dar) != USER_REGION_ID) + if (get_region_id(fault->dar) != USER_REGION_ID) access |= _PAGE_PRIVILEGED; local_irq_save(flags); @@ -179,12 +179,12 @@ ack: static irqreturn_t xsl_fault_handler(int irq, void *data) { - struct link *link = (struct link *) data; + struct ocxl_link *link = (struct ocxl_link *) data; struct spa *spa = link->spa; u64 dsisr, dar, pe_handle; struct pe_data *pe_data; struct ocxl_process_element *pe; - int lpid, pid, tid; + int pid; bool schedule = false; read_irq(spa, &dsisr, &dar, &pe_handle); @@ -192,9 +192,7 @@ static irqreturn_t xsl_fault_handler(int irq, void *data) WARN_ON(pe_handle > SPA_PE_MASK); pe = spa->spa_mem + pe_handle; - lpid = be32_to_cpu(pe->lpid); pid = be32_to_cpu(pe->pid); - tid = be32_to_cpu(pe->tid); /* We could be reading all null values here if the PE is being * removed while an interrupt kicks in. It's not supposed to * happen if the driver notified the AFU to terminate the @@ -256,7 +254,7 @@ static int map_irq_registers(struct pci_dev *dev, struct spa *spa) &spa->reg_tfc, &spa->reg_pe_handle); } -static int setup_xsl_irq(struct pci_dev *dev, struct link *link) +static int setup_xsl_irq(struct pci_dev *dev, struct ocxl_link *link) { struct spa *spa = link->spa; int rc; @@ -311,7 +309,7 @@ err_xsl: return rc; } -static void release_xsl_irq(struct link *link) +static void release_xsl_irq(struct ocxl_link *link) { struct spa *spa = link->spa; @@ -323,7 +321,7 @@ static void release_xsl_irq(struct link *link) unmap_irq_registers(spa); } -static int alloc_spa(struct pci_dev *dev, struct link *link) +static int alloc_spa(struct pci_dev *dev, struct ocxl_link *link) { struct spa *spa; @@ -350,7 +348,7 @@ static int alloc_spa(struct pci_dev *dev, struct link *link) return 0; } -static void free_spa(struct link *link) +static void free_spa(struct ocxl_link *link) { struct spa *spa = link->spa; @@ -364,12 +362,12 @@ static void free_spa(struct link *link) } } -static int alloc_link(struct pci_dev *dev, int PE_mask, struct link **out_link) +static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_link) { - struct link *link; + struct ocxl_link *link; int rc; - link = kzalloc(sizeof(struct link), GFP_KERNEL); + link = kzalloc(sizeof(struct ocxl_link), GFP_KERNEL); if (!link) return -ENOMEM; @@ -405,7 +403,7 @@ err_free: return rc; } -static void free_link(struct link *link) +static void free_link(struct ocxl_link *link) { release_xsl_irq(link); free_spa(link); @@ -415,7 +413,7 @@ static void free_link(struct link *link) int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle) { int rc = 0; - struct link *link; + struct ocxl_link *link; mutex_lock(&links_list_lock); list_for_each_entry(link, &links_list, list) { @@ -442,7 +440,7 @@ EXPORT_SYMBOL_GPL(ocxl_link_setup); static void release_xsl(struct kref *ref) { - struct link *link = container_of(ref, struct link, ref); + struct ocxl_link *link = container_of(ref, struct ocxl_link, ref); list_del(&link->list); /* call platform code before releasing data */ @@ -452,7 +450,7 @@ static void release_xsl(struct kref *ref) void ocxl_link_release(struct pci_dev *dev, void *link_handle) { - struct link *link = (struct link *) link_handle; + struct ocxl_link *link = (struct ocxl_link *) link_handle; mutex_lock(&links_list_lock); kref_put(&link->ref, release_xsl); @@ -488,7 +486,7 @@ int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr, void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr), void *xsl_err_data) { - struct link *link = (struct link *) link_handle; + struct ocxl_link *link = (struct ocxl_link *) link_handle; struct spa *spa = link->spa; struct ocxl_process_element *pe; int pe_handle, rc = 0; @@ -558,7 +556,7 @@ EXPORT_SYMBOL_GPL(ocxl_link_add_pe); int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid) { - struct link *link = (struct link *) link_handle; + struct ocxl_link *link = (struct ocxl_link *) link_handle; struct spa *spa = link->spa; struct ocxl_process_element *pe; int pe_handle, rc; @@ -594,7 +592,7 @@ int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid) int ocxl_link_remove_pe(void *link_handle, int pasid) { - struct link *link = (struct link *) link_handle; + struct ocxl_link *link = (struct ocxl_link *) link_handle; struct spa *spa = link->spa; struct ocxl_process_element *pe; struct pe_data *pe_data; @@ -666,7 +664,7 @@ EXPORT_SYMBOL_GPL(ocxl_link_remove_pe); int ocxl_link_irq_alloc(void *link_handle, int *hw_irq, u64 *trigger_addr) { - struct link *link = (struct link *) link_handle; + struct ocxl_link *link = (struct ocxl_link *) link_handle; int rc, irq; u64 addr; @@ -687,7 +685,7 @@ EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc); void ocxl_link_free_irq(void *link_handle, int hw_irq) { - struct link *link = (struct link *) link_handle; + struct ocxl_link *link = (struct ocxl_link *) link_handle; pnv_ocxl_free_xive_irq(hw_irq); atomic_inc(&link->irq_available); diff --git a/drivers/misc/ocxl/mmio.c b/drivers/misc/ocxl/mmio.c new file mode 100644 index 000000000000..aae713db4ebe --- /dev/null +++ b/drivers/misc/ocxl/mmio.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2019 IBM Corp. +#include <linux/sched/mm.h> +#include "trace.h" +#include "ocxl_internal.h" + +int ocxl_global_mmio_read32(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u32 *val) +{ + if (offset > afu->config.global_mmio_size - 4) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + *val = readl_be((char *)afu->global_mmio_ptr + offset); + break; + + default: + *val = readl((char *)afu->global_mmio_ptr + offset); + break; + } + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_read32); + +int ocxl_global_mmio_read64(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u64 *val) +{ + if (offset > afu->config.global_mmio_size - 8) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + *val = readq_be((char *)afu->global_mmio_ptr + offset); + break; + + default: + *val = readq((char *)afu->global_mmio_ptr + offset); + break; + } + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_read64); + +int ocxl_global_mmio_write32(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u32 val) +{ + if (offset > afu->config.global_mmio_size - 4) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + writel_be(val, (char *)afu->global_mmio_ptr + offset); + break; + + default: + writel(val, (char *)afu->global_mmio_ptr + offset); + break; + } + + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_write32); + +int ocxl_global_mmio_write64(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u64 val) +{ + if (offset > afu->config.global_mmio_size - 8) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + writeq_be(val, (char *)afu->global_mmio_ptr + offset); + break; + + default: + writeq(val, (char *)afu->global_mmio_ptr + offset); + break; + } + + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_write64); + +int ocxl_global_mmio_set32(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u32 mask) +{ + u32 tmp; + + if (offset > afu->config.global_mmio_size - 4) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + tmp = readl_be((char *)afu->global_mmio_ptr + offset); + tmp |= mask; + writel_be(tmp, (char *)afu->global_mmio_ptr + offset); + break; + + default: + tmp = readl((char *)afu->global_mmio_ptr + offset); + tmp |= mask; + writel(tmp, (char *)afu->global_mmio_ptr + offset); + break; + } + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_set32); + +int ocxl_global_mmio_set64(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u64 mask) +{ + u64 tmp; + + if (offset > afu->config.global_mmio_size - 8) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + tmp = readq_be((char *)afu->global_mmio_ptr + offset); + tmp |= mask; + writeq_be(tmp, (char *)afu->global_mmio_ptr + offset); + break; + + default: + tmp = readq((char *)afu->global_mmio_ptr + offset); + tmp |= mask; + writeq(tmp, (char *)afu->global_mmio_ptr + offset); + break; + } + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_set64); + +int ocxl_global_mmio_clear32(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u32 mask) +{ + u32 tmp; + + if (offset > afu->config.global_mmio_size - 4) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + tmp = readl_be((char *)afu->global_mmio_ptr + offset); + tmp &= ~mask; + writel_be(tmp, (char *)afu->global_mmio_ptr + offset); + break; + + default: + tmp = readl((char *)afu->global_mmio_ptr + offset); + tmp &= ~mask; + writel(tmp, (char *)afu->global_mmio_ptr + offset); + break; + } + + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_clear32); + +int ocxl_global_mmio_clear64(struct ocxl_afu *afu, size_t offset, + enum ocxl_endian endian, u64 mask) +{ + u64 tmp; + + if (offset > afu->config.global_mmio_size - 8) + return -EINVAL; + +#ifdef __BIG_ENDIAN__ + if (endian == OCXL_HOST_ENDIAN) + endian = OCXL_BIG_ENDIAN; +#endif + + switch (endian) { + case OCXL_BIG_ENDIAN: + tmp = readq_be((char *)afu->global_mmio_ptr + offset); + tmp &= ~mask; + writeq_be(tmp, (char *)afu->global_mmio_ptr + offset); + break; + + default: + tmp = readq((char *)afu->global_mmio_ptr + offset); + tmp &= ~mask; + writeq(tmp, (char *)afu->global_mmio_ptr + offset); + break; + } + + writeq(tmp, (char *)afu->global_mmio_ptr + offset); + + return 0; +} +EXPORT_SYMBOL_GPL(ocxl_global_mmio_clear64); diff --git a/drivers/misc/ocxl/ocxl_internal.h b/drivers/misc/ocxl/ocxl_internal.h index a32f2151029f..97415afd79f3 100644 --- a/drivers/misc/ocxl/ocxl_internal.h +++ b/drivers/misc/ocxl/ocxl_internal.h @@ -11,12 +11,8 @@ #define MAX_IRQ_PER_LINK 2000 #define MAX_IRQ_PER_CONTEXT MAX_IRQ_PER_LINK -#define to_ocxl_function(d) container_of(d, struct ocxl_fn, dev) -#define to_ocxl_afu(d) container_of(d, struct ocxl_afu, dev) - extern struct pci_driver ocxl_pci_driver; - struct ocxl_fn { struct device dev; int bar_used[3]; @@ -31,11 +27,17 @@ struct ocxl_fn { void *link; }; +struct ocxl_file_info { + struct ocxl_afu *afu; + struct device dev; + struct cdev cdev; + struct bin_attribute attr_global_mmio; +}; + struct ocxl_afu { + struct kref kref; struct ocxl_fn *fn; struct list_head list; - struct device dev; - struct cdev cdev; struct ocxl_afu_config config; int pasid_base; int pasid_count; /* opened contexts */ @@ -49,7 +51,7 @@ struct ocxl_afu { u64 irq_base_offset; void __iomem *global_mmio_ptr; u64 pp_mmio_start; - struct bin_attribute attr_global_mmio; + void *private; }; enum ocxl_context_status { @@ -92,41 +94,51 @@ struct ocxl_process_element { __be32 software_state; }; +int ocxl_create_cdev(struct ocxl_afu *afu); +void ocxl_destroy_cdev(struct ocxl_afu *afu); +int ocxl_file_register_afu(struct ocxl_afu *afu); +void ocxl_file_unregister_afu(struct ocxl_afu *afu); + +int ocxl_file_init(void); +void ocxl_file_exit(void); + +int ocxl_pasid_afu_alloc(struct ocxl_fn *fn, u32 size); +void ocxl_pasid_afu_free(struct ocxl_fn *fn, u32 start, u32 size); +int ocxl_actag_afu_alloc(struct ocxl_fn *fn, u32 size); +void ocxl_actag_afu_free(struct ocxl_fn *fn, u32 start, u32 size); + +/* + * Get the max PASID value that can be used by the function + */ +int ocxl_config_get_pasid_info(struct pci_dev *dev, int *count); + +/* + * Check if an AFU index is valid for the given function. + * + * AFU indexes can be sparse, so a driver should check all indexes up + * to the maximum found in the function description + */ +int ocxl_config_check_afu_index(struct pci_dev *dev, + struct ocxl_fn_config *fn, int afu_idx); + +/** + * Update values within a Process Element + * + * link_handle: the link handle associated with the process element + * pasid: the PASID for the AFU context + * tid: the new thread id for the process element + */ +int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid); + +int ocxl_context_mmap(struct ocxl_context *ctx, + struct vm_area_struct *vma); +void ocxl_context_detach_all(struct ocxl_afu *afu); -extern struct ocxl_afu *ocxl_afu_get(struct ocxl_afu *afu); -extern void ocxl_afu_put(struct ocxl_afu *afu); - -extern int ocxl_create_cdev(struct ocxl_afu *afu); -extern void ocxl_destroy_cdev(struct ocxl_afu *afu); -extern int ocxl_register_afu(struct ocxl_afu *afu); -extern void ocxl_unregister_afu(struct ocxl_afu *afu); - -extern int ocxl_file_init(void); -extern void ocxl_file_exit(void); - -extern int ocxl_pasid_afu_alloc(struct ocxl_fn *fn, u32 size); -extern void ocxl_pasid_afu_free(struct ocxl_fn *fn, u32 start, u32 size); -extern int ocxl_actag_afu_alloc(struct ocxl_fn *fn, u32 size); -extern void ocxl_actag_afu_free(struct ocxl_fn *fn, u32 start, u32 size); +int ocxl_sysfs_register_afu(struct ocxl_file_info *info); +void ocxl_sysfs_unregister_afu(struct ocxl_file_info *info); -extern struct ocxl_context *ocxl_context_alloc(void); -extern int ocxl_context_init(struct ocxl_context *ctx, struct ocxl_afu *afu, - struct address_space *mapping); -extern int ocxl_context_attach(struct ocxl_context *ctx, u64 amr); -extern int ocxl_context_mmap(struct ocxl_context *ctx, - struct vm_area_struct *vma); -extern int ocxl_context_detach(struct ocxl_context *ctx); -extern void ocxl_context_detach_all(struct ocxl_afu *afu); -extern void ocxl_context_free(struct ocxl_context *ctx); - -extern int ocxl_sysfs_add_afu(struct ocxl_afu *afu); -extern void ocxl_sysfs_remove_afu(struct ocxl_afu *afu); - -extern int ocxl_afu_irq_alloc(struct ocxl_context *ctx, u64 *irq_offset); -extern int ocxl_afu_irq_free(struct ocxl_context *ctx, u64 irq_offset); -extern void ocxl_afu_irq_free_all(struct ocxl_context *ctx); -extern int ocxl_afu_irq_set_fd(struct ocxl_context *ctx, u64 irq_offset, - int eventfd); -extern u64 ocxl_afu_irq_get_addr(struct ocxl_context *ctx, u64 irq_offset); +int ocxl_irq_offset_to_id(struct ocxl_context *ctx, u64 offset); +u64 ocxl_irq_id_to_offset(struct ocxl_context *ctx, int irq_id); +void ocxl_afu_irq_free_all(struct ocxl_context *ctx); #endif /* _OCXL_INTERNAL_H_ */ diff --git a/drivers/misc/ocxl/pci.c b/drivers/misc/ocxl/pci.c index 21f425472a82..f2a3ef4b9bdd 100644 --- a/drivers/misc/ocxl/pci.c +++ b/drivers/misc/ocxl/pci.c @@ -1,9 +1,6 @@ // SPDX-License-Identifier: GPL-2.0+ -// Copyright 2017 IBM Corp. +// Copyright 2019 IBM Corp. #include <linux/module.h> -#include <linux/pci.h> -#include <linux/idr.h> -#include <asm/pnv-ocxl.h> #include "ocxl_internal.h" /* @@ -17,563 +14,47 @@ static const struct pci_device_id ocxl_pci_tbl[] = { }; MODULE_DEVICE_TABLE(pci, ocxl_pci_tbl); - -static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn) -{ - return (get_device(&fn->dev) == NULL) ? NULL : fn; -} - -static void ocxl_fn_put(struct ocxl_fn *fn) -{ - put_device(&fn->dev); -} - -struct ocxl_afu *ocxl_afu_get(struct ocxl_afu *afu) -{ - return (get_device(&afu->dev) == NULL) ? NULL : afu; -} - -void ocxl_afu_put(struct ocxl_afu *afu) -{ - put_device(&afu->dev); -} - -static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn) -{ - struct ocxl_afu *afu; - - afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL); - if (!afu) - return NULL; - - mutex_init(&afu->contexts_lock); - mutex_init(&afu->afu_control_lock); - idr_init(&afu->contexts_idr); - afu->fn = fn; - ocxl_fn_get(fn); - return afu; -} - -static void free_afu(struct ocxl_afu *afu) -{ - idr_destroy(&afu->contexts_idr); - ocxl_fn_put(afu->fn); - kfree(afu); -} - -static void free_afu_dev(struct device *dev) -{ - struct ocxl_afu *afu = to_ocxl_afu(dev); - - ocxl_unregister_afu(afu); - free_afu(afu); -} - -static int set_afu_device(struct ocxl_afu *afu, const char *location) -{ - struct ocxl_fn *fn = afu->fn; - int rc; - - afu->dev.parent = &fn->dev; - afu->dev.release = free_afu_dev; - rc = dev_set_name(&afu->dev, "%s.%s.%hhu", afu->config.name, location, - afu->config.idx); - return rc; -} - -static int assign_afu_actag(struct ocxl_afu *afu, struct pci_dev *dev) -{ - struct ocxl_fn *fn = afu->fn; - int actag_count, actag_offset; - - /* - * if there were not enough actags for the function, each afu - * reduces its count as well - */ - actag_count = afu->config.actag_supported * - fn->actag_enabled / fn->actag_supported; - actag_offset = ocxl_actag_afu_alloc(fn, actag_count); - if (actag_offset < 0) { - dev_err(&afu->dev, "Can't allocate %d actags for AFU: %d\n", - actag_count, actag_offset); - return actag_offset; - } - afu->actag_base = fn->actag_base + actag_offset; - afu->actag_enabled = actag_count; - - ocxl_config_set_afu_actag(dev, afu->config.dvsec_afu_control_pos, - afu->actag_base, afu->actag_enabled); - dev_dbg(&afu->dev, "actag base=%d enabled=%d\n", - afu->actag_base, afu->actag_enabled); - return 0; -} - -static void reclaim_afu_actag(struct ocxl_afu *afu) -{ - struct ocxl_fn *fn = afu->fn; - int start_offset, size; - - start_offset = afu->actag_base - fn->actag_base; - size = afu->actag_enabled; - ocxl_actag_afu_free(afu->fn, start_offset, size); -} - -static int assign_afu_pasid(struct ocxl_afu *afu, struct pci_dev *dev) -{ - struct ocxl_fn *fn = afu->fn; - int pasid_count, pasid_offset; - - /* - * We only support the case where the function configuration - * requested enough PASIDs to cover all AFUs. - */ - pasid_count = 1 << afu->config.pasid_supported_log; - pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count); - if (pasid_offset < 0) { - dev_err(&afu->dev, "Can't allocate %d PASIDs for AFU: %d\n", - pasid_count, pasid_offset); - return pasid_offset; - } - afu->pasid_base = fn->pasid_base + pasid_offset; - afu->pasid_count = 0; - afu->pasid_max = pasid_count; - - ocxl_config_set_afu_pasid(dev, afu->config.dvsec_afu_control_pos, - afu->pasid_base, - afu->config.pasid_supported_log); - dev_dbg(&afu->dev, "PASID base=%d, enabled=%d\n", - afu->pasid_base, pasid_count); - return 0; -} - -static void reclaim_afu_pasid(struct ocxl_afu *afu) -{ - struct ocxl_fn *fn = afu->fn; - int start_offset, size; - - start_offset = afu->pasid_base - fn->pasid_base; - size = 1 << afu->config.pasid_supported_log; - ocxl_pasid_afu_free(afu->fn, start_offset, size); -} - -static int reserve_fn_bar(struct ocxl_fn *fn, int bar) -{ - struct pci_dev *dev = to_pci_dev(fn->dev.parent); - int rc, idx; - - if (bar != 0 && bar != 2 && bar != 4) - return -EINVAL; - - idx = bar >> 1; - if (fn->bar_used[idx]++ == 0) { - rc = pci_request_region(dev, bar, "ocxl"); - if (rc) - return rc; - } - return 0; -} - -static void release_fn_bar(struct ocxl_fn *fn, int bar) -{ - struct pci_dev *dev = to_pci_dev(fn->dev.parent); - int idx; - - if (bar != 0 && bar != 2 && bar != 4) - return; - - idx = bar >> 1; - if (--fn->bar_used[idx] == 0) - pci_release_region(dev, bar); - WARN_ON(fn->bar_used[idx] < 0); -} - -static int map_mmio_areas(struct ocxl_afu *afu, struct pci_dev *dev) -{ - int rc; - - rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar); - if (rc) - return rc; - - rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar); - if (rc) { - release_fn_bar(afu->fn, afu->config.global_mmio_bar); - return rc; - } - - afu->global_mmio_start = - pci_resource_start(dev, afu->config.global_mmio_bar) + - afu->config.global_mmio_offset; - afu->pp_mmio_start = - pci_resource_start(dev, afu->config.pp_mmio_bar) + - afu->config.pp_mmio_offset; - - afu->global_mmio_ptr = ioremap(afu->global_mmio_start, - afu->config.global_mmio_size); - if (!afu->global_mmio_ptr) { - release_fn_bar(afu->fn, afu->config.pp_mmio_bar); - release_fn_bar(afu->fn, afu->config.global_mmio_bar); - dev_err(&dev->dev, "Error mapping global mmio area\n"); - return -ENOMEM; - } - - /* - * Leave an empty page between the per-process mmio area and - * the AFU interrupt mappings - */ - afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE; - return 0; -} - -static void unmap_mmio_areas(struct ocxl_afu *afu) -{ - if (afu->global_mmio_ptr) { - iounmap(afu->global_mmio_ptr); - afu->global_mmio_ptr = NULL; - } - afu->global_mmio_start = 0; - afu->pp_mmio_start = 0; - release_fn_bar(afu->fn, afu->config.pp_mmio_bar); - release_fn_bar(afu->fn, afu->config.global_mmio_bar); -} - -static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev) -{ - int rc; - - rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx); - if (rc) - return rc; - - rc = set_afu_device(afu, dev_name(&dev->dev)); - if (rc) - return rc; - - rc = assign_afu_actag(afu, dev); - if (rc) - return rc; - - rc = assign_afu_pasid(afu, dev); - if (rc) { - reclaim_afu_actag(afu); - return rc; - } - - rc = map_mmio_areas(afu, dev); - if (rc) { - reclaim_afu_pasid(afu); - reclaim_afu_actag(afu); - return rc; - } - return 0; -} - -static void deconfigure_afu(struct ocxl_afu *afu) -{ - unmap_mmio_areas(afu); - reclaim_afu_pasid(afu); - reclaim_afu_actag(afu); -} - -static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu) -{ - int rc; - - ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1); - /* - * Char device creation is the last step, as processes can - * call our driver immediately, so all our inits must be finished. - */ - rc = ocxl_create_cdev(afu); - if (rc) - return rc; - return 0; -} - -static void deactivate_afu(struct ocxl_afu *afu) -{ - struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent); - - ocxl_destroy_cdev(afu); - ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0); -} - -static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx) +static int ocxl_probe(struct pci_dev *dev, const struct pci_device_id *id) { int rc; - struct ocxl_afu *afu; - - afu = alloc_afu(fn); - if (!afu) - return -ENOMEM; - - rc = configure_afu(afu, afu_idx, dev); - if (rc) { - free_afu(afu); - return rc; - } - - rc = ocxl_register_afu(afu); - if (rc) - goto err; - - rc = ocxl_sysfs_add_afu(afu); - if (rc) - goto err; - - rc = activate_afu(dev, afu); - if (rc) - goto err_sys; - - list_add_tail(&afu->list, &fn->afu_list); - return 0; - -err_sys: - ocxl_sysfs_remove_afu(afu); -err: - deconfigure_afu(afu); - device_unregister(&afu->dev); - return rc; -} - -static void remove_afu(struct ocxl_afu *afu) -{ - list_del(&afu->list); - ocxl_context_detach_all(afu); - deactivate_afu(afu); - ocxl_sysfs_remove_afu(afu); - deconfigure_afu(afu); - device_unregister(&afu->dev); -} - -static struct ocxl_fn *alloc_function(struct pci_dev *dev) -{ + struct ocxl_afu *afu, *tmp; struct ocxl_fn *fn; + struct list_head *afu_list; - fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL); - if (!fn) - return NULL; - - INIT_LIST_HEAD(&fn->afu_list); - INIT_LIST_HEAD(&fn->pasid_list); - INIT_LIST_HEAD(&fn->actag_list); - return fn; -} - -static void free_function(struct ocxl_fn *fn) -{ - WARN_ON(!list_empty(&fn->afu_list)); - WARN_ON(!list_empty(&fn->pasid_list)); - kfree(fn); -} - -static void free_function_dev(struct device *dev) -{ - struct ocxl_fn *fn = to_ocxl_function(dev); - - free_function(fn); -} - -static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev) -{ - int rc; + fn = ocxl_function_open(dev); + if (IS_ERR(fn)) + return PTR_ERR(fn); - fn->dev.parent = &dev->dev; - fn->dev.release = free_function_dev; - rc = dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev)); - if (rc) - return rc; pci_set_drvdata(dev, fn); - return 0; -} - -static int assign_function_actag(struct ocxl_fn *fn) -{ - struct pci_dev *dev = to_pci_dev(fn->dev.parent); - u16 base, enabled, supported; - int rc; - - rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported); - if (rc) - return rc; - - fn->actag_base = base; - fn->actag_enabled = enabled; - fn->actag_supported = supported; - - ocxl_config_set_actag(dev, fn->config.dvsec_function_pos, - fn->actag_base, fn->actag_enabled); - dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n", - fn->actag_base, fn->actag_enabled); - return 0; -} -static int set_function_pasid(struct ocxl_fn *fn) -{ - struct pci_dev *dev = to_pci_dev(fn->dev.parent); - int rc, desired_count, max_count; - - /* A function may not require any PASID */ - if (fn->config.max_pasid_log < 0) - return 0; - - rc = ocxl_config_get_pasid_info(dev, &max_count); - if (rc) - return rc; - - desired_count = 1 << fn->config.max_pasid_log; + afu_list = ocxl_function_afu_list(fn); - if (desired_count > max_count) { - dev_err(&fn->dev, - "Function requires more PASIDs than is available (%d vs. %d)\n", - desired_count, max_count); - return -ENOSPC; + list_for_each_entry_safe(afu, tmp, afu_list, list) { + // Cleanup handled within ocxl_file_register_afu() + rc = ocxl_file_register_afu(afu); + if (rc) { + dev_err(&dev->dev, "Failed to register AFU '%s' index %d", + afu->config.name, afu->config.idx); + } } - fn->pasid_base = 0; return 0; } -static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev) -{ - int rc; - - rc = pci_enable_device(dev); - if (rc) { - dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc); - return rc; - } - - /* - * Once it has been confirmed to work on our hardware, we - * should reset the function, to force the adapter to restart - * from scratch. - * A function reset would also reset all its AFUs. - * - * Some hints for implementation: - * - * - there's not status bit to know when the reset is done. We - * should try reading the config space to know when it's - * done. - * - probably something like: - * Reset - * wait 100ms - * issue config read - * allow device up to 1 sec to return success on config - * read before declaring it broken - * - * Some shared logic on the card (CFG, TLX) won't be reset, so - * there's no guarantee that it will be enough. - */ - rc = ocxl_config_read_function(dev, &fn->config); - if (rc) - return rc; - - rc = set_function_device(fn, dev); - if (rc) - return rc; - - rc = assign_function_actag(fn); - if (rc) - return rc; - - rc = set_function_pasid(fn); - if (rc) - return rc; - - rc = ocxl_link_setup(dev, 0, &fn->link); - if (rc) - return rc; - - rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos); - if (rc) { - ocxl_link_release(dev, fn->link); - return rc; - } - return 0; -} - -static void deconfigure_function(struct ocxl_fn *fn) -{ - struct pci_dev *dev = to_pci_dev(fn->dev.parent); - - ocxl_link_release(dev, fn->link); - pci_disable_device(dev); -} - -static struct ocxl_fn *init_function(struct pci_dev *dev) +void ocxl_remove(struct pci_dev *dev) { struct ocxl_fn *fn; - int rc; - - fn = alloc_function(dev); - if (!fn) - return ERR_PTR(-ENOMEM); - - rc = configure_function(fn, dev); - if (rc) { - free_function(fn); - return ERR_PTR(rc); - } - - rc = device_register(&fn->dev); - if (rc) { - deconfigure_function(fn); - put_device(&fn->dev); - return ERR_PTR(rc); - } - return fn; -} - -static void remove_function(struct ocxl_fn *fn) -{ - deconfigure_function(fn); - device_unregister(&fn->dev); -} - -static int ocxl_probe(struct pci_dev *dev, const struct pci_device_id *id) -{ - int rc, afu_count = 0; - u8 afu; - struct ocxl_fn *fn; - - if (!radix_enabled()) { - dev_err(&dev->dev, "Unsupported memory model (hash)\n"); - return -ENODEV; - } + struct ocxl_afu *afu; + struct list_head *afu_list; - fn = init_function(dev); - if (IS_ERR(fn)) { - dev_err(&dev->dev, "function init failed: %li\n", - PTR_ERR(fn)); - return PTR_ERR(fn); - } + fn = pci_get_drvdata(dev); + afu_list = ocxl_function_afu_list(fn); - for (afu = 0; afu <= fn->config.max_afu_index; afu++) { - rc = ocxl_config_check_afu_index(dev, &fn->config, afu); - if (rc > 0) { - rc = init_afu(dev, fn, afu); - if (rc) { - dev_err(&dev->dev, - "Can't initialize AFU index %d\n", afu); - continue; - } - afu_count++; - } + list_for_each_entry(afu, afu_list, list) { + ocxl_file_unregister_afu(afu); } - dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count); - return 0; -} - -static void ocxl_remove(struct pci_dev *dev) -{ - struct ocxl_afu *afu, *tmp; - struct ocxl_fn *fn = pci_get_drvdata(dev); - list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) { - remove_afu(afu); - } - remove_function(fn); + ocxl_function_close(fn); } struct pci_driver ocxl_pci_driver = { diff --git a/drivers/misc/ocxl/sysfs.c b/drivers/misc/ocxl/sysfs.c index 0ab1fd1b2682..58f1ba264206 100644 --- a/drivers/misc/ocxl/sysfs.c +++ b/drivers/misc/ocxl/sysfs.c @@ -3,11 +3,18 @@ #include <linux/sysfs.h> #include "ocxl_internal.h" +static inline struct ocxl_afu *to_afu(struct device *device) +{ + struct ocxl_file_info *info = container_of(device, struct ocxl_file_info, dev); + + return info->afu; +} + static ssize_t global_mmio_size_show(struct device *device, struct device_attribute *attr, char *buf) { - struct ocxl_afu *afu = to_ocxl_afu(device); + struct ocxl_afu *afu = to_afu(device); return scnprintf(buf, PAGE_SIZE, "%d\n", afu->config.global_mmio_size); @@ -17,7 +24,7 @@ static ssize_t pp_mmio_size_show(struct device *device, struct device_attribute *attr, char *buf) { - struct ocxl_afu *afu = to_ocxl_afu(device); + struct ocxl_afu *afu = to_afu(device); return scnprintf(buf, PAGE_SIZE, "%d\n", afu->config.pp_mmio_stride); @@ -27,7 +34,7 @@ static ssize_t afu_version_show(struct device *device, struct device_attribute *attr, char *buf) { - struct ocxl_afu *afu = to_ocxl_afu(device); + struct ocxl_afu *afu = to_afu(device); return scnprintf(buf, PAGE_SIZE, "%hhu:%hhu\n", afu->config.version_major, @@ -38,7 +45,7 @@ static ssize_t contexts_show(struct device *device, struct device_attribute *attr, char *buf) { - struct ocxl_afu *afu = to_ocxl_afu(device); + struct ocxl_afu *afu = to_afu(device); return scnprintf(buf, PAGE_SIZE, "%d/%d\n", afu->pasid_count, afu->pasid_max); @@ -55,7 +62,7 @@ static ssize_t global_mmio_read(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, char *buf, loff_t off, size_t count) { - struct ocxl_afu *afu = to_ocxl_afu(kobj_to_dev(kobj)); + struct ocxl_afu *afu = to_afu(kobj_to_dev(kobj)); if (count == 0 || off < 0 || off >= afu->config.global_mmio_size) @@ -86,7 +93,7 @@ static int global_mmio_mmap(struct file *filp, struct kobject *kobj, struct bin_attribute *bin_attr, struct vm_area_struct *vma) { - struct ocxl_afu *afu = to_ocxl_afu(kobj_to_dev(kobj)); + struct ocxl_afu *afu = to_afu(kobj_to_dev(kobj)); if ((vma_pages(vma) + vma->vm_pgoff) > (afu->config.global_mmio_size >> PAGE_SHIFT)) @@ -99,27 +106,25 @@ static int global_mmio_mmap(struct file *filp, struct kobject *kobj, return 0; } -int ocxl_sysfs_add_afu(struct ocxl_afu *afu) +int ocxl_sysfs_register_afu(struct ocxl_file_info *info) { int i, rc; for (i = 0; i < ARRAY_SIZE(afu_attrs); i++) { - rc = device_create_file(&afu->dev, &afu_attrs[i]); + rc = device_create_file(&info->dev, &afu_attrs[i]); if (rc) goto err; } - sysfs_attr_init(&afu->attr_global_mmio.attr); - afu->attr_global_mmio.attr.name = "global_mmio_area"; - afu->attr_global_mmio.attr.mode = 0600; - afu->attr_global_mmio.size = afu->config.global_mmio_size; - afu->attr_global_mmio.read = global_mmio_read; - afu->attr_global_mmio.mmap = global_mmio_mmap; - rc = device_create_bin_file(&afu->dev, &afu->attr_global_mmio); + sysfs_attr_init(&info->attr_global_mmio.attr); + info->attr_global_mmio.attr.name = "global_mmio_area"; + info->attr_global_mmio.attr.mode = 0600; + info->attr_global_mmio.size = info->afu->config.global_mmio_size; + info->attr_global_mmio.read = global_mmio_read; + info->attr_global_mmio.mmap = global_mmio_mmap; + rc = device_create_bin_file(&info->dev, &info->attr_global_mmio); if (rc) { - dev_err(&afu->dev, - "Unable to create global mmio attr for afu: %d\n", - rc); + dev_err(&info->dev, "Unable to create global mmio attr for afu: %d\n", rc); goto err; } @@ -127,15 +132,20 @@ int ocxl_sysfs_add_afu(struct ocxl_afu *afu) err: for (i--; i >= 0; i--) - device_remove_file(&afu->dev, &afu_attrs[i]); + device_remove_file(&info->dev, &afu_attrs[i]); + return rc; } -void ocxl_sysfs_remove_afu(struct ocxl_afu *afu) +void ocxl_sysfs_unregister_afu(struct ocxl_file_info *info) { int i; + /* + * device_remove_bin_file is safe to call if the file is not added as + * the files are removed by name, and early exit if not found + */ for (i = 0; i < ARRAY_SIZE(afu_attrs); i++) - device_remove_file(&afu->dev, &afu_attrs[i]); - device_remove_bin_file(&afu->dev, &afu->attr_global_mmio); + device_remove_file(&info->dev, &afu_attrs[i]); + device_remove_bin_file(&info->dev, &info->attr_global_mmio); } diff --git a/drivers/misc/ocxl/trace.h b/drivers/misc/ocxl/trace.h index bcb7ff330c1e..024f417e7e01 100644 --- a/drivers/misc/ocxl/trace.h +++ b/drivers/misc/ocxl/trace.h @@ -107,16 +107,14 @@ DEFINE_EVENT(ocxl_fault_handler, ocxl_fault_ack, ); TRACE_EVENT(ocxl_afu_irq_alloc, - TP_PROTO(int pasid, int irq_id, unsigned int virq, int hw_irq, - u64 irq_offset), - TP_ARGS(pasid, irq_id, virq, hw_irq, irq_offset), + TP_PROTO(int pasid, int irq_id, unsigned int virq, int hw_irq), + TP_ARGS(pasid, irq_id, virq, hw_irq), TP_STRUCT__entry( __field(int, pasid) __field(int, irq_id) __field(unsigned int, virq) __field(int, hw_irq) - __field(u64, irq_offset) ), TP_fast_assign( @@ -124,15 +122,13 @@ TRACE_EVENT(ocxl_afu_irq_alloc, __entry->irq_id = irq_id; __entry->virq = virq; __entry->hw_irq = hw_irq; - __entry->irq_offset = irq_offset; ), - TP_printk("pasid=0x%x irq_id=%d virq=%u hw_irq=%d irq_offset=0x%llx", + TP_printk("pasid=0x%x irq_id=%d virq=%u hw_irq=%d", __entry->pasid, __entry->irq_id, __entry->virq, - __entry->hw_irq, - __entry->irq_offset + __entry->hw_irq ) ); |