diff options
Diffstat (limited to 'drivers/scsi/cxlflash')
-rw-r--r-- | drivers/scsi/cxlflash/main.c | 81 | ||||
-rw-r--r-- | drivers/scsi/cxlflash/superpipe.c | 180 | ||||
-rw-r--r-- | drivers/scsi/cxlflash/superpipe.h | 3 | ||||
-rw-r--r-- | drivers/scsi/cxlflash/vlun.c | 13 |
4 files changed, 117 insertions, 160 deletions
diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c index 661bb94e2548..b301655f91cd 100644 --- a/drivers/scsi/cxlflash/main.c +++ b/drivers/scsi/cxlflash/main.c @@ -823,17 +823,6 @@ static void notify_shutdown(struct cxlflash_cfg *cfg, bool wait) } /** - * cxlflash_shutdown() - shutdown handler - * @pdev: PCI device associated with the host. - */ -static void cxlflash_shutdown(struct pci_dev *pdev) -{ - struct cxlflash_cfg *cfg = pci_get_drvdata(pdev); - - notify_shutdown(cfg, false); -} - -/** * cxlflash_remove() - PCI entry point to tear down host * @pdev: PCI device associated with the host. * @@ -844,6 +833,11 @@ static void cxlflash_remove(struct pci_dev *pdev) struct cxlflash_cfg *cfg = pci_get_drvdata(pdev); ulong lock_flags; + if (!pci_is_enabled(pdev)) { + pr_debug("%s: Device is disabled\n", __func__); + return; + } + /* If a Task Management Function is active, wait for it to complete * before continuing with remove. */ @@ -1046,6 +1040,8 @@ static int wait_port_online(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry) do { msleep(delay_us / 1000); status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]); + if (status == U64_MAX) + nretry /= 2; } while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_ONLINE && nretry--); @@ -1077,6 +1073,8 @@ static int wait_port_offline(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry) do { msleep(delay_us / 1000); status = readq_be(&fc_regs[FC_MTIP_STATUS / 8]); + if (status == U64_MAX) + nretry /= 2; } while ((status & FC_MTIP_STATUS_MASK) != FC_MTIP_STATUS_OFFLINE && nretry--); @@ -1095,42 +1093,25 @@ static int wait_port_offline(__be64 __iomem *fc_regs, u32 delay_us, u32 nretry) * online. This toggling action can cause this routine to delay up to a few * seconds. When configured to use the internal LUN feature of the AFU, a * failure to come online is overridden. - * - * Return: - * 0 when the WWPN is successfully written and the port comes back online - * -1 when the port fails to go offline or come back up online */ -static int afu_set_wwpn(struct afu *afu, int port, __be64 __iomem *fc_regs, - u64 wwpn) +static void afu_set_wwpn(struct afu *afu, int port, __be64 __iomem *fc_regs, + u64 wwpn) { - int rc = 0; - set_port_offline(fc_regs); - if (!wait_port_offline(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US, FC_PORT_STATUS_RETRY_CNT)) { pr_debug("%s: wait on port %d to go offline timed out\n", __func__, port); - rc = -1; /* but continue on to leave the port back online */ } - if (rc == 0) - writeq_be(wwpn, &fc_regs[FC_PNAME / 8]); - - /* Always return success after programming WWPN */ - rc = 0; + writeq_be(wwpn, &fc_regs[FC_PNAME / 8]); set_port_online(fc_regs); - if (!wait_port_online(fc_regs, FC_PORT_STATUS_RETRY_INTERVAL_US, FC_PORT_STATUS_RETRY_CNT)) { - pr_err("%s: wait on port %d to go online timed out\n", - __func__, port); + pr_debug("%s: wait on port %d to go online timed out\n", + __func__, port); } - - pr_debug("%s: returning rc=%d\n", __func__, rc); - - return rc; } /** @@ -1187,7 +1168,7 @@ static const struct asyc_intr_info ainfo[] = { {SISL_ASTATUS_FC0_LOGI_F, "login failed", 0, CLR_FC_ERROR}, {SISL_ASTATUS_FC0_LOGI_S, "login succeeded", 0, SCAN_HOST}, {SISL_ASTATUS_FC0_LINK_DN, "link down", 0, 0}, - {SISL_ASTATUS_FC0_LINK_UP, "link up", 0, SCAN_HOST}, + {SISL_ASTATUS_FC0_LINK_UP, "link up", 0, 0}, {SISL_ASTATUS_FC1_OTHER, "other error", 1, CLR_FC_ERROR | LINK_RESET}, {SISL_ASTATUS_FC1_LOGO, "target initiated LOGO", 1, 0}, {SISL_ASTATUS_FC1_CRC_T, "CRC threshold exceeded", 1, LINK_RESET}, @@ -1195,7 +1176,7 @@ static const struct asyc_intr_info ainfo[] = { {SISL_ASTATUS_FC1_LOGI_F, "login failed", 1, CLR_FC_ERROR}, {SISL_ASTATUS_FC1_LOGI_S, "login succeeded", 1, SCAN_HOST}, {SISL_ASTATUS_FC1_LINK_DN, "link down", 1, 0}, - {SISL_ASTATUS_FC1_LINK_UP, "link up", 1, SCAN_HOST}, + {SISL_ASTATUS_FC1_LINK_UP, "link up", 1, 0}, {0x0, "", 0, 0} /* terminator */ }; @@ -1631,15 +1612,10 @@ static int init_global(struct cxlflash_cfg *cfg) [FC_CRC_THRESH / 8]); /* Set WWPNs. If already programmed, wwpn[i] is 0 */ - if (wwpn[i] != 0 && - afu_set_wwpn(afu, i, - &afu->afu_map->global.fc_regs[i][0], - wwpn[i])) { - dev_err(dev, "%s: failed to set WWPN on port %d\n", - __func__, i); - rc = -EIO; - goto out; - } + if (wwpn[i] != 0) + afu_set_wwpn(afu, i, + &afu->afu_map->global.fc_regs[i][0], + wwpn[i]); /* Programming WWPN back to back causes additional * offline/online transitions and a PLOGI */ @@ -2048,6 +2024,11 @@ retry: * cxlflash_eh_host_reset_handler() - reset the host adapter * @scp: SCSI command from stack identifying host. * + * Following a reset, the state is evaluated again in case an EEH occurred + * during the reset. In such a scenario, the host reset will either yield + * until the EEH recovery is complete or return success or failure based + * upon the current device state. + * * Return: * SUCCESS as defined in scsi/scsi.h * FAILED as defined in scsi/scsi.h @@ -2080,7 +2061,8 @@ static int cxlflash_eh_host_reset_handler(struct scsi_cmnd *scp) } else cfg->state = STATE_NORMAL; wake_up_all(&cfg->reset_waitq); - break; + ssleep(1); + /* fall through */ case STATE_RESET: wait_event(cfg->reset_waitq, cfg->state != STATE_RESET); if (cfg->state == STATE_NORMAL) @@ -2596,6 +2578,9 @@ out_remove: * @pdev: PCI device struct. * @state: PCI channel state. * + * When an EEH occurs during an active reset, wait until the reset is + * complete and then take action based upon the device state. + * * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT */ static pci_ers_result_t cxlflash_pci_error_detected(struct pci_dev *pdev, @@ -2609,6 +2594,10 @@ static pci_ers_result_t cxlflash_pci_error_detected(struct pci_dev *pdev, switch (state) { case pci_channel_io_frozen: + wait_event(cfg->reset_waitq, cfg->state != STATE_RESET); + if (cfg->state == STATE_FAILTERM) + return PCI_ERS_RESULT_DISCONNECT; + cfg->state = STATE_RESET; scsi_block_requests(cfg->host); drain_ioctls(cfg); @@ -2685,7 +2674,7 @@ static struct pci_driver cxlflash_driver = { .id_table = cxlflash_pci_table, .probe = cxlflash_probe, .remove = cxlflash_remove, - .shutdown = cxlflash_shutdown, + .shutdown = cxlflash_remove, .err_handler = &cxlflash_err_handler, }; diff --git a/drivers/scsi/cxlflash/superpipe.c b/drivers/scsi/cxlflash/superpipe.c index ce1507023132..9636970d9611 100644 --- a/drivers/scsi/cxlflash/superpipe.c +++ b/drivers/scsi/cxlflash/superpipe.c @@ -709,14 +709,13 @@ int cxlflash_disk_release(struct scsi_device *sdev, * @cfg: Internal structure associated with the host. * @ctxi: Context to release. * - * This routine is safe to be called with a a non-initialized context - * and is tolerant of being called with the context's mutex held (it - * will be unlocked if necessary before freeing). Also note that the - * routine conditionally checks for the existence of the context control - * map before clearing the RHT registers and context capabilities because - * it is possible to destroy a context while the context is in the error - * state (previous mapping was removed [so there is no need to worry about - * clearing] and context is waiting for a new mapping). + * This routine is safe to be called with a a non-initialized context. + * Also note that the routine conditionally checks for the existence + * of the context control map before clearing the RHT registers and + * context capabilities because it is possible to destroy a context + * while the context is in the error state (previous mapping was + * removed [so there is no need to worry about clearing] and context + * is waiting for a new mapping). */ static void destroy_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi) @@ -732,9 +731,6 @@ static void destroy_context(struct cxlflash_cfg *cfg, writeq_be(0, &ctxi->ctrl_map->rht_cnt_id); writeq_be(0, &ctxi->ctrl_map->ctx_cap); } - - if (mutex_is_locked(&ctxi->mutex)) - mutex_unlock(&ctxi->mutex); } /* Free memory associated with context */ @@ -792,32 +788,58 @@ err: * @cfg: Internal structure associated with the host. * @ctx: Previously obtained CXL context reference. * @ctxid: Previously obtained process element associated with CXL context. - * @adap_fd: Previously obtained adapter fd associated with CXL context. * @file: Previously obtained file associated with CXL context. * @perms: User-specified permissions. - * - * Upon return, the context is marked as initialized and the context's mutex - * is locked. */ static void init_context(struct ctx_info *ctxi, struct cxlflash_cfg *cfg, - struct cxl_context *ctx, int ctxid, int adap_fd, - struct file *file, u32 perms) + struct cxl_context *ctx, int ctxid, struct file *file, + u32 perms) { struct afu *afu = cfg->afu; ctxi->rht_perms = perms; ctxi->ctrl_map = &afu->afu_map->ctrls[ctxid].ctrl; ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid); - ctxi->lfd = adap_fd; ctxi->pid = current->tgid; /* tgid = pid */ ctxi->ctx = ctx; + ctxi->cfg = cfg; ctxi->file = file; ctxi->initialized = true; mutex_init(&ctxi->mutex); + kref_init(&ctxi->kref); INIT_LIST_HEAD(&ctxi->luns); INIT_LIST_HEAD(&ctxi->list); /* initialize for list_empty() */ +} +/** + * remove_context() - context kref release handler + * @kref: Kernel reference associated with context to be removed. + * + * When a context no longer has any references it can safely be removed + * from global access and destroyed. Note that it is assumed the thread + * relinquishing access to the context holds its mutex. + */ +static void remove_context(struct kref *kref) +{ + struct ctx_info *ctxi = container_of(kref, struct ctx_info, kref); + struct cxlflash_cfg *cfg = ctxi->cfg; + u64 ctxid = DECODE_CTXID(ctxi->ctxid); + + /* Remove context from table/error list */ + WARN_ON(!mutex_is_locked(&ctxi->mutex)); + ctxi->unavail = true; + mutex_unlock(&ctxi->mutex); + mutex_lock(&cfg->ctx_tbl_list_mutex); mutex_lock(&ctxi->mutex); + + if (!list_empty(&ctxi->list)) + list_del(&ctxi->list); + cfg->ctx_tbl[ctxid] = NULL; + mutex_unlock(&cfg->ctx_tbl_list_mutex); + mutex_unlock(&ctxi->mutex); + + /* Context now completely uncoupled/unreachable */ + destroy_context(cfg, ctxi); } /** @@ -845,7 +867,6 @@ static int _cxlflash_disk_detach(struct scsi_device *sdev, int i; int rc = 0; - int lfd; u64 ctxid = DECODE_CTXID(detach->context_id), rctxid = detach->context_id; @@ -887,40 +908,13 @@ static int _cxlflash_disk_detach(struct scsi_device *sdev, break; } - /* Tear down context following last LUN cleanup */ - if (list_empty(&ctxi->luns)) { - ctxi->unavail = true; - mutex_unlock(&ctxi->mutex); - mutex_lock(&cfg->ctx_tbl_list_mutex); - mutex_lock(&ctxi->mutex); - - /* Might not have been in error list so conditionally remove */ - if (!list_empty(&ctxi->list)) - list_del(&ctxi->list); - cfg->ctx_tbl[ctxid] = NULL; - mutex_unlock(&cfg->ctx_tbl_list_mutex); - mutex_unlock(&ctxi->mutex); - - lfd = ctxi->lfd; - destroy_context(cfg, ctxi); - ctxi = NULL; + /* + * Release the context reference and the sdev reference that + * bound this LUN to the context. + */ + if (kref_put(&ctxi->kref, remove_context)) put_ctx = false; - - /* - * As a last step, clean up external resources when not - * already on an external cleanup thread, i.e.: close(adap_fd). - * - * NOTE: this will free up the context from the CXL services, - * allowing it to dole out the same context_id on a future - * (or even currently in-flight) disk_attach operation. - */ - if (lfd != -1) - sys_close(lfd); - } - - /* Release the sdev reference that bound this LUN to the context */ scsi_device_put(sdev); - out: if (put_ctx) put_context(ctxi); @@ -941,34 +935,18 @@ static int cxlflash_disk_detach(struct scsi_device *sdev, * * This routine is the release handler for the fops registered with * the CXL services on an initial attach for a context. It is called - * when a close is performed on the adapter file descriptor returned - * to the user. Programmatically, the user is not required to perform - * the close, as it is handled internally via the detach ioctl when - * a context is being removed. Note that nothing prevents the user - * from performing a close, but the user should be aware that doing - * so is considered catastrophic and subsequent usage of the superpipe - * API with previously saved off tokens will fail. - * - * When initiated from an external close (either by the user or via - * a process tear down), the routine derives the context reference - * and calls detach for each LUN associated with the context. The - * final detach operation will cause the context itself to be freed. - * Note that the saved off lfd is reset prior to calling detach to - * signify that the final detach should not perform a close. - * - * When initiated from a detach operation as part of the tear down - * of a context, the context is first completely freed and then the - * close is performed. This routine will fail to derive the context - * reference (due to the context having already been freed) and then - * call into the CXL release entry point. + * when a close (explicity by the user or as part of a process tear + * down) is performed on the adapter file descriptor returned to the + * user. The user should be aware that explicitly performing a close + * considered catastrophic and subsequent usage of the superpipe API + * with previously saved off tokens will fail. * - * Thus, with exception to when the CXL process element (context id) - * lookup fails (a case that should theoretically never occur), every - * call into this routine results in a complete freeing of a context. - * - * As part of the detach, all per-context resources associated with the LUN - * are cleaned up. When detaching the last LUN for a context, the context - * itself is cleaned up and released. + * This routine derives the context reference and calls detach for + * each LUN associated with the context.The final detach operation + * causes the context itself to be freed. With exception to when the + * CXL process element (context id) lookup fails (a case that should + * theoretically never occur), every call into this routine results + * in a complete freeing of a context. * * Return: 0 on success */ @@ -1006,11 +984,8 @@ static int cxlflash_cxl_release(struct inode *inode, struct file *file) goto out; } - dev_dbg(dev, "%s: close(%d) for context %d\n", - __func__, ctxi->lfd, ctxid); + dev_dbg(dev, "%s: close for context %d\n", __func__, ctxid); - /* Reset the file descriptor to indicate we're on a close() thread */ - ctxi->lfd = -1; detach.context_id = ctxi->ctxid; list_for_each_entry_safe(lun_access, t, &ctxi->luns, list) _cxlflash_disk_detach(lun_access->sdev, ctxi, &detach); @@ -1110,8 +1085,7 @@ static int cxlflash_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) goto err; } - dev_dbg(dev, "%s: fault(%d) for context %d\n", - __func__, ctxi->lfd, ctxid); + dev_dbg(dev, "%s: fault for context %d\n", __func__, ctxid); if (likely(!ctxi->err_recovery_active)) { vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); @@ -1186,8 +1160,7 @@ static int cxlflash_cxl_mmap(struct file *file, struct vm_area_struct *vma) goto out; } - dev_dbg(dev, "%s: mmap(%d) for context %d\n", - __func__, ctxi->lfd, ctxid); + dev_dbg(dev, "%s: mmap for context %d\n", __func__, ctxid); rc = cxl_fd_mmap(file, vma); if (likely(!rc)) { @@ -1377,12 +1350,12 @@ static int cxlflash_disk_attach(struct scsi_device *sdev, lun_access->lli = lli; lun_access->sdev = sdev; - /* Non-NULL context indicates reuse */ + /* Non-NULL context indicates reuse (another context reference) */ if (ctxi) { dev_dbg(dev, "%s: Reusing context for LUN! (%016llX)\n", __func__, rctxid); + kref_get(&ctxi->kref); list_add(&lun_access->list, &ctxi->luns); - fd = ctxi->lfd; goto out_attach; } @@ -1430,7 +1403,7 @@ static int cxlflash_disk_attach(struct scsi_device *sdev, perms = SISL_RHT_PERM(attach->hdr.flags + 1); /* Context mutex is locked upon return */ - init_context(ctxi, cfg, ctx, ctxid, fd, file, perms); + init_context(ctxi, cfg, ctx, ctxid, file, perms); rc = afu_attach(cfg, ctxi); if (unlikely(rc)) { @@ -1445,7 +1418,6 @@ static int cxlflash_disk_attach(struct scsi_device *sdev, * knows about us yet; we can be the only one holding our mutex. */ list_add(&lun_access->list, &ctxi->luns); - mutex_unlock(&ctxi->mutex); mutex_lock(&cfg->ctx_tbl_list_mutex); mutex_lock(&ctxi->mutex); cfg->ctx_tbl[ctxid] = ctxi; @@ -1453,7 +1425,11 @@ static int cxlflash_disk_attach(struct scsi_device *sdev, fd_install(fd, file); out_attach: - attach->hdr.return_flags = 0; + if (fd != -1) + attach->hdr.return_flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD; + else + attach->hdr.return_flags = 0; + attach->context_id = ctxi->ctxid; attach->block_size = gli->blk_len; attach->mmio_size = sizeof(afu->afu_map->hosts[0].harea); @@ -1494,7 +1470,7 @@ err: file = NULL; } - /* Cleanup our context; safe to call even with mutex locked */ + /* Cleanup our context */ if (ctxi) { destroy_context(cfg, ctxi); ctxi = NULL; @@ -1509,16 +1485,19 @@ err: * recover_context() - recovers a context in error * @cfg: Internal structure associated with the host. * @ctxi: Context to release. + * @adap_fd: Adapter file descriptor associated with new/recovered context. * * Restablishes the state for a context-in-error. * * Return: 0 on success, -errno on failure */ -static int recover_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi) +static int recover_context(struct cxlflash_cfg *cfg, + struct ctx_info *ctxi, + int *adap_fd) { struct device *dev = &cfg->dev->dev; int rc = 0; - int old_fd, fd = -1; + int fd = -1; int ctxid = -1; struct file *file; struct cxl_context *ctx; @@ -1566,9 +1545,7 @@ static int recover_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi) * No error paths after this point. Once the fd is installed it's * visible to user space and can't be undone safely on this thread. */ - old_fd = ctxi->lfd; ctxi->ctxid = ENCODE_CTXID(ctxi, ctxid); - ctxi->lfd = fd; ctxi->ctx = ctx; ctxi->file = file; @@ -1585,9 +1562,7 @@ static int recover_context(struct cxlflash_cfg *cfg, struct ctx_info *ctxi) cfg->ctx_tbl[ctxid] = ctxi; mutex_unlock(&cfg->ctx_tbl_list_mutex); fd_install(fd, file); - - /* Release the original adapter fd and associated CXL resources */ - sys_close(old_fd); + *adap_fd = fd; out: dev_dbg(dev, "%s: returning ctxid=%d fd=%d rc=%d\n", __func__, ctxid, fd, rc); @@ -1646,6 +1621,7 @@ static int cxlflash_afu_recover(struct scsi_device *sdev, rctxid = recover->context_id; long reg; int lretry = 20; /* up to 2 seconds */ + int new_adap_fd = -1; int rc = 0; atomic_inc(&cfg->recovery_threads); @@ -1675,7 +1651,7 @@ retry: if (ctxi->err_recovery_active) { retry_recover: - rc = recover_context(cfg, ctxi); + rc = recover_context(cfg, ctxi, &new_adap_fd); if (unlikely(rc)) { dev_err(dev, "%s: Recovery failed for context %llu (rc=%d)\n", __func__, ctxid, rc); @@ -1697,9 +1673,9 @@ retry_recover: ctxi->err_recovery_active = false; recover->context_id = ctxi->ctxid; - recover->adap_fd = ctxi->lfd; + recover->adap_fd = new_adap_fd; recover->mmio_size = sizeof(afu->afu_map->hosts[0].harea); - recover->hdr.return_flags |= + recover->hdr.return_flags = DK_CXLFLASH_APP_CLOSE_ADAP_FD | DK_CXLFLASH_RECOVER_AFU_CONTEXT_RESET; goto out; } diff --git a/drivers/scsi/cxlflash/superpipe.h b/drivers/scsi/cxlflash/superpipe.h index 5f9a091fda95..9e62ff304e4b 100644 --- a/drivers/scsi/cxlflash/superpipe.h +++ b/drivers/scsi/cxlflash/superpipe.h @@ -100,13 +100,14 @@ struct ctx_info { struct cxl_ioctl_start_work work; u64 ctxid; - int lfd; pid_t pid; bool initialized; bool unavail; bool err_recovery_active; struct mutex mutex; /* Context protection */ + struct kref kref; struct cxl_context *ctx; + struct cxlflash_cfg *cfg; struct list_head luns; /* LUNs attached to this context */ const struct vm_operations_struct *cxl_mmap_vmops; struct file *file; diff --git a/drivers/scsi/cxlflash/vlun.c b/drivers/scsi/cxlflash/vlun.c index 50f8e9300770..90c5d7f5278e 100644 --- a/drivers/scsi/cxlflash/vlun.c +++ b/drivers/scsi/cxlflash/vlun.c @@ -1135,14 +1135,13 @@ int cxlflash_disk_clone(struct scsi_device *sdev, ctxid_dst = DECODE_CTXID(clone->context_id_dst), rctxid_src = clone->context_id_src, rctxid_dst = clone->context_id_dst; - int adap_fd_src = clone->adap_fd_src; int i, j; int rc = 0; bool found; LIST_HEAD(sidecar); - pr_debug("%s: ctxid_src=%llu ctxid_dst=%llu adap_fd_src=%d\n", - __func__, ctxid_src, ctxid_dst, adap_fd_src); + pr_debug("%s: ctxid_src=%llu ctxid_dst=%llu\n", + __func__, ctxid_src, ctxid_dst); /* Do not clone yourself */ if (unlikely(rctxid_src == rctxid_dst)) { @@ -1166,13 +1165,6 @@ int cxlflash_disk_clone(struct scsi_device *sdev, goto out; } - if (unlikely(adap_fd_src != ctxi_src->lfd)) { - pr_debug("%s: Invalid source adapter fd! (%d)\n", - __func__, adap_fd_src); - rc = -EINVAL; - goto out; - } - /* Verify there is no open resource handle in the destination context */ for (i = 0; i < MAX_RHT_PER_CONTEXT; i++) if (ctxi_dst->rht_start[i].nmask != 0) { @@ -1257,7 +1249,6 @@ int cxlflash_disk_clone(struct scsi_device *sdev, out_success: list_splice(&sidecar, &ctxi_dst->luns); - sys_close(adap_fd_src); /* fall through */ out: |