diff options
Diffstat (limited to 'drivers/infiniband/hw')
51 files changed, 716 insertions, 1603 deletions
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c index ebfb117ba68b..d88077a21994 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_cm.c +++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c @@ -137,7 +137,7 @@ static void stop_ep_timer(struct iwch_ep *ep) put_ep(&ep->com); } -int iwch_l2t_send(struct t3cdev *tdev, struct sk_buff *skb, struct l2t_entry *l2e) +static int iwch_l2t_send(struct t3cdev *tdev, struct sk_buff *skb, struct l2t_entry *l2e) { int error = 0; struct cxio_rdev *rdev; @@ -1364,7 +1364,7 @@ static int pass_accept_req(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) __func__); goto reject; } - dst = &rt->u.dst; + dst = &rt->dst; l2t = t3_l2t_get(tdev, dst->neighbour, dst->neighbour->dev); if (!l2t) { printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", @@ -1932,7 +1932,7 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) err = -EHOSTUNREACH; goto fail3; } - ep->dst = &rt->u.dst; + ep->dst = &rt->dst; /* get a l2t entry */ ep->l2t = t3_l2t_get(ep->com.tdev, ep->dst->neighbour, diff --git a/drivers/infiniband/hw/cxgb3/iwch_qp.c b/drivers/infiniband/hw/cxgb3/iwch_qp.c index ae47bfd22bd5..c64d27bf2c15 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_qp.c +++ b/drivers/infiniband/hw/cxgb3/iwch_qp.c @@ -371,7 +371,7 @@ int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, } num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, qhp->wq.sq_size_log2); - if (num_wrs <= 0) { + if (num_wrs == 0) { spin_unlock_irqrestore(&qhp->lock, flag); err = -ENOMEM; goto out; @@ -554,7 +554,7 @@ int iwch_bind_mw(struct ib_qp *qp, } num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, qhp->wq.sq_size_log2); - if ((num_wrs) <= 0) { + if (num_wrs == 0) { spin_unlock_irqrestore(&qhp->lock, flag); return -ENOMEM; } @@ -816,7 +816,7 @@ static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) atomic_inc(&qhp->refcnt); spin_unlock_irqrestore(&qhp->lock, *flag); - /* locking heirarchy: cq lock first, then qp lock. */ + /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&rchp->lock, *flag); spin_lock(&qhp->lock); cxio_flush_hw_cq(&rchp->cq); @@ -827,7 +827,7 @@ static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) if (flushed) (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); - /* locking heirarchy: cq lock first, then qp lock. */ + /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&schp->lock, *flag); spin_lock(&qhp->lock); cxio_flush_hw_cq(&schp->cq); diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index 30ce0a8eca09..32d352a88d50 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -61,6 +61,10 @@ static char *states[] = { NULL, }; +static int dack_mode; +module_param(dack_mode, int, 0644); +MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=0)"); + int c4iw_max_read_depth = 8; module_param(c4iw_max_read_depth, int, 0644); MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); @@ -469,11 +473,12 @@ static int send_connect(struct c4iw_ep *ep) __func__); return -ENOMEM; } - set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx); + set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); wscale = compute_wscale(rcv_win); opt0 = KEEP_ALIVE(1) | + DELACK(1) | WND_SCALE(wscale) | MSS_IDX(mtu_idx) | L2T_IDX(ep->l2t->idx) | @@ -780,11 +785,11 @@ static void connect_reply_upcall(struct c4iw_ep *ep, int status) event.private_data_len = ep->plen; event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); } - if (ep->com.cm_id) { - PDBG("%s ep %p tid %u status %d\n", __func__, ep, - ep->hwtid, status); - ep->com.cm_id->event_handler(ep->com.cm_id, &event); - } + + PDBG("%s ep %p tid %u status %d\n", __func__, ep, + ep->hwtid, status); + ep->com.cm_id->event_handler(ep->com.cm_id, &event); + if (status < 0) { ep->com.cm_id->rem_ref(ep->com.cm_id); ep->com.cm_id = NULL; @@ -845,8 +850,10 @@ static int update_rx_credits(struct c4iw_ep *ep, u32 credits) INIT_TP_WR(req, ep->hwtid); OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, ep->hwtid)); - req->credit_dack = cpu_to_be32(credits); - set_wr_txq(skb, CPL_PRIORITY_ACK, ep->txq_idx); + req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | + F_RX_DACK_CHANGE | + V_RX_DACK_MODE(dack_mode)); + set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); c4iw_ofld_send(&ep->com.dev->rdev, skb); return credits; } @@ -969,7 +976,8 @@ static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) goto err; goto out; err: - abort_connection(ep, skb, GFP_KERNEL); + state_set(&ep->com, ABORTING); + send_abort(ep, skb, GFP_KERNEL); out: connect_reply_upcall(ep, err); return; @@ -1263,6 +1271,7 @@ static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb, cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); wscale = compute_wscale(rcv_win); opt0 = KEEP_ALIVE(1) | + DELACK(1) | WND_SCALE(wscale) | MSS_IDX(mtu_idx) | L2T_IDX(ep->l2t->idx) | @@ -1286,7 +1295,7 @@ static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb, ep->hwtid)); rpl->opt0 = cpu_to_be64(opt0); rpl->opt2 = cpu_to_be32(opt2); - set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->txq_idx); + set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); return; @@ -1343,7 +1352,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) u16 rss_qid; u32 mtu; int step; - int txq_idx; + int txq_idx, ctrlq_idx; parent_ep = lookup_stid(t, stid); PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid); @@ -1364,7 +1373,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) __func__); goto reject; } - dst = &rt->u.dst; + dst = &rt->dst; if (dst->neighbour->dev->flags & IFF_LOOPBACK) { pdev = ip_dev_find(&init_net, peer_ip); BUG_ON(!pdev); @@ -1372,9 +1381,10 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) pdev, 0); mtu = pdev->mtu; tx_chan = cxgb4_port_chan(pdev); - smac_idx = tx_chan << 1; + smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan; txq_idx = cxgb4_port_idx(pdev) * step; + ctrlq_idx = cxgb4_port_idx(pdev); step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan; rss_qid = dev->rdev.lldi.rxq_ids[cxgb4_port_idx(pdev) * step]; dev_put(pdev); @@ -1383,9 +1393,10 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) dst->neighbour->dev, 0); mtu = dst_mtu(dst); tx_chan = cxgb4_port_chan(dst->neighbour->dev); - smac_idx = tx_chan << 1; + smac_idx = (cxgb4_port_viid(dst->neighbour->dev) & 0x7F) << 1; step = dev->rdev.lldi.ntxq / dev->rdev.lldi.nchan; txq_idx = cxgb4_port_idx(dst->neighbour->dev) * step; + ctrlq_idx = cxgb4_port_idx(dst->neighbour->dev); step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan; rss_qid = dev->rdev.lldi.rxq_ids[ cxgb4_port_idx(dst->neighbour->dev) * step]; @@ -1425,6 +1436,7 @@ static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) child_ep->rss_qid = rss_qid; child_ep->mtu = mtu; child_ep->txq_idx = txq_idx; + child_ep->ctrlq_idx = ctrlq_idx; PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, tx_chan, smac_idx, rss_qid); @@ -1472,8 +1484,6 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) int closing = 0; struct tid_info *t = dev->rdev.lldi.tids; unsigned int tid = GET_TID(hdr); - int start_timer = 0; - int stop_timer = 0; ep = lookup_tid(t, tid); PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); @@ -1510,7 +1520,7 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) wake_up(&ep->com.waitq); break; case FPDU_MODE: - start_timer = 1; + start_ep_timer(ep); __state_set(&ep->com, CLOSING); closing = 1; peer_close_upcall(ep); @@ -1523,7 +1533,7 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) disconnect = 0; break; case MORIBUND: - stop_timer = 1; + stop_ep_timer(ep); if (ep->com.cm_id && ep->com.qp) { attrs.next_state = C4IW_QP_STATE_IDLE; c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, @@ -1546,10 +1556,6 @@ static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); } - if (start_timer) - start_ep_timer(ep); - if (stop_timer) - stop_ep_timer(ep); if (disconnect) c4iw_ep_disconnect(ep, 0, GFP_KERNEL); if (release) @@ -1578,7 +1584,6 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) unsigned long flags; struct tid_info *t = dev->rdev.lldi.tids; unsigned int tid = GET_TID(req); - int stop_timer = 0; ep = lookup_tid(t, tid); if (is_neg_adv_abort(req->status)) { @@ -1593,10 +1598,10 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) case CONNECTING: break; case MPA_REQ_WAIT: - stop_timer = 1; + stop_ep_timer(ep); break; case MPA_REQ_SENT: - stop_timer = 1; + stop_ep_timer(ep); connect_reply_upcall(ep, -ECONNRESET); break; case MPA_REP_SENT: @@ -1620,7 +1625,7 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) break; case MORIBUND: case CLOSING: - stop_timer = 1; + stop_ep_timer(ep); /*FALLTHROUGH*/ case FPDU_MODE: if (ep->com.cm_id && ep->com.qp) { @@ -1666,8 +1671,6 @@ static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) rpl->cmd = CPL_ABORT_NO_RST; c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); out: - if (stop_timer) - stop_ep_timer(ep); if (release) release_ep_resources(ep); return 0; @@ -1682,7 +1685,6 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) int release = 0; struct tid_info *t = dev->rdev.lldi.tids; unsigned int tid = GET_TID(rpl); - int stop_timer = 0; ep = lookup_tid(t, tid); @@ -1696,7 +1698,7 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) __state_set(&ep->com, MORIBUND); break; case MORIBUND: - stop_timer = 1; + stop_ep_timer(ep); if ((ep->com.cm_id) && (ep->com.qp)) { attrs.next_state = C4IW_QP_STATE_IDLE; c4iw_modify_qp(ep->com.qp->rhp, @@ -1716,8 +1718,6 @@ static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) break; } spin_unlock_irqrestore(&ep->com.lock, flags); - if (stop_timer) - stop_ep_timer(ep); if (release) release_ep_resources(ep); return 0; @@ -1938,7 +1938,7 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) err = -EHOSTUNREACH; goto fail3; } - ep->dst = &rt->u.dst; + ep->dst = &rt->dst; /* get a l2t entry */ if (ep->dst->neighbour->dev->flags & IFF_LOOPBACK) { @@ -1950,12 +1950,13 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) pdev, 0); ep->mtu = pdev->mtu; ep->tx_chan = cxgb4_port_chan(pdev); - ep->smac_idx = ep->tx_chan << 1; + ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; step = ep->com.dev->rdev.lldi.ntxq / ep->com.dev->rdev.lldi.nchan; ep->txq_idx = cxgb4_port_idx(pdev) * step; step = ep->com.dev->rdev.lldi.nrxq / ep->com.dev->rdev.lldi.nchan; + ep->ctrlq_idx = cxgb4_port_idx(pdev); ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[ cxgb4_port_idx(pdev) * step]; dev_put(pdev); @@ -1965,10 +1966,12 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) ep->dst->neighbour->dev, 0); ep->mtu = dst_mtu(ep->dst); ep->tx_chan = cxgb4_port_chan(ep->dst->neighbour->dev); - ep->smac_idx = ep->tx_chan << 1; + ep->smac_idx = (cxgb4_port_viid(ep->dst->neighbour->dev) & + 0x7F) << 1; step = ep->com.dev->rdev.lldi.ntxq / ep->com.dev->rdev.lldi.nchan; ep->txq_idx = cxgb4_port_idx(ep->dst->neighbour->dev) * step; + ep->ctrlq_idx = cxgb4_port_idx(ep->dst->neighbour->dev); step = ep->com.dev->rdev.lldi.nrxq / ep->com.dev->rdev.lldi.nchan; ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[ @@ -2047,8 +2050,15 @@ int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) goto fail3; /* wait for pass_open_rpl */ - wait_event(ep->com.waitq, ep->com.rpl_done); - err = ep->com.rpl_err; + wait_event_timeout(ep->com.waitq, ep->com.rpl_done, C4IW_WR_TO); + if (ep->com.rpl_done) + err = ep->com.rpl_err; + else { + printk(KERN_ERR MOD "Device %s not responding!\n", + pci_name(ep->com.dev->rdev.lldi.pdev)); + ep->com.dev->rdev.flags = T4_FATAL_ERROR; + err = -EIO; + } if (!err) { cm_id->provider_data = ep; goto out; @@ -2077,10 +2087,17 @@ int c4iw_destroy_listen(struct iw_cm_id *cm_id) err = listen_stop(ep); if (err) goto done; - wait_event(ep->com.waitq, ep->com.rpl_done); + wait_event_timeout(ep->com.waitq, ep->com.rpl_done, C4IW_WR_TO); + if (ep->com.rpl_done) + err = ep->com.rpl_err; + else { + printk(KERN_ERR MOD "Device %s not responding!\n", + pci_name(ep->com.dev->rdev.lldi.pdev)); + ep->com.dev->rdev.flags = T4_FATAL_ERROR; + err = -EIO; + } cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); done: - err = ep->com.rpl_err; cm_id->rem_ref(cm_id); c4iw_put_ep(&ep->com); return err; @@ -2093,8 +2110,6 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) int close = 0; int fatal = 0; struct c4iw_rdev *rdev; - int start_timer = 0; - int stop_timer = 0; spin_lock_irqsave(&ep->com.lock, flags); @@ -2118,7 +2133,7 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) ep->com.state = ABORTING; else { ep->com.state = CLOSING; - start_timer = 1; + start_ep_timer(ep); } set_bit(CLOSE_SENT, &ep->com.flags); break; @@ -2126,7 +2141,7 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { close = 1; if (abrupt) { - stop_timer = 1; + stop_ep_timer(ep); ep->com.state = ABORTING; } else ep->com.state = MORIBUND; @@ -2144,10 +2159,6 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) } spin_unlock_irqrestore(&ep->com.lock, flags); - if (start_timer) - start_ep_timer(ep); - if (stop_timer) - stop_ep_timer(ep); if (close) { if (abrupt) ret = abort_connection(ep, NULL, gfp); @@ -2242,7 +2253,7 @@ static void process_work(struct work_struct *work) { struct sk_buff *skb = NULL; struct c4iw_dev *dev; - struct cpl_act_establish *rpl = cplhdr(skb); + struct cpl_act_establish *rpl; unsigned int opcode; int ret; diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 2447f5295482..b3daf39eed4a 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -43,7 +43,7 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, int ret; wr_len = sizeof *res_wr + sizeof *res; - skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); + skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) return -ENOMEM; set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); @@ -77,7 +77,7 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, kfree(cq->sw_queue); dma_free_coherent(&(rdev->lldi.pdev->dev), cq->memsize, cq->queue, - pci_unmap_addr(cq, mapping)); + dma_unmap_addr(cq, mapping)); c4iw_put_cqid(rdev, cq->cqid, uctx); return ret; } @@ -112,13 +112,13 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, ret = -ENOMEM; goto err3; } - pci_unmap_addr_set(cq, mapping, cq->dma_addr); + dma_unmap_addr_set(cq, mapping, cq->dma_addr); memset(cq->queue, 0, cq->memsize); /* build fw_ri_res_wr */ wr_len = sizeof *res_wr + sizeof *res; - skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); + skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) { ret = -ENOMEM; goto err4; @@ -179,7 +179,7 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, return 0; err4: dma_free_coherent(&rdev->lldi.pdev->dev, cq->memsize, cq->queue, - pci_unmap_addr(cq, mapping)); + dma_unmap_addr(cq, mapping)); err3: kfree(cq->sw_queue); err2: @@ -764,7 +764,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int entries, struct c4iw_create_cq_resp uresp; struct c4iw_ucontext *ucontext = NULL; int ret; - size_t memsize; + size_t memsize, hwentries; struct c4iw_mm_entry *mm, *mm2; PDBG("%s ib_dev %p entries %d\n", __func__, ibdev, entries); @@ -788,14 +788,29 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int entries, * entries must be multiple of 16 for HW. */ entries = roundup(entries, 16); - memsize = entries * sizeof *chp->cq.queue; + + /* + * Make actual HW queue 2x to avoid cdix_inc overflows. + */ + hwentries = entries * 2; + + /* + * Make HW queue at least 64 entries so GTS updates aren't too + * frequent. + */ + if (hwentries < 64) + hwentries = 64; + + memsize = hwentries * sizeof *chp->cq.queue; /* * memsize must be a multiple of the page size if its a user cq. */ - if (ucontext) + if (ucontext) { memsize = roundup(memsize, PAGE_SIZE); - chp->cq.size = entries; + hwentries = memsize / sizeof *chp->cq.queue; + } + chp->cq.size = hwentries; chp->cq.memsize = memsize; ret = create_cq(&rhp->rdev, &chp->cq, @@ -805,7 +820,7 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, int entries, chp->rhp = rhp; chp->cq.size--; /* status page */ - chp->ibcq.cqe = chp->cq.size - 1; + chp->ibcq.cqe = entries - 2; spin_lock_init(&chp->lock); atomic_set(&chp->refcnt, 1); init_waitqueue_head(&chp->wait); diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 277ab589b44d..ed459b8f800f 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -261,7 +261,7 @@ static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw) struct c4iw_fr_page_list { struct ib_fast_reg_page_list ibpl; - DECLARE_PCI_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_ADDR(mapping); dma_addr_t dma_addr; struct c4iw_dev *dev; int size; @@ -619,6 +619,7 @@ struct c4iw_ep { u16 plen; u16 rss_qid; u16 txq_idx; + u16 ctrlq_idx; u8 tos; }; diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 7f94da1a2437..269373a62f22 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -59,7 +59,7 @@ static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, wr_len = roundup(sizeof *req + sizeof *sc + roundup(copy_len, T4_ULPTX_MIN_IO), 16); - skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); + skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) return -ENOMEM; set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); @@ -764,7 +764,7 @@ struct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(struct ib_device *device, if (!c4pl) return ERR_PTR(-ENOMEM); - pci_unmap_addr_set(c4pl, mapping, dma_addr); + dma_unmap_addr_set(c4pl, mapping, dma_addr); c4pl->dma_addr = dma_addr; c4pl->dev = dev; c4pl->size = size; @@ -779,7 +779,7 @@ void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *ibpl) struct c4iw_fr_page_list *c4pl = to_c4iw_fr_page_list(ibpl); dma_free_coherent(&c4pl->dev->rdev.lldi.pdev->dev, c4pl->size, - c4pl, pci_unmap_addr(c4pl, mapping)); + c4pl, dma_unmap_addr(c4pl, mapping)); } int c4iw_dereg_mr(struct ib_mr *ib_mr) diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index 0c28ed1eafa6..93f6e5bf0ec5 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -40,10 +40,10 @@ static int destroy_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, */ dma_free_coherent(&(rdev->lldi.pdev->dev), wq->rq.memsize, wq->rq.queue, - pci_unmap_addr(&wq->rq, mapping)); + dma_unmap_addr(&wq->rq, mapping)); dma_free_coherent(&(rdev->lldi.pdev->dev), wq->sq.memsize, wq->sq.queue, - pci_unmap_addr(&wq->sq, mapping)); + dma_unmap_addr(&wq->sq, mapping)); c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); kfree(wq->rq.sw_rq); kfree(wq->sq.sw_sq); @@ -99,7 +99,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, if (!wq->sq.queue) goto err5; memset(wq->sq.queue, 0, wq->sq.memsize); - pci_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr); + dma_unmap_addr_set(&wq->sq, mapping, wq->sq.dma_addr); wq->rq.queue = dma_alloc_coherent(&(rdev->lldi.pdev->dev), wq->rq.memsize, &(wq->rq.dma_addr), @@ -112,7 +112,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, wq->rq.queue, (unsigned long long)virt_to_phys(wq->rq.queue)); memset(wq->rq.queue, 0, wq->rq.memsize); - pci_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); + dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); wq->db = rdev->lldi.db_reg; wq->gts = rdev->lldi.gts_reg; @@ -130,7 +130,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, /* build fw_ri_res_wr */ wr_len = sizeof *res_wr + 2 * sizeof *res; - skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); + skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) { ret = -ENOMEM; goto err7; @@ -162,7 +162,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( V_FW_RI_RES_WR_DCAEN(0) | V_FW_RI_RES_WR_DCACPU(0) | - V_FW_RI_RES_WR_FBMIN(3) | + V_FW_RI_RES_WR_FBMIN(2) | V_FW_RI_RES_WR_FBMAX(3) | V_FW_RI_RES_WR_CIDXFTHRESHO(0) | V_FW_RI_RES_WR_CIDXFTHRESH(0) | @@ -185,7 +185,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, res->u.sqrq.dcaen_to_eqsize = cpu_to_be32( V_FW_RI_RES_WR_DCAEN(0) | V_FW_RI_RES_WR_DCACPU(0) | - V_FW_RI_RES_WR_FBMIN(3) | + V_FW_RI_RES_WR_FBMIN(2) | V_FW_RI_RES_WR_FBMAX(3) | V_FW_RI_RES_WR_CIDXFTHRESHO(0) | V_FW_RI_RES_WR_CIDXFTHRESH(0) | @@ -217,11 +217,11 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, err7: dma_free_coherent(&(rdev->lldi.pdev->dev), wq->rq.memsize, wq->rq.queue, - pci_unmap_addr(&wq->rq, mapping)); + dma_unmap_addr(&wq->rq, mapping)); err6: dma_free_coherent(&(rdev->lldi.pdev->dev), wq->sq.memsize, wq->sq.queue, - pci_unmap_addr(&wq->sq, mapping)); + dma_unmap_addr(&wq->sq, mapping)); err5: c4iw_rqtpool_free(rdev, wq->rq.rqt_hwaddr, wq->rq.rqt_size); err4: @@ -235,12 +235,78 @@ err1: return -ENOMEM; } -static int build_rdma_send(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) +static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp, + struct ib_send_wr *wr, int max, u32 *plenp) { + u8 *dstp, *srcp; + u32 plen = 0; int i; + int rem, len; + + dstp = (u8 *)immdp->data; + for (i = 0; i < wr->num_sge; i++) { + if ((plen + wr->sg_list[i].length) > max) + return -EMSGSIZE; + srcp = (u8 *)(unsigned long)wr->sg_list[i].addr; + plen += wr->sg_list[i].length; + rem = wr->sg_list[i].length; + while (rem) { + if (dstp == (u8 *)&sq->queue[sq->size]) + dstp = (u8 *)sq->queue; + if (rem <= (u8 *)&sq->queue[sq->size] - dstp) + len = rem; + else + len = (u8 *)&sq->queue[sq->size] - dstp; + memcpy(dstp, srcp, len); + dstp += len; + srcp += len; + rem -= len; + } + } + immdp->op = FW_RI_DATA_IMMD; + immdp->r1 = 0; + immdp->r2 = 0; + immdp->immdlen = cpu_to_be32(plen); + *plenp = plen; + return 0; +} + +static int build_isgl(__be64 *queue_start, __be64 *queue_end, + struct fw_ri_isgl *isglp, struct ib_sge *sg_list, + int num_sge, u32 *plenp) + +{ + int i; + u32 plen = 0; + __be64 *flitp = (__be64 *)isglp->sge; + + for (i = 0; i < num_sge; i++) { + if ((plen + sg_list[i].length) < plen) + return -EMSGSIZE; + plen += sg_list[i].length; + *flitp = cpu_to_be64(((u64)sg_list[i].lkey << 32) | + sg_list[i].length); + if (++flitp == queue_end) + flitp = queue_start; + *flitp = cpu_to_be64(sg_list[i].addr); + if (++flitp == queue_end) + flitp = queue_start; + } + isglp->op = FW_RI_DATA_ISGL; + isglp->r1 = 0; + isglp->nsge = cpu_to_be16(num_sge); + isglp->r2 = 0; + if (plenp) + *plenp = plen; + return 0; +} + +static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, + struct ib_send_wr *wr, u8 *len16) +{ u32 plen; int size; - u8 *datap; + int ret; if (wr->num_sge > T4_MAX_SEND_SGE) return -EINVAL; @@ -267,43 +333,23 @@ static int build_rdma_send(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) default: return -EINVAL; } + plen = 0; if (wr->num_sge) { if (wr->send_flags & IB_SEND_INLINE) { - datap = (u8 *)wqe->send.u.immd_src[0].data; - for (i = 0; i < wr->num_sge; i++) { - if ((plen + wr->sg_list[i].length) > - T4_MAX_SEND_INLINE) { - return -EMSGSIZE; - } - plen += wr->sg_list[i].length; - memcpy(datap, - (void *)(unsigned long)wr->sg_list[i].addr, - wr->sg_list[i].length); - datap += wr->sg_list[i].length; - } - wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD; - wqe->send.u.immd_src[0].r1 = 0; - wqe->send.u.immd_src[0].r2 = 0; - wqe->send.u.immd_src[0].immdlen = cpu_to_be32(plen); + ret = build_immd(sq, wqe->send.u.immd_src, wr, + T4_MAX_SEND_INLINE, &plen); + if (ret) + return ret; size = sizeof wqe->send + sizeof(struct fw_ri_immd) + plen; } else { - for (i = 0; i < wr->num_sge; i++) { - if ((plen + wr->sg_list[i].length) < plen) - return -EMSGSIZE; - plen += wr->sg_list[i].length; - wqe->send.u.isgl_src[0].sge[i].stag = - cpu_to_be32(wr->sg_list[i].lkey); - wqe->send.u.isgl_src[0].sge[i].len = - cpu_to_be32(wr->sg_list[i].length); - wqe->send.u.isgl_src[0].sge[i].to = - cpu_to_be64(wr->sg_list[i].addr); - } - wqe->send.u.isgl_src[0].op = FW_RI_DATA_ISGL; - wqe->send.u.isgl_src[0].r1 = 0; - wqe->send.u.isgl_src[0].nsge = cpu_to_be16(wr->num_sge); - wqe->send.u.isgl_src[0].r2 = 0; + ret = build_isgl((__be64 *)sq->queue, + (__be64 *)&sq->queue[sq->size], + wqe->send.u.isgl_src, + wr->sg_list, wr->num_sge, &plen); + if (ret) + return ret; size = sizeof wqe->send + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } @@ -313,62 +359,40 @@ static int build_rdma_send(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) wqe->send.u.immd_src[0].r2 = 0; wqe->send.u.immd_src[0].immdlen = 0; size = sizeof wqe->send + sizeof(struct fw_ri_immd); + plen = 0; } *len16 = DIV_ROUND_UP(size, 16); wqe->send.plen = cpu_to_be32(plen); return 0; } -static int build_rdma_write(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) +static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, + struct ib_send_wr *wr, u8 *len16) { - int i; u32 plen; int size; - u8 *datap; + int ret; - if (wr->num_sge > T4_MAX_WRITE_SGE) + if (wr->num_sge > T4_MAX_SEND_SGE) return -EINVAL; wqe->write.r2 = 0; wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); - plen = 0; if (wr->num_sge) { if (wr->send_flags & IB_SEND_INLINE) { - datap = (u8 *)wqe->write.u.immd_src[0].data; - for (i = 0; i < wr->num_sge; i++) { - if ((plen + wr->sg_list[i].length) > - T4_MAX_WRITE_INLINE) { - return -EMSGSIZE; - } - plen += wr->sg_list[i].length; - memcpy(datap, - (void *)(unsigned long)wr->sg_list[i].addr, - wr->sg_list[i].length); - datap += wr->sg_list[i].length; - } - wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD; - wqe->write.u.immd_src[0].r1 = 0; - wqe->write.u.immd_src[0].r2 = 0; - wqe->write.u.immd_src[0].immdlen = cpu_to_be32(plen); + ret = build_immd(sq, wqe->write.u.immd_src, wr, + T4_MAX_WRITE_INLINE, &plen); + if (ret) + return ret; size = sizeof wqe->write + sizeof(struct fw_ri_immd) + plen; } else { - for (i = 0; i < wr->num_sge; i++) { - if ((plen + wr->sg_list[i].length) < plen) - return -EMSGSIZE; - plen += wr->sg_list[i].length; - wqe->write.u.isgl_src[0].sge[i].stag = - cpu_to_be32(wr->sg_list[i].lkey); - wqe->write.u.isgl_src[0].sge[i].len = - cpu_to_be32(wr->sg_list[i].length); - wqe->write.u.isgl_src[0].sge[i].to = - cpu_to_be64(wr->sg_list[i].addr); - } - wqe->write.u.isgl_src[0].op = FW_RI_DATA_ISGL; - wqe->write.u.isgl_src[0].r1 = 0; - wqe->write.u.isgl_src[0].nsge = - cpu_to_be16(wr->num_sge); - wqe->write.u.isgl_src[0].r2 = 0; + ret = build_isgl((__be64 *)sq->queue, + (__be64 *)&sq->queue[sq->size], + wqe->write.u.isgl_src, + wr->sg_list, wr->num_sge, &plen); + if (ret) + return ret; size = sizeof wqe->write + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } @@ -378,6 +402,7 @@ static int build_rdma_write(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) wqe->write.u.immd_src[0].r2 = 0; wqe->write.u.immd_src[0].immdlen = 0; size = sizeof wqe->write + sizeof(struct fw_ri_immd); + plen = 0; } *len16 = DIV_ROUND_UP(size, 16); wqe->write.plen = cpu_to_be32(plen); @@ -416,29 +441,13 @@ static int build_rdma_read(union t4_wr *wqe, struct ib_send_wr *wr, u8 *len16) static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, struct ib_recv_wr *wr, u8 *len16) { - int i; - int plen = 0; + int ret; - for (i = 0; i < wr->num_sge; i++) { - if ((plen + wr->sg_list[i].length) < plen) - return -EMSGSIZE; - plen += wr->sg_list[i].length; - wqe->recv.isgl.sge[i].stag = - cpu_to_be32(wr->sg_list[i].lkey); - wqe->recv.isgl.sge[i].len = - cpu_to_be32(wr->sg_list[i].length); - wqe->recv.isgl.sge[i].to = - cpu_to_be64(wr->sg_list[i].addr); - } - for (; i < T4_MAX_RECV_SGE; i++) { - wqe->recv.isgl.sge[i].stag = 0; - wqe->recv.isgl.sge[i].len = 0; - wqe->recv.isgl.sge[i].to = 0; - } - wqe->recv.isgl.op = FW_RI_DATA_ISGL; - wqe->recv.isgl.r1 = 0; - wqe->recv.isgl.nsge = cpu_to_be16(wr->num_sge); - wqe->recv.isgl.r2 = 0; + ret = build_isgl((__be64 *)qhp->wq.rq.queue, + (__be64 *)&qhp->wq.rq.queue[qhp->wq.rq.size], + &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); + if (ret) + return ret; *len16 = DIV_ROUND_UP(sizeof wqe->recv + wr->num_sge * sizeof(struct fw_ri_sge), 16); return 0; @@ -547,7 +556,9 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, *bad_wr = wr; break; } - wqe = &qhp->wq.sq.queue[qhp->wq.sq.pidx]; + wqe = (union t4_wr *)((u8 *)qhp->wq.sq.queue + + qhp->wq.sq.wq_pidx * T4_EQ_ENTRY_SIZE); + fw_flags = 0; if (wr->send_flags & IB_SEND_SOLICITED) fw_flags |= FW_RI_SOLICITED_EVENT_FLAG; @@ -564,12 +575,12 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, swsqe->opcode = FW_RI_SEND; else swsqe->opcode = FW_RI_SEND_WITH_INV; - err = build_rdma_send(wqe, wr, &len16); + err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); break; case IB_WR_RDMA_WRITE: fw_opcode = FW_RI_RDMA_WRITE_WR; swsqe->opcode = FW_RI_RDMA_WRITE; - err = build_rdma_write(wqe, wr, &len16); + err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16); break; case IB_WR_RDMA_READ: case IB_WR_RDMA_READ_WITH_INV: @@ -619,8 +630,8 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, swsqe->opcode, swsqe->read_len); wr = wr->next; num_wrs--; - t4_sq_produce(&qhp->wq); - idx++; + t4_sq_produce(&qhp->wq, len16); + idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); } if (t4_wq_db_enabled(&qhp->wq)) t4_ring_sq_db(&qhp->wq, idx); @@ -656,7 +667,9 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, *bad_wr = wr; break; } - wqe = &qhp->wq.rq.queue[qhp->wq.rq.pidx]; + wqe = (union t4_recv_wr *)((u8 *)qhp->wq.rq.queue + + qhp->wq.rq.wq_pidx * + T4_EQ_ENTRY_SIZE); if (num_wrs) err = build_rdma_recv(qhp, wqe, wr, &len16); else @@ -675,15 +688,12 @@ int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, wqe->recv.r2[1] = 0; wqe->recv.r2[2] = 0; wqe->recv.len16 = len16; - if (len16 < 5) - wqe->flits[8] = 0; - PDBG("%s cookie 0x%llx pidx %u\n", __func__, (unsigned long long) wr->wr_id, qhp->wq.rq.pidx); - t4_rq_produce(&qhp->wq); + t4_rq_produce(&qhp->wq, len16); + idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); wr = wr->next; num_wrs--; - idx++; } if (t4_wq_db_enabled(&qhp->wq)) t4_ring_rq_db(&qhp->wq, idx); @@ -905,7 +915,7 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp, atomic_inc(&qhp->refcnt); spin_unlock_irqrestore(&qhp->lock, *flag); - /* locking heirarchy: cq lock first, then qp lock. */ + /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&rchp->lock, *flag); spin_lock(&qhp->lock); c4iw_flush_hw_cq(&rchp->cq); @@ -916,7 +926,7 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp, if (flushed) (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); - /* locking heirarchy: cq lock first, then qp lock. */ + /* locking hierarchy: cq lock first, then qp lock. */ spin_lock_irqsave(&schp->lock, *flag); spin_lock(&qhp->lock); c4iw_flush_hw_cq(&schp->cq); @@ -951,7 +961,8 @@ static void flush_qp(struct c4iw_qp *qhp, unsigned long *flag) __flush_qp(qhp, rchp, schp, flag); } -static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp) +static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, + struct c4iw_ep *ep) { struct fw_ri_wr *wqe; int ret; @@ -959,12 +970,12 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp) struct sk_buff *skb; PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid, - qhp->ep->hwtid); + ep->hwtid); - skb = alloc_skb(sizeof *wqe, GFP_KERNEL | __GFP_NOFAIL); + skb = alloc_skb(sizeof *wqe, GFP_KERNEL); if (!skb) return -ENOMEM; - set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx); + set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); wqe = (struct fw_ri_wr *)__skb_put(skb, sizeof(*wqe)); memset(wqe, 0, sizeof *wqe); @@ -972,7 +983,7 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp) FW_WR_OP(FW_RI_INIT_WR) | FW_WR_COMPL(1)); wqe->flowid_len16 = cpu_to_be32( - FW_WR_FLOWID(qhp->ep->hwtid) | + FW_WR_FLOWID(ep->hwtid) | FW_WR_LEN16(DIV_ROUND_UP(sizeof *wqe, 16))); wqe->cookie = (u64)&wr_wait; @@ -1035,7 +1046,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp) PDBG("%s qhp %p qid 0x%x tid %u\n", __func__, qhp, qhp->wq.sq.qid, qhp->ep->hwtid); - skb = alloc_skb(sizeof *wqe, GFP_KERNEL | __GFP_NOFAIL); + skb = alloc_skb(sizeof *wqe, GFP_KERNEL); if (!skb) return -ENOMEM; set_wr_txq(skb, CPL_PRIORITY_DATA, qhp->ep->txq_idx); @@ -1202,17 +1213,16 @@ int c4iw_modify_qp(struct c4iw_dev *rhp, struct c4iw_qp *qhp, case C4IW_QP_STATE_CLOSING: BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); qhp->attr.state = C4IW_QP_STATE_CLOSING; + ep = qhp->ep; if (!internal) { abort = 0; disconnect = 1; - ep = qhp->ep; c4iw_get_ep(&ep->com); } spin_unlock_irqrestore(&qhp->lock, flag); - ret = rdma_fini(rhp, qhp); + ret = rdma_fini(rhp, qhp, ep); spin_lock_irqsave(&qhp->lock, flag); if (ret) { - ep = qhp->ep; c4iw_get_ep(&ep->com); disconnect = abort = 1; goto err; diff --git a/drivers/infiniband/hw/cxgb4/t4.h b/drivers/infiniband/hw/cxgb4/t4.h index 1057cb96302e..aef55f42bea4 100644 --- a/drivers/infiniband/hw/cxgb4/t4.h +++ b/drivers/infiniband/hw/cxgb4/t4.h @@ -65,10 +65,10 @@ struct t4_status_page { u8 db_off; }; -#define T4_EQ_SIZE 64 +#define T4_EQ_ENTRY_SIZE 64 #define T4_SQ_NUM_SLOTS 4 -#define T4_SQ_NUM_BYTES (T4_EQ_SIZE * T4_SQ_NUM_SLOTS) +#define T4_SQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_SQ_NUM_SLOTS) #define T4_MAX_SEND_SGE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \ sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge)) #define T4_MAX_SEND_INLINE ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_send_wr) - \ @@ -84,7 +84,7 @@ struct t4_status_page { #define T4_MAX_FR_DEPTH (T4_MAX_FR_IMMD / sizeof(u64)) #define T4_RQ_NUM_SLOTS 2 -#define T4_RQ_NUM_BYTES (T4_EQ_SIZE * T4_RQ_NUM_SLOTS) +#define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS) #define T4_MAX_RECV_SGE 4 union t4_wr { @@ -97,20 +97,18 @@ union t4_wr { struct fw_ri_fr_nsmr_wr fr; struct fw_ri_inv_lstag_wr inv; struct t4_status_page status; - __be64 flits[T4_EQ_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS]; + __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS]; }; union t4_recv_wr { struct fw_ri_recv_wr recv; struct t4_status_page status; - __be64 flits[T4_EQ_SIZE / sizeof(__be64) * T4_RQ_NUM_SLOTS]; + __be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_RQ_NUM_SLOTS]; }; static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid, enum fw_wr_opcodes opcode, u8 flags, u8 len16) { - int slots_used; - wqe->send.opcode = (u8)opcode; wqe->send.flags = flags; wqe->send.wrid = wrid; @@ -118,12 +116,6 @@ static inline void init_wr_hdr(union t4_wr *wqe, u16 wrid, wqe->send.r1[1] = 0; wqe->send.r1[2] = 0; wqe->send.len16 = len16; - - slots_used = DIV_ROUND_UP(len16*16, T4_EQ_SIZE); - while (slots_used < T4_SQ_NUM_SLOTS) { - wqe->flits[slots_used * T4_EQ_SIZE / sizeof(__be64)] = 0; - slots_used++; - } } /* CQE/AE status codes */ @@ -279,7 +271,7 @@ struct t4_swsqe { struct t4_sq { union t4_wr *queue; dma_addr_t dma_addr; - DECLARE_PCI_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_ADDR(mapping); struct t4_swsqe *sw_sq; struct t4_swsqe *oldest_read; u64 udb; @@ -289,6 +281,7 @@ struct t4_sq { u16 size; u16 cidx; u16 pidx; + u16 wq_pidx; }; struct t4_swrqe { @@ -298,7 +291,7 @@ struct t4_swrqe { struct t4_rq { union t4_recv_wr *queue; dma_addr_t dma_addr; - DECLARE_PCI_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_ADDR(mapping); struct t4_swrqe *sw_rq; u64 udb; size_t memsize; @@ -310,6 +303,7 @@ struct t4_rq { u16 size; u16 cidx; u16 pidx; + u16 wq_pidx; }; struct t4_wq { @@ -340,11 +334,14 @@ static inline u32 t4_rq_avail(struct t4_wq *wq) return wq->rq.size - 1 - wq->rq.in_use; } -static inline void t4_rq_produce(struct t4_wq *wq) +static inline void t4_rq_produce(struct t4_wq *wq, u8 len16) { wq->rq.in_use++; if (++wq->rq.pidx == wq->rq.size) wq->rq.pidx = 0; + wq->rq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); + if (wq->rq.wq_pidx >= wq->rq.size * T4_RQ_NUM_SLOTS) + wq->rq.wq_pidx %= wq->rq.size * T4_RQ_NUM_SLOTS; } static inline void t4_rq_consume(struct t4_wq *wq) @@ -370,11 +367,14 @@ static inline u32 t4_sq_avail(struct t4_wq *wq) return wq->sq.size - 1 - wq->sq.in_use; } -static inline void t4_sq_produce(struct t4_wq *wq) +static inline void t4_sq_produce(struct t4_wq *wq, u8 len16) { wq->sq.in_use++; if (++wq->sq.pidx == wq->sq.size) wq->sq.pidx = 0; + wq->sq.wq_pidx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); + if (wq->sq.wq_pidx >= wq->sq.size * T4_SQ_NUM_SLOTS) + wq->sq.wq_pidx %= wq->sq.size * T4_SQ_NUM_SLOTS; } static inline void t4_sq_consume(struct t4_wq *wq) @@ -386,14 +386,12 @@ static inline void t4_sq_consume(struct t4_wq *wq) static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc) { - inc *= T4_SQ_NUM_SLOTS; wmb(); writel(QID(wq->sq.qid) | PIDX(inc), wq->db); } static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc) { - inc *= T4_RQ_NUM_SLOTS; wmb(); writel(QID(wq->rq.qid) | PIDX(inc), wq->db); } @@ -429,7 +427,7 @@ static inline int t4_wq_db_enabled(struct t4_wq *wq) struct t4_cq { struct t4_cqe *queue; dma_addr_t dma_addr; - DECLARE_PCI_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_ADDR(mapping); struct t4_cqe *sw_queue; void __iomem *gts; struct c4iw_rdev *rdev; diff --git a/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h b/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h index fc706bd07fae..dc193c292671 100644 --- a/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h +++ b/drivers/infiniband/hw/cxgb4/t4fw_ri_api.h @@ -826,4 +826,14 @@ struct ulptx_idata { #define S_ULPTX_NSGE 0 #define M_ULPTX_NSGE 0xFFFF #define V_ULPTX_NSGE(x) ((x) << S_ULPTX_NSGE) + +#define S_RX_DACK_MODE 29 +#define M_RX_DACK_MODE 0x3 +#define V_RX_DACK_MODE(x) ((x) << S_RX_DACK_MODE) +#define G_RX_DACK_MODE(x) (((x) >> S_RX_DACK_MODE) & M_RX_DACK_MODE) + +#define S_RX_DACK_CHANGE 31 +#define V_RX_DACK_CHANGE(x) ((x) << S_RX_DACK_CHANGE) +#define F_RX_DACK_CHANGE V_RX_DACK_CHANGE(1U) + #endif /* _T4FW_RI_API_H_ */ diff --git a/drivers/infiniband/hw/ehca/ehca_eq.c b/drivers/infiniband/hw/ehca/ehca_eq.c index 3b87589b8ea0..d9b1bb40f480 100644 --- a/drivers/infiniband/hw/ehca/ehca_eq.c +++ b/drivers/infiniband/hw/ehca/ehca_eq.c @@ -122,21 +122,21 @@ int ehca_create_eq(struct ehca_shca *shca, /* register interrupt handlers and initialize work queues */ if (type == EHCA_EQ) { + tasklet_init(&eq->interrupt_task, ehca_tasklet_eq, (long)shca); + ret = ibmebus_request_irq(eq->ist, ehca_interrupt_eq, IRQF_DISABLED, "ehca_eq", (void *)shca); if (ret < 0) ehca_err(ib_dev, "Can't map interrupt handler."); - - tasklet_init(&eq->interrupt_task, ehca_tasklet_eq, (long)shca); } else if (type == EHCA_NEQ) { + tasklet_init(&eq->interrupt_task, ehca_tasklet_neq, (long)shca); + ret = ibmebus_request_irq(eq->ist, ehca_interrupt_neq, IRQF_DISABLED, "ehca_neq", (void *)shca); if (ret < 0) ehca_err(ib_dev, "Can't map interrupt handler."); - - tasklet_init(&eq->interrupt_task, ehca_tasklet_neq, (long)shca); } eq->is_initialized = 1; diff --git a/drivers/infiniband/hw/ehca/ehca_main.c b/drivers/infiniband/hw/ehca/ehca_main.c index ecb51b396c42..cfc4de7a5da4 100644 --- a/drivers/infiniband/hw/ehca/ehca_main.c +++ b/drivers/infiniband/hw/ehca/ehca_main.c @@ -360,7 +360,8 @@ static int ehca_sense_attributes(struct ehca_shca *shca) * a firmware property, so it's valid across all adapters */ if (ehca_lock_hcalls == -1) - ehca_lock_hcalls = !(shca->hca_cap & HCA_CAP_H_ALLOC_RES_SYNC); + ehca_lock_hcalls = !EHCA_BMASK_GET(HCA_CAP_H_ALLOC_RES_SYNC, + shca->hca_cap); /* translate supported MR page sizes; always support 4K */ shca->hca_cap_mr_pgsize = EHCA_PAGESIZE; diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.c b/drivers/infiniband/hw/ehca/ehca_mrmw.c index 31a68b9c52d0..53f4cd4fc19a 100644 --- a/drivers/infiniband/hw/ehca/ehca_mrmw.c +++ b/drivers/infiniband/hw/ehca/ehca_mrmw.c @@ -933,11 +933,6 @@ int ehca_unmap_fmr(struct list_head *fmr_list) /* check all FMR belong to same SHCA, and check internal flag */ list_for_each_entry(ib_fmr, fmr_list, list) { prev_shca = shca; - if (!ib_fmr) { - ehca_gen_err("bad fmr=%p in list", ib_fmr); - ret = -EINVAL; - goto unmap_fmr_exit0; - } shca = container_of(ib_fmr->device, struct ehca_shca, ib_device); e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr); diff --git a/drivers/infiniband/hw/ehca/ehca_qp.c b/drivers/infiniband/hw/ehca/ehca_qp.c index 47d388ec1cde..32fb34201aba 100644 --- a/drivers/infiniband/hw/ehca/ehca_qp.c +++ b/drivers/infiniband/hw/ehca/ehca_qp.c @@ -251,7 +251,7 @@ static inline int ibqptype2servicetype(enum ib_qp_type ibqptype) return ST_UD; case IB_QPT_RAW_IPV6: return -EINVAL; - case IB_QPT_RAW_ETY: + case IB_QPT_RAW_ETHERTYPE: return -EINVAL; default: ehca_gen_err("Invalid ibqptype=%x", ibqptype); diff --git a/drivers/infiniband/hw/ehca/hcp_if.c b/drivers/infiniband/hw/ehca/hcp_if.c index 4d5dc3304d42..e6f9cdd94c7a 100644 --- a/drivers/infiniband/hw/ehca/hcp_if.c +++ b/drivers/infiniband/hw/ehca/hcp_if.c @@ -269,6 +269,7 @@ u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle, struct ehca_cq *cq, struct ehca_alloc_cq_parms *param) { + int rc; u64 ret; unsigned long outs[PLPAR_HCALL9_BUFSIZE]; @@ -283,8 +284,19 @@ u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle, param->act_nr_of_entries = (u32)outs[3]; param->act_pages = (u32)outs[4]; - if (ret == H_SUCCESS) - hcp_galpas_ctor(&cq->galpas, 0, outs[5], outs[6]); + if (ret == H_SUCCESS) { + rc = hcp_galpas_ctor(&cq->galpas, 0, outs[5], outs[6]); + if (rc) { + ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx", + rc, outs[5]); + + ehca_plpar_hcall_norets(H_FREE_RESOURCE, + adapter_handle.handle, /* r4 */ + cq->ipz_cq_handle.handle, /* r5 */ + 0, 0, 0, 0, 0); + ret = H_NO_MEM; + } + } if (ret == H_NOT_ENOUGH_RESOURCES) ehca_gen_err("Not enough resources. ret=%lli", ret); @@ -295,6 +307,7 @@ u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle, u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle, struct ehca_alloc_qp_parms *parms, int is_user) { + int rc; u64 ret; u64 allocate_controls, max_r10_reg, r11, r12; unsigned long outs[PLPAR_HCALL9_BUFSIZE]; @@ -358,8 +371,19 @@ u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle, parms->rqueue.queue_size = (u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]); - if (ret == H_SUCCESS) - hcp_galpas_ctor(&parms->galpas, is_user, outs[6], outs[6]); + if (ret == H_SUCCESS) { + rc = hcp_galpas_ctor(&parms->galpas, is_user, outs[6], outs[6]); + if (rc) { + ehca_gen_err("Could not establish HW access. rc=%d paddr=%#lx", + rc, outs[6]); + + ehca_plpar_hcall_norets(H_FREE_RESOURCE, + adapter_handle.handle, /* r4 */ + parms->qp_handle.handle, /* r5 */ + 0, 0, 0, 0, 0); + ret = H_NO_MEM; + } + } if (ret == H_NOT_ENOUGH_RESOURCES) ehca_gen_err("Not enough resources. ret=%lli", ret); diff --git a/drivers/infiniband/hw/ehca/hcp_if.h b/drivers/infiniband/hw/ehca/hcp_if.h index 39c1c3618ec7..a46e514c367b 100644 --- a/drivers/infiniband/hw/ehca/hcp_if.h +++ b/drivers/infiniband/hw/ehca/hcp_if.h @@ -49,7 +49,7 @@ #include "hipz_hw.h" /* - * hipz_h_alloc_resource_eq allocates EQ resources in HW and FW, initalize + * hipz_h_alloc_resource_eq allocates EQ resources in HW and FW, initialize * resources, create the empty EQPT (ring). */ u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle, diff --git a/drivers/infiniband/hw/ehca/hcp_phyp.c b/drivers/infiniband/hw/ehca/hcp_phyp.c index b3e0e72e8a73..077376ff3d28 100644 --- a/drivers/infiniband/hw/ehca/hcp_phyp.c +++ b/drivers/infiniband/hw/ehca/hcp_phyp.c @@ -42,10 +42,9 @@ #include "ehca_classes.h" #include "hipz_hw.h" -int hcall_map_page(u64 physaddr, u64 *mapaddr) +u64 hcall_map_page(u64 physaddr) { - *mapaddr = (u64)(ioremap(physaddr, EHCA_PAGESIZE)); - return 0; + return (u64)ioremap(physaddr, EHCA_PAGESIZE); } int hcall_unmap_page(u64 mapaddr) @@ -58,9 +57,9 @@ int hcp_galpas_ctor(struct h_galpas *galpas, int is_user, u64 paddr_kernel, u64 paddr_user) { if (!is_user) { - int ret = hcall_map_page(paddr_kernel, &galpas->kernel.fw_handle); - if (ret) - return ret; + galpas->kernel.fw_handle = hcall_map_page(paddr_kernel); + if (!galpas->kernel.fw_handle) + return -ENOMEM; } else galpas->kernel.fw_handle = 0; diff --git a/drivers/infiniband/hw/ehca/hcp_phyp.h b/drivers/infiniband/hw/ehca/hcp_phyp.h index 204227d5303a..d1b029910249 100644 --- a/drivers/infiniband/hw/ehca/hcp_phyp.h +++ b/drivers/infiniband/hw/ehca/hcp_phyp.h @@ -83,7 +83,7 @@ int hcp_galpas_ctor(struct h_galpas *galpas, int is_user, int hcp_galpas_dtor(struct h_galpas *galpas); -int hcall_map_page(u64 physaddr, u64 * mapaddr); +u64 hcall_map_page(u64 physaddr); int hcall_unmap_page(u64 mapaddr); diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c index 21337468c652..765f0fc1da76 100644 --- a/drivers/infiniband/hw/ipath/ipath_driver.c +++ b/drivers/infiniband/hw/ipath/ipath_driver.c @@ -390,6 +390,8 @@ done: ipath_enable_armlaunch(dd); } +static void cleanup_device(struct ipath_devdata *dd); + static int __devinit ipath_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { @@ -616,8 +618,13 @@ static int __devinit ipath_init_one(struct pci_dev *pdev, goto bail; bail_irqsetup: - if (pdev->irq) - free_irq(pdev->irq, dd); + cleanup_device(dd); + + if (dd->ipath_irq) + dd->ipath_f_free_irq(dd); + + if (dd->ipath_f_cleanup) + dd->ipath_f_cleanup(dd); bail_iounmap: iounmap((volatile void __iomem *) dd->ipath_kregbase); @@ -635,7 +642,7 @@ bail: return ret; } -static void __devexit cleanup_device(struct ipath_devdata *dd) +static void cleanup_device(struct ipath_devdata *dd) { int port; struct ipath_portdata **tmp; diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c index 9c5c66d16a23..65eb8929db22 100644 --- a/drivers/infiniband/hw/ipath/ipath_file_ops.c +++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c @@ -2055,7 +2055,7 @@ static int ipath_close(struct inode *in, struct file *fp) mutex_lock(&ipath_mutex); - fd = (struct ipath_filedata *) fp->private_data; + fd = fp->private_data; fp->private_data = NULL; pd = fd->pd; if (!pd) { diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c index 3603ae89b606..f4ceecd9684b 100644 --- a/drivers/infiniband/hw/mthca/mthca_cmd.c +++ b/drivers/infiniband/hw/mthca/mthca_cmd.c @@ -1817,7 +1817,7 @@ int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, case IB_QPT_RAW_IPV6: op_mod = 2; break; - case IB_QPT_RAW_ETY: + case IB_QPT_RAW_ETHERTYPE: op_mod = 3; break; default: diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c index de7b9d7166f3..0c9f0aa5d4ea 100644 --- a/drivers/infiniband/hw/nes/nes.c +++ b/drivers/infiniband/hw/nes/nes.c @@ -110,8 +110,8 @@ static unsigned int sysfs_nonidx_addr; static unsigned int sysfs_idx_addr; static struct pci_device_id nes_pci_table[] = { - {PCI_VENDOR_ID_NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020, PCI_ANY_ID, PCI_ANY_ID}, - {PCI_VENDOR_ID_NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020_KR, PCI_ANY_ID, PCI_ANY_ID}, + { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020), }, + { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020_KR), }, {0} }; @@ -259,13 +259,11 @@ static void nes_cqp_rem_ref_callback(struct nes_device *nesdev, struct nes_cqp_r unsigned long flags; struct nes_qp *nesqp = cqp_request->cqp_callback_pointer; struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 qp_id; atomic_inc(&qps_destroyed); /* Free the control structures */ - qp_id = nesqp->hwqp.qp_id; if (nesqp->pbl_vbase) { pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); @@ -441,7 +439,6 @@ static int __devinit nes_probe(struct pci_dev *pcidev, const struct pci_device_i struct net_device *netdev = NULL; struct nes_device *nesdev = NULL; int ret = 0; - struct nes_vnic *nesvnic = NULL; void __iomem *mmio_regs = NULL; u8 hw_rev; @@ -664,25 +661,21 @@ static int __devinit nes_probe(struct pci_dev *pcidev, const struct pci_device_i nes_notifiers_registered++; /* Initialize network devices */ - if ((netdev = nes_netdev_init(nesdev, mmio_regs)) == NULL) { - goto bail7; - } - - /* Register network device */ - ret = register_netdev(netdev); - if (ret) { - printk(KERN_ERR PFX "Unable to register netdev, ret = %d\n", ret); - nes_netdev_destroy(netdev); - goto bail7; - } + if ((netdev = nes_netdev_init(nesdev, mmio_regs)) == NULL) + goto bail7; - nes_print_macaddr(netdev); - /* create a CM core for this netdev */ - nesvnic = netdev_priv(netdev); + /* Register network device */ + ret = register_netdev(netdev); + if (ret) { + printk(KERN_ERR PFX "Unable to register netdev, ret = %d\n", ret); + nes_netdev_destroy(netdev); + goto bail7; + } - nesdev->netdev_count++; - nesdev->nesadapter->netdev_count++; + nes_print_macaddr(netdev); + nesdev->netdev_count++; + nesdev->nesadapter->netdev_count++; printk(KERN_ERR PFX "%s: NetEffect RNIC driver successfully loaded.\n", pci_name(pcidev)); @@ -1104,7 +1097,7 @@ static ssize_t nes_show_wqm_quanta(struct device_driver *ddp, char *buf) i++; } - return snprintf(buf, PAGE_SIZE, "0x%X\n", wqm_quanta); + return snprintf(buf, PAGE_SIZE, "0x%X\n", wqm_quanta_value); } diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h index cc78fee1dd51..b3d145e82b4c 100644 --- a/drivers/infiniband/hw/nes/nes.h +++ b/drivers/infiniband/hw/nes/nes.h @@ -262,6 +262,7 @@ struct nes_device { u16 base_doorbell_index; u16 currcq_count; u16 deepcq_count; + u8 iw_status; u8 msi_enabled; u8 netdev_count; u8 napi_isr_ran; @@ -527,6 +528,7 @@ void nes_cm_disconn_worker(void *); int nes_hw_modify_qp(struct nes_device *, struct nes_qp *, u32, u32, u32); int nes_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *); struct nes_ib_device *nes_init_ofa_device(struct net_device *); +void nes_port_ibevent(struct nes_vnic *nesvnic); void nes_destroy_ofa_device(struct nes_ib_device *); int nes_register_ofa_device(struct nes_ib_device *); diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c index 986d6f32dded..443cea55daac 100644 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ b/drivers/infiniband/hw/nes/nes_cm.c @@ -1146,7 +1146,7 @@ static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpi } if ((neigh == NULL) || (!(neigh->nud_state & NUD_VALID))) - neigh_event_send(rt->u.dst.neighbour, NULL); + neigh_event_send(rt->dst.neighbour, NULL); ip_rt_put(rt); return rc; @@ -1719,8 +1719,6 @@ static int handle_ack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, { int datasize = 0; u32 inc_sequence; - u32 rem_seq_ack; - u32 rem_seq; int ret = 0; int optionsize; optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); @@ -1730,8 +1728,6 @@ static int handle_ack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, skb_pull(skb, tcph->doff << 2); inc_sequence = ntohl(tcph->seq); - rem_seq = ntohl(tcph->seq); - rem_seq_ack = ntohl(tcph->ack_seq); datasize = skb->len; switch (cm_node->state) { case NES_CM_STATE_SYN_RCVD: @@ -2565,7 +2561,7 @@ static int nes_cm_disconn_true(struct nes_qp *nesqp) u16 last_ae; u8 original_hw_tcp_state; u8 original_ibqp_state; - enum iw_cm_event_type disconn_status = IW_CM_EVENT_STATUS_OK; + enum iw_cm_event_status disconn_status = IW_CM_EVENT_STATUS_OK; int issue_disconn = 0; int issue_close = 0; int issue_flush = 0; @@ -3128,17 +3124,15 @@ int nes_create_listen(struct iw_cm_id *cm_id, int backlog) struct nes_vnic *nesvnic; struct nes_cm_listener *cm_node; struct nes_cm_info cm_info; - struct nes_adapter *adapter; int err; - nes_debug(NES_DBG_CM, "cm_id = %p, local port = 0x%04X.\n", cm_id, ntohs(cm_id->local_addr.sin_port)); nesvnic = to_nesvnic(cm_id->device); if (!nesvnic) return -EINVAL; - adapter = nesvnic->nesdev->nesadapter; + nes_debug(NES_DBG_CM, "nesvnic=%p, netdev=%p, %s\n", nesvnic, nesvnic->netdev, nesvnic->netdev->name); diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c index 57874a165083..f8233c851c69 100644 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ b/drivers/infiniband/hw/nes/nes_hw.c @@ -1970,7 +1970,7 @@ void nes_destroy_nic_qp(struct nes_vnic *nesvnic) dev_kfree_skb( nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]); - nesvnic->nic.sq_tail = (++nesvnic->nic.sq_tail) + nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1) & (nesvnic->nic.sq_size - 1); } @@ -2737,9 +2737,9 @@ void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) nesnic->sq_tail &= nesnic->sq_size-1; if (sq_cqes > 128) { barrier(); - /* restart the queue if it had been stopped */ - if (netif_queue_stopped(nesvnic->netdev)) - netif_wake_queue(nesvnic->netdev); + /* restart the queue if it had been stopped */ + if (netif_queue_stopped(nesvnic->netdev)) + netif_wake_queue(nesvnic->netdev); sq_cqes = 0; } } else { @@ -2999,11 +2999,8 @@ static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq) static u8 *locate_mpa(u8 *pkt, u32 aeq_info) { - u16 pkt_len; - if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) { /* skip over ethernet header */ - pkt_len = be16_to_cpu(*(u16 *)(pkt + ETH_HLEN - 2)); pkt += ETH_HLEN; /* Skip over IP and TCP headers */ @@ -3283,9 +3280,15 @@ static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *n else mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG; - nes_terminate_start_timer(nesqp); - nesqp->term_flags |= NES_TERM_SENT; - nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0); + if (!nesdev->iw_status) { + nesqp->term_flags = NES_TERM_DONE; + nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0); + nes_cm_disconn(nesqp); + } else { + nes_terminate_start_timer(nesqp); + nesqp->term_flags |= NES_TERM_SENT; + nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0); + } } static void nes_terminate_send_fin(struct nes_device *nesdev, diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h index bbbfe9fc5a5a..aa9183db32b1 100644 --- a/drivers/infiniband/hw/nes/nes_hw.h +++ b/drivers/infiniband/hw/nes/nes_hw.h @@ -1100,11 +1100,12 @@ struct nes_adapter { u32 wqm_wat; u32 core_clock; u32 firmware_version; + u32 eeprom_version; u32 nic_rx_eth_route_err; u32 et_rx_coalesce_usecs; - u32 et_rx_max_coalesced_frames; + u32 et_rx_max_coalesced_frames; u32 et_rx_coalesce_usecs_irq; u32 et_rx_max_coalesced_frames_irq; u32 et_pkt_rate_low; diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c index 5cc0a9ae5bb1..6dfdd49cdbcf 100644 --- a/drivers/infiniband/hw/nes/nes_nic.c +++ b/drivers/infiniband/hw/nes/nes_nic.c @@ -232,6 +232,13 @@ static int nes_netdev_open(struct net_device *netdev) NES_MAC_INT_TX_UNDERFLOW | NES_MAC_INT_TX_ERROR)); first_nesvnic = nesvnic; } + + if (nesvnic->of_device_registered) { + nesdev->iw_status = 1; + nesdev->nesadapter->send_term_ok = 1; + nes_port_ibevent(nesvnic); + } + if (first_nesvnic->linkup) { /* Enable network packets */ nesvnic->linkup = 1; @@ -309,9 +316,9 @@ static int nes_netdev_stop(struct net_device *netdev) if (nesvnic->of_device_registered) { - nes_destroy_ofa_device(nesvnic->nesibdev); - nesvnic->nesibdev = NULL; - nesvnic->of_device_registered = 0; + nesdev->nesadapter->send_term_ok = 0; + nesdev->iw_status = 0; + nes_port_ibevent(nesvnic); } nes_destroy_nic_qp(nesvnic); @@ -463,7 +470,6 @@ static int nes_netdev_start_xmit(struct sk_buff *skb, struct net_device *netdev) u16 nhoffset; u16 wqes_needed; u16 wqes_available; - u32 old_head; u32 wqe_misc; /* @@ -503,7 +509,6 @@ sq_no_longer_full: if (skb_is_gso(skb)) { nesvnic->segmented_tso_requests++; nesvnic->tso_requests++; - old_head = nesnic->sq_head; /* Basically 4 fragments available per WQE with extended fragments */ wqes_needed = nr_frags >> 2; wqes_needed += (nr_frags&3)?1:0; @@ -1567,6 +1572,12 @@ static int nes_netdev_set_settings(struct net_device *netdev, struct ethtool_cmd } +static int nes_netdev_set_flags(struct net_device *netdev, u32 flags) +{ + return ethtool_op_set_flags(netdev, flags, ETH_FLAG_LRO); +} + + static const struct ethtool_ops nes_ethtool_ops = { .get_link = ethtool_op_get_link, .get_settings = nes_netdev_get_settings, @@ -1588,7 +1599,7 @@ static const struct ethtool_ops nes_ethtool_ops = { .get_tso = ethtool_op_get_tso, .set_tso = ethtool_op_set_tso, .get_flags = ethtool_op_get_flags, - .set_flags = ethtool_op_set_flags, + .set_flags = nes_netdev_set_flags, }; diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c index a9f5dd272f1a..f9c417c6b3b3 100644 --- a/drivers/infiniband/hw/nes/nes_utils.c +++ b/drivers/infiniband/hw/nes/nes_utils.c @@ -190,6 +190,11 @@ int nes_read_eeprom_values(struct nes_device *nesdev, struct nes_adapter *nesada nesadapter->firmware_version = (((u32)(u8)(eeprom_data>>8)) << 16) + (u32)((u8)eeprom_data); + eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 10); + printk(PFX "EEPROM version %u.%u\n", (u8)(eeprom_data>>8), (u8)eeprom_data); + nesadapter->eeprom_version = (((u32)(u8)(eeprom_data>>8)) << 16) + + (u32)((u8)eeprom_data); + no_fw_rev: /* eeprom is valid */ eeprom_offset = nesadapter->software_eeprom_offset; diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c index 9bc2d744b2ea..9046e6675686 100644 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ b/drivers/infiniband/hw/nes/nes_verbs.c @@ -518,7 +518,7 @@ static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *prop memset(props, 0, sizeof(*props)); memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6); - props->fw_ver = nesdev->nesadapter->fw_ver; + props->fw_ver = nesdev->nesadapter->firmware_version; props->device_cap_flags = nesdev->nesadapter->device_cap_flags; props->vendor_id = nesdev->nesadapter->vendor_id; props->vendor_part_id = nesdev->nesadapter->vendor_part_id; @@ -1941,7 +1941,7 @@ static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, u8 use_256_pbls = 0; u8 use_4k_pbls = 0; u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0; - struct nes_root_vpbl new_root = {0, 0, 0}; + struct nes_root_vpbl new_root = { 0, NULL, NULL }; u32 opcode = 0; u16 major_code; @@ -2112,13 +2112,12 @@ static struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, u32 driver_key = 0; u32 root_pbl_index = 0; u32 cur_pbl_index = 0; - int err = 0, pbl_depth = 0; + int err = 0; int ret = 0; u16 pbl_count = 0; u8 single_page = 1; u8 stag_key = 0; - pbl_depth = 0; region_length = 0; vpbl.pbl_vbase = NULL; root_vpbl.pbl_vbase = NULL; @@ -2931,7 +2930,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int ret; u16 original_last_aeq; u8 issue_modify_qp = 0; - u8 issue_disconnect = 0; u8 dont_wait = 0; nes_debug(NES_DBG_MOD_QP, "QP%u: QP State=%u, cur QP State=%u," @@ -3058,6 +3056,7 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, nesqp->hte_added = 0; } if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) && + (nesdev->iw_status) && (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) { next_iwarp_state |= NES_CQP_QP_RESET; } else { @@ -3082,7 +3081,6 @@ int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK; nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n", nesqp->iwarp_state); - issue_disconnect = 1; } else { nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK; nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n", @@ -3936,6 +3934,17 @@ struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev) return nesibdev; } +void nes_port_ibevent(struct nes_vnic *nesvnic) +{ + struct nes_ib_device *nesibdev = nesvnic->nesibdev; + struct nes_device *nesdev = nesvnic->nesdev; + struct ib_event event; + event.device = &nesibdev->ibdev; + event.element.port_num = nesvnic->logical_port + 1; + event.event = nesdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; + ib_dispatch_event(&event); +} + /** * nes_destroy_ofa_device diff --git a/drivers/infiniband/hw/qib/Makefile b/drivers/infiniband/hw/qib/Makefile index c6515a1b9a6a..f12d7bb8b39f 100644 --- a/drivers/infiniband/hw/qib/Makefile +++ b/drivers/infiniband/hw/qib/Makefile @@ -6,7 +6,7 @@ ib_qib-y := qib_cq.o qib_diag.o qib_dma.o qib_driver.o qib_eeprom.o \ qib_qp.o qib_qsfp.o qib_rc.o qib_ruc.o qib_sdma.o qib_srq.o \ qib_sysfs.o qib_twsi.o qib_tx.o qib_uc.o qib_ud.o \ qib_user_pages.o qib_user_sdma.o qib_verbs_mcast.o qib_iba7220.o \ - qib_sd7220.o qib_sd7220_img.o qib_iba7322.o qib_verbs.o + qib_sd7220.o qib_iba7322.o qib_verbs.o # 6120 has no fallback if no MSI interrupts, others can do INTx ib_qib-$(CONFIG_PCI_MSI) += qib_iba6120.o diff --git a/drivers/infiniband/hw/qib/qib.h b/drivers/infiniband/hw/qib/qib.h index 32d9208efcff..61de0654820e 100644 --- a/drivers/infiniband/hw/qib/qib.h +++ b/drivers/infiniband/hw/qib/qib.h @@ -45,6 +45,7 @@ #include <linux/mutex.h> #include <linux/list.h> #include <linux/scatterlist.h> +#include <linux/slab.h> #include <linux/io.h> #include <linux/fs.h> #include <linux/completion.h> @@ -326,6 +327,9 @@ struct qib_verbs_txreq { #define QIB_DEFAULT_MTU 4096 +/* max number of IB ports supported per HCA */ +#define QIB_MAX_IB_PORTS 2 + /* * Possible IB config parameters for f_get/set_ib_table() */ @@ -686,6 +690,7 @@ struct qib_devdata { void __iomem *piobase; /* mem-mapped pointer to base of user chip regs (if using WC PAT) */ u64 __iomem *userbase; + void __iomem *piovl15base; /* base of VL15 buffers, if not WC */ /* * points to area where PIOavail registers will be DMA'ed. * Has to be on a page of it's own, because the page will be diff --git a/drivers/infiniband/hw/qib/qib_7220.h b/drivers/infiniband/hw/qib/qib_7220.h index ea0bfd896f92..21f374aa0631 100644 --- a/drivers/infiniband/hw/qib/qib_7220.h +++ b/drivers/infiniband/hw/qib/qib_7220.h @@ -109,10 +109,6 @@ struct qib_chippport_specific { */ int qib_sd7220_presets(struct qib_devdata *dd); int qib_sd7220_init(struct qib_devdata *dd); -int qib_sd7220_prog_ld(struct qib_devdata *dd, int sdnum, u8 *img, - int len, int offset); -int qib_sd7220_prog_vfy(struct qib_devdata *dd, int sdnum, const u8 *img, - int len, int offset); void qib_sd7220_clr_ibpar(struct qib_devdata *); /* * Below used for sdnum parameter, selecting one of the two sections @@ -121,9 +117,6 @@ void qib_sd7220_clr_ibpar(struct qib_devdata *); */ #define IB_7220_SERDES 2 -int qib_sd7220_ib_load(struct qib_devdata *dd); -int qib_sd7220_ib_vfy(struct qib_devdata *dd); - static inline u32 qib_read_kreg32(const struct qib_devdata *dd, const u16 regno) { diff --git a/drivers/infiniband/hw/qib/qib_7322_regs.h b/drivers/infiniband/hw/qib/qib_7322_regs.h index a97440ba924c..32dc81ff8d4a 100644 --- a/drivers/infiniband/hw/qib/qib_7322_regs.h +++ b/drivers/infiniband/hw/qib/qib_7322_regs.h @@ -742,15 +742,15 @@ #define QIB_7322_HwErrMask_IBCBusFromSPCParityErrMask_1_LSB 0xF #define QIB_7322_HwErrMask_IBCBusFromSPCParityErrMask_1_MSB 0xF #define QIB_7322_HwErrMask_IBCBusFromSPCParityErrMask_1_RMASK 0x1 -#define QIB_7322_HwErrMask_statusValidNoEopMask_1_LSB 0xE -#define QIB_7322_HwErrMask_statusValidNoEopMask_1_MSB 0xE -#define QIB_7322_HwErrMask_statusValidNoEopMask_1_RMASK 0x1 +#define QIB_7322_HwErrMask_IBCBusToSPCParityErrMask_1_LSB 0xE +#define QIB_7322_HwErrMask_IBCBusToSPCParityErrMask_1_MSB 0xE +#define QIB_7322_HwErrMask_IBCBusToSPCParityErrMask_1_RMASK 0x1 #define QIB_7322_HwErrMask_IBCBusFromSPCParityErrMask_0_LSB 0xD #define QIB_7322_HwErrMask_IBCBusFromSPCParityErrMask_0_MSB 0xD #define QIB_7322_HwErrMask_IBCBusFromSPCParityErrMask_0_RMASK 0x1 -#define QIB_7322_HwErrMask_statusValidNoEopMask_0_LSB 0xC -#define QIB_7322_HwErrMask_statusValidNoEopMask_0_MSB 0xC -#define QIB_7322_HwErrMask_statusValidNoEopMask_0_RMASK 0x1 +#define QIB_7322_HwErrMask_statusValidNoEopMask_LSB 0xC +#define QIB_7322_HwErrMask_statusValidNoEopMask_MSB 0xC +#define QIB_7322_HwErrMask_statusValidNoEopMask_RMASK 0x1 #define QIB_7322_HwErrMask_LATriggeredMask_LSB 0xB #define QIB_7322_HwErrMask_LATriggeredMask_MSB 0xB #define QIB_7322_HwErrMask_LATriggeredMask_RMASK 0x1 @@ -796,15 +796,15 @@ #define QIB_7322_HwErrStatus_IBCBusFromSPCParityErr_1_LSB 0xF #define QIB_7322_HwErrStatus_IBCBusFromSPCParityErr_1_MSB 0xF #define QIB_7322_HwErrStatus_IBCBusFromSPCParityErr_1_RMASK 0x1 -#define QIB_7322_HwErrStatus_statusValidNoEop_1_LSB 0xE -#define QIB_7322_HwErrStatus_statusValidNoEop_1_MSB 0xE -#define QIB_7322_HwErrStatus_statusValidNoEop_1_RMASK 0x1 +#define QIB_7322_HwErrStatus_IBCBusToSPCParityErr_1_LSB 0xE +#define QIB_7322_HwErrStatus_IBCBusToSPCParityErr_1_MSB 0xE +#define QIB_7322_HwErrStatus_IBCBusToSPCParityErr_1_RMASK 0x1 #define QIB_7322_HwErrStatus_IBCBusFromSPCParityErr_0_LSB 0xD #define QIB_7322_HwErrStatus_IBCBusFromSPCParityErr_0_MSB 0xD #define QIB_7322_HwErrStatus_IBCBusFromSPCParityErr_0_RMASK 0x1 -#define QIB_7322_HwErrStatus_statusValidNoEop_0_LSB 0xC -#define QIB_7322_HwErrStatus_statusValidNoEop_0_MSB 0xC -#define QIB_7322_HwErrStatus_statusValidNoEop_0_RMASK 0x1 +#define QIB_7322_HwErrStatus_statusValidNoEop_LSB 0xC +#define QIB_7322_HwErrStatus_statusValidNoEop_MSB 0xC +#define QIB_7322_HwErrStatus_statusValidNoEop_RMASK 0x1 #define QIB_7322_HwErrStatus_LATriggered_LSB 0xB #define QIB_7322_HwErrStatus_LATriggered_MSB 0xB #define QIB_7322_HwErrStatus_LATriggered_RMASK 0x1 @@ -850,15 +850,15 @@ #define QIB_7322_HwErrClear_IBCBusFromSPCParityErrClear_1_LSB 0xF #define QIB_7322_HwErrClear_IBCBusFromSPCParityErrClear_1_MSB 0xF #define QIB_7322_HwErrClear_IBCBusFromSPCParityErrClear_1_RMASK 0x1 -#define QIB_7322_HwErrClear_IBCBusToSPCparityErrClear_1_LSB 0xE -#define QIB_7322_HwErrClear_IBCBusToSPCparityErrClear_1_MSB 0xE -#define QIB_7322_HwErrClear_IBCBusToSPCparityErrClear_1_RMASK 0x1 +#define QIB_7322_HwErrClear_IBCBusToSPCParityErrClear_1_LSB 0xE +#define QIB_7322_HwErrClear_IBCBusToSPCParityErrClear_1_MSB 0xE +#define QIB_7322_HwErrClear_IBCBusToSPCParityErrClear_1_RMASK 0x1 #define QIB_7322_HwErrClear_IBCBusFromSPCParityErrClear_0_LSB 0xD #define QIB_7322_HwErrClear_IBCBusFromSPCParityErrClear_0_MSB 0xD #define QIB_7322_HwErrClear_IBCBusFromSPCParityErrClear_0_RMASK 0x1 -#define QIB_7322_HwErrClear_IBCBusToSPCparityErrClear_0_LSB 0xC -#define QIB_7322_HwErrClear_IBCBusToSPCparityErrClear_0_MSB 0xC -#define QIB_7322_HwErrClear_IBCBusToSPCparityErrClear_0_RMASK 0x1 +#define QIB_7322_HwErrClear_statusValidNoEopClear_LSB 0xC +#define QIB_7322_HwErrClear_statusValidNoEopClear_MSB 0xC +#define QIB_7322_HwErrClear_statusValidNoEopClear_RMASK 0x1 #define QIB_7322_HwErrClear_LATriggeredClear_LSB 0xB #define QIB_7322_HwErrClear_LATriggeredClear_MSB 0xB #define QIB_7322_HwErrClear_LATriggeredClear_RMASK 0x1 @@ -880,15 +880,15 @@ #define QIB_7322_HwDiagCtrl_ForceIBCBusFromSPCParityErr_1_LSB 0xF #define QIB_7322_HwDiagCtrl_ForceIBCBusFromSPCParityErr_1_MSB 0xF #define QIB_7322_HwDiagCtrl_ForceIBCBusFromSPCParityErr_1_RMASK 0x1 -#define QIB_7322_HwDiagCtrl_ForcestatusValidNoEop_1_LSB 0xE -#define QIB_7322_HwDiagCtrl_ForcestatusValidNoEop_1_MSB 0xE -#define QIB_7322_HwDiagCtrl_ForcestatusValidNoEop_1_RMASK 0x1 +#define QIB_7322_HwDiagCtrl_ForceIBCBusToSPCParityErr_1_LSB 0xE +#define QIB_7322_HwDiagCtrl_ForceIBCBusToSPCParityErr_1_MSB 0xE +#define QIB_7322_HwDiagCtrl_ForceIBCBusToSPCParityErr_1_RMASK 0x1 #define QIB_7322_HwDiagCtrl_ForceIBCBusFromSPCParityErr_0_LSB 0xD #define QIB_7322_HwDiagCtrl_ForceIBCBusFromSPCParityErr_0_MSB 0xD #define QIB_7322_HwDiagCtrl_ForceIBCBusFromSPCParityErr_0_RMASK 0x1 -#define QIB_7322_HwDiagCtrl_ForcestatusValidNoEop_0_LSB 0xC -#define QIB_7322_HwDiagCtrl_ForcestatusValidNoEop_0_MSB 0xC -#define QIB_7322_HwDiagCtrl_ForcestatusValidNoEop_0_RMASK 0x1 +#define QIB_7322_HwDiagCtrl_ForceIBCBusToSPCParityErr_0_LSB 0xC +#define QIB_7322_HwDiagCtrl_ForceIBCBusToSPCParityErr_0_MSB 0xC +#define QIB_7322_HwDiagCtrl_ForceIBCBusToSPCParityErr_0_RMASK 0x1 #define QIB_7322_EXTStatus_OFFS 0xC0 #define QIB_7322_EXTStatus_DEF 0x000000000000X000 diff --git a/drivers/infiniband/hw/qib/qib_common.h b/drivers/infiniband/hw/qib/qib_common.h index b3955ed8f794..145da4040883 100644 --- a/drivers/infiniband/hw/qib/qib_common.h +++ b/drivers/infiniband/hw/qib/qib_common.h @@ -279,7 +279,7 @@ struct qib_base_info { * may not be implemented; the user code must deal with this if it * cares, or it must abort after initialization reports the difference. */ -#define QIB_USER_SWMINOR 10 +#define QIB_USER_SWMINOR 11 #define QIB_USER_SWVERSION ((QIB_USER_SWMAJOR << 16) | QIB_USER_SWMINOR) @@ -302,6 +302,18 @@ struct qib_base_info { #define QIB_KERN_SWVERSION ((QIB_KERN_TYPE << 31) | QIB_USER_SWVERSION) /* + * If the unit is specified via open, HCA choice is fixed. If port is + * specified, it's also fixed. Otherwise we try to spread contexts + * across ports and HCAs, using different algorithims. WITHIN is + * the old default, prior to this mechanism. + */ +#define QIB_PORT_ALG_ACROSS 0 /* round robin contexts across HCAs, then + * ports; this is the default */ +#define QIB_PORT_ALG_WITHIN 1 /* use all contexts on an HCA (round robin + * active ports within), then next HCA */ +#define QIB_PORT_ALG_COUNT 2 /* number of algorithm choices */ + +/* * This structure is passed to qib_userinit() to tell the driver where * user code buffers are, sizes, etc. The offsets and sizes of the * fields must remain unchanged, for binary compatibility. It can @@ -319,7 +331,7 @@ struct qib_user_info { /* size of struct base_info to write to */ __u32 spu_base_info_size; - __u32 _spu_unused3; + __u32 spu_port_alg; /* which QIB_PORT_ALG_*; unused user minor < 11 */ /* * If two or more processes wish to share a context, each process diff --git a/drivers/infiniband/hw/qib/qib_diag.c b/drivers/infiniband/hw/qib/qib_diag.c index ca98dd523752..05dcf0d9a7d3 100644 --- a/drivers/infiniband/hw/qib/qib_diag.c +++ b/drivers/infiniband/hw/qib/qib_diag.c @@ -233,6 +233,7 @@ static u32 __iomem *qib_remap_ioaddr32(struct qib_devdata *dd, u32 offset, u32 __iomem *krb32 = (u32 __iomem *)dd->kregbase; u32 __iomem *map = NULL; u32 cnt = 0; + u32 tot4k, offs4k; /* First, simplest case, offset is within the first map. */ kreglen = (dd->kregend - dd->kregbase) * sizeof(u64); @@ -250,7 +251,8 @@ static u32 __iomem *qib_remap_ioaddr32(struct qib_devdata *dd, u32 offset, if (dd->userbase) { /* If user regs mapped, they are after send, so set limit. */ u32 ulim = (dd->cfgctxts * dd->ureg_align) + dd->uregbase; - snd_lim = dd->uregbase; + if (!dd->piovl15base) + snd_lim = dd->uregbase; krb32 = (u32 __iomem *)dd->userbase; if (offset >= dd->uregbase && offset < ulim) { map = krb32 + (offset - dd->uregbase) / sizeof(u32); @@ -277,14 +279,14 @@ static u32 __iomem *qib_remap_ioaddr32(struct qib_devdata *dd, u32 offset, /* If 4k buffers exist, account for them by bumping * appropriate limit. */ + tot4k = dd->piobcnt4k * dd->align4k; + offs4k = dd->piobufbase >> 32; if (dd->piobcnt4k) { - u32 tot4k = dd->piobcnt4k * dd->align4k; - u32 offs4k = dd->piobufbase >> 32; if (snd_bottom > offs4k) snd_bottom = offs4k; else { /* 4k above 2k. Bump snd_lim, if needed*/ - if (!dd->userbase) + if (!dd->userbase || dd->piovl15base) snd_lim = offs4k + tot4k; } } @@ -298,6 +300,15 @@ static u32 __iomem *qib_remap_ioaddr32(struct qib_devdata *dd, u32 offset, cnt = snd_lim - offset; } + if (!map && offs4k && dd->piovl15base) { + snd_lim = offs4k + tot4k + 2 * dd->align4k; + if (offset >= (offs4k + tot4k) && offset < snd_lim) { + map = (u32 __iomem *)dd->piovl15base + + ((offset - (offs4k + tot4k)) / sizeof(u32)); + cnt = snd_lim - offset; + } + } + mapped: if (cntp) *cntp = cnt; diff --git a/drivers/infiniband/hw/qib/qib_driver.c b/drivers/infiniband/hw/qib/qib_driver.c index f15ce076ac49..9cd193603fb1 100644 --- a/drivers/infiniband/hw/qib/qib_driver.c +++ b/drivers/infiniband/hw/qib/qib_driver.c @@ -335,7 +335,7 @@ u32 qib_kreceive(struct qib_ctxtdata *rcd, u32 *llic, u32 *npkts) smp_rmb(); /* prevent speculative reads of dma'ed hdrq */ } - for (last = 0, i = 1; !last; i += !last) { + for (last = 0, i = 1; !last && i <= 64; i += !last) { hdr = dd->f_get_msgheader(dd, rhf_addr); eflags = qib_hdrget_err_flags(rhf_addr); etype = qib_hdrget_rcv_type(rhf_addr); diff --git a/drivers/infiniband/hw/qib/qib_file_ops.c b/drivers/infiniband/hw/qib/qib_file_ops.c index a142a9eb5226..6b11645edf35 100644 --- a/drivers/infiniband/hw/qib/qib_file_ops.c +++ b/drivers/infiniband/hw/qib/qib_file_ops.c @@ -1294,128 +1294,130 @@ bail: return ret; } -static inline int usable(struct qib_pportdata *ppd, int active_only) +static inline int usable(struct qib_pportdata *ppd) { struct qib_devdata *dd = ppd->dd; - u32 linkok = active_only ? QIBL_LINKACTIVE : - (QIBL_LINKINIT | QIBL_LINKARMED | QIBL_LINKACTIVE); return dd && (dd->flags & QIB_PRESENT) && dd->kregbase && ppd->lid && - (ppd->lflags & linkok); + (ppd->lflags & QIBL_LINKACTIVE); } -static int find_free_ctxt(int unit, struct file *fp, - const struct qib_user_info *uinfo) +/* + * Select a context on the given device, either using a requested port + * or the port based on the context number. + */ +static int choose_port_ctxt(struct file *fp, struct qib_devdata *dd, u32 port, + const struct qib_user_info *uinfo) { - struct qib_devdata *dd = qib_lookup(unit); struct qib_pportdata *ppd = NULL; - int ret; - u32 ctxt; + int ret, ctxt; - if (!dd || (uinfo->spu_port && uinfo->spu_port > dd->num_pports)) { - ret = -ENODEV; - goto bail; - } - - /* - * If users requests specific port, only try that one port, else - * select "best" port below, based on context. - */ - if (uinfo->spu_port) { - ppd = dd->pport + uinfo->spu_port - 1; - if (!usable(ppd, 0)) { + if (port) { + if (!usable(dd->pport + port - 1)) { ret = -ENETDOWN; - goto bail; - } + goto done; + } else + ppd = dd->pport + port - 1; } - - for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts; ctxt++) { - if (dd->rcd[ctxt]) - continue; - /* - * The setting and clearing of user context rcd[x] protected - * by the qib_mutex - */ - if (!ppd) { - /* choose port based on ctxt, if up, else 1st up */ - ppd = dd->pport + (ctxt % dd->num_pports); - if (!usable(ppd, 0)) { - int i; - for (i = 0; i < dd->num_pports; i++) { - ppd = dd->pport + i; - if (usable(ppd, 0)) - break; - } - if (i == dd->num_pports) { - ret = -ENETDOWN; - goto bail; - } - } + for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts && dd->rcd[ctxt]; + ctxt++) + ; + if (ctxt == dd->cfgctxts) { + ret = -EBUSY; + goto done; + } + if (!ppd) { + u32 pidx = ctxt % dd->num_pports; + if (usable(dd->pport + pidx)) + ppd = dd->pport + pidx; + else { + for (pidx = 0; pidx < dd->num_pports && !ppd; + pidx++) + if (usable(dd->pport + pidx)) + ppd = dd->pport + pidx; } - ret = setup_ctxt(ppd, ctxt, fp, uinfo); - goto bail; } - ret = -EBUSY; + ret = ppd ? setup_ctxt(ppd, ctxt, fp, uinfo) : -ENETDOWN; +done: + return ret; +} + +static int find_free_ctxt(int unit, struct file *fp, + const struct qib_user_info *uinfo) +{ + struct qib_devdata *dd = qib_lookup(unit); + int ret; + + if (!dd || (uinfo->spu_port && uinfo->spu_port > dd->num_pports)) + ret = -ENODEV; + else + ret = choose_port_ctxt(fp, dd, uinfo->spu_port, uinfo); -bail: return ret; } -static int get_a_ctxt(struct file *fp, const struct qib_user_info *uinfo) +static int get_a_ctxt(struct file *fp, const struct qib_user_info *uinfo, + unsigned alg) { - struct qib_pportdata *ppd; - int ret = 0, devmax; - int npresent, nup; - int ndev; + struct qib_devdata *udd = NULL; + int ret = 0, devmax, npresent, nup, ndev, dusable = 0, i; u32 port = uinfo->spu_port, ctxt; devmax = qib_count_units(&npresent, &nup); + if (!npresent) { + ret = -ENXIO; + goto done; + } + if (nup == 0) { + ret = -ENETDOWN; + goto done; + } - for (ndev = 0; ndev < devmax; ndev++) { - struct qib_devdata *dd = qib_lookup(ndev); - - /* device portion of usable() */ - if (!(dd && (dd->flags & QIB_PRESENT) && dd->kregbase)) - continue; - for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts; ctxt++) { - if (dd->rcd[ctxt]) + if (alg == QIB_PORT_ALG_ACROSS) { + unsigned inuse = ~0U; + /* find device (with ACTIVE ports) with fewest ctxts in use */ + for (ndev = 0; ndev < devmax; ndev++) { + struct qib_devdata *dd = qib_lookup(ndev); + unsigned cused = 0, cfree = 0; + if (!dd) continue; - if (port) { - if (port > dd->num_pports) - continue; - ppd = dd->pport + port - 1; - if (!usable(ppd, 0)) - continue; - } else { - /* - * choose port based on ctxt, if up, else - * first port that's up for multi-port HCA - */ - ppd = dd->pport + (ctxt % dd->num_pports); - if (!usable(ppd, 0)) { - int j; - - ppd = NULL; - for (j = 0; j < dd->num_pports && - !ppd; j++) - if (usable(dd->pport + j, 0)) - ppd = dd->pport + j; - if (!ppd) - continue; /* to next unit */ - } + if (port && port <= dd->num_pports && + usable(dd->pport + port - 1)) + dusable = 1; + else + for (i = 0; i < dd->num_pports; i++) + if (usable(dd->pport + i)) + dusable++; + if (!dusable) + continue; + for (ctxt = dd->first_user_ctxt; ctxt < dd->cfgctxts; + ctxt++) + if (dd->rcd[ctxt]) + cused++; + else + cfree++; + if (cfree && cused < inuse) { + udd = dd; + inuse = cused; } - ret = setup_ctxt(ppd, ctxt, fp, uinfo); + } + if (udd) { + ret = choose_port_ctxt(fp, udd, port, uinfo); goto done; } + } else { + for (ndev = 0; ndev < devmax; ndev++) { + struct qib_devdata *dd = qib_lookup(ndev); + if (dd) { + ret = choose_port_ctxt(fp, dd, port, uinfo); + if (!ret) + goto done; + if (ret == -EBUSY) + dusable++; + } + } } - - if (npresent) { - if (nup == 0) - ret = -ENETDOWN; - else - ret = -EBUSY; - } else - ret = -ENXIO; + ret = dusable ? -EBUSY : -ENETDOWN; done: return ret; @@ -1481,7 +1483,7 @@ static int qib_assign_ctxt(struct file *fp, const struct qib_user_info *uinfo) { int ret; int i_minor; - unsigned swmajor, swminor; + unsigned swmajor, swminor, alg = QIB_PORT_ALG_ACROSS; /* Check to be sure we haven't already initialized this file */ if (ctxt_fp(fp)) { @@ -1498,6 +1500,9 @@ static int qib_assign_ctxt(struct file *fp, const struct qib_user_info *uinfo) swminor = uinfo->spu_userversion & 0xffff; + if (swminor >= 11 && uinfo->spu_port_alg < QIB_PORT_ALG_COUNT) + alg = uinfo->spu_port_alg; + mutex_lock(&qib_mutex); if (qib_compatible_subctxts(swmajor, swminor) && @@ -1514,7 +1519,7 @@ static int qib_assign_ctxt(struct file *fp, const struct qib_user_info *uinfo) if (i_minor) ret = find_free_ctxt(i_minor - 1, fp, uinfo); else - ret = get_a_ctxt(fp, uinfo); + ret = get_a_ctxt(fp, uinfo, alg); done_chk_sdma: if (!ret) { @@ -1862,7 +1867,7 @@ static int disarm_req_delay(struct qib_ctxtdata *rcd) { int ret = 0; - if (!usable(rcd->ppd, 1)) { + if (!usable(rcd->ppd)) { int i; /* * if link is down, or otherwise not usable, delay @@ -1881,7 +1886,7 @@ static int disarm_req_delay(struct qib_ctxtdata *rcd) set_bit(_QIB_EVENT_DISARM_BUFS_BIT, &rcd->user_event_mask[i]); } - for (i = 0; !usable(rcd->ppd, 1) && i < 300; i++) + for (i = 0; !usable(rcd->ppd) && i < 300; i++) msleep(100); ret = -ENETDOWN; } diff --git a/drivers/infiniband/hw/qib/qib_fs.c b/drivers/infiniband/hw/qib/qib_fs.c index 844954bf417b..9f989c0ba9d3 100644 --- a/drivers/infiniband/hw/qib/qib_fs.c +++ b/drivers/infiniband/hw/qib/qib_fs.c @@ -135,8 +135,8 @@ static ssize_t driver_names_read(struct file *file, char __user *buf, } static const struct file_operations driver_ops[] = { - { .read = driver_stats_read, }, - { .read = driver_names_read, }, + { .read = driver_stats_read, .llseek = generic_file_llseek, }, + { .read = driver_names_read, .llseek = generic_file_llseek, }, }; /* read the per-device counters */ @@ -164,8 +164,8 @@ static ssize_t dev_names_read(struct file *file, char __user *buf, } static const struct file_operations cntr_ops[] = { - { .read = dev_counters_read, }, - { .read = dev_names_read, }, + { .read = dev_counters_read, .llseek = generic_file_llseek, }, + { .read = dev_names_read, .llseek = generic_file_llseek, }, }; /* @@ -210,9 +210,9 @@ static ssize_t portcntrs_2_read(struct file *file, char __user *buf, } static const struct file_operations portcntr_ops[] = { - { .read = portnames_read, }, - { .read = portcntrs_1_read, }, - { .read = portcntrs_2_read, }, + { .read = portnames_read, .llseek = generic_file_llseek, }, + { .read = portcntrs_1_read, .llseek = generic_file_llseek, }, + { .read = portcntrs_2_read, .llseek = generic_file_llseek, }, }; /* @@ -261,8 +261,8 @@ static ssize_t qsfp_2_read(struct file *file, char __user *buf, } static const struct file_operations qsfp_ops[] = { - { .read = qsfp_1_read, }, - { .read = qsfp_2_read, }, + { .read = qsfp_1_read, .llseek = generic_file_llseek, }, + { .read = qsfp_2_read, .llseek = generic_file_llseek, }, }; static ssize_t flash_read(struct file *file, char __user *buf, diff --git a/drivers/infiniband/hw/qib/qib_iba6120.c b/drivers/infiniband/hw/qib/qib_iba6120.c index 1eadadc13da8..a5e29dbb9537 100644 --- a/drivers/infiniband/hw/qib/qib_iba6120.c +++ b/drivers/infiniband/hw/qib/qib_iba6120.c @@ -1355,8 +1355,7 @@ static int qib_6120_bringup_serdes(struct qib_pportdata *ppd) hwstat = qib_read_kreg64(dd, kr_hwerrstatus); if (hwstat) { /* should just have PLL, clear all set, in an case */ - if (hwstat & ~QLOGIC_IB_HWE_SERDESPLLFAILED) - qib_write_kreg(dd, kr_hwerrclear, hwstat); + qib_write_kreg(dd, kr_hwerrclear, hwstat); qib_write_kreg(dd, kr_errclear, ERR_MASK(HardwareErr)); } diff --git a/drivers/infiniband/hw/qib/qib_iba7322.c b/drivers/infiniband/hw/qib/qib_iba7322.c index 503992d9c5ce..584d443b5335 100644 --- a/drivers/infiniband/hw/qib/qib_iba7322.c +++ b/drivers/infiniband/hw/qib/qib_iba7322.c @@ -543,7 +543,7 @@ struct vendor_txdds_ent { static void write_tx_serdes_param(struct qib_pportdata *, struct txdds_ent *); #define TXDDS_TABLE_SZ 16 /* number of entries per speed in onchip table */ -#define TXDDS_EXTRA_SZ 11 /* number of extra tx settings entries */ +#define TXDDS_EXTRA_SZ 13 /* number of extra tx settings entries */ #define SERDES_CHANS 4 /* yes, it's obvious, but one less magic number */ #define H1_FORCE_VAL 8 @@ -1100,9 +1100,9 @@ static const struct qib_hwerror_msgs qib_7322_hwerror_msgs[] = { HWE_AUTO_P(SDmaMemReadErr, 1), HWE_AUTO_P(SDmaMemReadErr, 0), HWE_AUTO_P(IBCBusFromSPCParityErr, 1), + HWE_AUTO_P(IBCBusToSPCParityErr, 1), HWE_AUTO_P(IBCBusFromSPCParityErr, 0), - HWE_AUTO_P(statusValidNoEop, 1), - HWE_AUTO_P(statusValidNoEop, 0), + HWE_AUTO(statusValidNoEop), HWE_AUTO(LATriggered), { .mask = 0 } }; @@ -4763,6 +4763,8 @@ static void qib_7322_mini_pcs_reset(struct qib_pportdata *ppd) SYM_MASK(IBPCSConfig_0, tx_rx_reset); val = qib_read_kreg_port(ppd, krp_ib_pcsconfig); + qib_write_kreg(dd, kr_hwerrmask, + dd->cspec->hwerrmask & ~HWE_MASK(statusValidNoEop)); qib_write_kreg_port(ppd, krp_ibcctrl_a, ppd->cpspec->ibcctrl_a & ~SYM_MASK(IBCCtrlA_0, IBLinkEn)); @@ -4772,6 +4774,9 @@ static void qib_7322_mini_pcs_reset(struct qib_pportdata *ppd) qib_write_kreg_port(ppd, krp_ib_pcsconfig, val & ~reset_bits); qib_write_kreg_port(ppd, krp_ibcctrl_a, ppd->cpspec->ibcctrl_a); qib_write_kreg(dd, kr_scratch, 0ULL); + qib_write_kreg(dd, kr_hwerrclear, + SYM_MASK(HwErrClear, statusValidNoEopClear)); + qib_write_kreg(dd, kr_hwerrmask, dd->cspec->hwerrmask); } /* @@ -5624,6 +5629,8 @@ static void set_no_qsfp_atten(struct qib_devdata *dd, int change) if (ppd->port != port || !ppd->link_speed_supported) continue; ppd->cpspec->no_eep = val; + if (seth1) + ppd->cpspec->h1_val = h1; /* now change the IBC and serdes, overriding generic */ init_txdds_table(ppd, 1); any++; @@ -5857,7 +5864,7 @@ static void write_7322_initregs(struct qib_devdata *dd) * Doesn't clear any of the error bits that might be set. */ val = TIDFLOW_ERRBITS; /* these are W1C */ - for (i = 0; i < dd->ctxtcnt; i++) { + for (i = 0; i < dd->cfgctxts; i++) { int flow; for (flow = 0; flow < NUM_TIDFLOWS_CTXT; flow++) qib_write_ureg(dd, ur_rcvflowtable+flow, val, i); @@ -6064,9 +6071,9 @@ static int qib_init_7322_variables(struct qib_devdata *dd) * the "cable info" setup here. Can be overridden * in adapter-specific routines. */ - if (!(ppd->dd->flags & QIB_HAS_QSFP)) { - if (!IS_QMH(ppd->dd) && !IS_QME(ppd->dd)) - qib_devinfo(ppd->dd->pcidev, "IB%u:%u: " + if (!(dd->flags & QIB_HAS_QSFP)) { + if (!IS_QMH(dd) && !IS_QME(dd)) + qib_devinfo(dd->pcidev, "IB%u:%u: " "Unknown mezzanine card type\n", dd->unit, ppd->port); cp->h1_val = IS_QMH(dd) ? H1_FORCE_QMH : H1_FORCE_QME; @@ -6119,9 +6126,25 @@ static int qib_init_7322_variables(struct qib_devdata *dd) qib_set_ctxtcnt(dd); if (qib_wc_pat) { - ret = init_chip_wc_pat(dd, NUM_VL15_BUFS * dd->align4k); + resource_size_t vl15off; + /* + * We do not set WC on the VL15 buffers to avoid + * a rare problem with unaligned writes from + * interrupt-flushed store buffers, so we need + * to map those separately here. We can't solve + * this for the rarely used mtrr case. + */ + ret = init_chip_wc_pat(dd, 0); if (ret) goto bail; + + /* vl15 buffers start just after the 4k buffers */ + vl15off = dd->physaddr + (dd->piobufbase >> 32) + + dd->piobcnt4k * dd->align4k; + dd->piovl15base = ioremap_nocache(vl15off, + NUM_VL15_BUFS * dd->align4k); + if (!dd->piovl15base) + goto bail; } qib_7322_set_baseaddrs(dd); /* set chip access pointers now */ @@ -6932,6 +6955,8 @@ static const struct txdds_ent txdds_extra_sdr[TXDDS_EXTRA_SZ] = { { 0, 0, 0, 11 }, /* QME7342 backplane settings */ { 0, 0, 0, 11 }, /* QME7342 backplane settings */ { 0, 0, 0, 11 }, /* QME7342 backplane settings */ + { 0, 0, 0, 3 }, /* QMH7342 backplane settings */ + { 0, 0, 0, 4 }, /* QMH7342 backplane settings */ }; static const struct txdds_ent txdds_extra_ddr[TXDDS_EXTRA_SZ] = { @@ -6947,6 +6972,8 @@ static const struct txdds_ent txdds_extra_ddr[TXDDS_EXTRA_SZ] = { { 0, 0, 0, 13 }, /* QME7342 backplane settings */ { 0, 0, 0, 13 }, /* QME7342 backplane settings */ { 0, 0, 0, 13 }, /* QME7342 backplane settings */ + { 0, 0, 0, 9 }, /* QMH7342 backplane settings */ + { 0, 0, 0, 10 }, /* QMH7342 backplane settings */ }; static const struct txdds_ent txdds_extra_qdr[TXDDS_EXTRA_SZ] = { @@ -6962,6 +6989,8 @@ static const struct txdds_ent txdds_extra_qdr[TXDDS_EXTRA_SZ] = { { 0, 1, 12, 6 }, /* QME7342 backplane setting */ { 0, 1, 12, 7 }, /* QME7342 backplane setting */ { 0, 1, 12, 8 }, /* QME7342 backplane setting */ + { 0, 1, 0, 10 }, /* QMH7342 backplane settings */ + { 0, 1, 0, 12 }, /* QMH7342 backplane settings */ }; static const struct txdds_ent *get_atten_table(const struct txdds_ent *txdds, @@ -7242,6 +7271,8 @@ static int serdes_7322_init(struct qib_pportdata *ppd) ibsd_wr_allchans(ppd, 20, (4 << 13), BMASK(15, 13)); /* SDR */ data = qib_read_kreg_port(ppd, krp_serdesctrl); + /* Turn off IB latency mode */ + data &= ~SYM_MASK(IBSerdesCtrl_0, IB_LAT_MODE); qib_write_kreg_port(ppd, krp_serdesctrl, data | SYM_MASK(IBSerdesCtrl_0, RXLOSEN)); diff --git a/drivers/infiniband/hw/qib/qib_init.c b/drivers/infiniband/hw/qib/qib_init.c index 9b40f345ac3f..f1d16d3a01f6 100644 --- a/drivers/infiniband/hw/qib/qib_init.c +++ b/drivers/infiniband/hw/qib/qib_init.c @@ -93,7 +93,7 @@ unsigned long *qib_cpulist; void qib_set_ctxtcnt(struct qib_devdata *dd) { if (!qib_cfgctxts) - dd->cfgctxts = dd->ctxtcnt; + dd->cfgctxts = dd->first_user_ctxt + num_online_cpus(); else if (qib_cfgctxts < dd->num_pports) dd->cfgctxts = dd->ctxtcnt; else if (qib_cfgctxts <= dd->ctxtcnt) @@ -1059,7 +1059,7 @@ static int __init qlogic_ib_init(void) goto bail_dev; } - qib_cq_wq = create_workqueue("qib_cq"); + qib_cq_wq = create_singlethread_workqueue("qib_cq"); if (!qib_cq_wq) { ret = -ENOMEM; goto bail_wq; @@ -1289,8 +1289,18 @@ static int __devinit qib_init_one(struct pci_dev *pdev, if (qib_mini_init || initfail || ret) { qib_stop_timers(dd); + flush_scheduled_work(); for (pidx = 0; pidx < dd->num_pports; ++pidx) dd->f_quiet_serdes(dd->pport + pidx); + if (qib_mini_init) + goto bail; + if (!j) { + (void) qibfs_remove(dd); + qib_device_remove(dd); + } + if (!ret) + qib_unregister_ib_device(dd); + qib_postinit_cleanup(dd); if (initfail) ret = initfail; goto bail; @@ -1472,6 +1482,9 @@ int qib_setup_eagerbufs(struct qib_ctxtdata *rcd) dma_addr_t pa = rcd->rcvegrbuf_phys[chunk]; unsigned i; + /* clear for security and sanity on each use */ + memset(rcd->rcvegrbuf[chunk], 0, size); + for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) { dd->f_put_tid(dd, e + egroff + (u64 __iomem *) @@ -1499,6 +1512,12 @@ bail: return -ENOMEM; } +/* + * Note: Changes to this routine should be mirrored + * for the diagnostics routine qib_remap_ioaddr32(). + * There is also related code for VL15 buffers in qib_init_7322_variables(). + * The teardown code that unmaps is in qib_pcie_ddcleanup() + */ int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen) { u64 __iomem *qib_kregbase = NULL; diff --git a/drivers/infiniband/hw/qib/qib_pcie.c b/drivers/infiniband/hw/qib/qib_pcie.c index c926bf4541df..7fa6e5592630 100644 --- a/drivers/infiniband/hw/qib/qib_pcie.c +++ b/drivers/infiniband/hw/qib/qib_pcie.c @@ -179,6 +179,8 @@ void qib_pcie_ddcleanup(struct qib_devdata *dd) iounmap(dd->piobase); if (dd->userbase) iounmap(dd->userbase); + if (dd->piovl15base) + iounmap(dd->piovl15base); pci_disable_device(dd->pcidev); pci_release_regions(dd->pcidev); diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c index e0f65e39076b..6c39851d2ded 100644 --- a/drivers/infiniband/hw/qib/qib_qp.c +++ b/drivers/infiniband/hw/qib/qib_qp.c @@ -450,7 +450,7 @@ static void clear_mr_refs(struct qib_qp *qp, int clr_sends) * * Flushes both send and receive work queues. * Returns true if last WQE event should be generated. - * The QP s_lock should be held and interrupts disabled. + * The QP r_lock and s_lock should be held and interrupts disabled. * If we are already in error state, just return. */ int qib_error_qp(struct qib_qp *qp, enum ib_wc_status err) diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c index 40c0a373719c..a0931119bd78 100644 --- a/drivers/infiniband/hw/qib/qib_rc.c +++ b/drivers/infiniband/hw/qib/qib_rc.c @@ -868,7 +868,7 @@ done: /* * Back up requester to resend the last un-ACKed request. - * The QP s_lock should be held and interrupts disabled. + * The QP r_lock and s_lock should be held and interrupts disabled. */ static void qib_restart_rc(struct qib_qp *qp, u32 psn, int wait) { @@ -911,7 +911,8 @@ static void rc_timeout(unsigned long arg) struct qib_ibport *ibp; unsigned long flags; - spin_lock_irqsave(&qp->s_lock, flags); + spin_lock_irqsave(&qp->r_lock, flags); + spin_lock(&qp->s_lock); if (qp->s_flags & QIB_S_TIMER) { ibp = to_iport(qp->ibqp.device, qp->port_num); ibp->n_rc_timeouts++; @@ -920,7 +921,8 @@ static void rc_timeout(unsigned long arg) qib_restart_rc(qp, qp->s_last_psn + 1, 1); qib_schedule_send(qp); } - spin_unlock_irqrestore(&qp->s_lock, flags); + spin_unlock(&qp->s_lock); + spin_unlock_irqrestore(&qp->r_lock, flags); } /* @@ -1414,10 +1416,6 @@ static void qib_rc_rcv_resp(struct qib_ibport *ibp, spin_lock_irqsave(&qp->s_lock, flags); - /* Double check we can process this now that we hold the s_lock. */ - if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) - goto ack_done; - /* Ignore invalid responses. */ if (qib_cmp24(psn, qp->s_next_psn) >= 0) goto ack_done; @@ -1661,9 +1659,6 @@ static int qib_rc_rcv_error(struct qib_other_headers *ohdr, ibp->n_rc_dupreq++; spin_lock_irqsave(&qp->s_lock, flags); - /* Double check we can process this now that we hold the s_lock. */ - if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) - goto unlock_done; for (i = qp->r_head_ack_queue; ; i = prev) { if (i == qp->s_tail_ack_queue) @@ -1878,9 +1873,6 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, psn = be32_to_cpu(ohdr->bth[2]); opcode >>= 24; - /* Prevent simultaneous processing after APM on different CPUs */ - spin_lock(&qp->r_lock); - /* * Process responses (ACKs) before anything else. Note that the * packet sequence number will be for something in the send work @@ -1891,14 +1883,14 @@ void qib_rc_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, opcode <= OP(ATOMIC_ACKNOWLEDGE)) { qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn, hdrsize, pmtu, rcd); - goto runlock; + return; } /* Compute 24 bits worth of difference. */ diff = qib_cmp24(psn, qp->r_psn); if (unlikely(diff)) { if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd)) - goto runlock; + return; goto send_ack; } @@ -2090,9 +2082,6 @@ send_last: if (next > QIB_MAX_RDMA_ATOMIC) next = 0; spin_lock_irqsave(&qp->s_lock, flags); - /* Double check we can process this while holding the s_lock. */ - if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) - goto srunlock; if (unlikely(next == qp->s_tail_ack_queue)) { if (!qp->s_ack_queue[next].sent) goto nack_inv_unlck; @@ -2146,7 +2135,7 @@ send_last: qp->s_flags |= QIB_S_RESP_PENDING; qib_schedule_send(qp); - goto srunlock; + goto sunlock; } case OP(COMPARE_SWAP): @@ -2165,9 +2154,6 @@ send_last: if (next > QIB_MAX_RDMA_ATOMIC) next = 0; spin_lock_irqsave(&qp->s_lock, flags); - /* Double check we can process this while holding the s_lock. */ - if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) - goto srunlock; if (unlikely(next == qp->s_tail_ack_queue)) { if (!qp->s_ack_queue[next].sent) goto nack_inv_unlck; @@ -2213,7 +2199,7 @@ send_last: qp->s_flags |= QIB_S_RESP_PENDING; qib_schedule_send(qp); - goto srunlock; + goto sunlock; } default: @@ -2227,7 +2213,7 @@ send_last: /* Send an ACK if requested or required. */ if (psn & (1 << 31)) goto send_ack; - goto runlock; + return; rnr_nak: qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; @@ -2238,7 +2224,7 @@ rnr_nak: atomic_inc(&qp->refcount); list_add_tail(&qp->rspwait, &rcd->qp_wait_list); } - goto runlock; + return; nack_op_err: qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); @@ -2250,7 +2236,7 @@ nack_op_err: atomic_inc(&qp->refcount); list_add_tail(&qp->rspwait, &rcd->qp_wait_list); } - goto runlock; + return; nack_inv_unlck: spin_unlock_irqrestore(&qp->s_lock, flags); @@ -2264,7 +2250,7 @@ nack_inv: atomic_inc(&qp->refcount); list_add_tail(&qp->rspwait, &rcd->qp_wait_list); } - goto runlock; + return; nack_acc_unlck: spin_unlock_irqrestore(&qp->s_lock, flags); @@ -2274,13 +2260,6 @@ nack_acc: qp->r_ack_psn = qp->r_psn; send_ack: qib_send_rc_ack(qp); -runlock: - spin_unlock(&qp->r_lock); - return; - -srunlock: - spin_unlock_irqrestore(&qp->s_lock, flags); - spin_unlock(&qp->r_lock); return; sunlock: diff --git a/drivers/infiniband/hw/qib/qib_sd7220.c b/drivers/infiniband/hw/qib/qib_sd7220.c index 0aeed0e74cb6..e9f9f8bc3204 100644 --- a/drivers/infiniband/hw/qib/qib_sd7220.c +++ b/drivers/infiniband/hw/qib/qib_sd7220.c @@ -1,5 +1,6 @@ /* - * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. + * Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation. + * All rights reserved. * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved. * * This software is available to you under a choice of one of two @@ -37,10 +38,14 @@ #include <linux/pci.h> #include <linux/delay.h> +#include <linux/firmware.h> #include "qib.h" #include "qib_7220.h" +#define SD7220_FW_NAME "qlogic/sd7220.fw" +MODULE_FIRMWARE(SD7220_FW_NAME); + /* * Same as in qib_iba7220.c, but just the registers needed here. * Could move whole set to qib_7220.h, but decided better to keep @@ -102,6 +107,10 @@ static int qib_internal_presets(struct qib_devdata *dd); /* Tweak the register (CMUCTRL5) that contains the TRIMSELF controls */ static int qib_sd_trimself(struct qib_devdata *dd, int val); static int epb_access(struct qib_devdata *dd, int sdnum, int claim); +static int qib_sd7220_ib_load(struct qib_devdata *dd, + const struct firmware *fw); +static int qib_sd7220_ib_vfy(struct qib_devdata *dd, + const struct firmware *fw); /* * Below keeps track of whether the "once per power-on" initialization has @@ -110,10 +119,13 @@ static int epb_access(struct qib_devdata *dd, int sdnum, int claim); * state of the reset "pin", is no longer valid. Instead, we check for the * actual uC code having been loaded. */ -static int qib_ibsd_ucode_loaded(struct qib_pportdata *ppd) +static int qib_ibsd_ucode_loaded(struct qib_pportdata *ppd, + const struct firmware *fw) { struct qib_devdata *dd = ppd->dd; - if (!dd->cspec->serdes_first_init_done && (qib_sd7220_ib_vfy(dd) > 0)) + + if (!dd->cspec->serdes_first_init_done && + qib_sd7220_ib_vfy(dd, fw) > 0) dd->cspec->serdes_first_init_done = 1; return dd->cspec->serdes_first_init_done; } @@ -377,6 +389,7 @@ static void qib_sd_trimdone_monitor(struct qib_devdata *dd, */ int qib_sd7220_init(struct qib_devdata *dd) { + const struct firmware *fw; int ret = 1; /* default to failure */ int first_reset, was_reset; @@ -387,8 +400,15 @@ int qib_sd7220_init(struct qib_devdata *dd) qib_ibsd_reset(dd, 1); qib_sd_trimdone_monitor(dd, "Driver-reload"); } + + ret = request_firmware(&fw, SD7220_FW_NAME, &dd->pcidev->dev); + if (ret) { + qib_dev_err(dd, "Failed to load IB SERDES image\n"); + goto done; + } + /* Substitute our deduced value for was_reset */ - ret = qib_ibsd_ucode_loaded(dd->pport); + ret = qib_ibsd_ucode_loaded(dd->pport, fw); if (ret < 0) goto bail; @@ -437,13 +457,13 @@ int qib_sd7220_init(struct qib_devdata *dd) int vfy; int trim_done; - ret = qib_sd7220_ib_load(dd); + ret = qib_sd7220_ib_load(dd, fw); if (ret < 0) { qib_dev_err(dd, "Failed to load IB SERDES image\n"); goto bail; } else { /* Loaded image, try to verify */ - vfy = qib_sd7220_ib_vfy(dd); + vfy = qib_sd7220_ib_vfy(dd, fw); if (vfy != ret) { qib_dev_err(dd, "SERDES PRAM VFY failed\n"); goto bail; @@ -506,6 +526,8 @@ bail: done: /* start relock timer regardless, but start at 1 second */ set_7220_relock_poll(dd, -1); + + release_firmware(fw); return ret; } @@ -829,8 +851,8 @@ static int qib_sd7220_ram_xfer(struct qib_devdata *dd, int sdnum, u32 loc, #define PROG_CHUNK 64 -int qib_sd7220_prog_ld(struct qib_devdata *dd, int sdnum, - u8 *img, int len, int offset) +static int qib_sd7220_prog_ld(struct qib_devdata *dd, int sdnum, + const u8 *img, int len, int offset) { int cnt, sofar, req; @@ -840,7 +862,7 @@ int qib_sd7220_prog_ld(struct qib_devdata *dd, int sdnum, if (req > PROG_CHUNK) req = PROG_CHUNK; cnt = qib_sd7220_ram_xfer(dd, sdnum, offset + sofar, - img + sofar, req, 0); + (u8 *)img + sofar, req, 0); if (cnt < req) { sofar = -1; break; @@ -853,8 +875,8 @@ int qib_sd7220_prog_ld(struct qib_devdata *dd, int sdnum, #define VFY_CHUNK 64 #define SD_PRAM_ERROR_LIMIT 42 -int qib_sd7220_prog_vfy(struct qib_devdata *dd, int sdnum, - const u8 *img, int len, int offset) +static int qib_sd7220_prog_vfy(struct qib_devdata *dd, int sdnum, + const u8 *img, int len, int offset) { int cnt, sofar, req, idx, errors; unsigned char readback[VFY_CHUNK]; @@ -881,6 +903,18 @@ int qib_sd7220_prog_vfy(struct qib_devdata *dd, int sdnum, return errors ? -errors : sofar; } +static int +qib_sd7220_ib_load(struct qib_devdata *dd, const struct firmware *fw) +{ + return qib_sd7220_prog_ld(dd, IB_7220_SERDES, fw->data, fw->size, 0); +} + +static int +qib_sd7220_ib_vfy(struct qib_devdata *dd, const struct firmware *fw) +{ + return qib_sd7220_prog_vfy(dd, IB_7220_SERDES, fw->data, fw->size, 0); +} + /* * IRQ not set up at this point in init, so we poll. */ diff --git a/drivers/infiniband/hw/qib/qib_sd7220_img.c b/drivers/infiniband/hw/qib/qib_sd7220_img.c deleted file mode 100644 index a1118fbd2370..000000000000 --- a/drivers/infiniband/hw/qib/qib_sd7220_img.c +++ /dev/null @@ -1,1081 +0,0 @@ -/* - * Copyright (c) 2007, 2008 QLogic Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -/* - * This file contains the memory image from the vendor, to be copied into - * the IB SERDES of the IBA7220 during initialization. - * The file also includes the two functions which use this image. - */ -#include <linux/pci.h> -#include <linux/delay.h> - -#include "qib.h" -#include "qib_7220.h" - -static unsigned char qib_sd7220_ib_img[] = { -/*0000*/0x02, 0x0A, 0x29, 0x02, 0x0A, 0x87, 0xE5, 0xE6, - 0x30, 0xE6, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, -/*0010*/0x00, 0xE5, 0xE2, 0x30, 0xE4, 0x04, 0x7E, 0x01, - 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x5F, 0x60, 0x08, -/*0020*/0x53, 0xF9, 0xF7, 0xE4, 0xF5, 0xFE, 0x80, 0x08, - 0x7F, 0x0A, 0x12, 0x17, 0x31, 0x12, 0x0E, 0xA2, -/*0030*/0x75, 0xFC, 0x08, 0xE4, 0xF5, 0xFD, 0xE5, 0xE7, - 0x20, 0xE7, 0x03, 0x43, 0xF9, 0x08, 0x22, 0x00, -/*0040*/0x01, 0x20, 0x11, 0x00, 0x04, 0x20, 0x00, 0x75, - 0x51, 0x01, 0xE4, 0xF5, 0x52, 0xF5, 0x53, 0xF5, -/*0050*/0x52, 0xF5, 0x7E, 0x7F, 0x04, 0x02, 0x04, 0x38, - 0xC2, 0x36, 0x05, 0x52, 0xE5, 0x52, 0xD3, 0x94, -/*0060*/0x0C, 0x40, 0x05, 0x75, 0x52, 0x01, 0xD2, 0x36, - 0x90, 0x07, 0x0C, 0x74, 0x07, 0xF0, 0xA3, 0x74, -/*0070*/0xFF, 0xF0, 0xE4, 0xF5, 0x0C, 0xA3, 0xF0, 0x90, - 0x07, 0x14, 0xF0, 0xA3, 0xF0, 0x75, 0x0B, 0x20, -/*0080*/0xF5, 0x09, 0xE4, 0xF5, 0x08, 0xE5, 0x08, 0xD3, - 0x94, 0x30, 0x40, 0x03, 0x02, 0x04, 0x04, 0x12, -/*0090*/0x00, 0x06, 0x15, 0x0B, 0xE5, 0x08, 0x70, 0x04, - 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x09, -/*00A0*/0x70, 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, - 0xEE, 0x5F, 0x60, 0x05, 0x12, 0x18, 0x71, 0xD2, -/*00B0*/0x35, 0x53, 0xE1, 0xF7, 0xE5, 0x08, 0x45, 0x09, - 0xFF, 0xE5, 0x0B, 0x25, 0xE0, 0x25, 0xE0, 0x24, -/*00C0*/0x83, 0xF5, 0x82, 0xE4, 0x34, 0x07, 0xF5, 0x83, - 0xEF, 0xF0, 0x85, 0xE2, 0x20, 0xE5, 0x52, 0xD3, -/*00D0*/0x94, 0x01, 0x40, 0x0D, 0x12, 0x19, 0xF3, 0xE0, - 0x54, 0xA0, 0x64, 0x40, 0x70, 0x03, 0x02, 0x03, -/*00E0*/0xFB, 0x53, 0xF9, 0xF8, 0x90, 0x94, 0x70, 0xE4, - 0xF0, 0xE0, 0xF5, 0x10, 0xAF, 0x09, 0x12, 0x1E, -/*00F0*/0xB3, 0xAF, 0x08, 0xEF, 0x44, 0x08, 0xF5, 0x82, - 0x75, 0x83, 0x80, 0xE0, 0xF5, 0x29, 0xEF, 0x44, -/*0100*/0x07, 0x12, 0x1A, 0x3C, 0xF5, 0x22, 0x54, 0x40, - 0xD3, 0x94, 0x00, 0x40, 0x1E, 0xE5, 0x29, 0x54, -/*0110*/0xF0, 0x70, 0x21, 0x12, 0x19, 0xF3, 0xE0, 0x44, - 0x80, 0xF0, 0xE5, 0x22, 0x54, 0x30, 0x65, 0x08, -/*0120*/0x70, 0x09, 0x12, 0x19, 0xF3, 0xE0, 0x54, 0xBF, - 0xF0, 0x80, 0x09, 0x12, 0x19, 0xF3, 0x74, 0x40, -/*0130*/0xF0, 0x02, 0x03, 0xFB, 0x12, 0x1A, 0x12, 0x75, - 0x83, 0xAE, 0x74, 0xFF, 0xF0, 0xAF, 0x08, 0x7E, -/*0140*/0x00, 0xEF, 0x44, 0x07, 0xF5, 0x82, 0xE0, 0xFD, - 0xE5, 0x0B, 0x25, 0xE0, 0x25, 0xE0, 0x24, 0x81, -/*0150*/0xF5, 0x82, 0xE4, 0x34, 0x07, 0xF5, 0x83, 0xED, - 0xF0, 0x90, 0x07, 0x0E, 0xE0, 0x04, 0xF0, 0xEF, -/*0160*/0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0x98, 0xE0, - 0xF5, 0x28, 0x12, 0x1A, 0x23, 0x40, 0x0C, 0x12, -/*0170*/0x19, 0xF3, 0xE0, 0x44, 0x01, 0x12, 0x1A, 0x32, - 0x02, 0x03, 0xF6, 0xAF, 0x08, 0x7E, 0x00, 0x74, -/*0180*/0x80, 0xCD, 0xEF, 0xCD, 0x8D, 0x82, 0xF5, 0x83, - 0xE0, 0x30, 0xE0, 0x0A, 0x12, 0x19, 0xF3, 0xE0, -/*0190*/0x44, 0x20, 0xF0, 0x02, 0x03, 0xFB, 0x12, 0x19, - 0xF3, 0xE0, 0x54, 0xDF, 0xF0, 0xEE, 0x44, 0xAE, -/*01A0*/0x12, 0x1A, 0x43, 0x30, 0xE4, 0x03, 0x02, 0x03, - 0xFB, 0x74, 0x9E, 0x12, 0x1A, 0x05, 0x20, 0xE0, -/*01B0*/0x03, 0x02, 0x03, 0xFB, 0x8F, 0x82, 0x8E, 0x83, - 0xE0, 0x20, 0xE0, 0x03, 0x02, 0x03, 0xFB, 0x12, -/*01C0*/0x19, 0xF3, 0xE0, 0x44, 0x10, 0xF0, 0xE5, 0xE3, - 0x20, 0xE7, 0x08, 0xE5, 0x08, 0x12, 0x1A, 0x3A, -/*01D0*/0x44, 0x04, 0xF0, 0xAF, 0x08, 0x7E, 0x00, 0xEF, - 0x12, 0x1A, 0x3A, 0x20, 0xE2, 0x34, 0x12, 0x19, -/*01E0*/0xF3, 0xE0, 0x44, 0x08, 0xF0, 0xE5, 0xE4, 0x30, - 0xE6, 0x04, 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00, -/*01F0*/0xE5, 0x7E, 0xC3, 0x94, 0x04, 0x50, 0x04, 0x7C, - 0x01, 0x80, 0x02, 0x7C, 0x00, 0xEC, 0x4D, 0x60, -/*0200*/0x05, 0xC2, 0x35, 0x02, 0x03, 0xFB, 0xEE, 0x44, - 0xD2, 0x12, 0x1A, 0x43, 0x44, 0x40, 0xF0, 0x02, -/*0210*/0x03, 0xFB, 0x12, 0x19, 0xF3, 0xE0, 0x54, 0xF7, - 0xF0, 0x12, 0x1A, 0x12, 0x75, 0x83, 0xD2, 0xE0, -/*0220*/0x54, 0xBF, 0xF0, 0x90, 0x07, 0x14, 0xE0, 0x04, - 0xF0, 0xE5, 0x7E, 0x70, 0x03, 0x75, 0x7E, 0x01, -/*0230*/0xAF, 0x08, 0x7E, 0x00, 0x12, 0x1A, 0x23, 0x40, - 0x12, 0x12, 0x19, 0xF3, 0xE0, 0x44, 0x01, 0x12, -/*0240*/0x19, 0xF2, 0xE0, 0x54, 0x02, 0x12, 0x1A, 0x32, - 0x02, 0x03, 0xFB, 0x12, 0x19, 0xF3, 0xE0, 0x44, -/*0250*/0x02, 0x12, 0x19, 0xF2, 0xE0, 0x54, 0xFE, 0xF0, - 0xC2, 0x35, 0xEE, 0x44, 0x8A, 0x8F, 0x82, 0xF5, -/*0260*/0x83, 0xE0, 0xF5, 0x17, 0x54, 0x8F, 0x44, 0x40, - 0xF0, 0x74, 0x90, 0xFC, 0xE5, 0x08, 0x44, 0x07, -/*0270*/0xFD, 0xF5, 0x82, 0x8C, 0x83, 0xE0, 0x54, 0x3F, - 0x90, 0x07, 0x02, 0xF0, 0xE0, 0x54, 0xC0, 0x8D, -/*0280*/0x82, 0x8C, 0x83, 0xF0, 0x74, 0x92, 0x12, 0x1A, - 0x05, 0x90, 0x07, 0x03, 0x12, 0x1A, 0x19, 0x74, -/*0290*/0x82, 0x12, 0x1A, 0x05, 0x90, 0x07, 0x04, 0x12, - 0x1A, 0x19, 0x74, 0xB4, 0x12, 0x1A, 0x05, 0x90, -/*02A0*/0x07, 0x05, 0x12, 0x1A, 0x19, 0x74, 0x94, 0xFE, - 0xE5, 0x08, 0x44, 0x06, 0x12, 0x1A, 0x0A, 0xF5, -/*02B0*/0x10, 0x30, 0xE0, 0x04, 0xD2, 0x37, 0x80, 0x02, - 0xC2, 0x37, 0xE5, 0x10, 0x54, 0x7F, 0x8F, 0x82, -/*02C0*/0x8E, 0x83, 0xF0, 0x30, 0x44, 0x30, 0x12, 0x1A, - 0x03, 0x54, 0x80, 0xD3, 0x94, 0x00, 0x40, 0x04, -/*02D0*/0xD2, 0x39, 0x80, 0x02, 0xC2, 0x39, 0x8F, 0x82, - 0x8E, 0x83, 0xE0, 0x44, 0x80, 0xF0, 0x12, 0x1A, -/*02E0*/0x03, 0x54, 0x40, 0xD3, 0x94, 0x00, 0x40, 0x04, - 0xD2, 0x3A, 0x80, 0x02, 0xC2, 0x3A, 0x8F, 0x82, -/*02F0*/0x8E, 0x83, 0xE0, 0x44, 0x40, 0xF0, 0x74, 0x92, - 0xFE, 0xE5, 0x08, 0x44, 0x06, 0x12, 0x1A, 0x0A, -/*0300*/0x30, 0xE7, 0x04, 0xD2, 0x38, 0x80, 0x02, 0xC2, - 0x38, 0x8F, 0x82, 0x8E, 0x83, 0xE0, 0x54, 0x7F, -/*0310*/0xF0, 0x12, 0x1E, 0x46, 0xE4, 0xF5, 0x0A, 0x20, - 0x03, 0x02, 0x80, 0x03, 0x30, 0x43, 0x03, 0x12, -/*0320*/0x19, 0x95, 0x20, 0x02, 0x02, 0x80, 0x03, 0x30, - 0x42, 0x03, 0x12, 0x0C, 0x8F, 0x30, 0x30, 0x06, -/*0330*/0x12, 0x19, 0x95, 0x12, 0x0C, 0x8F, 0x12, 0x0D, - 0x47, 0x12, 0x19, 0xF3, 0xE0, 0x54, 0xFB, 0xF0, -/*0340*/0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x40, 0x46, 0x43, - 0xE1, 0x08, 0x12, 0x19, 0xF3, 0xE0, 0x44, 0x04, -/*0350*/0xF0, 0xE5, 0xE4, 0x20, 0xE7, 0x2A, 0x12, 0x1A, - 0x12, 0x75, 0x83, 0xD2, 0xE0, 0x54, 0x08, 0xD3, -/*0360*/0x94, 0x00, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02, - 0x7F, 0x00, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x40, -/*0370*/0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEF, - 0x5E, 0x60, 0x05, 0x12, 0x1D, 0xD7, 0x80, 0x17, -/*0380*/0x12, 0x1A, 0x12, 0x75, 0x83, 0xD2, 0xE0, 0x44, - 0x08, 0xF0, 0x02, 0x03, 0xFB, 0x12, 0x1A, 0x12, -/*0390*/0x75, 0x83, 0xD2, 0xE0, 0x54, 0xF7, 0xF0, 0x12, - 0x1E, 0x46, 0x7F, 0x08, 0x12, 0x17, 0x31, 0x74, -/*03A0*/0x8E, 0xFE, 0x12, 0x1A, 0x12, 0x8E, 0x83, 0xE0, - 0xF5, 0x10, 0x54, 0xFE, 0xF0, 0xE5, 0x10, 0x44, -/*03B0*/0x01, 0xFF, 0xE5, 0x08, 0xFD, 0xED, 0x44, 0x07, - 0xF5, 0x82, 0xEF, 0xF0, 0xE5, 0x10, 0x54, 0xFE, -/*03C0*/0xFF, 0xED, 0x44, 0x07, 0xF5, 0x82, 0xEF, 0x12, - 0x1A, 0x11, 0x75, 0x83, 0x86, 0xE0, 0x44, 0x10, -/*03D0*/0x12, 0x1A, 0x11, 0xE0, 0x44, 0x10, 0xF0, 0x12, - 0x19, 0xF3, 0xE0, 0x54, 0xFD, 0x44, 0x01, 0xFF, -/*03E0*/0x12, 0x19, 0xF3, 0xEF, 0x12, 0x1A, 0x32, 0x30, - 0x32, 0x0C, 0xE5, 0x08, 0x44, 0x08, 0xF5, 0x82, -/*03F0*/0x75, 0x83, 0x82, 0x74, 0x05, 0xF0, 0xAF, 0x0B, - 0x12, 0x18, 0xD7, 0x74, 0x10, 0x25, 0x08, 0xF5, -/*0400*/0x08, 0x02, 0x00, 0x85, 0x05, 0x09, 0xE5, 0x09, - 0xD3, 0x94, 0x07, 0x50, 0x03, 0x02, 0x00, 0x82, -/*0410*/0xE5, 0x7E, 0xD3, 0x94, 0x00, 0x40, 0x04, 0x7F, - 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x7E, 0xC3, -/*0420*/0x94, 0xFA, 0x50, 0x04, 0x7E, 0x01, 0x80, 0x02, - 0x7E, 0x00, 0xEE, 0x5F, 0x60, 0x02, 0x05, 0x7E, -/*0430*/0x30, 0x35, 0x0B, 0x43, 0xE1, 0x01, 0x7F, 0x09, - 0x12, 0x17, 0x31, 0x02, 0x00, 0x58, 0x53, 0xE1, -/*0440*/0xFE, 0x02, 0x00, 0x58, 0x8E, 0x6A, 0x8F, 0x6B, - 0x8C, 0x6C, 0x8D, 0x6D, 0x75, 0x6E, 0x01, 0x75, -/*0450*/0x6F, 0x01, 0x75, 0x70, 0x01, 0xE4, 0xF5, 0x73, - 0xF5, 0x74, 0xF5, 0x75, 0x90, 0x07, 0x2F, 0xF0, -/*0460*/0xF5, 0x3C, 0xF5, 0x3E, 0xF5, 0x46, 0xF5, 0x47, - 0xF5, 0x3D, 0xF5, 0x3F, 0xF5, 0x6F, 0xE5, 0x6F, -/*0470*/0x70, 0x0F, 0xE5, 0x6B, 0x45, 0x6A, 0x12, 0x07, - 0x2A, 0x75, 0x83, 0x80, 0x74, 0x3A, 0xF0, 0x80, -/*0480*/0x09, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x80, 0x74, - 0x1A, 0xF0, 0xE4, 0xF5, 0x6E, 0xC3, 0x74, 0x3F, -/*0490*/0x95, 0x6E, 0xFF, 0x12, 0x08, 0x65, 0x75, 0x83, - 0x82, 0xEF, 0xF0, 0x12, 0x1A, 0x4D, 0x12, 0x08, -/*04A0*/0xC6, 0xE5, 0x33, 0xF0, 0x12, 0x08, 0xFA, 0x12, - 0x08, 0xB1, 0x40, 0xE1, 0xE5, 0x6F, 0x70, 0x0B, -/*04B0*/0x12, 0x07, 0x2A, 0x75, 0x83, 0x80, 0x74, 0x36, - 0xF0, 0x80, 0x09, 0x12, 0x07, 0x2A, 0x75, 0x83, -/*04C0*/0x80, 0x74, 0x16, 0xF0, 0x75, 0x6E, 0x01, 0x12, - 0x07, 0x2A, 0x75, 0x83, 0xB4, 0xE5, 0x6E, 0xF0, -/*04D0*/0x12, 0x1A, 0x4D, 0x74, 0x3F, 0x25, 0x6E, 0xF5, - 0x82, 0xE4, 0x34, 0x00, 0xF5, 0x83, 0xE5, 0x33, -/*04E0*/0xF0, 0x74, 0xBF, 0x25, 0x6E, 0xF5, 0x82, 0xE4, - 0x34, 0x00, 0x12, 0x08, 0xB1, 0x40, 0xD8, 0xE4, -/*04F0*/0xF5, 0x70, 0xF5, 0x46, 0xF5, 0x47, 0xF5, 0x6E, - 0x12, 0x08, 0xFA, 0xF5, 0x83, 0xE0, 0xFE, 0x12, -/*0500*/0x08, 0xC6, 0xE0, 0x7C, 0x00, 0x24, 0x00, 0xFF, - 0xEC, 0x3E, 0xFE, 0xAD, 0x3B, 0xD3, 0xEF, 0x9D, -/*0510*/0xEE, 0x9C, 0x50, 0x04, 0x7B, 0x01, 0x80, 0x02, - 0x7B, 0x00, 0xE5, 0x70, 0x70, 0x04, 0x7A, 0x01, -/*0520*/0x80, 0x02, 0x7A, 0x00, 0xEB, 0x5A, 0x60, 0x06, - 0x85, 0x6E, 0x46, 0x75, 0x70, 0x01, 0xD3, 0xEF, -/*0530*/0x9D, 0xEE, 0x9C, 0x50, 0x04, 0x7F, 0x01, 0x80, - 0x02, 0x7F, 0x00, 0xE5, 0x70, 0xB4, 0x01, 0x04, -/*0540*/0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEF, 0x5E, - 0x60, 0x03, 0x85, 0x6E, 0x47, 0x05, 0x6E, 0xE5, -/*0550*/0x6E, 0x64, 0x7F, 0x70, 0xA3, 0xE5, 0x46, 0x60, - 0x05, 0xE5, 0x47, 0xB4, 0x7E, 0x03, 0x85, 0x46, -/*0560*/0x47, 0xE5, 0x6F, 0x70, 0x08, 0x85, 0x46, 0x76, - 0x85, 0x47, 0x77, 0x80, 0x0E, 0xC3, 0x74, 0x7F, -/*0570*/0x95, 0x46, 0xF5, 0x78, 0xC3, 0x74, 0x7F, 0x95, - 0x47, 0xF5, 0x79, 0xE5, 0x6F, 0x70, 0x37, 0xE5, -/*0580*/0x46, 0x65, 0x47, 0x70, 0x0C, 0x75, 0x73, 0x01, - 0x75, 0x74, 0x01, 0xF5, 0x3C, 0xF5, 0x3D, 0x80, -/*0590*/0x35, 0xE4, 0xF5, 0x4E, 0xC3, 0xE5, 0x47, 0x95, - 0x46, 0xF5, 0x3C, 0xC3, 0x13, 0xF5, 0x71, 0x25, -/*05A0*/0x46, 0xF5, 0x72, 0xC3, 0x94, 0x3F, 0x40, 0x05, - 0xE4, 0xF5, 0x3D, 0x80, 0x40, 0xC3, 0x74, 0x3F, -/*05B0*/0x95, 0x72, 0xF5, 0x3D, 0x80, 0x37, 0xE5, 0x46, - 0x65, 0x47, 0x70, 0x0F, 0x75, 0x73, 0x01, 0x75, -/*05C0*/0x75, 0x01, 0xF5, 0x3E, 0xF5, 0x3F, 0x75, 0x4E, - 0x01, 0x80, 0x22, 0xE4, 0xF5, 0x4E, 0xC3, 0xE5, -/*05D0*/0x47, 0x95, 0x46, 0xF5, 0x3E, 0xC3, 0x13, 0xF5, - 0x71, 0x25, 0x46, 0xF5, 0x72, 0xD3, 0x94, 0x3F, -/*05E0*/0x50, 0x05, 0xE4, 0xF5, 0x3F, 0x80, 0x06, 0xE5, - 0x72, 0x24, 0xC1, 0xF5, 0x3F, 0x05, 0x6F, 0xE5, -/*05F0*/0x6F, 0xC3, 0x94, 0x02, 0x50, 0x03, 0x02, 0x04, - 0x6E, 0xE5, 0x6D, 0x45, 0x6C, 0x70, 0x02, 0x80, -/*0600*/0x04, 0xE5, 0x74, 0x45, 0x75, 0x90, 0x07, 0x2F, - 0xF0, 0x7F, 0x01, 0xE5, 0x3E, 0x60, 0x04, 0xE5, -/*0610*/0x3C, 0x70, 0x14, 0xE4, 0xF5, 0x3C, 0xF5, 0x3D, - 0xF5, 0x3E, 0xF5, 0x3F, 0x12, 0x08, 0xD2, 0x70, -/*0620*/0x04, 0xF0, 0x02, 0x06, 0xA4, 0x80, 0x7A, 0xE5, - 0x3C, 0xC3, 0x95, 0x3E, 0x40, 0x07, 0xE5, 0x3C, -/*0630*/0x95, 0x3E, 0xFF, 0x80, 0x06, 0xC3, 0xE5, 0x3E, - 0x95, 0x3C, 0xFF, 0xE5, 0x76, 0xD3, 0x95, 0x79, -/*0640*/0x40, 0x05, 0x85, 0x76, 0x7A, 0x80, 0x03, 0x85, - 0x79, 0x7A, 0xE5, 0x77, 0xC3, 0x95, 0x78, 0x50, -/*0650*/0x05, 0x85, 0x77, 0x7B, 0x80, 0x03, 0x85, 0x78, - 0x7B, 0xE5, 0x7B, 0xD3, 0x95, 0x7A, 0x40, 0x30, -/*0660*/0xE5, 0x7B, 0x95, 0x7A, 0xF5, 0x3C, 0xF5, 0x3E, - 0xC3, 0xE5, 0x7B, 0x95, 0x7A, 0x90, 0x07, 0x19, -/*0670*/0xF0, 0xE5, 0x3C, 0xC3, 0x13, 0xF5, 0x71, 0x25, - 0x7A, 0xF5, 0x72, 0xC3, 0x94, 0x3F, 0x40, 0x05, -/*0680*/0xE4, 0xF5, 0x3D, 0x80, 0x1F, 0xC3, 0x74, 0x3F, - 0x95, 0x72, 0xF5, 0x3D, 0xF5, 0x3F, 0x80, 0x14, -/*0690*/0xE4, 0xF5, 0x3C, 0xF5, 0x3E, 0x90, 0x07, 0x19, - 0xF0, 0x12, 0x08, 0xD2, 0x70, 0x03, 0xF0, 0x80, -/*06A0*/0x03, 0x74, 0x01, 0xF0, 0x12, 0x08, 0x65, 0x75, - 0x83, 0xD0, 0xE0, 0x54, 0x0F, 0xFE, 0xAD, 0x3C, -/*06B0*/0x70, 0x02, 0x7E, 0x07, 0xBE, 0x0F, 0x02, 0x7E, - 0x80, 0xEE, 0xFB, 0xEF, 0xD3, 0x9B, 0x74, 0x80, -/*06C0*/0xF8, 0x98, 0x40, 0x1F, 0xE4, 0xF5, 0x3C, 0xF5, - 0x3E, 0x12, 0x08, 0xD2, 0x70, 0x03, 0xF0, 0x80, -/*06D0*/0x12, 0x74, 0x01, 0xF0, 0xE5, 0x08, 0xFB, 0xEB, - 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0xD2, 0xE0, -/*06E0*/0x44, 0x10, 0xF0, 0xE5, 0x08, 0xFB, 0xEB, 0x44, - 0x09, 0xF5, 0x82, 0x75, 0x83, 0x9E, 0xED, 0xF0, -/*06F0*/0xEB, 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0xCA, - 0xED, 0xF0, 0x12, 0x08, 0x65, 0x75, 0x83, 0xCC, -/*0700*/0xEF, 0xF0, 0x22, 0xE5, 0x08, 0x44, 0x07, 0xF5, - 0x82, 0x75, 0x83, 0xBC, 0xE0, 0x54, 0xF0, 0xF0, -/*0710*/0xE5, 0x08, 0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, - 0xBE, 0xE0, 0x54, 0xF0, 0xF0, 0xE5, 0x08, 0x44, -/*0720*/0x07, 0xF5, 0x82, 0x75, 0x83, 0xC0, 0xE0, 0x54, - 0xF0, 0xF0, 0xE5, 0x08, 0x44, 0x07, 0xF5, 0x82, -/*0730*/0x22, 0xF0, 0x90, 0x07, 0x28, 0xE0, 0xFE, 0xA3, - 0xE0, 0xF5, 0x82, 0x8E, 0x83, 0x22, 0x85, 0x42, -/*0740*/0x42, 0x85, 0x41, 0x41, 0x85, 0x40, 0x40, 0x74, - 0xC0, 0x2F, 0xF5, 0x82, 0x74, 0x02, 0x3E, 0xF5, -/*0750*/0x83, 0xE5, 0x42, 0xF0, 0x74, 0xE0, 0x2F, 0xF5, - 0x82, 0x74, 0x02, 0x3E, 0xF5, 0x83, 0x22, 0xE5, -/*0760*/0x42, 0x29, 0xFD, 0xE4, 0x33, 0xFC, 0xE5, 0x3C, - 0xC3, 0x9D, 0xEC, 0x64, 0x80, 0xF8, 0x74, 0x80, -/*0770*/0x98, 0x22, 0xF5, 0x83, 0xE0, 0x90, 0x07, 0x22, - 0x54, 0x1F, 0xFD, 0xE0, 0xFA, 0xA3, 0xE0, 0xF5, -/*0780*/0x82, 0x8A, 0x83, 0xED, 0xF0, 0x22, 0x90, 0x07, - 0x22, 0xE0, 0xFC, 0xA3, 0xE0, 0xF5, 0x82, 0x8C, -/*0790*/0x83, 0x22, 0x90, 0x07, 0x24, 0xFF, 0xED, 0x44, - 0x07, 0xCF, 0xF0, 0xA3, 0xEF, 0xF0, 0x22, 0x85, -/*07A0*/0x38, 0x38, 0x85, 0x39, 0x39, 0x85, 0x3A, 0x3A, - 0x74, 0xC0, 0x2F, 0xF5, 0x82, 0x74, 0x02, 0x3E, -/*07B0*/0xF5, 0x83, 0x22, 0x90, 0x07, 0x26, 0xFF, 0xED, - 0x44, 0x07, 0xCF, 0xF0, 0xA3, 0xEF, 0xF0, 0x22, -/*07C0*/0xF0, 0x74, 0xA0, 0x2F, 0xF5, 0x82, 0x74, 0x02, - 0x3E, 0xF5, 0x83, 0x22, 0x74, 0xC0, 0x25, 0x11, -/*07D0*/0xF5, 0x82, 0xE4, 0x34, 0x01, 0xF5, 0x83, 0x22, - 0x74, 0x00, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34, -/*07E0*/0x02, 0xF5, 0x83, 0x22, 0x74, 0x60, 0x25, 0x11, - 0xF5, 0x82, 0xE4, 0x34, 0x03, 0xF5, 0x83, 0x22, -/*07F0*/0x74, 0x80, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34, - 0x03, 0xF5, 0x83, 0x22, 0x74, 0xE0, 0x25, 0x11, -/*0800*/0xF5, 0x82, 0xE4, 0x34, 0x03, 0xF5, 0x83, 0x22, - 0x74, 0x40, 0x25, 0x11, 0xF5, 0x82, 0xE4, 0x34, -/*0810*/0x06, 0xF5, 0x83, 0x22, 0x74, 0x80, 0x2F, 0xF5, - 0x82, 0x74, 0x02, 0x3E, 0xF5, 0x83, 0x22, 0xAF, -/*0820*/0x08, 0x7E, 0x00, 0xEF, 0x44, 0x07, 0xF5, 0x82, - 0x22, 0xF5, 0x83, 0xE5, 0x82, 0x44, 0x07, 0xF5, -/*0830*/0x82, 0xE5, 0x40, 0xF0, 0x22, 0x74, 0x40, 0x25, - 0x11, 0xF5, 0x82, 0xE4, 0x34, 0x02, 0xF5, 0x83, -/*0840*/0x22, 0x74, 0xC0, 0x25, 0x11, 0xF5, 0x82, 0xE4, - 0x34, 0x03, 0xF5, 0x83, 0x22, 0x74, 0x00, 0x25, -/*0850*/0x11, 0xF5, 0x82, 0xE4, 0x34, 0x06, 0xF5, 0x83, - 0x22, 0x74, 0x20, 0x25, 0x11, 0xF5, 0x82, 0xE4, -/*0860*/0x34, 0x06, 0xF5, 0x83, 0x22, 0xE5, 0x08, 0xFD, - 0xED, 0x44, 0x07, 0xF5, 0x82, 0x22, 0xE5, 0x41, -/*0870*/0xF0, 0xE5, 0x65, 0x64, 0x01, 0x45, 0x64, 0x22, - 0x7E, 0x00, 0xFB, 0x7A, 0x00, 0xFD, 0x7C, 0x00, -/*0880*/0x22, 0x74, 0x20, 0x25, 0x11, 0xF5, 0x82, 0xE4, - 0x34, 0x02, 0x22, 0x74, 0xA0, 0x25, 0x11, 0xF5, -/*0890*/0x82, 0xE4, 0x34, 0x03, 0x22, 0x85, 0x3E, 0x42, - 0x85, 0x3F, 0x41, 0x8F, 0x40, 0x22, 0x85, 0x3C, -/*08A0*/0x42, 0x85, 0x3D, 0x41, 0x8F, 0x40, 0x22, 0x75, - 0x45, 0x3F, 0x90, 0x07, 0x20, 0xE4, 0xF0, 0xA3, -/*08B0*/0x22, 0xF5, 0x83, 0xE5, 0x32, 0xF0, 0x05, 0x6E, - 0xE5, 0x6E, 0xC3, 0x94, 0x40, 0x22, 0xF0, 0xE5, -/*08C0*/0x08, 0x44, 0x06, 0xF5, 0x82, 0x22, 0x74, 0x00, - 0x25, 0x6E, 0xF5, 0x82, 0xE4, 0x34, 0x00, 0xF5, -/*08D0*/0x83, 0x22, 0xE5, 0x6D, 0x45, 0x6C, 0x90, 0x07, - 0x2F, 0x22, 0xE4, 0xF9, 0xE5, 0x3C, 0xD3, 0x95, -/*08E0*/0x3E, 0x22, 0x74, 0x80, 0x2E, 0xF5, 0x82, 0xE4, - 0x34, 0x02, 0xF5, 0x83, 0xE0, 0x22, 0x74, 0xA0, -/*08F0*/0x2E, 0xF5, 0x82, 0xE4, 0x34, 0x02, 0xF5, 0x83, - 0xE0, 0x22, 0x74, 0x80, 0x25, 0x6E, 0xF5, 0x82, -/*0900*/0xE4, 0x34, 0x00, 0x22, 0x25, 0x42, 0xFD, 0xE4, - 0x33, 0xFC, 0x22, 0x85, 0x42, 0x42, 0x85, 0x41, -/*0910*/0x41, 0x85, 0x40, 0x40, 0x22, 0xED, 0x4C, 0x60, - 0x03, 0x02, 0x09, 0xE5, 0xEF, 0x4E, 0x70, 0x37, -/*0920*/0x90, 0x07, 0x26, 0x12, 0x07, 0x89, 0xE0, 0xFD, - 0x12, 0x07, 0xCC, 0xED, 0xF0, 0x90, 0x07, 0x28, -/*0930*/0x12, 0x07, 0x89, 0xE0, 0xFD, 0x12, 0x07, 0xD8, - 0xED, 0xF0, 0x12, 0x07, 0x86, 0xE0, 0x54, 0x1F, -/*0940*/0xFD, 0x12, 0x08, 0x81, 0xF5, 0x83, 0xED, 0xF0, - 0x90, 0x07, 0x24, 0x12, 0x07, 0x89, 0xE0, 0x54, -/*0950*/0x1F, 0xFD, 0x12, 0x08, 0x35, 0xED, 0xF0, 0xEF, - 0x64, 0x04, 0x4E, 0x70, 0x37, 0x90, 0x07, 0x26, -/*0960*/0x12, 0x07, 0x89, 0xE0, 0xFD, 0x12, 0x07, 0xE4, - 0xED, 0xF0, 0x90, 0x07, 0x28, 0x12, 0x07, 0x89, -/*0970*/0xE0, 0xFD, 0x12, 0x07, 0xF0, 0xED, 0xF0, 0x12, - 0x07, 0x86, 0xE0, 0x54, 0x1F, 0xFD, 0x12, 0x08, -/*0980*/0x8B, 0xF5, 0x83, 0xED, 0xF0, 0x90, 0x07, 0x24, - 0x12, 0x07, 0x89, 0xE0, 0x54, 0x1F, 0xFD, 0x12, -/*0990*/0x08, 0x41, 0xED, 0xF0, 0xEF, 0x64, 0x01, 0x4E, - 0x70, 0x04, 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00, -/*09A0*/0xEF, 0x64, 0x02, 0x4E, 0x70, 0x04, 0x7F, 0x01, - 0x80, 0x02, 0x7F, 0x00, 0xEF, 0x4D, 0x60, 0x78, -/*09B0*/0x90, 0x07, 0x26, 0x12, 0x07, 0x35, 0xE0, 0xFF, - 0x12, 0x07, 0xFC, 0xEF, 0x12, 0x07, 0x31, 0xE0, -/*09C0*/0xFF, 0x12, 0x08, 0x08, 0xEF, 0xF0, 0x90, 0x07, - 0x22, 0x12, 0x07, 0x35, 0xE0, 0x54, 0x1F, 0xFF, -/*09D0*/0x12, 0x08, 0x4D, 0xEF, 0xF0, 0x90, 0x07, 0x24, - 0x12, 0x07, 0x35, 0xE0, 0x54, 0x1F, 0xFF, 0x12, -/*09E0*/0x08, 0x59, 0xEF, 0xF0, 0x22, 0x12, 0x07, 0xCC, - 0xE4, 0xF0, 0x12, 0x07, 0xD8, 0xE4, 0xF0, 0x12, -/*09F0*/0x08, 0x81, 0xF5, 0x83, 0xE4, 0xF0, 0x12, 0x08, - 0x35, 0x74, 0x14, 0xF0, 0x12, 0x07, 0xE4, 0xE4, -/*0A00*/0xF0, 0x12, 0x07, 0xF0, 0xE4, 0xF0, 0x12, 0x08, - 0x8B, 0xF5, 0x83, 0xE4, 0xF0, 0x12, 0x08, 0x41, -/*0A10*/0x74, 0x14, 0xF0, 0x12, 0x07, 0xFC, 0xE4, 0xF0, - 0x12, 0x08, 0x08, 0xE4, 0xF0, 0x12, 0x08, 0x4D, -/*0A20*/0xE4, 0xF0, 0x12, 0x08, 0x59, 0x74, 0x14, 0xF0, - 0x22, 0x53, 0xF9, 0xF7, 0x75, 0xFC, 0x10, 0xE4, -/*0A30*/0xF5, 0xFD, 0x75, 0xFE, 0x30, 0xF5, 0xFF, 0xE5, - 0xE7, 0x20, 0xE7, 0x03, 0x43, 0xF9, 0x08, 0xE5, -/*0A40*/0xE6, 0x20, 0xE7, 0x0B, 0x78, 0xFF, 0xE4, 0xF6, - 0xD8, 0xFD, 0x53, 0xE6, 0xFE, 0x80, 0x09, 0x78, -/*0A50*/0x08, 0xE4, 0xF6, 0xD8, 0xFD, 0x53, 0xE6, 0xFE, - 0x75, 0x81, 0x80, 0xE4, 0xF5, 0xA8, 0xD2, 0xA8, -/*0A60*/0xC2, 0xA9, 0xD2, 0xAF, 0xE5, 0xE2, 0x20, 0xE5, - 0x05, 0x20, 0xE6, 0x02, 0x80, 0x03, 0x43, 0xE1, -/*0A70*/0x02, 0xE5, 0xE2, 0x20, 0xE0, 0x0E, 0x90, 0x00, - 0x00, 0x7F, 0x00, 0x7E, 0x08, 0xE4, 0xF0, 0xA3, -/*0A80*/0xDF, 0xFC, 0xDE, 0xFA, 0x02, 0x0A, 0xDB, 0x43, - 0xFA, 0x01, 0xC0, 0xE0, 0xC0, 0xF0, 0xC0, 0x83, -/*0A90*/0xC0, 0x82, 0xC0, 0xD0, 0x12, 0x1C, 0xE7, 0xD0, - 0xD0, 0xD0, 0x82, 0xD0, 0x83, 0xD0, 0xF0, 0xD0, -/*0AA0*/0xE0, 0x53, 0xFA, 0xFE, 0x32, 0x02, 0x1B, 0x55, - 0xE4, 0x93, 0xA3, 0xF8, 0xE4, 0x93, 0xA3, 0xF6, -/*0AB0*/0x08, 0xDF, 0xF9, 0x80, 0x29, 0xE4, 0x93, 0xA3, - 0xF8, 0x54, 0x07, 0x24, 0x0C, 0xC8, 0xC3, 0x33, -/*0AC0*/0xC4, 0x54, 0x0F, 0x44, 0x20, 0xC8, 0x83, 0x40, - 0x04, 0xF4, 0x56, 0x80, 0x01, 0x46, 0xF6, 0xDF, -/*0AD0*/0xE4, 0x80, 0x0B, 0x01, 0x02, 0x04, 0x08, 0x10, - 0x20, 0x40, 0x80, 0x90, 0x00, 0x3F, 0xE4, 0x7E, -/*0AE0*/0x01, 0x93, 0x60, 0xC1, 0xA3, 0xFF, 0x54, 0x3F, - 0x30, 0xE5, 0x09, 0x54, 0x1F, 0xFE, 0xE4, 0x93, -/*0AF0*/0xA3, 0x60, 0x01, 0x0E, 0xCF, 0x54, 0xC0, 0x25, - 0xE0, 0x60, 0xAD, 0x40, 0xB8, 0x80, 0xFE, 0x8C, -/*0B00*/0x64, 0x8D, 0x65, 0x8A, 0x66, 0x8B, 0x67, 0xE4, - 0xF5, 0x69, 0xEF, 0x4E, 0x70, 0x03, 0x02, 0x1D, -/*0B10*/0x55, 0xE4, 0xF5, 0x68, 0xE5, 0x67, 0x45, 0x66, - 0x70, 0x32, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x90, -/*0B20*/0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2, 0xE4, - 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0xE4, 0x12, -/*0B30*/0x08, 0x70, 0x70, 0x29, 0x12, 0x07, 0x2A, 0x75, - 0x83, 0x92, 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, -/*0B40*/0xC6, 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC8, - 0xE4, 0xF0, 0x80, 0x11, 0x90, 0x07, 0x26, 0x12, -/*0B50*/0x07, 0x35, 0xE4, 0x12, 0x08, 0x70, 0x70, 0x05, - 0x12, 0x07, 0x32, 0xE4, 0xF0, 0x12, 0x1D, 0x55, -/*0B60*/0x12, 0x1E, 0xBF, 0xE5, 0x67, 0x45, 0x66, 0x70, - 0x33, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x90, 0xE5, -/*0B70*/0x41, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2, 0xE5, - 0x41, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0x12, -/*0B80*/0x08, 0x6E, 0x70, 0x29, 0x12, 0x07, 0x2A, 0x75, - 0x83, 0x92, 0xE5, 0x40, 0x12, 0x07, 0x29, 0x75, -/*0B90*/0x83, 0xC6, 0xE5, 0x40, 0x12, 0x07, 0x29, 0x75, - 0x83, 0xC8, 0x80, 0x0E, 0x90, 0x07, 0x26, 0x12, -/*0BA0*/0x07, 0x35, 0x12, 0x08, 0x6E, 0x70, 0x06, 0x12, - 0x07, 0x32, 0xE5, 0x40, 0xF0, 0xAF, 0x69, 0x7E, -/*0BB0*/0x00, 0xAD, 0x67, 0xAC, 0x66, 0x12, 0x04, 0x44, - 0x12, 0x07, 0x2A, 0x75, 0x83, 0xCA, 0xE0, 0xD3, -/*0BC0*/0x94, 0x00, 0x50, 0x0C, 0x05, 0x68, 0xE5, 0x68, - 0xC3, 0x94, 0x05, 0x50, 0x03, 0x02, 0x0B, 0x14, -/*0BD0*/0x22, 0x8C, 0x60, 0x8D, 0x61, 0x12, 0x08, 0xDA, - 0x74, 0x20, 0x40, 0x0D, 0x2F, 0xF5, 0x82, 0x74, -/*0BE0*/0x03, 0x3E, 0xF5, 0x83, 0xE5, 0x3E, 0xF0, 0x80, - 0x0B, 0x2F, 0xF5, 0x82, 0x74, 0x03, 0x3E, 0xF5, -/*0BF0*/0x83, 0xE5, 0x3C, 0xF0, 0xE5, 0x3C, 0xD3, 0x95, - 0x3E, 0x40, 0x3C, 0xE5, 0x61, 0x45, 0x60, 0x70, -/*0C00*/0x10, 0xE9, 0x12, 0x09, 0x04, 0xE5, 0x3E, 0x12, - 0x07, 0x68, 0x40, 0x3B, 0x12, 0x08, 0x95, 0x80, -/*0C10*/0x18, 0xE5, 0x3E, 0xC3, 0x95, 0x38, 0x40, 0x1D, - 0x85, 0x3E, 0x38, 0xE5, 0x3E, 0x60, 0x05, 0x85, -/*0C20*/0x3F, 0x39, 0x80, 0x03, 0x85, 0x39, 0x39, 0x8F, - 0x3A, 0x12, 0x08, 0x14, 0xE5, 0x3E, 0x12, 0x07, -/*0C30*/0xC0, 0xE5, 0x3F, 0xF0, 0x22, 0x80, 0x43, 0xE5, - 0x61, 0x45, 0x60, 0x70, 0x19, 0x12, 0x07, 0x5F, -/*0C40*/0x40, 0x05, 0x12, 0x08, 0x9E, 0x80, 0x27, 0x12, - 0x09, 0x0B, 0x12, 0x08, 0x14, 0xE5, 0x42, 0x12, -/*0C50*/0x07, 0xC0, 0xE5, 0x41, 0xF0, 0x22, 0xE5, 0x3C, - 0xC3, 0x95, 0x38, 0x40, 0x1D, 0x85, 0x3C, 0x38, -/*0C60*/0xE5, 0x3C, 0x60, 0x05, 0x85, 0x3D, 0x39, 0x80, - 0x03, 0x85, 0x39, 0x39, 0x8F, 0x3A, 0x12, 0x08, -/*0C70*/0x14, 0xE5, 0x3C, 0x12, 0x07, 0xC0, 0xE5, 0x3D, - 0xF0, 0x22, 0x85, 0x38, 0x38, 0x85, 0x39, 0x39, -/*0C80*/0x85, 0x3A, 0x3A, 0x12, 0x08, 0x14, 0xE5, 0x38, - 0x12, 0x07, 0xC0, 0xE5, 0x39, 0xF0, 0x22, 0x7F, -/*0C90*/0x06, 0x12, 0x17, 0x31, 0x12, 0x1D, 0x23, 0x12, - 0x0E, 0x04, 0x12, 0x0E, 0x33, 0xE0, 0x44, 0x0A, -/*0CA0*/0xF0, 0x74, 0x8E, 0xFE, 0x12, 0x0E, 0x04, 0x12, - 0x0E, 0x0B, 0xEF, 0xF0, 0xE5, 0x28, 0x30, 0xE5, -/*0CB0*/0x03, 0xD3, 0x80, 0x01, 0xC3, 0x40, 0x05, 0x75, - 0x14, 0x20, 0x80, 0x03, 0x75, 0x14, 0x08, 0x12, -/*0CC0*/0x0E, 0x04, 0x75, 0x83, 0x8A, 0xE5, 0x14, 0xF0, - 0xB4, 0xFF, 0x05, 0x75, 0x12, 0x80, 0x80, 0x06, -/*0CD0*/0xE5, 0x14, 0xC3, 0x13, 0xF5, 0x12, 0xE4, 0xF5, - 0x16, 0xF5, 0x7F, 0x12, 0x19, 0x36, 0x12, 0x13, -/*0CE0*/0xA3, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x50, 0x09, - 0x05, 0x16, 0xE5, 0x16, 0xC3, 0x94, 0x14, 0x40, -/*0CF0*/0xEA, 0xE5, 0xE4, 0x20, 0xE7, 0x28, 0x12, 0x0E, - 0x04, 0x75, 0x83, 0xD2, 0xE0, 0x54, 0x08, 0xD3, -/*0D00*/0x94, 0x00, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02, - 0x7F, 0x00, 0xE5, 0x0A, 0xC3, 0x94, 0x01, 0x40, -/*0D10*/0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEF, - 0x5E, 0x60, 0x03, 0x12, 0x1D, 0xD7, 0xE5, 0x7F, -/*0D20*/0xC3, 0x94, 0x11, 0x40, 0x14, 0x12, 0x0E, 0x04, - 0x75, 0x83, 0xD2, 0xE0, 0x44, 0x80, 0xF0, 0xE5, -/*0D30*/0xE4, 0x20, 0xE7, 0x0F, 0x12, 0x1D, 0xD7, 0x80, - 0x0A, 0x12, 0x0E, 0x04, 0x75, 0x83, 0xD2, 0xE0, -/*0D40*/0x54, 0x7F, 0xF0, 0x12, 0x1D, 0x23, 0x22, 0x74, - 0x8A, 0x85, 0x08, 0x82, 0xF5, 0x83, 0xE5, 0x17, -/*0D50*/0xF0, 0x12, 0x0E, 0x3A, 0xE4, 0xF0, 0x90, 0x07, - 0x02, 0xE0, 0x12, 0x0E, 0x17, 0x75, 0x83, 0x90, -/*0D60*/0xEF, 0xF0, 0x74, 0x92, 0xFE, 0xE5, 0x08, 0x44, - 0x07, 0xFF, 0xF5, 0x82, 0x8E, 0x83, 0xE0, 0x54, -/*0D70*/0xC0, 0xFD, 0x90, 0x07, 0x03, 0xE0, 0x54, 0x3F, - 0x4D, 0x8F, 0x82, 0x8E, 0x83, 0xF0, 0x90, 0x07, -/*0D80*/0x04, 0xE0, 0x12, 0x0E, 0x17, 0x75, 0x83, 0x82, - 0xEF, 0xF0, 0x90, 0x07, 0x05, 0xE0, 0xFF, 0xED, -/*0D90*/0x44, 0x07, 0xF5, 0x82, 0x75, 0x83, 0xB4, 0xEF, - 0x12, 0x0E, 0x03, 0x75, 0x83, 0x80, 0xE0, 0x54, -/*0DA0*/0xBF, 0xF0, 0x30, 0x37, 0x0A, 0x12, 0x0E, 0x91, - 0x75, 0x83, 0x94, 0xE0, 0x44, 0x80, 0xF0, 0x30, -/*0DB0*/0x38, 0x0A, 0x12, 0x0E, 0x91, 0x75, 0x83, 0x92, - 0xE0, 0x44, 0x80, 0xF0, 0xE5, 0x28, 0x30, 0xE4, -/*0DC0*/0x1A, 0x20, 0x39, 0x0A, 0x12, 0x0E, 0x04, 0x75, - 0x83, 0x88, 0xE0, 0x54, 0x7F, 0xF0, 0x20, 0x3A, -/*0DD0*/0x0A, 0x12, 0x0E, 0x04, 0x75, 0x83, 0x88, 0xE0, - 0x54, 0xBF, 0xF0, 0x74, 0x8C, 0xFE, 0x12, 0x0E, -/*0DE0*/0x04, 0x8E, 0x83, 0xE0, 0x54, 0x0F, 0x12, 0x0E, - 0x03, 0x75, 0x83, 0x86, 0xE0, 0x54, 0xBF, 0xF0, -/*0DF0*/0xE5, 0x08, 0x44, 0x06, 0x12, 0x0D, 0xFD, 0x75, - 0x83, 0x8A, 0xE4, 0xF0, 0x22, 0xF5, 0x82, 0x75, -/*0E00*/0x83, 0x82, 0xE4, 0xF0, 0xE5, 0x08, 0x44, 0x07, - 0xF5, 0x82, 0x22, 0x8E, 0x83, 0xE0, 0xF5, 0x10, -/*0E10*/0x54, 0xFE, 0xF0, 0xE5, 0x10, 0x44, 0x01, 0xFF, - 0xE5, 0x08, 0xFD, 0xED, 0x44, 0x07, 0xF5, 0x82, -/*0E20*/0x22, 0xE5, 0x15, 0xC4, 0x54, 0x07, 0xFF, 0xE5, - 0x08, 0xFD, 0xED, 0x44, 0x08, 0xF5, 0x82, 0x75, -/*0E30*/0x83, 0x82, 0x22, 0x75, 0x83, 0x80, 0xE0, 0x44, - 0x40, 0xF0, 0xE5, 0x08, 0x44, 0x08, 0xF5, 0x82, -/*0E40*/0x75, 0x83, 0x8A, 0x22, 0xE5, 0x16, 0x25, 0xE0, - 0x25, 0xE0, 0x24, 0xAF, 0xF5, 0x82, 0xE4, 0x34, -/*0E50*/0x1A, 0xF5, 0x83, 0xE4, 0x93, 0xF5, 0x0D, 0x22, - 0x43, 0xE1, 0x10, 0x43, 0xE1, 0x80, 0x53, 0xE1, -/*0E60*/0xFD, 0x85, 0xE1, 0x10, 0x22, 0xE5, 0x16, 0x25, - 0xE0, 0x25, 0xE0, 0x24, 0xB2, 0xF5, 0x82, 0xE4, -/*0E70*/0x34, 0x1A, 0xF5, 0x83, 0xE4, 0x93, 0x22, 0x85, - 0x55, 0x82, 0x85, 0x54, 0x83, 0xE5, 0x15, 0xF0, -/*0E80*/0x22, 0xE5, 0xE2, 0x54, 0x20, 0xD3, 0x94, 0x00, - 0x22, 0xE5, 0xE2, 0x54, 0x40, 0xD3, 0x94, 0x00, -/*0E90*/0x22, 0xE5, 0x08, 0x44, 0x06, 0xF5, 0x82, 0x22, - 0xFD, 0xE5, 0x08, 0xFB, 0xEB, 0x44, 0x07, 0xF5, -/*0EA0*/0x82, 0x22, 0x53, 0xF9, 0xF7, 0x75, 0xFE, 0x30, - 0x22, 0xEF, 0x4E, 0x70, 0x26, 0x12, 0x07, 0xCC, -/*0EB0*/0xE0, 0xFD, 0x90, 0x07, 0x26, 0x12, 0x07, 0x7B, - 0x12, 0x07, 0xD8, 0xE0, 0xFD, 0x90, 0x07, 0x28, -/*0EC0*/0x12, 0x07, 0x7B, 0x12, 0x08, 0x81, 0x12, 0x07, - 0x72, 0x12, 0x08, 0x35, 0xE0, 0x90, 0x07, 0x24, -/*0ED0*/0x12, 0x07, 0x78, 0xEF, 0x64, 0x04, 0x4E, 0x70, - 0x29, 0x12, 0x07, 0xE4, 0xE0, 0xFD, 0x90, 0x07, -/*0EE0*/0x26, 0x12, 0x07, 0x7B, 0x12, 0x07, 0xF0, 0xE0, - 0xFD, 0x90, 0x07, 0x28, 0x12, 0x07, 0x7B, 0x12, -/*0EF0*/0x08, 0x8B, 0x12, 0x07, 0x72, 0x12, 0x08, 0x41, - 0xE0, 0x54, 0x1F, 0xFD, 0x90, 0x07, 0x24, 0x12, -/*0F00*/0x07, 0x7B, 0xEF, 0x64, 0x01, 0x4E, 0x70, 0x04, - 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00, 0xEF, 0x64, -/*0F10*/0x02, 0x4E, 0x70, 0x04, 0x7F, 0x01, 0x80, 0x02, - 0x7F, 0x00, 0xEF, 0x4D, 0x60, 0x35, 0x12, 0x07, -/*0F20*/0xFC, 0xE0, 0xFF, 0x90, 0x07, 0x26, 0x12, 0x07, - 0x89, 0xEF, 0xF0, 0x12, 0x08, 0x08, 0xE0, 0xFF, -/*0F30*/0x90, 0x07, 0x28, 0x12, 0x07, 0x89, 0xEF, 0xF0, - 0x12, 0x08, 0x4D, 0xE0, 0x54, 0x1F, 0xFF, 0x12, -/*0F40*/0x07, 0x86, 0xEF, 0xF0, 0x12, 0x08, 0x59, 0xE0, - 0x54, 0x1F, 0xFF, 0x90, 0x07, 0x24, 0x12, 0x07, -/*0F50*/0x89, 0xEF, 0xF0, 0x22, 0xE4, 0xF5, 0x53, 0x12, - 0x0E, 0x81, 0x40, 0x04, 0x7F, 0x01, 0x80, 0x02, -/*0F60*/0x7F, 0x00, 0x12, 0x0E, 0x89, 0x40, 0x04, 0x7E, - 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F, 0x70, -/*0F70*/0x03, 0x02, 0x0F, 0xF6, 0x85, 0xE1, 0x10, 0x43, - 0xE1, 0x02, 0x53, 0xE1, 0x0F, 0x85, 0xE1, 0x10, -/*0F80*/0xE4, 0xF5, 0x51, 0xE5, 0xE3, 0x54, 0x3F, 0xF5, - 0x52, 0x12, 0x0E, 0x89, 0x40, 0x1D, 0xAD, 0x52, -/*0F90*/0xAF, 0x51, 0x12, 0x11, 0x18, 0xEF, 0x60, 0x08, - 0x85, 0xE1, 0x10, 0x43, 0xE1, 0x40, 0x80, 0x0B, -/*0FA0*/0x53, 0xE1, 0xBF, 0x12, 0x0E, 0x58, 0x12, 0x00, - 0x06, 0x80, 0xFB, 0xE5, 0xE3, 0x54, 0x3F, 0xF5, -/*0FB0*/0x51, 0xE5, 0xE4, 0x54, 0x3F, 0xF5, 0x52, 0x12, - 0x0E, 0x81, 0x40, 0x1D, 0xAD, 0x52, 0xAF, 0x51, -/*0FC0*/0x12, 0x11, 0x18, 0xEF, 0x60, 0x08, 0x85, 0xE1, - 0x10, 0x43, 0xE1, 0x20, 0x80, 0x0B, 0x53, 0xE1, -/*0FD0*/0xDF, 0x12, 0x0E, 0x58, 0x12, 0x00, 0x06, 0x80, - 0xFB, 0x12, 0x0E, 0x81, 0x40, 0x04, 0x7F, 0x01, -/*0FE0*/0x80, 0x02, 0x7F, 0x00, 0x12, 0x0E, 0x89, 0x40, - 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, -/*0FF0*/0x4F, 0x60, 0x03, 0x12, 0x0E, 0x5B, 0x22, 0x12, - 0x0E, 0x21, 0xEF, 0xF0, 0x12, 0x10, 0x91, 0x22, -/*1000*/0x02, 0x11, 0x00, 0x02, 0x10, 0x40, 0x02, 0x10, - 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1010*/0x01, 0x20, 0x01, 0x20, 0xE4, 0xF5, 0x57, 0x12, - 0x16, 0xBD, 0x12, 0x16, 0x44, 0xE4, 0x12, 0x10, -/*1020*/0x56, 0x12, 0x14, 0xB7, 0x90, 0x07, 0x26, 0x12, - 0x07, 0x35, 0xE4, 0x12, 0x07, 0x31, 0xE4, 0xF0, -/*1030*/0x12, 0x10, 0x56, 0x12, 0x14, 0xB7, 0x90, 0x07, - 0x26, 0x12, 0x07, 0x35, 0xE5, 0x41, 0x12, 0x07, -/*1040*/0x31, 0xE5, 0x40, 0xF0, 0xAF, 0x57, 0x7E, 0x00, - 0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x44, 0xAF, -/*1050*/0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE, 0xFF, 0x90, - 0x07, 0x20, 0xA3, 0xE0, 0xFD, 0xE4, 0xF5, 0x56, -/*1060*/0xF5, 0x40, 0xFE, 0xFC, 0xAB, 0x56, 0xFA, 0x12, - 0x11, 0x51, 0x7F, 0x0F, 0x7D, 0x18, 0xE4, 0xF5, -/*1070*/0x56, 0xF5, 0x40, 0xFE, 0xFC, 0xAB, 0x56, 0xFA, - 0x12, 0x15, 0x41, 0xAF, 0x56, 0x7E, 0x00, 0x12, -/*1080*/0x1A, 0xFF, 0xE4, 0xFF, 0xF5, 0x56, 0x7D, 0x1F, - 0xF5, 0x40, 0xFE, 0xFC, 0xAB, 0x56, 0xFA, 0x22, -/*1090*/0x22, 0xE4, 0xF5, 0x55, 0xE5, 0x08, 0xFD, 0x74, - 0xA0, 0xF5, 0x56, 0xED, 0x44, 0x07, 0xF5, 0x57, -/*10A0*/0xE5, 0x28, 0x30, 0xE5, 0x03, 0xD3, 0x80, 0x01, - 0xC3, 0x40, 0x05, 0x7F, 0x28, 0xEF, 0x80, 0x04, -/*10B0*/0x7F, 0x14, 0xEF, 0xC3, 0x13, 0xF5, 0x54, 0xE4, - 0xF9, 0x12, 0x0E, 0x18, 0x75, 0x83, 0x8E, 0xE0, -/*10C0*/0xF5, 0x10, 0xCE, 0xEF, 0xCE, 0xEE, 0xD3, 0x94, - 0x00, 0x40, 0x26, 0xE5, 0x10, 0x54, 0xFE, 0x12, -/*10D0*/0x0E, 0x98, 0x75, 0x83, 0x8E, 0xED, 0xF0, 0xE5, - 0x10, 0x44, 0x01, 0xFD, 0xEB, 0x44, 0x07, 0xF5, -/*10E0*/0x82, 0xED, 0xF0, 0x85, 0x57, 0x82, 0x85, 0x56, - 0x83, 0xE0, 0x30, 0xE3, 0x01, 0x09, 0x1E, 0x80, -/*10F0*/0xD4, 0xC2, 0x34, 0xE9, 0xC3, 0x95, 0x54, 0x40, - 0x02, 0xD2, 0x34, 0x22, 0x02, 0x00, 0x06, 0x22, -/*1100*/0x30, 0x30, 0x11, 0x90, 0x10, 0x00, 0xE4, 0x93, - 0xF5, 0x10, 0x90, 0x10, 0x10, 0xE4, 0x93, 0xF5, -/*1110*/0x10, 0x12, 0x10, 0x90, 0x12, 0x11, 0x50, 0x22, - 0xE4, 0xFC, 0xC3, 0xED, 0x9F, 0xFA, 0xEF, 0xF5, -/*1120*/0x83, 0x75, 0x82, 0x00, 0x79, 0xFF, 0xE4, 0x93, - 0xCC, 0x6C, 0xCC, 0xA3, 0xD9, 0xF8, 0xDA, 0xF6, -/*1130*/0xE5, 0xE2, 0x30, 0xE4, 0x02, 0x8C, 0xE5, 0xED, - 0x24, 0xFF, 0xFF, 0xEF, 0x75, 0x82, 0xFF, 0xF5, -/*1140*/0x83, 0xE4, 0x93, 0x6C, 0x70, 0x03, 0x7F, 0x01, - 0x22, 0x7F, 0x00, 0x22, 0x22, 0x11, 0x00, 0x00, -/*1150*/0x22, 0x8E, 0x58, 0x8F, 0x59, 0x8C, 0x5A, 0x8D, - 0x5B, 0x8A, 0x5C, 0x8B, 0x5D, 0x75, 0x5E, 0x01, -/*1160*/0xE4, 0xF5, 0x5F, 0xF5, 0x60, 0xF5, 0x62, 0x12, - 0x07, 0x2A, 0x75, 0x83, 0xD0, 0xE0, 0xFF, 0xC4, -/*1170*/0x54, 0x0F, 0xF5, 0x61, 0x12, 0x1E, 0xA5, 0x85, - 0x59, 0x5E, 0xD3, 0xE5, 0x5E, 0x95, 0x5B, 0xE5, -/*1180*/0x5A, 0x12, 0x07, 0x6B, 0x50, 0x4B, 0x12, 0x07, - 0x03, 0x75, 0x83, 0xBC, 0xE0, 0x45, 0x5E, 0x12, -/*1190*/0x07, 0x29, 0x75, 0x83, 0xBE, 0xE0, 0x45, 0x5E, - 0x12, 0x07, 0x29, 0x75, 0x83, 0xC0, 0xE0, 0x45, -/*11A0*/0x5E, 0xF0, 0xAF, 0x5F, 0xE5, 0x60, 0x12, 0x08, - 0x78, 0x12, 0x0A, 0xFF, 0xAF, 0x62, 0x7E, 0x00, -/*11B0*/0xAD, 0x5D, 0xAC, 0x5C, 0x12, 0x04, 0x44, 0xE5, - 0x61, 0xAF, 0x5E, 0x7E, 0x00, 0xB4, 0x03, 0x05, -/*11C0*/0x12, 0x1E, 0x21, 0x80, 0x07, 0xAD, 0x5D, 0xAC, - 0x5C, 0x12, 0x13, 0x17, 0x05, 0x5E, 0x02, 0x11, -/*11D0*/0x7A, 0x12, 0x07, 0x03, 0x75, 0x83, 0xBC, 0xE0, - 0x45, 0x40, 0x12, 0x07, 0x29, 0x75, 0x83, 0xBE, -/*11E0*/0xE0, 0x45, 0x40, 0x12, 0x07, 0x29, 0x75, 0x83, - 0xC0, 0xE0, 0x45, 0x40, 0xF0, 0x22, 0x8E, 0x58, -/*11F0*/0x8F, 0x59, 0x75, 0x5A, 0x01, 0x79, 0x01, 0x75, - 0x5B, 0x01, 0xE4, 0xFB, 0x12, 0x07, 0x2A, 0x75, -/*1200*/0x83, 0xAE, 0xE0, 0x54, 0x1A, 0xFF, 0x12, 0x08, - 0x65, 0xE0, 0xC4, 0x13, 0x54, 0x07, 0xFE, 0xEF, -/*1210*/0x70, 0x0C, 0xEE, 0x65, 0x35, 0x70, 0x07, 0x90, - 0x07, 0x2F, 0xE0, 0xB4, 0x01, 0x0D, 0xAF, 0x35, -/*1220*/0x7E, 0x00, 0x12, 0x0E, 0xA9, 0xCF, 0xEB, 0xCF, - 0x02, 0x1E, 0x60, 0xE5, 0x59, 0x64, 0x02, 0x45, -/*1230*/0x58, 0x70, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, - 0x00, 0xE5, 0x59, 0x45, 0x58, 0x70, 0x04, 0x7E, -/*1240*/0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F, 0x60, - 0x23, 0x85, 0x41, 0x49, 0x85, 0x40, 0x4B, 0xE5, -/*1250*/0x59, 0x45, 0x58, 0x70, 0x2C, 0xAF, 0x5A, 0xFE, - 0xCD, 0xE9, 0xCD, 0xFC, 0xAB, 0x59, 0xAA, 0x58, -/*1260*/0x12, 0x0A, 0xFF, 0xAF, 0x5B, 0x7E, 0x00, 0x12, - 0x1E, 0x60, 0x80, 0x15, 0xAF, 0x5B, 0x7E, 0x00, -/*1270*/0x12, 0x1E, 0x60, 0x90, 0x07, 0x26, 0x12, 0x07, - 0x35, 0xE5, 0x49, 0x12, 0x07, 0x31, 0xE5, 0x4B, -/*1280*/0xF0, 0xE4, 0xFD, 0xAF, 0x35, 0xFE, 0xFC, 0x12, - 0x09, 0x15, 0x22, 0x8C, 0x64, 0x8D, 0x65, 0x12, -/*1290*/0x08, 0xDA, 0x40, 0x3C, 0xE5, 0x65, 0x45, 0x64, - 0x70, 0x10, 0x12, 0x09, 0x04, 0xC3, 0xE5, 0x3E, -/*12A0*/0x12, 0x07, 0x69, 0x40, 0x3B, 0x12, 0x08, 0x95, - 0x80, 0x18, 0xE5, 0x3E, 0xC3, 0x95, 0x38, 0x40, -/*12B0*/0x1D, 0x85, 0x3E, 0x38, 0xE5, 0x3E, 0x60, 0x05, - 0x85, 0x3F, 0x39, 0x80, 0x03, 0x85, 0x39, 0x39, -/*12C0*/0x8F, 0x3A, 0x12, 0x07, 0xA8, 0xE5, 0x3E, 0x12, - 0x07, 0x53, 0xE5, 0x3F, 0xF0, 0x22, 0x80, 0x3B, -/*12D0*/0xE5, 0x65, 0x45, 0x64, 0x70, 0x11, 0x12, 0x07, - 0x5F, 0x40, 0x05, 0x12, 0x08, 0x9E, 0x80, 0x1F, -/*12E0*/0x12, 0x07, 0x3E, 0xE5, 0x41, 0xF0, 0x22, 0xE5, - 0x3C, 0xC3, 0x95, 0x38, 0x40, 0x1D, 0x85, 0x3C, -/*12F0*/0x38, 0xE5, 0x3C, 0x60, 0x05, 0x85, 0x3D, 0x39, - 0x80, 0x03, 0x85, 0x39, 0x39, 0x8F, 0x3A, 0x12, -/*1300*/0x07, 0xA8, 0xE5, 0x3C, 0x12, 0x07, 0x53, 0xE5, - 0x3D, 0xF0, 0x22, 0x12, 0x07, 0x9F, 0xE5, 0x38, -/*1310*/0x12, 0x07, 0x53, 0xE5, 0x39, 0xF0, 0x22, 0x8C, - 0x63, 0x8D, 0x64, 0x12, 0x08, 0xDA, 0x40, 0x3C, -/*1320*/0xE5, 0x64, 0x45, 0x63, 0x70, 0x10, 0x12, 0x09, - 0x04, 0xC3, 0xE5, 0x3E, 0x12, 0x07, 0x69, 0x40, -/*1330*/0x3B, 0x12, 0x08, 0x95, 0x80, 0x18, 0xE5, 0x3E, - 0xC3, 0x95, 0x38, 0x40, 0x1D, 0x85, 0x3E, 0x38, -/*1340*/0xE5, 0x3E, 0x60, 0x05, 0x85, 0x3F, 0x39, 0x80, - 0x03, 0x85, 0x39, 0x39, 0x8F, 0x3A, 0x12, 0x07, -/*1350*/0xA8, 0xE5, 0x3E, 0x12, 0x07, 0x53, 0xE5, 0x3F, - 0xF0, 0x22, 0x80, 0x3B, 0xE5, 0x64, 0x45, 0x63, -/*1360*/0x70, 0x11, 0x12, 0x07, 0x5F, 0x40, 0x05, 0x12, - 0x08, 0x9E, 0x80, 0x1F, 0x12, 0x07, 0x3E, 0xE5, -/*1370*/0x41, 0xF0, 0x22, 0xE5, 0x3C, 0xC3, 0x95, 0x38, - 0x40, 0x1D, 0x85, 0x3C, 0x38, 0xE5, 0x3C, 0x60, -/*1380*/0x05, 0x85, 0x3D, 0x39, 0x80, 0x03, 0x85, 0x39, - 0x39, 0x8F, 0x3A, 0x12, 0x07, 0xA8, 0xE5, 0x3C, -/*1390*/0x12, 0x07, 0x53, 0xE5, 0x3D, 0xF0, 0x22, 0x12, - 0x07, 0x9F, 0xE5, 0x38, 0x12, 0x07, 0x53, 0xE5, -/*13A0*/0x39, 0xF0, 0x22, 0xE5, 0x0D, 0xFE, 0xE5, 0x08, - 0x8E, 0x54, 0x44, 0x05, 0xF5, 0x55, 0x75, 0x15, -/*13B0*/0x0F, 0xF5, 0x82, 0x12, 0x0E, 0x7A, 0x12, 0x17, - 0xA3, 0x20, 0x31, 0x05, 0x75, 0x15, 0x03, 0x80, -/*13C0*/0x03, 0x75, 0x15, 0x0B, 0xE5, 0x0A, 0xC3, 0x94, - 0x01, 0x50, 0x38, 0x12, 0x14, 0x20, 0x20, 0x31, -/*13D0*/0x06, 0x05, 0x15, 0x05, 0x15, 0x80, 0x04, 0x15, - 0x15, 0x15, 0x15, 0xE5, 0x0A, 0xC3, 0x94, 0x01, -/*13E0*/0x50, 0x21, 0x12, 0x14, 0x20, 0x20, 0x31, 0x04, - 0x05, 0x15, 0x80, 0x02, 0x15, 0x15, 0xE5, 0x0A, -/*13F0*/0xC3, 0x94, 0x01, 0x50, 0x0E, 0x12, 0x0E, 0x77, - 0x12, 0x17, 0xA3, 0x20, 0x31, 0x05, 0x05, 0x15, -/*1400*/0x12, 0x0E, 0x77, 0xE5, 0x15, 0xB4, 0x08, 0x04, - 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x15, -/*1410*/0xB4, 0x07, 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, - 0x00, 0xEE, 0x4F, 0x60, 0x02, 0x05, 0x7F, 0x22, -/*1420*/0x85, 0x55, 0x82, 0x85, 0x54, 0x83, 0xE5, 0x15, - 0xF0, 0x12, 0x17, 0xA3, 0x22, 0x12, 0x07, 0x2A, -/*1430*/0x75, 0x83, 0xAE, 0x74, 0xFF, 0x12, 0x07, 0x29, - 0xE0, 0x54, 0x1A, 0xF5, 0x34, 0xE0, 0xC4, 0x13, -/*1440*/0x54, 0x07, 0xF5, 0x35, 0x24, 0xFE, 0x60, 0x24, - 0x24, 0xFE, 0x60, 0x3C, 0x24, 0x04, 0x70, 0x63, -/*1450*/0x75, 0x31, 0x2D, 0xE5, 0x08, 0xFD, 0x74, 0xB6, - 0x12, 0x07, 0x92, 0x74, 0xBC, 0x90, 0x07, 0x22, -/*1460*/0x12, 0x07, 0x95, 0x74, 0x90, 0x12, 0x07, 0xB3, - 0x74, 0x92, 0x80, 0x3C, 0x75, 0x31, 0x3A, 0xE5, -/*1470*/0x08, 0xFD, 0x74, 0xBA, 0x12, 0x07, 0x92, 0x74, - 0xC0, 0x90, 0x07, 0x22, 0x12, 0x07, 0xB6, 0x74, -/*1480*/0xC4, 0x12, 0x07, 0xB3, 0x74, 0xC8, 0x80, 0x20, - 0x75, 0x31, 0x35, 0xE5, 0x08, 0xFD, 0x74, 0xB8, -/*1490*/0x12, 0x07, 0x92, 0x74, 0xBE, 0xFF, 0xED, 0x44, - 0x07, 0x90, 0x07, 0x22, 0xCF, 0xF0, 0xA3, 0xEF, -/*14A0*/0xF0, 0x74, 0xC2, 0x12, 0x07, 0xB3, 0x74, 0xC6, - 0xFF, 0xED, 0x44, 0x07, 0xA3, 0xCF, 0xF0, 0xA3, -/*14B0*/0xEF, 0xF0, 0x22, 0x75, 0x34, 0x01, 0x22, 0x8E, - 0x58, 0x8F, 0x59, 0x8C, 0x5A, 0x8D, 0x5B, 0x8A, -/*14C0*/0x5C, 0x8B, 0x5D, 0x75, 0x5E, 0x01, 0xE4, 0xF5, - 0x5F, 0x12, 0x1E, 0xA5, 0x85, 0x59, 0x5E, 0xD3, -/*14D0*/0xE5, 0x5E, 0x95, 0x5B, 0xE5, 0x5A, 0x12, 0x07, - 0x6B, 0x50, 0x57, 0xE5, 0x5D, 0x45, 0x5C, 0x70, -/*14E0*/0x30, 0x12, 0x07, 0x2A, 0x75, 0x83, 0x92, 0xE5, - 0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC6, 0xE5, -/*14F0*/0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC8, 0xE5, - 0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0x90, 0xE5, -/*1500*/0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2, 0xE5, - 0x5E, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0x80, -/*1510*/0x03, 0x12, 0x07, 0x32, 0xE5, 0x5E, 0xF0, 0xAF, - 0x5F, 0x7E, 0x00, 0xAD, 0x5D, 0xAC, 0x5C, 0x12, -/*1520*/0x04, 0x44, 0xAF, 0x5E, 0x7E, 0x00, 0xAD, 0x5D, - 0xAC, 0x5C, 0x12, 0x0B, 0xD1, 0x05, 0x5E, 0x02, -/*1530*/0x14, 0xCF, 0xAB, 0x5D, 0xAA, 0x5C, 0xAD, 0x5B, - 0xAC, 0x5A, 0xAF, 0x59, 0xAE, 0x58, 0x02, 0x1B, -/*1540*/0xFB, 0x8C, 0x5C, 0x8D, 0x5D, 0x8A, 0x5E, 0x8B, - 0x5F, 0x75, 0x60, 0x01, 0xE4, 0xF5, 0x61, 0xF5, -/*1550*/0x62, 0xF5, 0x63, 0x12, 0x1E, 0xA5, 0x8F, 0x60, - 0xD3, 0xE5, 0x60, 0x95, 0x5D, 0xE5, 0x5C, 0x12, -/*1560*/0x07, 0x6B, 0x50, 0x61, 0xE5, 0x5F, 0x45, 0x5E, - 0x70, 0x27, 0x12, 0x07, 0x2A, 0x75, 0x83, 0xB6, -/*1570*/0xE5, 0x60, 0x12, 0x07, 0x29, 0x75, 0x83, 0xB8, - 0xE5, 0x60, 0x12, 0x07, 0x29, 0x75, 0x83, 0xBA, -/*1580*/0xE5, 0x60, 0xF0, 0xAF, 0x61, 0x7E, 0x00, 0xE5, - 0x62, 0x12, 0x08, 0x7A, 0x12, 0x0A, 0xFF, 0x80, -/*1590*/0x19, 0x90, 0x07, 0x24, 0x12, 0x07, 0x35, 0xE5, - 0x60, 0x12, 0x07, 0x29, 0x75, 0x83, 0x8E, 0xE4, -/*15A0*/0x12, 0x07, 0x29, 0x74, 0x01, 0x12, 0x07, 0x29, - 0xE4, 0xF0, 0xAF, 0x63, 0x7E, 0x00, 0xAD, 0x5F, -/*15B0*/0xAC, 0x5E, 0x12, 0x04, 0x44, 0xAF, 0x60, 0x7E, - 0x00, 0xAD, 0x5F, 0xAC, 0x5E, 0x12, 0x12, 0x8B, -/*15C0*/0x05, 0x60, 0x02, 0x15, 0x58, 0x22, 0x90, 0x11, - 0x4D, 0xE4, 0x93, 0x90, 0x07, 0x2E, 0xF0, 0x12, -/*15D0*/0x08, 0x1F, 0x75, 0x83, 0xAE, 0xE0, 0x54, 0x1A, - 0xF5, 0x34, 0x70, 0x67, 0xEF, 0x44, 0x07, 0xF5, -/*15E0*/0x82, 0x75, 0x83, 0xCE, 0xE0, 0xFF, 0x13, 0x13, - 0x13, 0x54, 0x07, 0xF5, 0x36, 0x54, 0x0F, 0xD3, -/*15F0*/0x94, 0x00, 0x40, 0x06, 0x12, 0x14, 0x2D, 0x12, - 0x1B, 0xA9, 0xE5, 0x36, 0x54, 0x0F, 0x24, 0xFE, -/*1600*/0x60, 0x0C, 0x14, 0x60, 0x0C, 0x14, 0x60, 0x19, - 0x24, 0x03, 0x70, 0x37, 0x80, 0x10, 0x02, 0x1E, -/*1610*/0x91, 0x12, 0x1E, 0x91, 0x12, 0x07, 0x2A, 0x75, - 0x83, 0xCE, 0xE0, 0x54, 0xEF, 0xF0, 0x02, 0x1D, -/*1620*/0xAE, 0x12, 0x10, 0x14, 0xE4, 0xF5, 0x55, 0x12, - 0x1D, 0x85, 0x05, 0x55, 0xE5, 0x55, 0xC3, 0x94, -/*1630*/0x05, 0x40, 0xF4, 0x12, 0x07, 0x2A, 0x75, 0x83, - 0xCE, 0xE0, 0x54, 0xC7, 0x12, 0x07, 0x29, 0xE0, -/*1640*/0x44, 0x08, 0xF0, 0x22, 0xE4, 0xF5, 0x58, 0xF5, - 0x59, 0xAF, 0x08, 0xEF, 0x44, 0x07, 0xF5, 0x82, -/*1650*/0x75, 0x83, 0xD0, 0xE0, 0xFD, 0xC4, 0x54, 0x0F, - 0xF5, 0x5A, 0xEF, 0x44, 0x07, 0xF5, 0x82, 0x75, -/*1660*/0x83, 0x80, 0x74, 0x01, 0xF0, 0x12, 0x08, 0x21, - 0x75, 0x83, 0x82, 0xE5, 0x45, 0xF0, 0xEF, 0x44, -/*1670*/0x07, 0xF5, 0x82, 0x75, 0x83, 0x8A, 0x74, 0xFF, - 0xF0, 0x12, 0x1A, 0x4D, 0x12, 0x07, 0x2A, 0x75, -/*1680*/0x83, 0xBC, 0xE0, 0x54, 0xEF, 0x12, 0x07, 0x29, - 0x75, 0x83, 0xBE, 0xE0, 0x54, 0xEF, 0x12, 0x07, -/*1690*/0x29, 0x75, 0x83, 0xC0, 0xE0, 0x54, 0xEF, 0x12, - 0x07, 0x29, 0x75, 0x83, 0xBC, 0xE0, 0x44, 0x10, -/*16A0*/0x12, 0x07, 0x29, 0x75, 0x83, 0xBE, 0xE0, 0x44, - 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC0, 0xE0, -/*16B0*/0x44, 0x10, 0xF0, 0xAF, 0x58, 0xE5, 0x59, 0x12, - 0x08, 0x78, 0x02, 0x0A, 0xFF, 0xE4, 0xF5, 0x58, -/*16C0*/0x7D, 0x01, 0xF5, 0x59, 0xAF, 0x35, 0xFE, 0xFC, - 0x12, 0x09, 0x15, 0x12, 0x07, 0x2A, 0x75, 0x83, -/*16D0*/0xB6, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, - 0xB8, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, -/*16E0*/0xBA, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, - 0xBC, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, -/*16F0*/0xBE, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, - 0xC0, 0x74, 0x10, 0x12, 0x07, 0x29, 0x75, 0x83, -/*1700*/0x90, 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC2, - 0xE4, 0x12, 0x07, 0x29, 0x75, 0x83, 0xC4, 0xE4, -/*1710*/0x12, 0x07, 0x29, 0x75, 0x83, 0x92, 0xE4, 0x12, - 0x07, 0x29, 0x75, 0x83, 0xC6, 0xE4, 0x12, 0x07, -/*1720*/0x29, 0x75, 0x83, 0xC8, 0xE4, 0xF0, 0xAF, 0x58, - 0xFE, 0xE5, 0x59, 0x12, 0x08, 0x7A, 0x02, 0x0A, -/*1730*/0xFF, 0xE5, 0xE2, 0x30, 0xE4, 0x6C, 0xE5, 0xE7, - 0x54, 0xC0, 0x64, 0x40, 0x70, 0x64, 0xE5, 0x09, -/*1740*/0xC4, 0x54, 0x30, 0xFE, 0xE5, 0x08, 0x25, 0xE0, - 0x25, 0xE0, 0x54, 0xC0, 0x4E, 0xFE, 0xEF, 0x54, -/*1750*/0x3F, 0x4E, 0xFD, 0xE5, 0x2B, 0xAE, 0x2A, 0x78, - 0x02, 0xC3, 0x33, 0xCE, 0x33, 0xCE, 0xD8, 0xF9, -/*1760*/0xF5, 0x82, 0x8E, 0x83, 0xED, 0xF0, 0xE5, 0x2B, - 0xAE, 0x2A, 0x78, 0x02, 0xC3, 0x33, 0xCE, 0x33, -/*1770*/0xCE, 0xD8, 0xF9, 0xFF, 0xF5, 0x82, 0x8E, 0x83, - 0xA3, 0xE5, 0xFE, 0xF0, 0x8F, 0x82, 0x8E, 0x83, -/*1780*/0xA3, 0xA3, 0xE5, 0xFD, 0xF0, 0x8F, 0x82, 0x8E, - 0x83, 0xA3, 0xA3, 0xA3, 0xE5, 0xFC, 0xF0, 0xC3, -/*1790*/0xE5, 0x2B, 0x94, 0xFA, 0xE5, 0x2A, 0x94, 0x00, - 0x50, 0x08, 0x05, 0x2B, 0xE5, 0x2B, 0x70, 0x02, -/*17A0*/0x05, 0x2A, 0x22, 0xE4, 0xFF, 0xE4, 0xF5, 0x58, - 0xF5, 0x56, 0xF5, 0x57, 0x74, 0x82, 0xFC, 0x12, -/*17B0*/0x0E, 0x04, 0x8C, 0x83, 0xE0, 0xF5, 0x10, 0x54, - 0x7F, 0xF0, 0xE5, 0x10, 0x44, 0x80, 0x12, 0x0E, -/*17C0*/0x98, 0xED, 0xF0, 0x7E, 0x0A, 0x12, 0x0E, 0x04, - 0x75, 0x83, 0xA0, 0xE0, 0x20, 0xE0, 0x26, 0xDE, -/*17D0*/0xF4, 0x05, 0x57, 0xE5, 0x57, 0x70, 0x02, 0x05, - 0x56, 0xE5, 0x14, 0x24, 0x01, 0xFD, 0xE4, 0x33, -/*17E0*/0xFC, 0xD3, 0xE5, 0x57, 0x9D, 0xE5, 0x56, 0x9C, - 0x40, 0xD9, 0xE5, 0x0A, 0x94, 0x20, 0x50, 0x02, -/*17F0*/0x05, 0x0A, 0x43, 0xE1, 0x08, 0xC2, 0x31, 0x12, - 0x0E, 0x04, 0x75, 0x83, 0xA6, 0xE0, 0x55, 0x12, -/*1800*/0x65, 0x12, 0x70, 0x03, 0xD2, 0x31, 0x22, 0xC2, - 0x31, 0x22, 0x90, 0x07, 0x26, 0xE0, 0xFA, 0xA3, -/*1810*/0xE0, 0xF5, 0x82, 0x8A, 0x83, 0xE0, 0xF5, 0x41, - 0xE5, 0x39, 0xC3, 0x95, 0x41, 0x40, 0x26, 0xE5, -/*1820*/0x39, 0x95, 0x41, 0xC3, 0x9F, 0xEE, 0x12, 0x07, - 0x6B, 0x40, 0x04, 0x7C, 0x01, 0x80, 0x02, 0x7C, -/*1830*/0x00, 0xE5, 0x41, 0x64, 0x3F, 0x60, 0x04, 0x7B, - 0x01, 0x80, 0x02, 0x7B, 0x00, 0xEC, 0x5B, 0x60, -/*1840*/0x29, 0x05, 0x41, 0x80, 0x28, 0xC3, 0xE5, 0x41, - 0x95, 0x39, 0xC3, 0x9F, 0xEE, 0x12, 0x07, 0x6B, -/*1850*/0x40, 0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, - 0xE5, 0x41, 0x60, 0x04, 0x7E, 0x01, 0x80, 0x02, -/*1860*/0x7E, 0x00, 0xEF, 0x5E, 0x60, 0x04, 0x15, 0x41, - 0x80, 0x03, 0x85, 0x39, 0x41, 0x85, 0x3A, 0x40, -/*1870*/0x22, 0xE5, 0xE2, 0x30, 0xE4, 0x60, 0xE5, 0xE1, - 0x30, 0xE2, 0x5B, 0xE5, 0x09, 0x70, 0x04, 0x7F, -/*1880*/0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, 0x08, 0x70, - 0x04, 0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, -/*1890*/0x5F, 0x60, 0x43, 0x53, 0xF9, 0xF8, 0xE5, 0xE2, - 0x30, 0xE4, 0x3B, 0xE5, 0xE1, 0x30, 0xE2, 0x2E, -/*18A0*/0x43, 0xFA, 0x02, 0x53, 0xFA, 0xFB, 0xE4, 0xF5, - 0x10, 0x90, 0x94, 0x70, 0xE5, 0x10, 0xF0, 0xE5, -/*18B0*/0xE1, 0x30, 0xE2, 0xE7, 0x90, 0x94, 0x70, 0xE0, - 0x65, 0x10, 0x60, 0x03, 0x43, 0xFA, 0x04, 0x05, -/*18C0*/0x10, 0x90, 0x94, 0x70, 0xE5, 0x10, 0xF0, 0x70, - 0xE6, 0x12, 0x00, 0x06, 0x80, 0xE1, 0x53, 0xFA, -/*18D0*/0xFD, 0x53, 0xFA, 0xFB, 0x80, 0xC0, 0x22, 0x8F, - 0x54, 0x12, 0x00, 0x06, 0xE5, 0xE1, 0x30, 0xE0, -/*18E0*/0x04, 0x7F, 0x01, 0x80, 0x02, 0x7F, 0x00, 0xE5, - 0x7E, 0xD3, 0x94, 0x05, 0x40, 0x04, 0x7E, 0x01, -/*18F0*/0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F, 0x60, 0x3D, - 0x85, 0x54, 0x11, 0xE5, 0xE2, 0x20, 0xE1, 0x32, -/*1900*/0x74, 0xCE, 0x12, 0x1A, 0x05, 0x30, 0xE7, 0x04, - 0x7D, 0x01, 0x80, 0x02, 0x7D, 0x00, 0x8F, 0x82, -/*1910*/0x8E, 0x83, 0xE0, 0x30, 0xE6, 0x04, 0x7F, 0x01, - 0x80, 0x02, 0x7F, 0x00, 0xEF, 0x5D, 0x70, 0x15, -/*1920*/0x12, 0x15, 0xC6, 0x74, 0xCE, 0x12, 0x1A, 0x05, - 0x30, 0xE6, 0x07, 0xE0, 0x44, 0x80, 0xF0, 0x43, -/*1930*/0xF9, 0x80, 0x12, 0x18, 0x71, 0x22, 0x12, 0x0E, - 0x44, 0xE5, 0x16, 0x25, 0xE0, 0x25, 0xE0, 0x24, -/*1940*/0xB0, 0xF5, 0x82, 0xE4, 0x34, 0x1A, 0xF5, 0x83, - 0xE4, 0x93, 0xF5, 0x0F, 0xE5, 0x16, 0x25, 0xE0, -/*1950*/0x25, 0xE0, 0x24, 0xB1, 0xF5, 0x82, 0xE4, 0x34, - 0x1A, 0xF5, 0x83, 0xE4, 0x93, 0xF5, 0x0E, 0x12, -/*1960*/0x0E, 0x65, 0xF5, 0x10, 0xE5, 0x0F, 0x54, 0xF0, - 0x12, 0x0E, 0x17, 0x75, 0x83, 0x8C, 0xEF, 0xF0, -/*1970*/0xE5, 0x0F, 0x30, 0xE0, 0x0C, 0x12, 0x0E, 0x04, - 0x75, 0x83, 0x86, 0xE0, 0x44, 0x40, 0xF0, 0x80, -/*1980*/0x0A, 0x12, 0x0E, 0x04, 0x75, 0x83, 0x86, 0xE0, - 0x54, 0xBF, 0xF0, 0x12, 0x0E, 0x91, 0x75, 0x83, -/*1990*/0x82, 0xE5, 0x0E, 0xF0, 0x22, 0x7F, 0x05, 0x12, - 0x17, 0x31, 0x12, 0x0E, 0x04, 0x12, 0x0E, 0x33, -/*19A0*/0x74, 0x02, 0xF0, 0x74, 0x8E, 0xFE, 0x12, 0x0E, - 0x04, 0x12, 0x0E, 0x0B, 0xEF, 0xF0, 0x75, 0x15, -/*19B0*/0x70, 0x12, 0x0F, 0xF7, 0x20, 0x34, 0x05, 0x75, - 0x15, 0x10, 0x80, 0x03, 0x75, 0x15, 0x50, 0x12, -/*19C0*/0x0F, 0xF7, 0x20, 0x34, 0x04, 0x74, 0x10, 0x80, - 0x02, 0x74, 0xF0, 0x25, 0x15, 0xF5, 0x15, 0x12, -/*19D0*/0x0E, 0x21, 0xEF, 0xF0, 0x12, 0x10, 0x91, 0x20, - 0x34, 0x17, 0xE5, 0x15, 0x64, 0x30, 0x60, 0x0C, -/*19E0*/0x74, 0x10, 0x25, 0x15, 0xF5, 0x15, 0xB4, 0x80, - 0x03, 0xE4, 0xF5, 0x15, 0x12, 0x0E, 0x21, 0xEF, -/*19F0*/0xF0, 0x22, 0xF0, 0xE5, 0x0B, 0x25, 0xE0, 0x25, - 0xE0, 0x24, 0x82, 0xF5, 0x82, 0xE4, 0x34, 0x07, -/*1A00*/0xF5, 0x83, 0x22, 0x74, 0x88, 0xFE, 0xE5, 0x08, - 0x44, 0x07, 0xFF, 0xF5, 0x82, 0x8E, 0x83, 0xE0, -/*1A10*/0x22, 0xF0, 0xE5, 0x08, 0x44, 0x07, 0xF5, 0x82, - 0x22, 0xF0, 0xE0, 0x54, 0xC0, 0x8F, 0x82, 0x8E, -/*1A20*/0x83, 0xF0, 0x22, 0xEF, 0x44, 0x07, 0xF5, 0x82, - 0x75, 0x83, 0x86, 0xE0, 0x54, 0x10, 0xD3, 0x94, -/*1A30*/0x00, 0x22, 0xF0, 0x90, 0x07, 0x15, 0xE0, 0x04, - 0xF0, 0x22, 0x44, 0x06, 0xF5, 0x82, 0x75, 0x83, -/*1A40*/0x9E, 0xE0, 0x22, 0xFE, 0xEF, 0x44, 0x07, 0xF5, - 0x82, 0x8E, 0x83, 0xE0, 0x22, 0xE4, 0x90, 0x07, -/*1A50*/0x2A, 0xF0, 0xA3, 0xF0, 0x12, 0x07, 0x2A, 0x75, - 0x83, 0x82, 0xE0, 0x54, 0x7F, 0x12, 0x07, 0x29, -/*1A60*/0xE0, 0x44, 0x80, 0xF0, 0x12, 0x10, 0xFC, 0x12, - 0x08, 0x1F, 0x75, 0x83, 0xA0, 0xE0, 0x20, 0xE0, -/*1A70*/0x1A, 0x90, 0x07, 0x2B, 0xE0, 0x04, 0xF0, 0x70, - 0x06, 0x90, 0x07, 0x2A, 0xE0, 0x04, 0xF0, 0x90, -/*1A80*/0x07, 0x2A, 0xE0, 0xB4, 0x10, 0xE1, 0xA3, 0xE0, - 0xB4, 0x00, 0xDC, 0xEE, 0x44, 0xA6, 0xFC, 0xEF, -/*1A90*/0x44, 0x07, 0xF5, 0x82, 0x8C, 0x83, 0xE0, 0xF5, - 0x32, 0xEE, 0x44, 0xA8, 0xFE, 0xEF, 0x44, 0x07, -/*1AA0*/0xF5, 0x82, 0x8E, 0x83, 0xE0, 0xF5, 0x33, 0x22, - 0x01, 0x20, 0x11, 0x00, 0x04, 0x20, 0x00, 0x90, -/*1AB0*/0x00, 0x20, 0x0F, 0x92, 0x00, 0x21, 0x0F, 0x94, - 0x00, 0x22, 0x0F, 0x96, 0x00, 0x23, 0x0F, 0x98, -/*1AC0*/0x00, 0x24, 0x0F, 0x9A, 0x00, 0x25, 0x0F, 0x9C, - 0x00, 0x26, 0x0F, 0x9E, 0x00, 0x27, 0x0F, 0xA0, -/*1AD0*/0x01, 0x20, 0x01, 0xA2, 0x01, 0x21, 0x01, 0xA4, - 0x01, 0x22, 0x01, 0xA6, 0x01, 0x23, 0x01, 0xA8, -/*1AE0*/0x01, 0x24, 0x01, 0xAA, 0x01, 0x25, 0x01, 0xAC, - 0x01, 0x26, 0x01, 0xAE, 0x01, 0x27, 0x01, 0xB0, -/*1AF0*/0x01, 0x28, 0x01, 0xB4, 0x00, 0x28, 0x0F, 0xB6, - 0x40, 0x28, 0x0F, 0xB8, 0x61, 0x28, 0x01, 0xCB, -/*1B00*/0xEF, 0xCB, 0xCA, 0xEE, 0xCA, 0x7F, 0x01, 0xE4, - 0xFD, 0xEB, 0x4A, 0x70, 0x24, 0xE5, 0x08, 0xF5, -/*1B10*/0x82, 0x74, 0xB6, 0x12, 0x08, 0x29, 0xE5, 0x08, - 0xF5, 0x82, 0x74, 0xB8, 0x12, 0x08, 0x29, 0xE5, -/*1B20*/0x08, 0xF5, 0x82, 0x74, 0xBA, 0x12, 0x08, 0x29, - 0x7E, 0x00, 0x7C, 0x00, 0x12, 0x0A, 0xFF, 0x80, -/*1B30*/0x12, 0x90, 0x07, 0x26, 0x12, 0x07, 0x35, 0xE5, - 0x41, 0xF0, 0x90, 0x07, 0x24, 0x12, 0x07, 0x35, -/*1B40*/0xE5, 0x40, 0xF0, 0x12, 0x07, 0x2A, 0x75, 0x83, - 0x8E, 0xE4, 0x12, 0x07, 0x29, 0x74, 0x01, 0x12, -/*1B50*/0x07, 0x29, 0xE4, 0xF0, 0x22, 0xE4, 0xF5, 0x26, - 0xF5, 0x27, 0x53, 0xE1, 0xFE, 0xF5, 0x2A, 0x75, -/*1B60*/0x2B, 0x01, 0xF5, 0x08, 0x7F, 0x01, 0x12, 0x17, - 0x31, 0x30, 0x30, 0x1C, 0x90, 0x1A, 0xA9, 0xE4, -/*1B70*/0x93, 0xF5, 0x10, 0x90, 0x1F, 0xF9, 0xE4, 0x93, - 0xF5, 0x10, 0x90, 0x00, 0x41, 0xE4, 0x93, 0xF5, -/*1B80*/0x10, 0x90, 0x1E, 0xCA, 0xE4, 0x93, 0xF5, 0x10, - 0x7F, 0x02, 0x12, 0x17, 0x31, 0x12, 0x0F, 0x54, -/*1B90*/0x7F, 0x03, 0x12, 0x17, 0x31, 0x12, 0x00, 0x06, - 0xE5, 0xE2, 0x30, 0xE7, 0x09, 0x12, 0x10, 0x00, -/*1BA0*/0x30, 0x30, 0x03, 0x12, 0x11, 0x00, 0x02, 0x00, - 0x47, 0x12, 0x08, 0x1F, 0x75, 0x83, 0xD0, 0xE0, -/*1BB0*/0xC4, 0x54, 0x0F, 0xFD, 0x75, 0x43, 0x01, 0x75, - 0x44, 0xFF, 0x12, 0x08, 0xAA, 0x74, 0x04, 0xF0, -/*1BC0*/0x75, 0x3B, 0x01, 0xED, 0x14, 0x60, 0x0C, 0x14, - 0x60, 0x0B, 0x14, 0x60, 0x0F, 0x24, 0x03, 0x70, -/*1BD0*/0x0B, 0x80, 0x09, 0x80, 0x00, 0x12, 0x08, 0xA7, - 0x04, 0xF0, 0x80, 0x06, 0x12, 0x08, 0xA7, 0x74, -/*1BE0*/0x04, 0xF0, 0xEE, 0x44, 0x82, 0xFE, 0xEF, 0x44, - 0x07, 0xF5, 0x82, 0x8E, 0x83, 0xE5, 0x45, 0x12, -/*1BF0*/0x08, 0xBE, 0x75, 0x83, 0x82, 0xE5, 0x31, 0xF0, - 0x02, 0x11, 0x4C, 0x8E, 0x60, 0x8F, 0x61, 0x12, -/*1C00*/0x1E, 0xA5, 0xE4, 0xFF, 0xCE, 0xED, 0xCE, 0xEE, - 0xD3, 0x95, 0x61, 0xE5, 0x60, 0x12, 0x07, 0x6B, -/*1C10*/0x40, 0x39, 0x74, 0x20, 0x2E, 0xF5, 0x82, 0xE4, - 0x34, 0x03, 0xF5, 0x83, 0xE0, 0x70, 0x03, 0xFF, -/*1C20*/0x80, 0x26, 0x12, 0x08, 0xE2, 0xFD, 0xC3, 0x9F, - 0x40, 0x1E, 0xCF, 0xED, 0xCF, 0xEB, 0x4A, 0x70, -/*1C30*/0x0B, 0x8D, 0x42, 0x12, 0x08, 0xEE, 0xF5, 0x41, - 0x8E, 0x40, 0x80, 0x0C, 0x12, 0x08, 0xE2, 0xF5, -/*1C40*/0x38, 0x12, 0x08, 0xEE, 0xF5, 0x39, 0x8E, 0x3A, - 0x1E, 0x80, 0xBC, 0x22, 0x75, 0x58, 0x01, 0xE5, -/*1C50*/0x35, 0x70, 0x0C, 0x12, 0x07, 0xCC, 0xE0, 0xF5, - 0x4A, 0x12, 0x07, 0xD8, 0xE0, 0xF5, 0x4C, 0xE5, -/*1C60*/0x35, 0xB4, 0x04, 0x0C, 0x12, 0x07, 0xE4, 0xE0, - 0xF5, 0x4A, 0x12, 0x07, 0xF0, 0xE0, 0xF5, 0x4C, -/*1C70*/0xE5, 0x35, 0xB4, 0x01, 0x04, 0x7F, 0x01, 0x80, - 0x02, 0x7F, 0x00, 0xE5, 0x35, 0xB4, 0x02, 0x04, -/*1C80*/0x7E, 0x01, 0x80, 0x02, 0x7E, 0x00, 0xEE, 0x4F, - 0x60, 0x0C, 0x12, 0x07, 0xFC, 0xE0, 0xF5, 0x4A, -/*1C90*/0x12, 0x08, 0x08, 0xE0, 0xF5, 0x4C, 0x85, 0x41, - 0x49, 0x85, 0x40, 0x4B, 0x22, 0x75, 0x5B, 0x01, -/*1CA0*/0x90, 0x07, 0x24, 0x12, 0x07, 0x35, 0xE0, 0x54, - 0x1F, 0xFF, 0xD3, 0x94, 0x02, 0x50, 0x04, 0x8F, -/*1CB0*/0x58, 0x80, 0x05, 0xEF, 0x24, 0xFE, 0xF5, 0x58, - 0xEF, 0xC3, 0x94, 0x18, 0x40, 0x05, 0x75, 0x59, -/*1CC0*/0x18, 0x80, 0x04, 0xEF, 0x04, 0xF5, 0x59, 0x85, - 0x43, 0x5A, 0xAF, 0x58, 0x7E, 0x00, 0xAD, 0x59, -/*1CD0*/0x7C, 0x00, 0xAB, 0x5B, 0x7A, 0x00, 0x12, 0x15, - 0x41, 0xAF, 0x5A, 0x7E, 0x00, 0x12, 0x18, 0x0A, -/*1CE0*/0xAF, 0x5B, 0x7E, 0x00, 0x02, 0x1A, 0xFF, 0xE5, - 0xE2, 0x30, 0xE7, 0x0E, 0x12, 0x10, 0x03, 0xC2, -/*1CF0*/0x30, 0x30, 0x30, 0x03, 0x12, 0x10, 0xFF, 0x20, - 0x33, 0x28, 0xE5, 0xE7, 0x30, 0xE7, 0x05, 0x12, -/*1D00*/0x0E, 0xA2, 0x80, 0x0D, 0xE5, 0xFE, 0xC3, 0x94, - 0x20, 0x50, 0x06, 0x12, 0x0E, 0xA2, 0x43, 0xF9, -/*1D10*/0x08, 0xE5, 0xF2, 0x30, 0xE7, 0x03, 0x53, 0xF9, - 0x7F, 0xE5, 0xF1, 0x54, 0x70, 0xD3, 0x94, 0x00, -/*1D20*/0x50, 0xD8, 0x22, 0x12, 0x0E, 0x04, 0x75, 0x83, - 0x80, 0xE4, 0xF0, 0xE5, 0x08, 0x44, 0x07, 0x12, -/*1D30*/0x0D, 0xFD, 0x75, 0x83, 0x84, 0x12, 0x0E, 0x02, - 0x75, 0x83, 0x86, 0x12, 0x0E, 0x02, 0x75, 0x83, -/*1D40*/0x8C, 0xE0, 0x54, 0xF3, 0x12, 0x0E, 0x03, 0x75, - 0x83, 0x8E, 0x12, 0x0E, 0x02, 0x75, 0x83, 0x94, -/*1D50*/0xE0, 0x54, 0xFB, 0xF0, 0x22, 0x12, 0x07, 0x2A, - 0x75, 0x83, 0x8E, 0xE4, 0x12, 0x07, 0x29, 0x74, -/*1D60*/0x01, 0x12, 0x07, 0x29, 0xE4, 0x12, 0x08, 0xBE, - 0x75, 0x83, 0x8C, 0xE0, 0x44, 0x20, 0x12, 0x08, -/*1D70*/0xBE, 0xE0, 0x54, 0xDF, 0xF0, 0x74, 0x84, 0x85, - 0x08, 0x82, 0xF5, 0x83, 0xE0, 0x54, 0x7F, 0xF0, -/*1D80*/0xE0, 0x44, 0x80, 0xF0, 0x22, 0x75, 0x56, 0x01, - 0xE4, 0xFD, 0xF5, 0x57, 0xAF, 0x35, 0xFE, 0xFC, -/*1D90*/0x12, 0x09, 0x15, 0x12, 0x1C, 0x9D, 0x12, 0x1E, - 0x7A, 0x12, 0x1C, 0x4C, 0xAF, 0x57, 0x7E, 0x00, -/*1DA0*/0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x44, 0xAF, - 0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE, 0x75, 0x56, -/*1DB0*/0x01, 0xE4, 0xFD, 0xF5, 0x57, 0xAF, 0x35, 0xFE, - 0xFC, 0x12, 0x09, 0x15, 0x12, 0x1C, 0x9D, 0x12, -/*1DC0*/0x1E, 0x7A, 0x12, 0x1C, 0x4C, 0xAF, 0x57, 0x7E, - 0x00, 0xAD, 0x56, 0x7C, 0x00, 0x12, 0x04, 0x44, -/*1DD0*/0xAF, 0x56, 0x7E, 0x00, 0x02, 0x11, 0xEE, 0xE4, - 0xF5, 0x16, 0x12, 0x0E, 0x44, 0xFE, 0xE5, 0x08, -/*1DE0*/0x44, 0x05, 0xFF, 0x12, 0x0E, 0x65, 0x8F, 0x82, - 0x8E, 0x83, 0xF0, 0x05, 0x16, 0xE5, 0x16, 0xC3, -/*1DF0*/0x94, 0x14, 0x40, 0xE6, 0xE5, 0x08, 0x12, 0x0E, - 0x2B, 0xE4, 0xF0, 0x22, 0xE4, 0xF5, 0x58, 0xF5, -/*1E00*/0x59, 0xF5, 0x5A, 0xFF, 0xFE, 0xAD, 0x58, 0xFC, - 0x12, 0x09, 0x15, 0x7F, 0x04, 0x7E, 0x00, 0xAD, -/*1E10*/0x58, 0x7C, 0x00, 0x12, 0x09, 0x15, 0x7F, 0x02, - 0x7E, 0x00, 0xAD, 0x58, 0x7C, 0x00, 0x02, 0x09, -/*1E20*/0x15, 0xE5, 0x3C, 0x25, 0x3E, 0xFC, 0xE5, 0x42, - 0x24, 0x00, 0xFB, 0xE4, 0x33, 0xFA, 0xEC, 0xC3, -/*1E30*/0x9B, 0xEA, 0x12, 0x07, 0x6B, 0x40, 0x0B, 0x8C, - 0x42, 0xE5, 0x3D, 0x25, 0x3F, 0xF5, 0x41, 0x8F, -/*1E40*/0x40, 0x22, 0x12, 0x09, 0x0B, 0x22, 0x74, 0x84, - 0xF5, 0x18, 0x85, 0x08, 0x19, 0x85, 0x19, 0x82, -/*1E50*/0x85, 0x18, 0x83, 0xE0, 0x54, 0x7F, 0xF0, 0xE0, - 0x44, 0x80, 0xF0, 0xE0, 0x44, 0x80, 0xF0, 0x22, -/*1E60*/0xEF, 0x4E, 0x70, 0x0B, 0x12, 0x07, 0x2A, 0x75, - 0x83, 0xD2, 0xE0, 0x54, 0xDF, 0xF0, 0x22, 0x12, -/*1E70*/0x07, 0x2A, 0x75, 0x83, 0xD2, 0xE0, 0x44, 0x20, - 0xF0, 0x22, 0x75, 0x58, 0x01, 0x90, 0x07, 0x26, -/*1E80*/0x12, 0x07, 0x35, 0xE0, 0x54, 0x3F, 0xF5, 0x41, - 0x12, 0x07, 0x32, 0xE0, 0x54, 0x3F, 0xF5, 0x40, -/*1E90*/0x22, 0x75, 0x56, 0x02, 0xE4, 0xF5, 0x57, 0x12, - 0x1D, 0xFC, 0xAF, 0x57, 0x7E, 0x00, 0xAD, 0x56, -/*1EA0*/0x7C, 0x00, 0x02, 0x04, 0x44, 0xE4, 0xF5, 0x42, - 0xF5, 0x41, 0xF5, 0x40, 0xF5, 0x38, 0xF5, 0x39, -/*1EB0*/0xF5, 0x3A, 0x22, 0xEF, 0x54, 0x07, 0xFF, 0xE5, - 0xF9, 0x54, 0xF8, 0x4F, 0xF5, 0xF9, 0x22, 0x7F, -/*1EC0*/0x01, 0xE4, 0xFE, 0x0F, 0x0E, 0xBE, 0xFF, 0xFB, - 0x22, 0x01, 0x20, 0x00, 0x01, 0x04, 0x20, 0x00, -/*1ED0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1EE0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1EF0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F00*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F10*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F20*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F30*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F40*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F50*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F60*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F70*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F80*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1F90*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1FA0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1FB0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1FC0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1FD0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1FE0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -/*1FF0*/0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x20, 0x11, 0x00, 0x04, 0x20, 0x00, 0x81 -}; - -int qib_sd7220_ib_load(struct qib_devdata *dd) -{ - return qib_sd7220_prog_ld(dd, IB_7220_SERDES, qib_sd7220_ib_img, - sizeof(qib_sd7220_ib_img), 0); -} - -int qib_sd7220_ib_vfy(struct qib_devdata *dd) -{ - return qib_sd7220_prog_vfy(dd, IB_7220_SERDES, qib_sd7220_ib_img, - sizeof(qib_sd7220_ib_img), 0); -} diff --git a/drivers/infiniband/hw/qib/qib_sdma.c b/drivers/infiniband/hw/qib/qib_sdma.c index b8456881f7f6..cad44491320b 100644 --- a/drivers/infiniband/hw/qib/qib_sdma.c +++ b/drivers/infiniband/hw/qib/qib_sdma.c @@ -656,6 +656,7 @@ unmap: } qp = tx->qp; qib_put_txreq(tx); + spin_lock(&qp->r_lock); spin_lock(&qp->s_lock); if (qp->ibqp.qp_type == IB_QPT_RC) { /* XXX what about error sending RDMA read responses? */ @@ -664,6 +665,7 @@ unmap: } else if (qp->s_wqe) qib_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR); spin_unlock(&qp->s_lock); + spin_unlock(&qp->r_lock); /* return zero to process the next send work request */ goto unlock; diff --git a/drivers/infiniband/hw/qib/qib_sysfs.c b/drivers/infiniband/hw/qib/qib_sysfs.c index dab4d9f4a2cc..d50a33fe8bbc 100644 --- a/drivers/infiniband/hw/qib/qib_sysfs.c +++ b/drivers/infiniband/hw/qib/qib_sysfs.c @@ -347,7 +347,7 @@ static struct kobj_type qib_sl2vl_ktype = { #define QIB_DIAGC_ATTR(N) \ static struct qib_diagc_attr qib_diagc_attr_##N = { \ - .attr = { .name = __stringify(N), .mode = 0444 }, \ + .attr = { .name = __stringify(N), .mode = 0664 }, \ .counter = offsetof(struct qib_ibport, n_##N) \ } @@ -403,8 +403,27 @@ static ssize_t diagc_attr_show(struct kobject *kobj, struct attribute *attr, return sprintf(buf, "%u\n", *(u32 *)((char *)qibp + dattr->counter)); } +static ssize_t diagc_attr_store(struct kobject *kobj, struct attribute *attr, + const char *buf, size_t size) +{ + struct qib_diagc_attr *dattr = + container_of(attr, struct qib_diagc_attr, attr); + struct qib_pportdata *ppd = + container_of(kobj, struct qib_pportdata, diagc_kobj); + struct qib_ibport *qibp = &ppd->ibport_data; + char *endp; + long val = simple_strtol(buf, &endp, 0); + + if (val < 0 || endp == buf) + return -EINVAL; + + *(u32 *)((char *) qibp + dattr->counter) = val; + return size; +} + static const struct sysfs_ops qib_diagc_ops = { .show = diagc_attr_show, + .store = diagc_attr_store, }; static struct kobj_type qib_diagc_ktype = { diff --git a/drivers/infiniband/hw/qib/qib_tx.c b/drivers/infiniband/hw/qib/qib_tx.c index f7eb1ddff5f3..7f36454c225e 100644 --- a/drivers/infiniband/hw/qib/qib_tx.c +++ b/drivers/infiniband/hw/qib/qib_tx.c @@ -170,7 +170,7 @@ static int find_ctxt(struct qib_devdata *dd, unsigned bufn) void qib_disarm_piobufs_set(struct qib_devdata *dd, unsigned long *mask, unsigned cnt) { - struct qib_pportdata *ppd, *pppd[dd->num_pports]; + struct qib_pportdata *ppd, *pppd[QIB_MAX_IB_PORTS]; unsigned i; unsigned long flags; @@ -340,9 +340,13 @@ rescan: if (i < dd->piobcnt2k) buf = (u32 __iomem *)(dd->pio2kbase + i * dd->palign); - else + else if (i < dd->piobcnt2k + dd->piobcnt4k || !dd->piovl15base) buf = (u32 __iomem *)(dd->pio4kbase + (i - dd->piobcnt2k) * dd->align4k); + else + buf = (u32 __iomem *)(dd->piovl15base + + (i - (dd->piobcnt2k + dd->piobcnt4k)) * + dd->align4k); if (pbufnum) *pbufnum = i; dd->upd_pio_shadow = 0; diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c index 6c7fe78cca64..b9c8b6346c1b 100644 --- a/drivers/infiniband/hw/qib/qib_uc.c +++ b/drivers/infiniband/hw/qib/qib_uc.c @@ -272,9 +272,6 @@ void qib_uc_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr, opcode >>= 24; memset(&wc, 0, sizeof wc); - /* Prevent simultaneous processing after APM on different CPUs */ - spin_lock(&qp->r_lock); - /* Compare the PSN verses the expected PSN. */ if (unlikely(qib_cmp24(psn, qp->r_psn) != 0)) { /* @@ -534,7 +531,6 @@ rdma_last: } qp->r_psn++; qp->r_state = opcode; - spin_unlock(&qp->r_lock); return; rewind: @@ -542,12 +538,10 @@ rewind: qp->r_sge.num_sge = 0; drop: ibp->n_pkt_drops++; - spin_unlock(&qp->r_lock); return; op_err: qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); - spin_unlock(&qp->r_lock); return; sunlock: diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c index c838cda73347..e1b3da2a1f85 100644 --- a/drivers/infiniband/hw/qib/qib_ud.c +++ b/drivers/infiniband/hw/qib/qib_ud.c @@ -535,13 +535,6 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr, wc.byte_len = tlen + sizeof(struct ib_grh); /* - * We need to serialize getting a receive work queue entry and - * generating a completion for it against QPs sending to this QP - * locally. - */ - spin_lock(&qp->r_lock); - - /* * Get the next work request entry to find where to put the data. */ if (qp->r_flags & QIB_R_REUSE_SGE) @@ -552,19 +545,19 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr, ret = qib_get_rwqe(qp, 0); if (ret < 0) { qib_rc_error(qp, IB_WC_LOC_QP_OP_ERR); - goto bail_unlock; + return; } if (!ret) { if (qp->ibqp.qp_num == 0) ibp->n_vl15_dropped++; - goto bail_unlock; + return; } } /* Silently drop packets which are too big. */ if (unlikely(wc.byte_len > qp->r_len)) { qp->r_flags |= QIB_R_REUSE_SGE; ibp->n_pkt_drops++; - goto bail_unlock; + return; } if (has_grh) { qib_copy_sge(&qp->r_sge, &hdr->u.l.grh, @@ -579,7 +572,7 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr, qp->r_sge.sge = *qp->r_sge.sg_list++; } if (!test_and_clear_bit(QIB_R_WRID_VALID, &qp->r_aflags)) - goto bail_unlock; + return; wc.wr_id = qp->r_wr_id; wc.status = IB_WC_SUCCESS; wc.opcode = IB_WC_RECV; @@ -601,7 +594,5 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct qib_ib_header *hdr, qib_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, (ohdr->bth[0] & cpu_to_be32(IB_BTH_SOLICITED)) != 0); -bail_unlock: - spin_unlock(&qp->r_lock); bail:; } diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c index cda8f4173d23..9fab40488850 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.c +++ b/drivers/infiniband/hw/qib/qib_verbs.c @@ -550,10 +550,12 @@ static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, { struct qib_ibport *ibp = &rcd->ppd->ibport_data; + spin_lock(&qp->r_lock); + /* Check for valid receive state. */ if (!(ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK)) { ibp->n_pkt_drops++; - return; + goto unlock; } switch (qp->ibqp.qp_type) { @@ -577,6 +579,9 @@ static void qib_qp_rcv(struct qib_ctxtdata *rcd, struct qib_ib_header *hdr, default: break; } + +unlock: + spin_unlock(&qp->r_lock); } /** |