From 7826f304b1efa5ab839cf029742290f51c4fa009 Mon Sep 17 00:00:00 2001 From: Krishna Gudipati Date: Wed, 20 Jul 2011 16:59:13 -0700 Subject: [SCSI] bfa: Add FC-transport based Asynchronous Event Notification support. - Added support to post vendor unique events on fc_host. - Supports adapter, port, ioc, flash and remote port based AEN events. Signed-off-by: Krishna Gudipati Signed-off-by: James Bottomley --- drivers/scsi/bfa/bfa_ioc.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/scsi/bfa/bfa_ioc.h') diff --git a/drivers/scsi/bfa/bfa_ioc.h b/drivers/scsi/bfa/bfa_ioc.h index c5ecd2edc95d..28bf6e18f33e 100644 --- a/drivers/scsi/bfa/bfa_ioc.h +++ b/drivers/scsi/bfa/bfa_ioc.h @@ -327,6 +327,7 @@ struct bfa_ioc_s { enum bfa_mode_s port_mode; u8 ad_cap_bm; /* adapter cap bit mask */ u8 port_mode_cfg; /* config port mode */ + int ioc_aen_seq; }; struct bfa_ioc_hwif_s { @@ -803,6 +804,7 @@ void bfa_ioc_fwver_get(struct bfa_ioc_s *ioc, struct bfi_ioc_image_hdr_s *fwhdr); bfa_boolean_t bfa_ioc_fwver_cmp(struct bfa_ioc_s *ioc, struct bfi_ioc_image_hdr_s *fwhdr); +void bfa_ioc_aen_post(struct bfa_ioc_s *ioc, enum bfa_ioc_aen_event event); bfa_status_t bfa_ioc_fw_stats_get(struct bfa_ioc_s *ioc, void *stats); bfa_status_t bfa_ioc_fw_stats_clear(struct bfa_ioc_s *ioc); -- cgit v1.2.3 From 37ea0558b87abb151245bc750070a4dcda490aca Mon Sep 17 00:00:00 2001 From: Krishna Gudipati Date: Wed, 20 Jul 2011 17:02:11 -0700 Subject: [SCSI] bfa: Added support to collect and reset fcport stats - Added support to collect and reset fcport stats. - Modified design to allow multiple requests for fcport stats. - fcport will remember the stats request in its stats_pending queue and service each of the queued requests after receiving a firmware response for the prior request. Signed-off-by: Krishna Gudipati Signed-off-by: James Bottomley --- drivers/scsi/bfa/bfa.h | 21 +++++++ drivers/scsi/bfa/bfa_core.c | 10 +++- drivers/scsi/bfa/bfa_ioc.h | 2 + drivers/scsi/bfa/bfa_svc.c | 130 ++++++++++++++++++++++++-------------------- drivers/scsi/bfa/bfa_svc.h | 14 ++--- drivers/scsi/bfa/bfad_bsg.c | 55 +++++++++++++++++++ drivers/scsi/bfa/bfad_bsg.h | 9 +++ 7 files changed, 172 insertions(+), 69 deletions(-) (limited to 'drivers/scsi/bfa/bfa_ioc.h') diff --git a/drivers/scsi/bfa/bfa.h b/drivers/scsi/bfa/bfa.h index 6e3869a41cb9..32f46265daa5 100644 --- a/drivers/scsi/bfa/bfa.h +++ b/drivers/scsi/bfa/bfa.h @@ -27,6 +27,7 @@ struct bfa_s; typedef void (*bfa_isr_func_t) (struct bfa_s *bfa, struct bfi_msg_s *m); +typedef void (*bfa_cb_cbfn_status_t) (void *cbarg, bfa_status_t status); /* * Interrupt message handlers @@ -121,6 +122,7 @@ bfa_reqq_winit(struct bfa_reqq_wait_s *wqe, void (*qresume) (void *cbarg), #define bfa_cb_queue(__bfa, __hcb_qe, __cbfn, __cbarg) do { \ (__hcb_qe)->cbfn = (__cbfn); \ (__hcb_qe)->cbarg = (__cbarg); \ + (__hcb_qe)->pre_rmv = BFA_FALSE; \ list_add_tail(&(__hcb_qe)->qe, &(__bfa)->comp_q); \ } while (0) @@ -135,6 +137,11 @@ bfa_reqq_winit(struct bfa_reqq_wait_s *wqe, void (*qresume) (void *cbarg), } \ } while (0) +#define bfa_cb_queue_status(__bfa, __hcb_qe, __status) do { \ + (__hcb_qe)->fw_status = (__status); \ + list_add_tail(&(__hcb_qe)->qe, &(__bfa)->comp_q); \ +} while (0) + #define bfa_cb_queue_done(__hcb_qe) do { \ (__hcb_qe)->once = BFA_FALSE; \ } while (0) @@ -408,4 +415,18 @@ void bfa_iocfc_disable(struct bfa_s *bfa); #define bfa_timer_start(_bfa, _timer, _timercb, _arg, _timeout) \ bfa_timer_begin(&(_bfa)->timer_mod, _timer, _timercb, _arg, _timeout) +struct bfa_cb_pending_q_s { + struct bfa_cb_qe_s hcb_qe; + void *data; /* Driver buffer */ +}; + +/* Common macros to operate on pending stats/attr apis */ +#define bfa_pending_q_init(__qe, __cbfn, __cbarg, __data) do { \ + bfa_q_qe_init(&((__qe)->hcb_qe.qe)); \ + (__qe)->hcb_qe.cbfn = (__cbfn); \ + (__qe)->hcb_qe.cbarg = (__cbarg); \ + (__qe)->hcb_qe.pre_rmv = BFA_TRUE; \ + (__qe)->data = (__data); \ +} while (0) + #endif /* __BFA_H__ */ diff --git a/drivers/scsi/bfa/bfa_core.c b/drivers/scsi/bfa/bfa_core.c index 0c236964c67e..4613bddd58e4 100644 --- a/drivers/scsi/bfa/bfa_core.c +++ b/drivers/scsi/bfa/bfa_core.c @@ -1533,10 +1533,17 @@ bfa_comp_process(struct bfa_s *bfa, struct list_head *comp_q) struct list_head *qe; struct list_head *qen; struct bfa_cb_qe_s *hcb_qe; + bfa_cb_cbfn_status_t cbfn; list_for_each_safe(qe, qen, comp_q) { hcb_qe = (struct bfa_cb_qe_s *) qe; - hcb_qe->cbfn(hcb_qe->cbarg, BFA_TRUE); + if (hcb_qe->pre_rmv) { + /* qe is invalid after return, dequeue before cbfn() */ + list_del(qe); + cbfn = (bfa_cb_cbfn_status_t)(hcb_qe->cbfn); + cbfn(hcb_qe->cbarg, hcb_qe->fw_status); + } else + hcb_qe->cbfn(hcb_qe->cbarg, BFA_TRUE); } } @@ -1549,6 +1556,7 @@ bfa_comp_free(struct bfa_s *bfa, struct list_head *comp_q) while (!list_empty(comp_q)) { bfa_q_deq(comp_q, &qe); hcb_qe = (struct bfa_cb_qe_s *) qe; + WARN_ON(hcb_qe->pre_rmv); hcb_qe->cbfn(hcb_qe->cbarg, BFA_FALSE); } } diff --git a/drivers/scsi/bfa/bfa_ioc.h b/drivers/scsi/bfa/bfa_ioc.h index 28bf6e18f33e..a7463c955117 100644 --- a/drivers/scsi/bfa/bfa_ioc.h +++ b/drivers/scsi/bfa/bfa_ioc.h @@ -367,6 +367,8 @@ struct bfa_cb_qe_s { struct list_head qe; bfa_cb_cbfn_t cbfn; bfa_boolean_t once; + bfa_boolean_t pre_rmv; /* set for stack based qe(s) */ + bfa_status_t fw_status; /* to access fw status in comp proc */ void *cbarg; }; diff --git a/drivers/scsi/bfa/bfa_svc.c b/drivers/scsi/bfa/bfa_svc.c index 6e1ed9a9aa2a..9884d6edfb57 100644 --- a/drivers/scsi/bfa/bfa_svc.c +++ b/drivers/scsi/bfa/bfa_svc.c @@ -2910,6 +2910,9 @@ bfa_fcport_attach(struct bfa_s *bfa, void *bfad, struct bfa_iocfc_cfg_s *cfg, port_cfg->trl_def_speed = BFA_PORT_SPEED_1GBPS; + INIT_LIST_HEAD(&fcport->stats_pending_q); + INIT_LIST_HEAD(&fcport->statsclr_pending_q); + bfa_reqq_winit(&fcport->reqq_wait, bfa_fcport_qresume, fcport); } @@ -3138,30 +3141,38 @@ bfa_fcport_fcoe_stats_swap(struct bfa_fcoe_stats_s *d, static void __bfa_cb_fcport_stats_get(void *cbarg, bfa_boolean_t complete) { - struct bfa_fcport_s *fcport = cbarg; + struct bfa_fcport_s *fcport = (struct bfa_fcport_s *)cbarg; + struct bfa_cb_pending_q_s *cb; + struct list_head *qe, *qen; + union bfa_fcport_stats_u *ret; if (complete) { - if (fcport->stats_status == BFA_STATUS_OK) { - struct timeval tv; - - /* Swap FC QoS or FCoE stats */ - if (bfa_ioc_get_fcmode(&fcport->bfa->ioc)) { - bfa_fcport_qos_stats_swap( - &fcport->stats_ret->fcqos, - &fcport->stats->fcqos); - } else { - bfa_fcport_fcoe_stats_swap( - &fcport->stats_ret->fcoe, - &fcport->stats->fcoe); - - do_gettimeofday(&tv); - fcport->stats_ret->fcoe.secs_reset = + struct timeval tv; + if (fcport->stats_status == BFA_STATUS_OK) + do_gettimeofday(&tv); + + list_for_each_safe(qe, qen, &fcport->stats_pending_q) { + bfa_q_deq(&fcport->stats_pending_q, &qe); + cb = (struct bfa_cb_pending_q_s *)qe; + if (fcport->stats_status == BFA_STATUS_OK) { + ret = (union bfa_fcport_stats_u *)cb->data; + /* Swap FC QoS or FCoE stats */ + if (bfa_ioc_get_fcmode(&fcport->bfa->ioc)) + bfa_fcport_qos_stats_swap(&ret->fcqos, + &fcport->stats->fcqos); + else { + bfa_fcport_fcoe_stats_swap(&ret->fcoe, + &fcport->stats->fcoe); + ret->fcoe.secs_reset = tv.tv_sec - fcport->stats_reset_time; + } } + bfa_cb_queue_status(fcport->bfa, &cb->hcb_qe, + fcport->stats_status); } - fcport->stats_cbfn(fcport->stats_cbarg, fcport->stats_status); + fcport->stats_status = BFA_STATUS_OK; } else { - fcport->stats_busy = BFA_FALSE; + INIT_LIST_HEAD(&fcport->stats_pending_q); fcport->stats_status = BFA_STATUS_OK; } } @@ -3179,8 +3190,7 @@ bfa_fcport_stats_get_timeout(void *cbarg) } fcport->stats_status = BFA_STATUS_ETIMER; - bfa_cb_queue(fcport->bfa, &fcport->hcb_qe, __bfa_cb_fcport_stats_get, - fcport); + __bfa_cb_fcport_stats_get(fcport, BFA_TRUE); } static void @@ -3210,7 +3220,9 @@ bfa_fcport_send_stats_get(void *cbarg) static void __bfa_cb_fcport_stats_clr(void *cbarg, bfa_boolean_t complete) { - struct bfa_fcport_s *fcport = cbarg; + struct bfa_fcport_s *fcport = (struct bfa_fcport_s *) cbarg; + struct bfa_cb_pending_q_s *cb; + struct list_head *qe, *qen; if (complete) { struct timeval tv; @@ -3220,10 +3232,15 @@ __bfa_cb_fcport_stats_clr(void *cbarg, bfa_boolean_t complete) */ do_gettimeofday(&tv); fcport->stats_reset_time = tv.tv_sec; - - fcport->stats_cbfn(fcport->stats_cbarg, fcport->stats_status); + list_for_each_safe(qe, qen, &fcport->statsclr_pending_q) { + bfa_q_deq(&fcport->statsclr_pending_q, &qe); + cb = (struct bfa_cb_pending_q_s *)qe; + bfa_cb_queue_status(fcport->bfa, &cb->hcb_qe, + fcport->stats_status); + } + fcport->stats_status = BFA_STATUS_OK; } else { - fcport->stats_busy = BFA_FALSE; + INIT_LIST_HEAD(&fcport->statsclr_pending_q); fcport->stats_status = BFA_STATUS_OK; } } @@ -3241,8 +3258,7 @@ bfa_fcport_stats_clr_timeout(void *cbarg) } fcport->stats_status = BFA_STATUS_ETIMER; - bfa_cb_queue(fcport->bfa, &fcport->hcb_qe, - __bfa_cb_fcport_stats_clr, fcport); + __bfa_cb_fcport_stats_clr(fcport, BFA_TRUE); } static void @@ -3462,28 +3478,26 @@ bfa_fcport_isr(struct bfa_s *bfa, struct bfi_msg_s *msg) /* * check for timer pop before processing the rsp */ - if (fcport->stats_busy == BFA_FALSE || - fcport->stats_status == BFA_STATUS_ETIMER) + if (list_empty(&fcport->stats_pending_q) || + (fcport->stats_status == BFA_STATUS_ETIMER)) break; bfa_timer_stop(&fcport->timer); fcport->stats_status = i2hmsg.pstatsget_rsp->status; - bfa_cb_queue(fcport->bfa, &fcport->hcb_qe, - __bfa_cb_fcport_stats_get, fcport); + __bfa_cb_fcport_stats_get(fcport, BFA_TRUE); break; case BFI_FCPORT_I2H_STATS_CLEAR_RSP: /* * check for timer pop before processing the rsp */ - if (fcport->stats_busy == BFA_FALSE || - fcport->stats_status == BFA_STATUS_ETIMER) + if (list_empty(&fcport->statsclr_pending_q) || + (fcport->stats_status == BFA_STATUS_ETIMER)) break; bfa_timer_stop(&fcport->timer); fcport->stats_status = BFA_STATUS_OK; - bfa_cb_queue(fcport->bfa, &fcport->hcb_qe, - __bfa_cb_fcport_stats_clr, fcport); + __bfa_cb_fcport_stats_clr(fcport, BFA_TRUE); break; case BFI_FCPORT_I2H_ENABLE_AEN: @@ -3815,25 +3829,25 @@ bfa_fcport_get_attr(struct bfa_s *bfa, struct bfa_port_attr_s *attr) * Fetch port statistics (FCQoS or FCoE). */ bfa_status_t -bfa_fcport_get_stats(struct bfa_s *bfa, union bfa_fcport_stats_u *stats, - bfa_cb_port_t cbfn, void *cbarg) +bfa_fcport_get_stats(struct bfa_s *bfa, struct bfa_cb_pending_q_s *cb) { struct bfa_fcport_s *fcport = BFA_FCPORT_MOD(bfa); - if (fcport->stats_busy) { - bfa_trc(bfa, fcport->stats_busy); - return BFA_STATUS_DEVBUSY; - } + if (bfa_ioc_is_disabled(&bfa->ioc)) + return BFA_STATUS_IOC_DISABLED; - fcport->stats_busy = BFA_TRUE; - fcport->stats_ret = stats; - fcport->stats_cbfn = cbfn; - fcport->stats_cbarg = cbarg; + if (!list_empty(&fcport->statsclr_pending_q)) + return BFA_STATUS_DEVBUSY; - bfa_fcport_send_stats_get(fcport); + if (list_empty(&fcport->stats_pending_q)) { + list_add_tail(&cb->hcb_qe.qe, &fcport->stats_pending_q); + bfa_fcport_send_stats_get(fcport); + bfa_timer_start(bfa, &fcport->timer, + bfa_fcport_stats_get_timeout, + fcport, BFA_FCPORT_STATS_TOV); + } else + list_add_tail(&cb->hcb_qe.qe, &fcport->stats_pending_q); - bfa_timer_start(bfa, &fcport->timer, bfa_fcport_stats_get_timeout, - fcport, BFA_FCPORT_STATS_TOV); return BFA_STATUS_OK; } @@ -3841,27 +3855,25 @@ bfa_fcport_get_stats(struct bfa_s *bfa, union bfa_fcport_stats_u *stats, * Reset port statistics (FCQoS or FCoE). */ bfa_status_t -bfa_fcport_clear_stats(struct bfa_s *bfa, bfa_cb_port_t cbfn, void *cbarg) +bfa_fcport_clear_stats(struct bfa_s *bfa, struct bfa_cb_pending_q_s *cb) { struct bfa_fcport_s *fcport = BFA_FCPORT_MOD(bfa); - if (fcport->stats_busy) { - bfa_trc(bfa, fcport->stats_busy); + if (!list_empty(&fcport->stats_pending_q)) return BFA_STATUS_DEVBUSY; - } - fcport->stats_busy = BFA_TRUE; - fcport->stats_cbfn = cbfn; - fcport->stats_cbarg = cbarg; + if (list_empty(&fcport->statsclr_pending_q)) { + list_add_tail(&cb->hcb_qe.qe, &fcport->statsclr_pending_q); + bfa_fcport_send_stats_clear(fcport); + bfa_timer_start(bfa, &fcport->timer, + bfa_fcport_stats_clr_timeout, + fcport, BFA_FCPORT_STATS_TOV); + } else + list_add_tail(&cb->hcb_qe.qe, &fcport->statsclr_pending_q); - bfa_fcport_send_stats_clear(fcport); - - bfa_timer_start(bfa, &fcport->timer, bfa_fcport_stats_clr_timeout, - fcport, BFA_FCPORT_STATS_TOV); return BFA_STATUS_OK; } - /* * Fetch port attributes. */ diff --git a/drivers/scsi/bfa/bfa_svc.h b/drivers/scsi/bfa/bfa_svc.h index fbe513a671b5..01db3520070f 100644 --- a/drivers/scsi/bfa/bfa_svc.h +++ b/drivers/scsi/bfa/bfa_svc.h @@ -441,7 +441,6 @@ void bfa_lps_isr(struct bfa_s *bfa, struct bfi_msg_s *msg); */ #define BFA_FCPORT(_bfa) (&((_bfa)->modules.port)) -typedef void (*bfa_cb_port_t) (void *cbarg, enum bfa_status status); /* * Link notification data structure @@ -495,13 +494,11 @@ struct bfa_fcport_s { u8 *stats_kva; u64 stats_pa; union bfa_fcport_stats_u *stats; - union bfa_fcport_stats_u *stats_ret; /* driver stats location */ bfa_status_t stats_status; /* stats/statsclr status */ - bfa_boolean_t stats_busy; /* outstanding stats/statsclr */ + struct list_head stats_pending_q; + struct list_head statsclr_pending_q; bfa_boolean_t stats_qfull; u32 stats_reset_time; /* stats reset time stamp */ - bfa_cb_port_t stats_cbfn; /* driver callback function */ - void *stats_cbarg; /* *!< user callback arg */ bfa_boolean_t diag_busy; /* diag busy status */ bfa_boolean_t beacon; /* port beacon status */ bfa_boolean_t link_e2e_beacon; /* link beacon status */ @@ -552,10 +549,9 @@ void bfa_fcport_beacon(void *dev, bfa_boolean_t beacon, bfa_boolean_t link_e2e_beacon); bfa_boolean_t bfa_fcport_is_linkup(struct bfa_s *bfa); bfa_status_t bfa_fcport_get_stats(struct bfa_s *bfa, - union bfa_fcport_stats_u *stats, - bfa_cb_port_t cbfn, void *cbarg); -bfa_status_t bfa_fcport_clear_stats(struct bfa_s *bfa, bfa_cb_port_t cbfn, - void *cbarg); + struct bfa_cb_pending_q_s *cb); +bfa_status_t bfa_fcport_clear_stats(struct bfa_s *bfa, + struct bfa_cb_pending_q_s *cb); bfa_boolean_t bfa_fcport_is_qos_enabled(struct bfa_s *bfa); bfa_boolean_t bfa_fcport_is_trunk_enabled(struct bfa_s *bfa); bfa_status_t bfa_fcport_is_pbcdisabled(struct bfa_s *bfa); diff --git a/drivers/scsi/bfa/bfad_bsg.c b/drivers/scsi/bfa/bfad_bsg.c index 26def664cb83..0d920edebac9 100644 --- a/drivers/scsi/bfa/bfad_bsg.c +++ b/drivers/scsi/bfa/bfad_bsg.c @@ -2023,6 +2023,55 @@ bfad_iocmd_itnim_get_ioprofile(struct bfad_s *bfad, void *cmd) return 0; } +int +bfad_iocmd_fcport_get_stats(struct bfad_s *bfad, void *cmd) +{ + struct bfa_bsg_fcport_stats_s *iocmd = + (struct bfa_bsg_fcport_stats_s *)cmd; + struct bfad_hal_comp fcomp; + unsigned long flags; + struct bfa_cb_pending_q_s cb_qe; + + init_completion(&fcomp.comp); + bfa_pending_q_init(&cb_qe, (bfa_cb_cbfn_t)bfad_hcb_comp, + &fcomp, &iocmd->stats); + spin_lock_irqsave(&bfad->bfad_lock, flags); + iocmd->status = bfa_fcport_get_stats(&bfad->bfa, &cb_qe); + spin_unlock_irqrestore(&bfad->bfad_lock, flags); + if (iocmd->status != BFA_STATUS_OK) { + bfa_trc(bfad, iocmd->status); + goto out; + } + wait_for_completion(&fcomp.comp); + iocmd->status = fcomp.status; +out: + return 0; +} + +int +bfad_iocmd_fcport_reset_stats(struct bfad_s *bfad, void *cmd) +{ + struct bfa_bsg_gen_s *iocmd = (struct bfa_bsg_gen_s *)cmd; + struct bfad_hal_comp fcomp; + unsigned long flags; + struct bfa_cb_pending_q_s cb_qe; + + init_completion(&fcomp.comp); + bfa_pending_q_init(&cb_qe, (bfa_cb_cbfn_t)bfad_hcb_comp, &fcomp, NULL); + + spin_lock_irqsave(&bfad->bfad_lock, flags); + iocmd->status = bfa_fcport_clear_stats(&bfad->bfa, &cb_qe); + spin_unlock_irqrestore(&bfad->bfad_lock, flags); + if (iocmd->status != BFA_STATUS_OK) { + bfa_trc(bfad, iocmd->status); + goto out; + } + wait_for_completion(&fcomp.comp); + iocmd->status = fcomp.status; +out: + return 0; +} + static int bfad_iocmd_handler(struct bfad_s *bfad, unsigned int cmd, void *iocmd, unsigned int payload_len) @@ -2291,6 +2340,12 @@ bfad_iocmd_handler(struct bfad_s *bfad, unsigned int cmd, void *iocmd, case IOCMD_ITNIM_GET_IOPROFILE: rc = bfad_iocmd_itnim_get_ioprofile(bfad, iocmd); break; + case IOCMD_FCPORT_GET_STATS: + rc = bfad_iocmd_fcport_get_stats(bfad, iocmd); + break; + case IOCMD_FCPORT_RESET_STATS: + rc = bfad_iocmd_fcport_reset_stats(bfad, iocmd); + break; default: rc = -EINVAL; break; diff --git a/drivers/scsi/bfa/bfad_bsg.h b/drivers/scsi/bfa/bfad_bsg.h index 91cb2123dec0..f64f3eee6dbf 100644 --- a/drivers/scsi/bfa/bfad_bsg.h +++ b/drivers/scsi/bfa/bfad_bsg.h @@ -119,6 +119,8 @@ enum { IOCMD_FCPIM_PROFILE_ON, IOCMD_FCPIM_PROFILE_OFF, IOCMD_ITNIM_GET_IOPROFILE, + IOCMD_FCPORT_GET_STATS, + IOCMD_FCPORT_RESET_STATS, }; struct bfa_bsg_gen_s { @@ -150,6 +152,13 @@ struct bfa_bsg_itnim_ioprofile_s { struct bfa_itnim_ioprofile_s ioprofile; }; +struct bfa_bsg_fcport_stats_s { + bfa_status_t status; + u16 bfad_num; + u16 rsvd; + union bfa_fcport_stats_u stats; +}; + struct bfa_bsg_ioc_name_s { bfa_status_t status; u16 bfad_num; -- cgit v1.2.3 From 45c5dc1d3f42d4f54a5ab5f45ee55f0ffe1099f1 Mon Sep 17 00:00:00 2001 From: Krishna Gudipati Date: Wed, 20 Jul 2011 17:03:46 -0700 Subject: [SCSI] bfa: Add support to store driver configuration in flash. - Added dconf (Driver Config) BFA sub-module. - The dconf sub-module provides interfaces and manages flash writes to the flash DRV parition. - dconf sub-module also ensures that the whole 64K DRV partition is updated on a flash write. Signed-off-by: Krishna Gudipati Signed-off-by: James Bottomley --- drivers/scsi/bfa/bfa.h | 1 + drivers/scsi/bfa/bfa_core.c | 25 ++- drivers/scsi/bfa/bfa_defs_svc.h | 21 +++ drivers/scsi/bfa/bfa_fc.h | 1 + drivers/scsi/bfa/bfa_ioc.c | 393 ++++++++++++++++++++++++++++++++++++++++ drivers/scsi/bfa/bfa_ioc.h | 44 ++++- drivers/scsi/bfa/bfa_modules.h | 2 + 7 files changed, 481 insertions(+), 6 deletions(-) (limited to 'drivers/scsi/bfa/bfa_ioc.h') diff --git a/drivers/scsi/bfa/bfa.h b/drivers/scsi/bfa/bfa.h index 32f46265daa5..76300befffbe 100644 --- a/drivers/scsi/bfa/bfa.h +++ b/drivers/scsi/bfa/bfa.h @@ -412,6 +412,7 @@ bfa_status_t bfa_iocfc_israttr_set(struct bfa_s *bfa, void bfa_iocfc_enable(struct bfa_s *bfa); void bfa_iocfc_disable(struct bfa_s *bfa); +void bfa_iocfc_cb_dconf_modinit(struct bfa_s *bfa, bfa_status_t status); #define bfa_timer_start(_bfa, _timer, _timercb, _arg, _timeout) \ bfa_timer_begin(&(_bfa)->timer_mod, _timer, _timercb, _arg, _timeout) diff --git a/drivers/scsi/bfa/bfa_core.c b/drivers/scsi/bfa/bfa_core.c index 4613bddd58e4..4bd546bcc240 100644 --- a/drivers/scsi/bfa/bfa_core.c +++ b/drivers/scsi/bfa/bfa_core.c @@ -33,6 +33,7 @@ static struct bfa_module_s *hal_mods[] = { &hal_mod_uf, &hal_mod_rport, &hal_mod_fcp, + &hal_mod_dconf, NULL }; @@ -702,7 +703,7 @@ bfa_iocfc_init_cb(void *bfa_arg, bfa_boolean_t complete) struct bfa_s *bfa = bfa_arg; if (complete) { - if (bfa->iocfc.cfgdone) + if (bfa->iocfc.cfgdone && BFA_DCONF_MOD(bfa)->flashdone) bfa_cb_init(bfa->bfad, BFA_STATUS_OK); else bfa_cb_init(bfa->bfad, BFA_STATUS_FAILED); @@ -815,9 +816,11 @@ bfa_iocfc_cfgrsp(struct bfa_s *bfa) */ bfa_fcport_init(bfa); - if (iocfc->action == BFA_IOCFC_ACT_INIT) - bfa_cb_queue(bfa, &iocfc->init_hcb_qe, bfa_iocfc_init_cb, bfa); - else { + if (iocfc->action == BFA_IOCFC_ACT_INIT) { + if (BFA_DCONF_MOD(bfa)->flashdone == BFA_TRUE) + bfa_cb_queue(bfa, &iocfc->init_hcb_qe, + bfa_iocfc_init_cb, bfa); + } else { if (bfa->iocfc.action == BFA_IOCFC_ACT_ENABLE) bfa_cb_queue(bfa, &bfa->iocfc.en_hcb_qe, bfa_iocfc_enable_cb, bfa); @@ -1038,6 +1041,7 @@ bfa_iocfc_enable_cbfn(void *bfa_arg, enum bfa_status status) } bfa_iocfc_send_cfg(bfa); + bfa_dconf_modinit(bfa); } /* @@ -1200,7 +1204,9 @@ bfa_iocfc_stop(struct bfa_s *bfa) bfa->iocfc.action = BFA_IOCFC_ACT_STOP; bfa->queue_process = BFA_FALSE; - bfa_ioc_disable(&bfa->ioc); + bfa_dconf_modexit(bfa); + if (BFA_DCONF_MOD(bfa)->flashdone == BFA_TRUE) + bfa_ioc_disable(&bfa->ioc); } void @@ -1561,6 +1567,15 @@ bfa_comp_free(struct bfa_s *bfa, struct list_head *comp_q) } } +void +bfa_iocfc_cb_dconf_modinit(struct bfa_s *bfa, bfa_status_t status) +{ + if (bfa->iocfc.action == BFA_IOCFC_ACT_INIT) { + if (bfa->iocfc.cfgdone == BFA_TRUE) + bfa_cb_queue(bfa, &bfa->iocfc.init_hcb_qe, + bfa_iocfc_init_cb, bfa); + } +} /* * Return the list of PCI vendor/device id lists supported by this diff --git a/drivers/scsi/bfa/bfa_defs_svc.h b/drivers/scsi/bfa/bfa_defs_svc.h index 263438bd8b6e..202cb692f981 100644 --- a/drivers/scsi/bfa/bfa_defs_svc.h +++ b/drivers/scsi/bfa/bfa_defs_svc.h @@ -788,6 +788,27 @@ enum bfa_port_linkstate_rsn { CEE_ISCSI_PRI_OVERLAP_FCOE_PRI = 43 }; #pragma pack(1) +/* + * LUN mask configuration + */ +struct bfa_lun_mask_s { + wwn_t lp_wwn; + wwn_t rp_wwn; + lun_t lun; + u8 ua; + u8 rsvd[3]; + u16 rp_tag; + u8 lp_tag; + u8 state; +}; + +#define MAX_LUN_MASK_CFG 16 +struct bfa_lunmask_cfg_s { + u32 status; + u32 rsvd; + struct bfa_lun_mask_s lun_list[MAX_LUN_MASK_CFG]; +}; + /* * Physical port configuration */ diff --git a/drivers/scsi/bfa/bfa_fc.h b/drivers/scsi/bfa/bfa_fc.h index 8d0b88f67a38..a007345e472a 100644 --- a/drivers/scsi/bfa/bfa_fc.h +++ b/drivers/scsi/bfa/bfa_fc.h @@ -21,6 +21,7 @@ #include "bfad_drv.h" typedef u64 wwn_t; +typedef u64 lun_t; #define WWN_NULL (0) #define FC_SYMNAME_MAX 256 /* max name server symbolic name size */ diff --git a/drivers/scsi/bfa/bfa_ioc.c b/drivers/scsi/bfa/bfa_ioc.c index 27c5565ba2c3..1ac5aecf25a6 100644 --- a/drivers/scsi/bfa/bfa_ioc.c +++ b/drivers/scsi/bfa/bfa_ioc.c @@ -5588,3 +5588,396 @@ bfa_phy_intr(void *phyarg, struct bfi_mbmsg_s *msg) WARN_ON(1); } } + +/* + * DCONF module specific + */ + +BFA_MODULE(dconf); + +/* + * DCONF state machine events + */ +enum bfa_dconf_event { + BFA_DCONF_SM_INIT = 1, /* dconf Init */ + BFA_DCONF_SM_FLASH_COMP = 2, /* read/write to flash */ + BFA_DCONF_SM_WR = 3, /* binding change, map */ + BFA_DCONF_SM_TIMEOUT = 4, /* Start timer */ + BFA_DCONF_SM_EXIT = 5, /* exit dconf module */ + BFA_DCONF_SM_IOCDISABLE = 6, /* IOC disable event */ +}; + +/* forward declaration of DCONF state machine */ +static void bfa_dconf_sm_uninit(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event); +static void bfa_dconf_sm_flash_read(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event); +static void bfa_dconf_sm_ready(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event); +static void bfa_dconf_sm_dirty(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event); +static void bfa_dconf_sm_sync(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event); +static void bfa_dconf_sm_final_sync(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event); +static void bfa_dconf_sm_iocdown_dirty(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event); + +static void bfa_dconf_cbfn(void *dconf, bfa_status_t status); +static void bfa_dconf_timer(void *cbarg); +static bfa_status_t bfa_dconf_flash_write(struct bfa_dconf_mod_s *dconf); +static void bfa_dconf_init_cb(void *arg, bfa_status_t status); + +/* + * Begining state of dconf module. Waiting for an event to start. + */ +static void +bfa_dconf_sm_uninit(struct bfa_dconf_mod_s *dconf, enum bfa_dconf_event event) +{ + bfa_status_t bfa_status; + bfa_trc(dconf->bfa, event); + + switch (event) { + case BFA_DCONF_SM_INIT: + if (dconf->min_cfg) { + bfa_trc(dconf->bfa, dconf->min_cfg); + return; + } + bfa_sm_set_state(dconf, bfa_dconf_sm_flash_read); + dconf->flashdone = BFA_FALSE; + bfa_trc(dconf->bfa, dconf->flashdone); + bfa_status = bfa_flash_read_part(BFA_FLASH(dconf->bfa), + BFA_FLASH_PART_DRV, dconf->instance, + dconf->dconf, + sizeof(struct bfa_dconf_s), 0, + bfa_dconf_init_cb, dconf->bfa); + if (bfa_status != BFA_STATUS_OK) { + bfa_dconf_init_cb(dconf->bfa, BFA_STATUS_FAILED); + bfa_sm_set_state(dconf, bfa_dconf_sm_uninit); + return; + } + break; + case BFA_DCONF_SM_EXIT: + dconf->flashdone = BFA_TRUE; + case BFA_DCONF_SM_IOCDISABLE: + case BFA_DCONF_SM_WR: + case BFA_DCONF_SM_FLASH_COMP: + break; + default: + bfa_sm_fault(dconf->bfa, event); + } +} + +/* + * Read flash for dconf entries and make a call back to the driver once done. + */ +static void +bfa_dconf_sm_flash_read(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event) +{ + bfa_trc(dconf->bfa, event); + + switch (event) { + case BFA_DCONF_SM_FLASH_COMP: + bfa_sm_set_state(dconf, bfa_dconf_sm_ready); + break; + case BFA_DCONF_SM_TIMEOUT: + bfa_sm_set_state(dconf, bfa_dconf_sm_ready); + break; + case BFA_DCONF_SM_EXIT: + dconf->flashdone = BFA_TRUE; + bfa_trc(dconf->bfa, dconf->flashdone); + case BFA_DCONF_SM_IOCDISABLE: + bfa_sm_set_state(dconf, bfa_dconf_sm_uninit); + break; + default: + bfa_sm_fault(dconf->bfa, event); + } +} + +/* + * DCONF Module is in ready state. Has completed the initialization. + */ +static void +bfa_dconf_sm_ready(struct bfa_dconf_mod_s *dconf, enum bfa_dconf_event event) +{ + bfa_trc(dconf->bfa, event); + + switch (event) { + case BFA_DCONF_SM_WR: + bfa_timer_start(dconf->bfa, &dconf->timer, + bfa_dconf_timer, dconf, BFA_DCONF_UPDATE_TOV); + bfa_sm_set_state(dconf, bfa_dconf_sm_dirty); + break; + case BFA_DCONF_SM_EXIT: + dconf->flashdone = BFA_TRUE; + bfa_trc(dconf->bfa, dconf->flashdone); + bfa_sm_set_state(dconf, bfa_dconf_sm_uninit); + break; + case BFA_DCONF_SM_INIT: + case BFA_DCONF_SM_IOCDISABLE: + break; + default: + bfa_sm_fault(dconf->bfa, event); + } +} + +/* + * entries are dirty, write back to the flash. + */ + +static void +bfa_dconf_sm_dirty(struct bfa_dconf_mod_s *dconf, enum bfa_dconf_event event) +{ + bfa_trc(dconf->bfa, event); + + switch (event) { + case BFA_DCONF_SM_TIMEOUT: + bfa_sm_set_state(dconf, bfa_dconf_sm_sync); + bfa_dconf_flash_write(dconf); + break; + case BFA_DCONF_SM_WR: + bfa_timer_stop(&dconf->timer); + bfa_timer_start(dconf->bfa, &dconf->timer, + bfa_dconf_timer, dconf, BFA_DCONF_UPDATE_TOV); + break; + case BFA_DCONF_SM_EXIT: + bfa_timer_stop(&dconf->timer); + bfa_timer_start(dconf->bfa, &dconf->timer, + bfa_dconf_timer, dconf, BFA_DCONF_UPDATE_TOV); + bfa_sm_set_state(dconf, bfa_dconf_sm_final_sync); + bfa_dconf_flash_write(dconf); + break; + case BFA_DCONF_SM_FLASH_COMP: + break; + case BFA_DCONF_SM_IOCDISABLE: + bfa_timer_stop(&dconf->timer); + bfa_sm_set_state(dconf, bfa_dconf_sm_iocdown_dirty); + break; + default: + bfa_sm_fault(dconf->bfa, event); + } +} + +/* + * Sync the dconf entries to the flash. + */ +static void +bfa_dconf_sm_final_sync(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event) +{ + bfa_trc(dconf->bfa, event); + + switch (event) { + case BFA_DCONF_SM_IOCDISABLE: + case BFA_DCONF_SM_FLASH_COMP: + bfa_timer_stop(&dconf->timer); + case BFA_DCONF_SM_TIMEOUT: + bfa_sm_set_state(dconf, bfa_dconf_sm_uninit); + dconf->flashdone = BFA_TRUE; + bfa_trc(dconf->bfa, dconf->flashdone); + bfa_ioc_disable(&dconf->bfa->ioc); + break; + default: + bfa_sm_fault(dconf->bfa, event); + } +} + +static void +bfa_dconf_sm_sync(struct bfa_dconf_mod_s *dconf, enum bfa_dconf_event event) +{ + bfa_trc(dconf->bfa, event); + + switch (event) { + case BFA_DCONF_SM_FLASH_COMP: + bfa_sm_set_state(dconf, bfa_dconf_sm_ready); + break; + case BFA_DCONF_SM_WR: + bfa_timer_start(dconf->bfa, &dconf->timer, + bfa_dconf_timer, dconf, BFA_DCONF_UPDATE_TOV); + bfa_sm_set_state(dconf, bfa_dconf_sm_dirty); + break; + case BFA_DCONF_SM_EXIT: + bfa_timer_start(dconf->bfa, &dconf->timer, + bfa_dconf_timer, dconf, BFA_DCONF_UPDATE_TOV); + bfa_sm_set_state(dconf, bfa_dconf_sm_final_sync); + break; + case BFA_DCONF_SM_IOCDISABLE: + bfa_sm_set_state(dconf, bfa_dconf_sm_iocdown_dirty); + break; + default: + bfa_sm_fault(dconf->bfa, event); + } +} + +static void +bfa_dconf_sm_iocdown_dirty(struct bfa_dconf_mod_s *dconf, + enum bfa_dconf_event event) +{ + bfa_trc(dconf->bfa, event); + + switch (event) { + case BFA_DCONF_SM_INIT: + bfa_timer_start(dconf->bfa, &dconf->timer, + bfa_dconf_timer, dconf, BFA_DCONF_UPDATE_TOV); + bfa_sm_set_state(dconf, bfa_dconf_sm_dirty); + break; + case BFA_DCONF_SM_EXIT: + dconf->flashdone = BFA_TRUE; + bfa_sm_set_state(dconf, bfa_dconf_sm_uninit); + break; + case BFA_DCONF_SM_IOCDISABLE: + break; + default: + bfa_sm_fault(dconf->bfa, event); + } +} + +/* + * Compute and return memory needed by DRV_CFG module. + */ +static void +bfa_dconf_meminfo(struct bfa_iocfc_cfg_s *cfg, struct bfa_meminfo_s *meminfo, + struct bfa_s *bfa) +{ + struct bfa_mem_kva_s *dconf_kva = BFA_MEM_DCONF_KVA(bfa); + + if (cfg->drvcfg.min_cfg) + bfa_mem_kva_setup(meminfo, dconf_kva, + sizeof(struct bfa_dconf_hdr_s)); + else + bfa_mem_kva_setup(meminfo, dconf_kva, + sizeof(struct bfa_dconf_s)); +} + +static void +bfa_dconf_attach(struct bfa_s *bfa, void *bfad, struct bfa_iocfc_cfg_s *cfg, + struct bfa_pcidev_s *pcidev) +{ + struct bfa_dconf_mod_s *dconf = BFA_DCONF_MOD(bfa); + + dconf->bfad = bfad; + dconf->bfa = bfa; + dconf->instance = bfa->ioc.port_id; + bfa_trc(bfa, dconf->instance); + + dconf->dconf = (struct bfa_dconf_s *) bfa_mem_kva_curp(dconf); + if (cfg->drvcfg.min_cfg) { + bfa_mem_kva_curp(dconf) += sizeof(struct bfa_dconf_hdr_s); + dconf->min_cfg = BFA_TRUE; + /* + * Set the flashdone flag to TRUE explicitly as no flash + * write will happen in min_cfg mode. + */ + dconf->flashdone = BFA_TRUE; + } else { + dconf->min_cfg = BFA_FALSE; + bfa_mem_kva_curp(dconf) += sizeof(struct bfa_dconf_s); + } + + bfa_dconf_read_data_valid(bfa) = BFA_FALSE; + bfa_sm_set_state(dconf, bfa_dconf_sm_uninit); +} + +static void +bfa_dconf_init_cb(void *arg, bfa_status_t status) +{ + struct bfa_s *bfa = arg; + struct bfa_dconf_mod_s *dconf = BFA_DCONF_MOD(bfa); + + dconf->flashdone = BFA_TRUE; + bfa_trc(bfa, dconf->flashdone); + bfa_iocfc_cb_dconf_modinit(bfa, status); + if (status == BFA_STATUS_OK) { + bfa_dconf_read_data_valid(bfa) = BFA_TRUE; + if (dconf->dconf->hdr.signature != BFI_DCONF_SIGNATURE) + dconf->dconf->hdr.signature = BFI_DCONF_SIGNATURE; + if (dconf->dconf->hdr.version != BFI_DCONF_VERSION) + dconf->dconf->hdr.version = BFI_DCONF_VERSION; + } + bfa_sm_send_event(dconf, BFA_DCONF_SM_FLASH_COMP); +} + +void +bfa_dconf_modinit(struct bfa_s *bfa) +{ + struct bfa_dconf_mod_s *dconf = BFA_DCONF_MOD(bfa); + bfa_sm_send_event(dconf, BFA_DCONF_SM_INIT); +} +static void +bfa_dconf_start(struct bfa_s *bfa) +{ +} + +static void +bfa_dconf_stop(struct bfa_s *bfa) +{ +} + +static void bfa_dconf_timer(void *cbarg) +{ + struct bfa_dconf_mod_s *dconf = cbarg; + bfa_sm_send_event(dconf, BFA_DCONF_SM_TIMEOUT); +} +static void +bfa_dconf_iocdisable(struct bfa_s *bfa) +{ + struct bfa_dconf_mod_s *dconf = BFA_DCONF_MOD(bfa); + bfa_sm_send_event(dconf, BFA_DCONF_SM_IOCDISABLE); +} + +static void +bfa_dconf_detach(struct bfa_s *bfa) +{ +} + +static bfa_status_t +bfa_dconf_flash_write(struct bfa_dconf_mod_s *dconf) +{ + bfa_status_t bfa_status; + bfa_trc(dconf->bfa, 0); + + bfa_status = bfa_flash_update_part(BFA_FLASH(dconf->bfa), + BFA_FLASH_PART_DRV, dconf->instance, + dconf->dconf, sizeof(struct bfa_dconf_s), 0, + bfa_dconf_cbfn, dconf); + if (bfa_status != BFA_STATUS_OK) + WARN_ON(bfa_status); + bfa_trc(dconf->bfa, bfa_status); + + return bfa_status; +} + +bfa_status_t +bfa_dconf_update(struct bfa_s *bfa) +{ + struct bfa_dconf_mod_s *dconf = BFA_DCONF_MOD(bfa); + bfa_trc(dconf->bfa, 0); + if (bfa_sm_cmp_state(dconf, bfa_dconf_sm_iocdown_dirty)) + return BFA_STATUS_FAILED; + + if (dconf->min_cfg) { + bfa_trc(dconf->bfa, dconf->min_cfg); + return BFA_STATUS_FAILED; + } + + bfa_sm_send_event(dconf, BFA_DCONF_SM_WR); + return BFA_STATUS_OK; +} + +static void +bfa_dconf_cbfn(void *arg, bfa_status_t status) +{ + struct bfa_dconf_mod_s *dconf = arg; + WARN_ON(status); + bfa_sm_send_event(dconf, BFA_DCONF_SM_FLASH_COMP); +} + +void +bfa_dconf_modexit(struct bfa_s *bfa) +{ + struct bfa_dconf_mod_s *dconf = BFA_DCONF_MOD(bfa); + BFA_DCONF_MOD(bfa)->flashdone = BFA_FALSE; + bfa_trc(bfa, BFA_DCONF_MOD(bfa)->flashdone); + bfa_sm_send_event(dconf, BFA_DCONF_SM_EXIT); +} diff --git a/drivers/scsi/bfa/bfa_ioc.h b/drivers/scsi/bfa/bfa_ioc.h index a7463c955117..546d46b37101 100644 --- a/drivers/scsi/bfa/bfa_ioc.h +++ b/drivers/scsi/bfa/bfa_ioc.h @@ -661,7 +661,6 @@ struct bfa_phy_s { struct bfa_ioc_notify_s ioc_notify; /* ioc event notify */ struct bfa_mem_dma_s phy_dma; }; - #define BFA_PHY(__bfa) (&(__bfa)->modules.phy) #define BFA_MEM_PHY_DMA(__bfa) (&(BFA_PHY(__bfa)->phy_dma)) @@ -686,6 +685,49 @@ void bfa_phy_memclaim(struct bfa_phy_s *phy, u8 *dm_kva, u64 dm_pa, bfa_boolean_t mincfg); void bfa_phy_intr(void *phyarg, struct bfi_mbmsg_s *msg); +/* + * Driver Config( dconf) specific + */ +#define BFI_DCONF_SIGNATURE 0xabcdabcd +#define BFI_DCONF_VERSION 1 + +#pragma pack(1) +struct bfa_dconf_hdr_s { + u32 signature; + u32 version; +}; + +struct bfa_dconf_s { + struct bfa_dconf_hdr_s hdr; + struct bfa_lunmask_cfg_s lun_mask; +}; +#pragma pack() + +struct bfa_dconf_mod_s { + bfa_sm_t sm; + u8 instance; + bfa_boolean_t flashdone; + bfa_boolean_t read_data_valid; + bfa_boolean_t min_cfg; + struct bfa_timer_s timer; + struct bfa_s *bfa; + void *bfad; + void *trcmod; + struct bfa_dconf_s *dconf; + struct bfa_mem_kva_s kva_seg; +}; + +#define BFA_DCONF_MOD(__bfa) \ + (&(__bfa)->modules.dconf_mod) +#define BFA_MEM_DCONF_KVA(__bfa) (&(BFA_DCONF_MOD(__bfa)->kva_seg)) +#define bfa_dconf_read_data_valid(__bfa) \ + (BFA_DCONF_MOD(__bfa)->read_data_valid) +#define BFA_DCONF_UPDATE_TOV 5000 /* memtest timeout in msec */ + +void bfa_dconf_modinit(struct bfa_s *bfa); +void bfa_dconf_modexit(struct bfa_s *bfa); +bfa_status_t bfa_dconf_update(struct bfa_s *bfa); + /* * IOC specfic macros */ diff --git a/drivers/scsi/bfa/bfa_modules.h b/drivers/scsi/bfa/bfa_modules.h index cd51ddf64ae5..2d36e4823835 100644 --- a/drivers/scsi/bfa/bfa_modules.h +++ b/drivers/scsi/bfa/bfa_modules.h @@ -44,6 +44,7 @@ struct bfa_modules_s { struct bfa_flash_s flash; /* flash module */ struct bfa_diag_s diag_mod; /* diagnostics module */ struct bfa_phy_s phy; /* phy module */ + struct bfa_dconf_mod_s dconf_mod; /* DCONF common module */ }; /* @@ -131,5 +132,6 @@ extern struct bfa_module_s hal_mod_lps; extern struct bfa_module_s hal_mod_uf; extern struct bfa_module_s hal_mod_rport; extern struct bfa_module_s hal_mod_fcp; +extern struct bfa_module_s hal_mod_dconf; #endif /* __BFA_MODULES_H__ */ -- cgit v1.2.3