summaryrefslogtreecommitdiff
path: root/drivers/usb/host/xhci-ring.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/usb/host/xhci-ring.c')
-rw-r--r--drivers/usb/host/xhci-ring.c189
1 files changed, 125 insertions, 64 deletions
diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c
index 7113d16e2d3a..940321b3ec68 100644
--- a/drivers/usb/host/xhci-ring.c
+++ b/drivers/usb/host/xhci-ring.c
@@ -185,7 +185,7 @@ static void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring, bool consumer
* prepare_transfer()?
*/
static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
- bool consumer, bool more_trbs_coming)
+ bool consumer, bool more_trbs_coming, bool isoc)
{
u32 chain;
union xhci_trb *next;
@@ -212,11 +212,13 @@ static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
if (!chain && !more_trbs_coming)
break;
- /* If we're not dealing with 0.95 hardware,
+ /* If we're not dealing with 0.95 hardware or
+ * isoc rings on AMD 0.96 host,
* carry over the chain bit of the previous TRB
* (which may mean the chain bit is cleared).
*/
- if (!xhci_link_trb_quirk(xhci)) {
+ if (!(isoc && (xhci->quirks & XHCI_AMD_0x96_HOST))
+ && !xhci_link_trb_quirk(xhci)) {
next->link.control &=
cpu_to_le32(~TRB_CHAIN);
next->link.control |=
@@ -514,8 +516,12 @@ void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
(unsigned long long) addr);
}
+/* flip_cycle means flip the cycle bit of all but the first and last TRB.
+ * (The last TRB actually points to the ring enqueue pointer, which is not part
+ * of this TD.) This is used to remove partially enqueued isoc TDs from a ring.
+ */
static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
- struct xhci_td *cur_td)
+ struct xhci_td *cur_td, bool flip_cycle)
{
struct xhci_segment *cur_seg;
union xhci_trb *cur_trb;
@@ -528,6 +534,12 @@ static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
* leave the pointers intact.
*/
cur_trb->generic.field[3] &= cpu_to_le32(~TRB_CHAIN);
+ /* Flip the cycle bit (link TRBs can't be the first
+ * or last TRB).
+ */
+ if (flip_cycle)
+ cur_trb->generic.field[3] ^=
+ cpu_to_le32(TRB_CYCLE);
xhci_dbg(xhci, "Cancel (unchain) link TRB\n");
xhci_dbg(xhci, "Address = %p (0x%llx dma); "
"in seg %p (0x%llx dma)\n",
@@ -541,6 +553,11 @@ static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
cur_trb->generic.field[2] = 0;
/* Preserve only the cycle bit of this TRB */
cur_trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
+ /* Flip the cycle bit except on the first or last TRB */
+ if (flip_cycle && cur_trb != cur_td->first_trb &&
+ cur_trb != cur_td->last_trb)
+ cur_trb->generic.field[3] ^=
+ cpu_to_le32(TRB_CYCLE);
cur_trb->generic.field[3] |= cpu_to_le32(
TRB_TYPE(TRB_TR_NOOP));
xhci_dbg(xhci, "Cancel TRB %p (0x%llx dma) "
@@ -719,14 +736,14 @@ static void handle_stopped_endpoint(struct xhci_hcd *xhci,
cur_td->urb->stream_id,
cur_td, &deq_state);
else
- td_to_noop(xhci, ep_ring, cur_td);
+ td_to_noop(xhci, ep_ring, cur_td, false);
remove_finished_td:
/*
* The event handler won't see a completion for this TD anymore,
* so remove it from the endpoint ring's TD list. Keep it in
* the cancelled TD list for URB completion later.
*/
- list_del(&cur_td->td_list);
+ list_del_init(&cur_td->td_list);
}
last_unlinked_td = cur_td;
xhci_stop_watchdog_timer_in_irq(xhci, ep);
@@ -754,7 +771,7 @@ remove_finished_td:
do {
cur_td = list_entry(ep->cancelled_td_list.next,
struct xhci_td, cancelled_td_list);
- list_del(&cur_td->cancelled_td_list);
+ list_del_init(&cur_td->cancelled_td_list);
/* Clean up the cancelled URB */
/* Doesn't matter what we pass for status, since the core will
@@ -862,9 +879,9 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg)
cur_td = list_first_entry(&ring->td_list,
struct xhci_td,
td_list);
- list_del(&cur_td->td_list);
+ list_del_init(&cur_td->td_list);
if (!list_empty(&cur_td->cancelled_td_list))
- list_del(&cur_td->cancelled_td_list);
+ list_del_init(&cur_td->cancelled_td_list);
xhci_giveback_urb_in_irq(xhci, cur_td,
-ESHUTDOWN, "killed");
}
@@ -873,7 +890,7 @@ void xhci_stop_endpoint_command_watchdog(unsigned long arg)
&temp_ep->cancelled_td_list,
struct xhci_td,
cancelled_td_list);
- list_del(&cur_td->cancelled_td_list);
+ list_del_init(&cur_td->cancelled_td_list);
xhci_giveback_urb_in_irq(xhci, cur_td,
-ESHUTDOWN, "killed");
}
@@ -1314,10 +1331,8 @@ static void handle_port_status(struct xhci_hcd *xhci,
if (DEV_SUPERSPEED(temp)) {
xhci_dbg(xhci, "resume SS port %d\n", port_id);
- temp = xhci_port_state_to_neutral(temp);
- temp &= ~PORT_PLS_MASK;
- temp |= PORT_LINK_STROBE | XDEV_U0;
- xhci_writel(xhci, temp, port_array[faked_port_index]);
+ xhci_set_link_state(xhci, port_array, faked_port_index,
+ XDEV_U0);
slot_id = xhci_find_slot_id_by_port(hcd, xhci,
faked_port_index);
if (!slot_id) {
@@ -1327,10 +1342,8 @@ static void handle_port_status(struct xhci_hcd *xhci,
xhci_ring_device(xhci, slot_id);
xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
/* Clear PORT_PLC */
- temp = xhci_readl(xhci, port_array[faked_port_index]);
- temp = xhci_port_state_to_neutral(temp);
- temp |= PORT_PLC;
- xhci_writel(xhci, temp, port_array[faked_port_index]);
+ xhci_test_and_clear_bit(xhci, port_array,
+ faked_port_index, PORT_PLC);
} else {
xhci_dbg(xhci, "resume HS port %d\n", port_id);
bus_state->resume_done[faked_port_index] = jiffies +
@@ -1341,6 +1354,10 @@ static void handle_port_status(struct xhci_hcd *xhci,
}
}
+ if (hcd->speed != HCD_USB3)
+ xhci_test_and_clear_bit(xhci, port_array, faked_port_index,
+ PORT_PLC);
+
cleanup:
/* Update event ring dequeue pointer before dropping the lock */
inc_deq(xhci, xhci->event_ring, true);
@@ -1565,10 +1582,10 @@ td_cleanup:
else
*status = 0;
}
- list_del(&td->td_list);
+ list_del_init(&td->td_list);
/* Was this TD slated to be cancelled but completed anyway? */
if (!list_empty(&td->cancelled_td_list))
- list_del(&td->cancelled_td_list);
+ list_del_init(&td->cancelled_td_list);
urb_priv->td_cnt++;
/* Giveback the urb when all the tds are completed */
@@ -1919,8 +1936,10 @@ static int handle_tx_event(struct xhci_hcd *xhci,
int status = -EINPROGRESS;
struct urb_priv *urb_priv;
struct xhci_ep_ctx *ep_ctx;
+ struct list_head *tmp;
u32 trb_comp_code;
int ret = 0;
+ int td_num = 0;
slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
xdev = xhci->devs[slot_id];
@@ -1942,6 +1961,12 @@ static int handle_tx_event(struct xhci_hcd *xhci,
return -ENODEV;
}
+ /* Count current td numbers if ep->skip is set */
+ if (ep->skip) {
+ list_for_each(tmp, &ep_ring->td_list)
+ td_num++;
+ }
+
event_dma = le64_to_cpu(event->buffer);
trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
/* Look for common error cases */
@@ -2053,7 +2078,18 @@ static int handle_tx_event(struct xhci_hcd *xhci,
goto cleanup;
}
+ /* We've skipped all the TDs on the ep ring when ep->skip set */
+ if (ep->skip && td_num == 0) {
+ ep->skip = false;
+ xhci_dbg(xhci, "All tds on the ep_ring skipped. "
+ "Clear skip flag.\n");
+ ret = 0;
+ goto cleanup;
+ }
+
td = list_entry(ep_ring->td_list.next, struct xhci_td, td_list);
+ if (ep->skip)
+ td_num--;
/* Is this a TRB in the currently executing TD? */
event_seg = trb_in_td(ep_ring->deq_seg, ep_ring->dequeue,
@@ -2158,7 +2194,8 @@ cleanup:
if ((urb->actual_length != urb->transfer_buffer_length &&
(urb->transfer_flags &
URB_SHORT_NOT_OK)) ||
- status != 0)
+ (status != 0 &&
+ !usb_endpoint_xfer_isoc(&urb->ep->desc)))
xhci_dbg(xhci, "Giveback URB %p, len = %d, "
"expected = %x, status = %d\n",
urb, urb->actual_length,
@@ -2375,7 +2412,7 @@ irqreturn_t xhci_msi_irq(int irq, struct usb_hcd *hcd)
* prepare_transfer()?
*/
static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
- bool consumer, bool more_trbs_coming,
+ bool consumer, bool more_trbs_coming, bool isoc,
u32 field1, u32 field2, u32 field3, u32 field4)
{
struct xhci_generic_trb *trb;
@@ -2385,7 +2422,7 @@ static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
trb->field[1] = cpu_to_le32(field2);
trb->field[2] = cpu_to_le32(field3);
trb->field[3] = cpu_to_le32(field4);
- inc_enq(xhci, ring, consumer, more_trbs_coming);
+ inc_enq(xhci, ring, consumer, more_trbs_coming, isoc);
}
/*
@@ -2393,7 +2430,7 @@ static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
* FIXME allocate segments if the ring is full.
*/
static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
- u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
+ u32 ep_state, unsigned int num_trbs, bool isoc, gfp_t mem_flags)
{
/* Make sure the endpoint has been added to xHC schedule */
switch (ep_state) {
@@ -2435,10 +2472,11 @@ static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
next = ring->enqueue;
while (last_trb(xhci, ring, ring->enq_seg, next)) {
- /* If we're not dealing with 0.95 hardware,
- * clear the chain bit.
+ /* If we're not dealing with 0.95 hardware or isoc rings
+ * on AMD 0.96 host, clear the chain bit.
*/
- if (!xhci_link_trb_quirk(xhci))
+ if (!xhci_link_trb_quirk(xhci) && !(isoc &&
+ (xhci->quirks & XHCI_AMD_0x96_HOST)))
next->link.control &= cpu_to_le32(~TRB_CHAIN);
else
next->link.control |= cpu_to_le32(TRB_CHAIN);
@@ -2471,6 +2509,7 @@ static int prepare_transfer(struct xhci_hcd *xhci,
unsigned int num_trbs,
struct urb *urb,
unsigned int td_index,
+ bool isoc,
gfp_t mem_flags)
{
int ret;
@@ -2488,7 +2527,7 @@ static int prepare_transfer(struct xhci_hcd *xhci,
ret = prepare_ring(xhci, ep_ring,
le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
- num_trbs, mem_flags);
+ num_trbs, isoc, mem_flags);
if (ret)
return ret;
@@ -2500,11 +2539,8 @@ static int prepare_transfer(struct xhci_hcd *xhci,
if (td_index == 0) {
ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb);
- if (unlikely(ret)) {
- xhci_urb_free_priv(xhci, urb_priv);
- urb->hcpriv = NULL;
+ if (unlikely(ret))
return ret;
- }
}
td->urb = urb;
@@ -2672,11 +2708,15 @@ static u32 xhci_v1_0_td_remainder(int running_total, int trb_buff_len,
{
int packets_transferred;
+ /* One TRB with a zero-length data packet. */
+ if (running_total == 0 && trb_buff_len == 0)
+ return 0;
+
/* All the TRB queueing functions don't count the current TRB in
* running_total.
*/
packets_transferred = (running_total + trb_buff_len) /
- le16_to_cpu(urb->ep->desc.wMaxPacketSize);
+ usb_endpoint_maxp(&urb->ep->desc);
return xhci_td_remainder(total_packet_count - packets_transferred);
}
@@ -2706,11 +2746,11 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
num_trbs = count_sg_trbs_needed(xhci, urb);
num_sgs = urb->num_sgs;
total_packet_count = roundup(urb->transfer_buffer_length,
- le16_to_cpu(urb->ep->desc.wMaxPacketSize));
+ usb_endpoint_maxp(&urb->ep->desc));
trb_buff_len = prepare_transfer(xhci, xhci->devs[slot_id],
ep_index, urb->stream_id,
- num_trbs, urb, 0, mem_flags);
+ num_trbs, urb, 0, false, mem_flags);
if (trb_buff_len < 0)
return trb_buff_len;
@@ -2805,7 +2845,7 @@ static int queue_bulk_sg_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
more_trbs_coming = true;
else
more_trbs_coming = false;
- queue_trb(xhci, ep_ring, false, more_trbs_coming,
+ queue_trb(xhci, ep_ring, false, more_trbs_coming, false,
lower_32_bits(addr),
upper_32_bits(addr),
length_field,
@@ -2896,7 +2936,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
ret = prepare_transfer(xhci, xhci->devs[slot_id],
ep_index, urb->stream_id,
- num_trbs, urb, 0, mem_flags);
+ num_trbs, urb, 0, false, mem_flags);
if (ret < 0)
return ret;
@@ -2913,7 +2953,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
running_total = 0;
total_packet_count = roundup(urb->transfer_buffer_length,
- le16_to_cpu(urb->ep->desc.wMaxPacketSize));
+ usb_endpoint_maxp(&urb->ep->desc));
/* How much data is in the first TRB? */
addr = (u64) urb->transfer_dma;
trb_buff_len = TRB_MAX_BUFF_SIZE -
@@ -2968,7 +3008,7 @@ int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
more_trbs_coming = true;
else
more_trbs_coming = false;
- queue_trb(xhci, ep_ring, false, more_trbs_coming,
+ queue_trb(xhci, ep_ring, false, more_trbs_coming, false,
lower_32_bits(addr),
upper_32_bits(addr),
length_field,
@@ -3028,7 +3068,7 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
num_trbs++;
ret = prepare_transfer(xhci, xhci->devs[slot_id],
ep_index, urb->stream_id,
- num_trbs, urb, 0, mem_flags);
+ num_trbs, urb, 0, false, mem_flags);
if (ret < 0)
return ret;
@@ -3061,7 +3101,7 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
}
}
- queue_trb(xhci, ep_ring, false, true,
+ queue_trb(xhci, ep_ring, false, true, false,
setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16,
le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16,
TRB_LEN(8) | TRB_INTR_TARGET(0),
@@ -3081,7 +3121,7 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
if (urb->transfer_buffer_length > 0) {
if (setup->bRequestType & USB_DIR_IN)
field |= TRB_DIR_IN;
- queue_trb(xhci, ep_ring, false, true,
+ queue_trb(xhci, ep_ring, false, true, false,
lower_32_bits(urb->transfer_dma),
upper_32_bits(urb->transfer_dma),
length_field,
@@ -3097,7 +3137,7 @@ int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
field = 0;
else
field = TRB_DIR_IN;
- queue_trb(xhci, ep_ring, false, false,
+ queue_trb(xhci, ep_ring, false, false, false,
0,
0,
TRB_INTR_TARGET(0),
@@ -3113,21 +3153,16 @@ static int count_isoc_trbs_needed(struct xhci_hcd *xhci,
struct urb *urb, int i)
{
int num_trbs = 0;
- u64 addr, td_len, running_total;
+ u64 addr, td_len;
addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);
td_len = urb->iso_frame_desc[i].length;
- running_total = TRB_MAX_BUFF_SIZE - (addr & (TRB_MAX_BUFF_SIZE - 1));
- running_total &= TRB_MAX_BUFF_SIZE - 1;
- if (running_total != 0)
+ num_trbs = DIV_ROUND_UP(td_len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
+ TRB_MAX_BUFF_SIZE);
+ if (num_trbs == 0)
num_trbs++;
- while (running_total < td_len) {
- num_trbs++;
- running_total += TRB_MAX_BUFF_SIZE;
- }
-
return num_trbs;
}
@@ -3226,6 +3261,7 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
start_trb = &ep_ring->enqueue->generic;
start_cycle = ep_ring->cycle_state;
+ urb_priv = urb->hcpriv;
/* Queue the first TRB, even if it's zero-length */
for (i = 0; i < num_tds; i++) {
unsigned int total_packet_count;
@@ -3237,9 +3273,11 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
addr = start_addr + urb->iso_frame_desc[i].offset;
td_len = urb->iso_frame_desc[i].length;
td_remain_len = td_len;
- /* FIXME: Ignoring zero-length packets, can those happen? */
total_packet_count = roundup(td_len,
- le16_to_cpu(urb->ep->desc.wMaxPacketSize));
+ usb_endpoint_maxp(&urb->ep->desc));
+ /* A zero-length transfer still involves at least one packet. */
+ if (total_packet_count == 0)
+ total_packet_count++;
burst_count = xhci_get_burst_count(xhci, urb->dev, urb,
total_packet_count);
residue = xhci_get_last_burst_packet_count(xhci,
@@ -3248,13 +3286,15 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
trbs_per_td = count_isoc_trbs_needed(xhci, urb, i);
ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index,
- urb->stream_id, trbs_per_td, urb, i, mem_flags);
- if (ret < 0)
- return ret;
+ urb->stream_id, trbs_per_td, urb, i, true,
+ mem_flags);
+ if (ret < 0) {
+ if (i == 0)
+ return ret;
+ goto cleanup;
+ }
- urb_priv = urb->hcpriv;
td = urb_priv->td[i];
-
for (j = 0; j < trbs_per_td; j++) {
u32 remainder = 0;
field = TRB_TBC(burst_count) | TRB_TLBPC(residue);
@@ -3317,7 +3357,7 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
remainder |
TRB_INTR_TARGET(0);
- queue_trb(xhci, ep_ring, false, more_trbs_coming,
+ queue_trb(xhci, ep_ring, false, more_trbs_coming, true,
lower_32_bits(addr),
upper_32_bits(addr),
length_field,
@@ -3344,6 +3384,27 @@ static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
start_cycle, start_trb);
return 0;
+cleanup:
+ /* Clean up a partially enqueued isoc transfer. */
+
+ for (i--; i >= 0; i--)
+ list_del_init(&urb_priv->td[i]->td_list);
+
+ /* Use the first TD as a temporary variable to turn the TDs we've queued
+ * into No-ops with a software-owned cycle bit. That way the hardware
+ * won't accidentally start executing bogus TDs when we partially
+ * overwrite them. td->first_trb and td->start_seg are already set.
+ */
+ urb_priv->td[0]->last_trb = ep_ring->enqueue;
+ /* Every TRB except the first & last will have its cycle bit flipped. */
+ td_to_noop(xhci, ep_ring, urb_priv->td[0], true);
+
+ /* Reset the ring enqueue back to the first TRB and its cycle bit. */
+ ep_ring->enqueue = urb_priv->td[0]->first_trb;
+ ep_ring->enq_seg = urb_priv->td[0]->start_seg;
+ ep_ring->cycle_state = start_cycle;
+ usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
+ return ret;
}
/*
@@ -3378,7 +3439,7 @@ int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
* Do not insert any td of the urb to the ring if the check failed.
*/
ret = prepare_ring(xhci, ep_ring, le32_to_cpu(ep_ctx->ep_info) & EP_STATE_MASK,
- num_trbs, mem_flags);
+ num_trbs, true, mem_flags);
if (ret)
return ret;
@@ -3437,7 +3498,7 @@ static int queue_command(struct xhci_hcd *xhci, u32 field1, u32 field2,
reserved_trbs++;
ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
- reserved_trbs, GFP_ATOMIC);
+ reserved_trbs, false, GFP_ATOMIC);
if (ret < 0) {
xhci_err(xhci, "ERR: No room for command on command ring\n");
if (command_must_succeed)
@@ -3445,8 +3506,8 @@ static int queue_command(struct xhci_hcd *xhci, u32 field1, u32 field2,
"unfailable commands failed.\n");
return ret;
}
- queue_trb(xhci, xhci->cmd_ring, false, false, field1, field2, field3,
- field4 | xhci->cmd_ring->cycle_state);
+ queue_trb(xhci, xhci->cmd_ring, false, false, false, field1, field2,
+ field3, field4 | xhci->cmd_ring->cycle_state);
return 0;
}
generated by cgit v1.2.3 (git 2.39.0) at 2025-02-13 05:58:53 +0300