igbvf: refactor Interrupt Throttle Rate code

The existing ITR code is broken and confusing, with lots of similarly-named
variables that do different things. Additionally, after the driver carefully
determines the optimal interrupt rate for the adapter, it then
ignores it and always writes a fixed, suboptimal value.

This patch refactors that code to make variable names more descriptive of
what they actually do, and then actually writes the calculated result to
the hardware.

Preliminary testing shows that netperf TCP_STREAM tests goes from ~918Mbps
to ~940Mbps, and TCP_RR goes from ~2k transactions/sec up to > 8k.

Signed-off-by: Mitch Williams <mitch.a.williams@intel.com>
Tested-by: Robert E Garrett <robertX.e.garrett@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>

3 files changed, 99 insertions, 78 deletions

diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c
index db7dce2351c2..8ce67064b9c5 100644
--- a/drivers/net/ethernet/intel/igbvf/ethtool.c
+++ b/drivers/net/ethernet/intel/igbvf/ethtool.c
@@ -343,10 +343,10 @@ static int igbvf_get_coalesce(struct net_device *netdev,
 {
 	struct igbvf_adapter *adapter = netdev_priv(netdev);
 
-	if (adapter->itr_setting <= 3)
-		ec->rx_coalesce_usecs = adapter->itr_setting;
+	if (adapter->requested_itr <= 3)
+		ec->rx_coalesce_usecs = adapter->requested_itr;
 	else
-		ec->rx_coalesce_usecs = adapter->itr_setting >> 2;
+		ec->rx_coalesce_usecs = adapter->current_itr >> 2;
 
 	return 0;
 }
@@ -365,15 +365,16 @@ static int igbvf_set_coalesce(struct net_device *netdev,
 
 	/* convert to rate of irq's per second */
 	if (ec->rx_coalesce_usecs && ec->rx_coalesce_usecs <= 3) {
-		adapter->itr = IGBVF_START_ITR;
-		adapter->itr_setting = ec->rx_coalesce_usecs;
+		adapter->current_itr = IGBVF_START_ITR;
+		adapter->requested_itr = ec->rx_coalesce_usecs;
 	} else {
-		adapter->itr = ec->rx_coalesce_usecs << 2;
-		adapter->itr_setting = adapter->itr;
+		adapter->current_itr = ec->rx_coalesce_usecs << 2;
+		adapter->requested_itr = 1000000000 /
+					(adapter->current_itr * 256);
 	}
 
-	writel(adapter->itr,
-	       hw->hw_addr + adapter->rx_ring[0].itr_register);
+	writel(adapter->current_itr,
+	       hw->hw_addr + adapter->rx_ring->itr_register);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/intel/igbvf/igbvf.h b/drivers/net/ethernet/intel/igbvf/igbvf.h
index 2c6d87e4d3d9..a895e2f7b34d 100644
--- a/drivers/net/ethernet/intel/igbvf/igbvf.h
+++ b/drivers/net/ethernet/intel/igbvf/igbvf.h
@@ -43,7 +43,18 @@ struct igbvf_info;
 struct igbvf_adapter;
 
 /* Interrupt defines */
-#define IGBVF_START_ITR                 648 /* ~6000 ints/sec */
+#define IGBVF_START_ITR                    488 /* ~8000 ints/sec */
+#define IGBVF_4K_ITR                       980
+#define IGBVF_20K_ITR                      196
+#define IGBVF_70K_ITR                       56
+
+enum latency_range {
+	lowest_latency = 0,
+	low_latency = 1,
+	bulk_latency = 2,
+	latency_invalid = 255
+};
+
 
 /* Interrupt modes, as used by the IntMode parameter */
 #define IGBVF_INT_MODE_LEGACY           0
@@ -155,6 +166,7 @@ struct igbvf_ring {
 	char name[IFNAMSIZ + 5];
 	u32 eims_value;
 	u32 itr_val;
+	enum latency_range itr_range;
 	u16 itr_register;
 	int set_itr;
 
@@ -187,10 +199,8 @@ struct igbvf_adapter {
 	unsigned long state;
 
 	/* Interrupt Throttle Rate */
-	u32 itr;
-	u32 itr_setting;
-	u16 tx_itr;
-	u16 rx_itr;
+	u32 requested_itr; /* ints/sec or adaptive */
+	u32 current_itr; /* Actual ITR register value, not ints/sec */
 
 	/*
 	 * Tx
@@ -299,13 +309,6 @@ enum igbvf_state_t {
 	__IGBVF_DOWN
 };
 
-enum latency_range {
-	lowest_latency = 0,
-	low_latency = 1,
-	bulk_latency = 2,
-	latency_invalid = 255
-};
-
 extern char igbvf_driver_name[];
 extern const char igbvf_driver_version[];
 
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index 4e9141cfe81d..446297ff0104 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -632,14 +632,13 @@ void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)
  *      traffic pattern.  Constants in this function were computed
  *      based on theoretical maximum wire speed and thresholds were set based
  *      on testing data as well as attempting to minimize response time
- *      while increasing bulk throughput.  This functionality is controlled
- *      by the InterruptThrottleRate module parameter.
+ *      while increasing bulk throughput.
  **/
-static unsigned int igbvf_update_itr(struct igbvf_adapter *adapter,
-                                     u16 itr_setting, int packets,
-                                     int bytes)
+static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,
+					   enum latency_range itr_setting,
+					   int packets, int bytes)
 {
-	unsigned int retval = itr_setting;
+	enum latency_range retval = itr_setting;
 
 	if (packets == 0)
 		goto update_itr_done;
@@ -675,65 +674,87 @@ static unsigned int igbvf_update_itr(struct igbvf_adapter *adapter,
 			retval = low_latency;
 		}
 		break;
+	default:
+		break;
 	}
 
 update_itr_done:
 	return retval;
 }
 
-static void igbvf_set_itr(struct igbvf_adapter *adapter)
+static int igbvf_range_to_itr(enum latency_range current_range)
 {
-	struct e1000_hw *hw = &adapter->hw;
-	u16 current_itr;
-	u32 new_itr = adapter->itr;
-
-	adapter->tx_itr = igbvf_update_itr(adapter, adapter->tx_itr,
-	                                   adapter->total_tx_packets,
-	                                   adapter->total_tx_bytes);
-	/* conservative mode (itr 3) eliminates the lowest_latency setting */
-	if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
-		adapter->tx_itr = low_latency;
-
-	adapter->rx_itr = igbvf_update_itr(adapter, adapter->rx_itr,
-	                                   adapter->total_rx_packets,
-	                                   adapter->total_rx_bytes);
-	/* conservative mode (itr 3) eliminates the lowest_latency setting */
-	if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
-		adapter->rx_itr = low_latency;
+	int new_itr;
 
-	current_itr = max(adapter->rx_itr, adapter->tx_itr);
-
-	switch (current_itr) {
+	switch (current_range) {
 	/* counts and packets in update_itr are dependent on these numbers */
 	case lowest_latency:
-		new_itr = 70000;
+		new_itr = IGBVF_70K_ITR;
 		break;
 	case low_latency:
-		new_itr = 20000; /* aka hwitr = ~200 */
+		new_itr = IGBVF_20K_ITR;
 		break;
 	case bulk_latency:
-		new_itr = 4000;
+		new_itr = IGBVF_4K_ITR;
 		break;
 	default:
+		new_itr = IGBVF_START_ITR;
 		break;
 	}
+	return new_itr;
+}
+
+static void igbvf_set_itr(struct igbvf_adapter *adapter)
+{
+	u32 new_itr;
+
+	adapter->tx_ring->itr_range =
+			igbvf_update_itr(adapter,
+					 adapter->tx_ring->itr_val,
+					 adapter->total_tx_packets,
+					 adapter->total_tx_bytes);
+
+	/* conservative mode (itr 3) eliminates the lowest_latency setting */
+	if (adapter->requested_itr == 3 &&
+	    adapter->tx_ring->itr_range == lowest_latency)
+		adapter->tx_ring->itr_range = low_latency;
 
-	if (new_itr != adapter->itr) {
+	new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);
+
+
+	if (new_itr != adapter->tx_ring->itr_val) {
+		u32 current_itr = adapter->tx_ring->itr_val;
 		/*
 		 * this attempts to bias the interrupt rate towards Bulk
 		 * by adding intermediate steps when interrupt rate is
 		 * increasing
 		 */
-		new_itr = new_itr > adapter->itr ?
-		             min(adapter->itr + (new_itr >> 2), new_itr) :
-		             new_itr;
-		adapter->itr = new_itr;
-		adapter->rx_ring->itr_val = 1952;
-
-		if (adapter->msix_entries)
-			adapter->rx_ring->set_itr = 1;
-		else
-			ew32(ITR, 1952);
+		new_itr = new_itr > current_itr ?
+			     min(current_itr + (new_itr >> 2), new_itr) :
+			     new_itr;
+		adapter->tx_ring->itr_val = new_itr;
+
+		adapter->tx_ring->set_itr = 1;
+	}
+
+	adapter->rx_ring->itr_range =
+			igbvf_update_itr(adapter, adapter->rx_ring->itr_val,
+					 adapter->total_rx_packets,
+					 adapter->total_rx_bytes);
+	if (adapter->requested_itr == 3 &&
+	    adapter->rx_ring->itr_range == lowest_latency)
+		adapter->rx_ring->itr_range = low_latency;
+
+	new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);
+
+	if (new_itr != adapter->rx_ring->itr_val) {
+		u32 current_itr = adapter->rx_ring->itr_val;
+		new_itr = new_itr > current_itr ?
+			     min(current_itr + (new_itr >> 2), new_itr) :
+			     new_itr;
+		adapter->rx_ring->itr_val = new_itr;
+
+		adapter->rx_ring->set_itr = 1;
 	}
 }
 
@@ -835,6 +856,11 @@ static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)
 	struct e1000_hw *hw = &adapter->hw;
 	struct igbvf_ring *tx_ring = adapter->tx_ring;
 
+	if (tx_ring->set_itr) {
+		writel(tx_ring->itr_val,
+		       adapter->hw.hw_addr + tx_ring->itr_register);
+		adapter->tx_ring->set_itr = 0;
+	}
 
 	adapter->total_tx_bytes = 0;
 	adapter->total_tx_packets = 0;
@@ -937,19 +963,10 @@ static void igbvf_configure_msix(struct igbvf_adapter *adapter)
 
 	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);
 	adapter->eims_enable_mask |= tx_ring->eims_value;
-	if (tx_ring->itr_val)
-		writel(tx_ring->itr_val,
-		       hw->hw_addr + tx_ring->itr_register);
-	else
-		writel(1952, hw->hw_addr + tx_ring->itr_register);
-
+	writel(tx_ring->itr_val, hw->hw_addr + tx_ring->itr_register);
 	igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);
 	adapter->eims_enable_mask |= rx_ring->eims_value;
-	if (rx_ring->itr_val)
-		writel(rx_ring->itr_val,
-		       hw->hw_addr + rx_ring->itr_register);
-	else
-		writel(1952, hw->hw_addr + rx_ring->itr_register);
+	writel(rx_ring->itr_val, hw->hw_addr + rx_ring->itr_register);
 
 	/* set vector for other causes, i.e. link changes */
 
@@ -1027,7 +1044,7 @@ static int igbvf_request_msix(struct igbvf_adapter *adapter)
 		goto out;
 
 	adapter->tx_ring->itr_register = E1000_EITR(vector);
-	adapter->tx_ring->itr_val = 1952;
+	adapter->tx_ring->itr_val = adapter->current_itr;
 	vector++;
 
 	err = request_irq(adapter->msix_entries[vector].vector,
@@ -1037,7 +1054,7 @@ static int igbvf_request_msix(struct igbvf_adapter *adapter)
 		goto out;
 
 	adapter->rx_ring->itr_register = E1000_EITR(vector);
-	adapter->rx_ring->itr_val = 1952;
+	adapter->rx_ring->itr_val = adapter->current_itr;
 	vector++;
 
 	err = request_irq(adapter->msix_entries[vector].vector,
@@ -1151,7 +1168,7 @@ static int igbvf_poll(struct napi_struct *napi, int budget)
 	if (work_done < budget) {
 		napi_complete(napi);
 
-		if (adapter->itr_setting & 3)
+		if (adapter->requested_itr & 3)
 			igbvf_set_itr(adapter);
 
 		if (!test_bit(__IGBVF_DOWN, &adapter->state))
@@ -1521,8 +1538,8 @@ static int __devinit igbvf_sw_init(struct igbvf_adapter *adapter)
 	adapter->tx_abs_int_delay = 32;
 	adapter->rx_int_delay = 0;
 	adapter->rx_abs_int_delay = 8;
-	adapter->itr_setting = 3;
-	adapter->itr = 20000;
+	adapter->requested_itr = 3;
+	adapter->current_itr = IGBVF_START_ITR;
 
 	/* Set various function pointers */
 	adapter->ei->init_ops(&adapter->hw);