1 files changed, 75 insertions, 17 deletions

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
index 4984b41cd372..b1a6eb349bb3 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c
@@ -31,6 +31,7 @@
 #define OVER_SUBSCRIPTION_PROCESS_COUNT (1 << 0)
 #define OVER_SUBSCRIPTION_COMPUTE_QUEUE_COUNT (1 << 1)
 #define OVER_SUBSCRIPTION_GWS_QUEUE_COUNT (1 << 2)
+#define OVER_SUBSCRIPTION_XNACK_CONFLICT (1 << 3)
 
 static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
 				unsigned int buffer_size_bytes)
@@ -44,7 +45,8 @@ static inline void inc_wptr(unsigned int *wptr, unsigned int increment_bytes,
 
 static void pm_calc_rlib_size(struct packet_manager *pm,
 				unsigned int *rlib_size,
-				int *over_subscription)
+				int *over_subscription,
+				int xnack_conflict)
 {
 	unsigned int process_count, queue_count, compute_queue_count, gws_queue_count;
 	unsigned int map_queue_size;
@@ -73,6 +75,8 @@ static void pm_calc_rlib_size(struct packet_manager *pm,
 		*over_subscription |= OVER_SUBSCRIPTION_COMPUTE_QUEUE_COUNT;
 	if (gws_queue_count > 1)
 		*over_subscription |= OVER_SUBSCRIPTION_GWS_QUEUE_COUNT;
+	if (xnack_conflict && (node->adev->gmc.xnack_flags & AMDGPU_GMC_XNACK_FLAG_CHAIN))
+		*over_subscription |= OVER_SUBSCRIPTION_XNACK_CONFLICT;
 
 	if (*over_subscription)
 		dev_dbg(dev, "Over subscribed runlist\n");
@@ -96,7 +100,8 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 				unsigned int **rl_buffer,
 				uint64_t *rl_gpu_buffer,
 				unsigned int *rl_buffer_size,
-				int *is_over_subscription)
+				int *is_over_subscription,
+				int xnack_conflict)
 {
 	struct kfd_node *node = pm->dqm->dev;
 	struct device *dev = node->adev->dev;
@@ -105,7 +110,8 @@ static int pm_allocate_runlist_ib(struct packet_manager *pm,
 	if (WARN_ON(pm->allocated))
 		return -EINVAL;
 
-	pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription);
+	pm_calc_rlib_size(pm, rl_buffer_size, is_over_subscription,
+				xnack_conflict);
 
 	mutex_lock(&pm->lock);
 
@@ -142,11 +148,27 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 	struct queue *q;
 	struct kernel_queue *kq;
 	int is_over_subscription;
+	int xnack_enabled = -1;
+	bool xnack_conflict = 0;
 
 	rl_wptr = retval = processes_mapped = 0;
 
+	/* Check if processes set different xnack modes */
+	list_for_each_entry(cur, queues, list) {
+		qpd = cur->qpd;
+		if (xnack_enabled < 0)
+			/* First process */
+			xnack_enabled = qpd->pqm->process->xnack_enabled;
+		else if (qpd->pqm->process->xnack_enabled != xnack_enabled) {
+			/* Found a process with a different xnack mode */
+			xnack_conflict = 1;
+			break;
+		}
+	}
+
 	retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,
-				&alloc_size_bytes, &is_over_subscription);
+				&alloc_size_bytes, &is_over_subscription,
+				xnack_conflict);
 	if (retval)
 		return retval;
 
@@ -156,9 +178,13 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 	dev_dbg(dev, "Building runlist ib process count: %d queues count %d\n",
 		pm->dqm->processes_count, pm->dqm->active_queue_count);
 
+build_runlist_ib:
 	/* build the run list ib packet */
 	list_for_each_entry(cur, queues, list) {
 		qpd = cur->qpd;
+		/* group processes with the same xnack mode together */
+		if (qpd->pqm->process->xnack_enabled != xnack_enabled)
+			continue;
 		/* build map process packet */
 		if (processes_mapped >= pm->dqm->processes_count) {
 			dev_dbg(dev, "Not enough space left in runlist IB\n");
@@ -215,18 +241,26 @@ static int pm_create_runlist_ib(struct packet_manager *pm,
 				alloc_size_bytes);
 		}
 	}
+	if (xnack_conflict) {
+		/* pick up processes with the other xnack mode */
+		xnack_enabled = !xnack_enabled;
+		xnack_conflict = 0;
+		goto build_runlist_ib;
+	}
 
 	dev_dbg(dev, "Finished map process and queues to runlist\n");
 
 	if (is_over_subscription) {
 		if (!pm->is_over_subscription)
-			dev_warn(dev, "Runlist is getting oversubscribed due to%s%s%s. Expect reduced ROCm performance.\n",
-				 is_over_subscription & OVER_SUBSCRIPTION_PROCESS_COUNT ?
-				 " too many processes." : "",
-				 is_over_subscription & OVER_SUBSCRIPTION_COMPUTE_QUEUE_COUNT ?
-				 " too many queues." : "",
-				 is_over_subscription & OVER_SUBSCRIPTION_GWS_QUEUE_COUNT ?
-				 " multiple processes using cooperative launch." : "");
+			dev_warn(dev, "Runlist is getting oversubscribed due to%s%s%s%s. Expect reduced ROCm performance.\n",
+				is_over_subscription & OVER_SUBSCRIPTION_PROCESS_COUNT ?
+				" too many processes" : "",
+				is_over_subscription & OVER_SUBSCRIPTION_COMPUTE_QUEUE_COUNT ?
+				" too many queues" : "",
+				is_over_subscription & OVER_SUBSCRIPTION_GWS_QUEUE_COUNT ?
+				" multiple processes using cooperative launch" : "",
+				is_over_subscription & OVER_SUBSCRIPTION_XNACK_CONFLICT ?
+				" xnack on/off processes mixed on gfx9" : "");
 
 		retval = pm->pmf->runlist(pm, &rl_buffer[rl_wptr],
 					*rl_gpu_addr,
@@ -396,14 +430,33 @@ out:
 	return retval;
 }
 
-int pm_update_grace_period(struct packet_manager *pm, uint32_t grace_period)
+/* pm_config_dequeue_wait_counts: Configure dequeue timer Wait Counts
+ *  by writing to CP_IQ_WAIT_TIME2 registers.
+ *
+ *  @cmd: See emum kfd_config_dequeue_wait_counts_cmd definition
+ *  @value: Depends on the cmd. This parameter is unused for
+ *    KFD_DEQUEUE_WAIT_INIT and KFD_DEQUEUE_WAIT_RESET. For
+ *    KFD_DEQUEUE_WAIT_SET_SCH_WAVE it holds value to be set
+ *
+ */
+int pm_config_dequeue_wait_counts(struct packet_manager *pm,
+		enum kfd_config_dequeue_wait_counts_cmd cmd,
+		uint32_t value)
 {
 	struct kfd_node *node = pm->dqm->dev;
 	struct device *dev = node->adev->dev;
 	int retval = 0;
 	uint32_t *buffer, size;
 
-	size = pm->pmf->set_grace_period_size;
+	if (!pm->pmf->config_dequeue_wait_counts ||
+	    !pm->pmf->config_dequeue_wait_counts_size)
+		return 0;
+
+	if (cmd == KFD_DEQUEUE_WAIT_INIT && (KFD_GC_VERSION(pm->dqm->dev) < IP_VERSION(9, 4, 1) ||
+	   KFD_GC_VERSION(pm->dqm->dev) >= IP_VERSION(10, 0, 0)))
+		return 0;
+
+	size = pm->pmf->config_dequeue_wait_counts_size;
 
 	mutex_lock(&pm->lock);
 
@@ -419,13 +472,18 @@ int pm_update_grace_period(struct packet_manager *pm, uint32_t grace_period)
 			goto out;
 		}
 
-		retval = pm->pmf->set_grace_period(pm, buffer, grace_period);
-		if (!retval)
+		retval = pm->pmf->config_dequeue_wait_counts(pm, buffer,
+							     cmd, value);
+		if (!retval) {
 			retval = kq_submit_packet(pm->priv_queue);
-		else
+
+			/* If default value is modified, cache that in dqm->wait_times */
+			if (!retval && cmd == KFD_DEQUEUE_WAIT_INIT)
+				update_dqm_wait_times(pm->dqm);
+		} else {
 			kq_rollback_packet(pm->priv_queue);
+		}
 	}
-
 out:
 	mutex_unlock(&pm->lock);
 	return retval;