1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
|
// SPDX-License-Identifier: MIT
/*
* Copyright © 2021 Intel Corporation
*/
#include "xe_hw_fence.h"
#include <linux/device.h>
#include <linux/slab.h>
#include "xe_bo.h"
#include "xe_device.h"
#include "xe_gt.h"
#include "xe_hw_engine.h"
#include "xe_macros.h"
#include "xe_map.h"
#include "xe_trace.h"
static struct kmem_cache *xe_hw_fence_slab;
int __init xe_hw_fence_module_init(void)
{
xe_hw_fence_slab = kmem_cache_create("xe_hw_fence",
sizeof(struct xe_hw_fence), 0,
SLAB_HWCACHE_ALIGN, NULL);
if (!xe_hw_fence_slab)
return -ENOMEM;
return 0;
}
void xe_hw_fence_module_exit(void)
{
rcu_barrier();
kmem_cache_destroy(xe_hw_fence_slab);
}
static struct xe_hw_fence *fence_alloc(void)
{
return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL);
}
static void fence_free(struct rcu_head *rcu)
{
struct xe_hw_fence *fence =
container_of(rcu, struct xe_hw_fence, dma.rcu);
if (!WARN_ON_ONCE(!fence))
kmem_cache_free(xe_hw_fence_slab, fence);
}
static void hw_fence_irq_run_cb(struct irq_work *work)
{
struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work);
struct xe_hw_fence *fence, *next;
bool tmp;
tmp = dma_fence_begin_signalling();
spin_lock(&irq->lock);
if (irq->enabled) {
list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
struct dma_fence *dma_fence = &fence->dma;
trace_xe_hw_fence_try_signal(fence);
if (dma_fence_is_signaled_locked(dma_fence)) {
trace_xe_hw_fence_signal(fence);
list_del_init(&fence->irq_link);
dma_fence_put(dma_fence);
}
}
}
spin_unlock(&irq->lock);
dma_fence_end_signalling(tmp);
}
void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq)
{
spin_lock_init(&irq->lock);
init_irq_work(&irq->work, hw_fence_irq_run_cb);
INIT_LIST_HEAD(&irq->pending);
irq->enabled = true;
}
void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq)
{
struct xe_hw_fence *fence, *next;
unsigned long flags;
int err;
bool tmp;
if (XE_WARN_ON(!list_empty(&irq->pending))) {
tmp = dma_fence_begin_signalling();
spin_lock_irqsave(&irq->lock, flags);
list_for_each_entry_safe(fence, next, &irq->pending, irq_link) {
list_del_init(&fence->irq_link);
err = dma_fence_signal_locked(&fence->dma);
dma_fence_put(&fence->dma);
XE_WARN_ON(err);
}
spin_unlock_irqrestore(&irq->lock, flags);
dma_fence_end_signalling(tmp);
}
}
void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq)
{
irq_work_queue(&irq->work);
}
void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq)
{
spin_lock_irq(&irq->lock);
irq->enabled = false;
spin_unlock_irq(&irq->lock);
}
void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq)
{
spin_lock_irq(&irq->lock);
irq->enabled = true;
spin_unlock_irq(&irq->lock);
irq_work_queue(&irq->work);
}
void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt,
struct xe_hw_fence_irq *irq, const char *name)
{
ctx->gt = gt;
ctx->irq = irq;
ctx->dma_fence_ctx = dma_fence_context_alloc(1);
ctx->next_seqno = XE_FENCE_INITIAL_SEQNO;
sprintf(ctx->name, "%s", name);
}
void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx)
{
}
static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence);
static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence)
{
return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock);
}
static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence)
{
struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
return dev_name(gt_to_xe(fence->ctx->gt)->drm.dev);
}
static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence)
{
struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
return fence->ctx->name;
}
static bool xe_hw_fence_signaled(struct dma_fence *dma_fence)
{
struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
struct xe_device *xe = gt_to_xe(fence->ctx->gt);
u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32);
return dma_fence->error ||
!__dma_fence_is_later(dma_fence->seqno, seqno, dma_fence->ops);
}
static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence)
{
struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence);
dma_fence_get(dma_fence);
list_add_tail(&fence->irq_link, &irq->pending);
/* SW completed (no HW IRQ) so kick handler to signal fence */
if (xe_hw_fence_signaled(dma_fence))
xe_hw_fence_irq_run(irq);
return true;
}
static void xe_hw_fence_release(struct dma_fence *dma_fence)
{
struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence);
trace_xe_hw_fence_free(fence);
XE_WARN_ON(!list_empty(&fence->irq_link));
call_rcu(&dma_fence->rcu, fence_free);
}
static const struct dma_fence_ops xe_hw_fence_ops = {
.get_driver_name = xe_hw_fence_get_driver_name,
.get_timeline_name = xe_hw_fence_get_timeline_name,
.enable_signaling = xe_hw_fence_enable_signaling,
.signaled = xe_hw_fence_signaled,
.release = xe_hw_fence_release,
};
static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence)
{
if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops))
return NULL;
return container_of(fence, struct xe_hw_fence, dma);
}
struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx,
struct iosys_map seqno_map)
{
struct xe_hw_fence *fence;
fence = fence_alloc();
if (!fence)
return ERR_PTR(-ENOMEM);
dma_fence_init(&fence->dma, &xe_hw_fence_ops, &ctx->irq->lock,
ctx->dma_fence_ctx, ctx->next_seqno++);
fence->ctx = ctx;
fence->seqno_map = seqno_map;
INIT_LIST_HEAD(&fence->irq_link);
trace_xe_hw_fence_create(fence);
return fence;
}
|
