summaryrefslogtreecommitdiff
path: root/linux-core/radeon_ioc32.c
blob: b980bedbdf147a4a130296bf6d5f3ff45629e58c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
/**
 * \file radeon_ioc32.c
 *
 * 32-bit ioctl compatibility routines for the Radeon DRM.
 *
 * \author Paul Mackerras <paulus@samba.org>
 *
 * Copyright (C) Paul Mackerras 2005
 * All Rights Reserved.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */
#include <linux/compat.h>
#include <linux/ioctl32.h>

#include "drmP.h"
#include "drm.h"
#include "radeon_drm.h"
#include "radeon_drv.h"

typedef struct drm_radeon_init32 {
	int func;
	u32 sarea_priv_offset;
	int is_pci;
	int cp_mode;
	int gart_size;
	int ring_size;
	int usec_timeout;

	unsigned int fb_bpp;
	unsigned int front_offset, front_pitch;
	unsigned int back_offset, back_pitch;
	unsigned int depth_bpp;
	unsigned int depth_offset, depth_pitch;

	u32 fb_offset;
	u32 mmio_offset;
	u32 ring_offset;
	u32 ring_rptr_offset;
	u32 buffers_offset;
	u32 gart_textures_offset;
} drm_radeon_init32_t;

static int compat_radeon_cp_init(struct file *file, unsigned int cmd,
				 unsigned long arg)
{
	drm_radeon_init32_t init32;
	drm_radeon_init_t __user *init;

	if (copy_from_user(&init32, (void __user *)arg, sizeof(init32)))
		return -EFAULT;

	init = compat_alloc_user_space(sizeof(*init));
	if (!access_ok(VERIFY_WRITE, init, sizeof(*init))
	    || __put_user(init32.func, &init->func)
	    || __put_user(init32.sarea_priv_offset, &init->sarea_priv_offset)
	    || __put_user(init32.is_pci, &init->is_pci)
	    || __put_user(init32.cp_mode, &init->cp_mode)
	    || __put_user(init32.gart_size, &init->gart_size)
	    || __put_user(init32.ring_size, &init->ring_size)
	    || __put_user(init32.usec_timeout, &init->usec_timeout)
	    || __put_user(init32.fb_bpp, &init->fb_bpp)
	    || __put_user(init32.front_offset, &init->front_offset)
	    || __put_user(init32.front_pitch, &init->front_pitch)
	    || __put_user(init32.back_offset, &init->back_offset)
	    || __put_user(init32.back_pitch, &init->back_pitch)
	    || __put_user(init32.depth_bpp, &init->depth_bpp)
	    || __put_user(init32.depth_offset, &init->depth_offset)
	    || __put_user(init32.depth_pitch, &init->depth_pitch)
	    || __put_user(init32.fb_offset, &init->fb_offset)
	    || __put_user(init32.mmio_offset, &init->mmio_offset)
	    || __put_user(init32.ring_offset, &init->ring_offset)
	    || __put_user(init32.ring_rptr_offset, &init->ring_rptr_offset)
	    || __put_user(init32.buffers_offset, &init->buffers_offset)
	    || __put_user(init32.gart_textures_offset,
			  &init->gart_textures_offset))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_CP_INIT, (unsigned long) init);
}

typedef struct drm_radeon_clear32 {
	unsigned int flags;
	unsigned int clear_color;
	unsigned int clear_depth;
	unsigned int color_mask;
	unsigned int depth_mask;   /* misnamed field:  should be stencil */
	u32	     depth_boxes;
} drm_radeon_clear32_t;

static int compat_radeon_cp_clear(struct file *file, unsigned int cmd,
				  unsigned long arg)
{
	drm_radeon_clear32_t clr32;
	drm_radeon_clear_t __user *clr;

	if (copy_from_user(&clr32, (void __user *)arg, sizeof(clr32)))
		return -EFAULT;

	clr = compat_alloc_user_space(sizeof(*clr));
	if (!access_ok(VERIFY_WRITE, clr, sizeof(*clr))
	    || __put_user(clr32.flags, &clr->flags)
	    || __put_user(clr32.clear_color, &clr->clear_color)
	    || __put_user(clr32.clear_depth, &clr->clear_depth)
	    || __put_user(clr32.color_mask, &clr->color_mask)
	    || __put_user(clr32.depth_mask, &clr->depth_mask)
	    || __put_user((void __user *)(unsigned long)clr32.depth_boxes,
			  &clr->depth_boxes))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_CLEAR, (unsigned long) clr);
}

typedef struct drm_radeon_stipple32 {
	u32 mask;
} drm_radeon_stipple32_t;

static int compat_radeon_cp_stipple(struct file *file, unsigned int cmd,
				    unsigned long arg)
{
	drm_radeon_stipple32_t __user *argp = (void __user *) arg;
	drm_radeon_stipple_t __user *request;
	u32 mask;

	if (get_user(mask, &argp->mask))
		return -EFAULT;

	request = compat_alloc_user_space(sizeof(*request));
	if (!access_ok(VERIFY_WRITE, request, sizeof(*request))
	    || __put_user((unsigned int __user *)(unsigned long) mask,
			  &request->mask))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_STIPPLE, (unsigned long) request);
}

typedef struct drm_radeon_tex_image32 {
	unsigned int x, y;		/* Blit coordinates */
	unsigned int width, height;
	u32 data;
} drm_radeon_tex_image32_t;

typedef struct drm_radeon_texture32 {
	unsigned int offset;
	int pitch;
	int format;
	int width;			/* Texture image coordinates */
	int height;
	u32 image;
} drm_radeon_texture32_t;

static int compat_radeon_cp_texture(struct file *file, unsigned int cmd,
				    unsigned long arg)
{
	drm_radeon_texture32_t req32;
	drm_radeon_texture_t __user *request;
	drm_radeon_tex_image32_t img32;
	drm_radeon_tex_image_t __user *image;

	if (copy_from_user(&req32, (void __user *) arg, sizeof(req32)))
		return -EFAULT;
	if (req32.image == 0)
		return -EINVAL;
	if (copy_from_user(&img32, (void __user *)(unsigned long)req32.image,
			   sizeof(img32)))
		return -EFAULT;

	request = compat_alloc_user_space(sizeof(*request) + sizeof(*image));
	if (!access_ok(VERIFY_WRITE, request,
		       sizeof(*request) + sizeof(*image)))
		return -EFAULT;
	image = (drm_radeon_tex_image_t __user *) (request + 1);

	if (__put_user(req32.offset, &request->offset)
	    || __put_user(req32.pitch, &request->pitch)
	    || __put_user(req32.format, &request->format)
	    || __put_user(req32.width, &request->width)
	    || __put_user(req32.height, &request->height)
	    || __put_user(image, &request->image)
	    || __put_user(img32.x, &image->x)
	    || __put_user(img32.y, &image->y)
	    || __put_user(img32.width, &image->width)
	    || __put_user(img32.height, &image->height)
	    || __put_user((const void __user *)(unsigned long)img32.data,
			  &image->data))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_TEXTURE, (unsigned long) request);
}

typedef struct drm_radeon_vertex2_32 {
	int idx;			/* Index of vertex buffer */
	int discard;			/* Client finished with buffer? */
	int nr_states;
	u32 state;
	int nr_prims;
	u32 prim;
} drm_radeon_vertex2_32_t;

static int compat_radeon_cp_vertex2(struct file *file, unsigned int cmd,
				    unsigned long arg)
{
	drm_radeon_vertex2_32_t req32;
	drm_radeon_vertex2_t __user *request;

	if (copy_from_user(&req32, (void __user *) arg, sizeof(req32)))
		return -EFAULT;

	request = compat_alloc_user_space(sizeof(*request));
	if (!access_ok(VERIFY_WRITE, request, sizeof(*request))
	    || __put_user(req32.idx, &request->idx)
	    || __put_user(req32.discard, &request->discard)
	    || __put_user(req32.nr_states, &request->nr_states)
	    || __put_user((void __user *)(unsigned long)req32.state,
			  &request->state)
	    || __put_user(req32.nr_prims, &request->nr_prims)
	    || __put_user((void __user *)(unsigned long)req32.prim,
			  &request->prim))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_VERTEX2, (unsigned long) request);
}

typedef struct drm_radeon_cmd_buffer32 {
	int bufsz;
	u32 buf;
	int nbox;
	u32 boxes;
} drm_radeon_cmd_buffer32_t;

static int compat_radeon_cp_cmdbuf(struct file *file, unsigned int cmd,
				   unsigned long arg)
{
	drm_radeon_cmd_buffer32_t req32;
	drm_radeon_cmd_buffer_t __user *request;

	if (copy_from_user(&req32, (void __user *) arg, sizeof(req32)))
		return -EFAULT;

	request = compat_alloc_user_space(sizeof(*request));
	if (!access_ok(VERIFY_WRITE, request, sizeof(*request))
	    || __put_user(req32.bufsz, &request->bufsz)
	    || __put_user((void __user *)(unsigned long)req32.buf,
			  &request->buf)
	    || __put_user(req32.nbox, &request->nbox)
	    || __put_user((void __user *)(unsigned long)req32.boxes,
			  &request->boxes))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_CMDBUF, (unsigned long) request);
}

typedef struct drm_radeon_getparam32 {
	int param;
	u32 value;
} drm_radeon_getparam32_t;

static int compat_radeon_cp_getparam(struct file *file, unsigned int cmd,
				     unsigned long arg)
{
	drm_radeon_getparam32_t req32;
	drm_radeon_getparam_t __user *request;

	if (copy_from_user(&req32, (void __user *) arg, sizeof(req32)))
		return -EFAULT;

	request = compat_alloc_user_space(sizeof(*request));
	if (!access_ok(VERIFY_WRITE, request, sizeof(*request))
	    || __put_user(req32.param, &request->param)
	    || __put_user((void __user *)(unsigned long)req32.value,
			  &request->value))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_GETPARAM, (unsigned long) request);
}

typedef struct drm_radeon_mem_alloc32 {
	int region;
	int alignment;
	int size;
	u32 region_offset;	/* offset from start of fb or GART */
} drm_radeon_mem_alloc32_t;

static int compat_radeon_mem_alloc(struct file *file, unsigned int cmd,
				   unsigned long arg)
{
	drm_radeon_mem_alloc32_t req32;
	drm_radeon_mem_alloc_t __user *request;

	if (copy_from_user(&req32, (void __user *) arg, sizeof(req32)))
		return -EFAULT;

	request = compat_alloc_user_space(sizeof(*request));
	if (!access_ok(VERIFY_WRITE, request, sizeof(*request))
	    || __put_user(req32.region, &request->region)
	    || __put_user(req32.alignment, &request->alignment)
	    || __put_user(req32.size, &request->size)
	    || __put_user((int __user *)(unsigned long)req32.region_offset,
			  &request->region_offset))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_ALLOC, (unsigned long) request);
}

typedef struct drm_radeon_irq_emit32 {
	u32 irq_seq;
} drm_radeon_irq_emit32_t;

static int compat_radeon_irq_emit(struct file *file, unsigned int cmd,
				  unsigned long arg)
{
	drm_radeon_irq_emit32_t req32;
	drm_radeon_irq_emit_t __user *request;

	if (copy_from_user(&req32, (void __user *) arg, sizeof(req32)))
		return -EFAULT;

	request = compat_alloc_user_space(sizeof(*request));
	if (!access_ok(VERIFY_WRITE, request, sizeof(*request))
	    || __put_user((int __user *)(unsigned long)req32.irq_seq,
			  &request->irq_seq))
		return -EFAULT;

	return drm_ioctl(file->f_dentry->d_inode, file,
			 DRM_IOCTL_RADEON_IRQ_EMIT, (unsigned long) request);
}

drm_ioctl_compat_t *radeon_compat_ioctls[] = {
	[DRM_RADEON_CP_INIT] = compat_radeon_cp_init,
	[DRM_RADEON_CLEAR] = compat_radeon_cp_clear,
	[DRM_RADEON_STIPPLE] = compat_radeon_cp_stipple,
	[DRM_RADEON_TEXTURE] = compat_radeon_cp_texture,
	[DRM_RADEON_VERTEX2] = compat_radeon_cp_vertex2,
	[DRM_RADEON_CMDBUF] = compat_radeon_cp_cmdbuf,
	[DRM_RADEON_GETPARAM] = compat_radeon_cp_getparam,
	[DRM_RADEON_ALLOC] = compat_radeon_mem_alloc,
	[DRM_RADEON_IRQ_EMIT] = compat_radeon_irq_emit,
};

/**
 * Called whenever a 32-bit process running under a 64-bit kernel
 * performs an ioctl on /dev/dri/card<n>.
 *
 * \param filp file pointer.
 * \param cmd command.
 * \param arg user argument.
 * \return zero on success or negative number on failure.
 */
long radeon_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
	unsigned int nr = DRM_IOCTL_NR(cmd);
	drm_ioctl_compat_t *fn = NULL;
	int ret;

	if (nr < DRM_COMMAND_BASE)
		return drm_compat_ioctl(filp, cmd, arg);

	if (nr < DRM_COMMAND_BASE + DRM_ARRAY_SIZE(radeon_compat_ioctls))
		fn = radeon_compat_ioctls[nr - DRM_COMMAND_BASE];

	lock_kernel();		/* XXX for now */
	if (fn != NULL)
		ret = (*fn)(filp, cmd, arg);
	else
		ret = drm_ioctl(filp->f_dentry->d_inode, filp, cmd, arg);
	unlock_kernel();

	return ret;
}
t buffer is read into. */ uint32_t read_domains; /** Cache domain the target buffer will have dirty cachelines in. */ uint32_t write_domain; }; struct block { struct block *next, *prev; struct mem_block *mem; /* BM_MEM_AGP */ /** * Marks that the block is currently in the aperture and has yet to be * fenced. */ unsigned on_hardware:1; /** * Marks that the block is currently fenced (being used by rendering) * and can't be freed until @fence is passed. */ unsigned fenced:1; /** Fence cookie for the block. */ unsigned fence; /* Split to read_fence, write_fence */ drm_intel_bo *bo; void *virtual; }; typedef struct _bufmgr_fake { drm_intel_bufmgr bufmgr; pthread_mutex_t lock; unsigned long low_offset; unsigned long size; void *virtual; struct mem_block *heap; unsigned buf_nr; /* for generating ids */ /** * List of blocks which are currently in the GART but haven't been * fenced yet. */ struct block on_hardware; /** * List of blocks which are in the GART and have an active fence on * them. */ struct block fenced; /** * List of blocks which have an expired fence and are ready to be * evicted. */ struct block lru; unsigned int last_fence; unsigned fail:1; unsigned need_fence:1; int thrashing; /** * Driver callback to emit a fence, returning the cookie. * * This allows the driver to hook in a replacement for the DRM usage in * bufmgr_fake. * * Currently, this also requires that a write flush be emitted before * emitting the fence, but this should change. */ unsigned int (*fence_emit) (void *private); /** Driver callback to wait for a fence cookie to have passed. */ void (*fence_wait) (unsigned int fence, void *private); void *fence_priv; /** * Driver callback to execute a buffer. * * This allows the driver to hook in a replacement for the DRM usage in * bufmgr_fake. */ int (*exec) (drm_intel_bo *bo, unsigned int used, void *priv); void *exec_priv; /** Driver-supplied argument to driver callbacks */ void *driver_priv; /** * Pointer to kernel-updated sarea data for the last completed user irq */ volatile int *last_dispatch; int fd; int debug; int performed_rendering; } drm_intel_bufmgr_fake; typedef struct _drm_intel_bo_fake { drm_intel_bo bo; unsigned id; /* debug only */ const char *name; unsigned dirty:1; /** * has the card written to this buffer - we make need to copy it back */ unsigned card_dirty:1; unsigned int refcount; /* Flags may consist of any of the DRM_BO flags, plus * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the * first two driver private flags. */ uint64_t flags; /** Cache domains the target buffer is read into. */ uint32_t read_domains; /** Cache domain the target buffer will have dirty cachelines in. */ uint32_t write_domain; unsigned int alignment; int is_static, validated; unsigned int map_count; /** relocation list */ struct fake_buffer_reloc *relocs; int nr_relocs; /** * Total size of the target_bos of this buffer. * * Used for estimation in check_aperture. */ unsigned int child_size; struct block *block; void *backing_store; void (*invalidate_cb) (drm_intel_bo *bo, void *ptr); void *invalidate_ptr; } drm_intel_bo_fake; static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie); #define MAXFENCE 0x7fffffff static int FENCE_LTE(unsigned a, unsigned b) { if (a == b) return 1; if (a < b && b - a < (1 << 24)) return 1; if (a > b && MAXFENCE - a + b < (1 << 24)) return 1; return 0; } void drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr, unsigned int (*emit) (void *priv), void (*wait) (unsigned int fence, void *priv), void *priv) { drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr; bufmgr_fake->fence_emit = emit; bufmgr_fake->fence_wait = wait; bufmgr_fake->fence_priv = priv; } static unsigned int _fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake) { struct drm_i915_irq_emit ie; int ret, seq = 1; if (bufmgr_fake->fence_emit != NULL) { seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv); return seq; } ie.irq_seq = &seq; ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT, &ie, sizeof(ie)); if (ret) { drmMsg("%s: drm_i915_irq_emit: %d\n", __FUNCTION__, ret); abort(); } DBG("emit 0x%08x\n", seq); return seq; } static void _fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq) { struct drm_i915_irq_wait iw; int hw_seq, busy_count = 0; int ret; int kernel_lied; if (bufmgr_fake->fence_wait != NULL) { bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv); clear_fenced(bufmgr_fake, seq); return; } iw.irq_seq = seq; DBG("wait 0x%08x\n", iw.irq_seq); /* The kernel IRQ_WAIT implementation is all sorts of broken. * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit * unsigned range. * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit * signed range. * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit * signed range. * 4) It returns -EBUSY in 3 seconds even if the hardware is still * successfully chewing through buffers. * * Assume that in userland we treat sequence numbers as ints, which * makes some of the comparisons convenient, since the sequence * numbers are all postive signed integers. * * From this we get several cases we need to handle. Here's a timeline. * 0x2 0x7 0x7ffffff8 0x7ffffffd * | | | | * ------------------------------------------------------------ * * A) Normal wait for hw to catch up * hw_seq seq * | | * ------------------------------------------------------------ * seq - hw_seq = 5. If we call IRQ_WAIT, it will wait for hw to * catch up. * * B) Normal wait for a sequence number that's already passed. * seq hw_seq * | | * ------------------------------------------------------------ * seq - hw_seq = -5. If we call IRQ_WAIT, it returns 0 quickly. * * C) Hardware has already wrapped around ahead of us * hw_seq seq * | | * ------------------------------------------------------------ * seq - hw_seq = 0x80000000 - 5. If we called IRQ_WAIT, it would wait * for hw_seq >= seq, which may never occur. Thus, we want to catch * this in userland and return 0. * * D) We've wrapped around ahead of the hardware. * seq hw_seq * | | * ------------------------------------------------------------ * seq - hw_seq = -(0x80000000 - 5). If we called IRQ_WAIT, it would * return 0 quickly because hw_seq >= seq, even though the hardware * isn't caught up. Thus, we need to catch this early return in * userland and bother the kernel until the hardware really does * catch up. * * E) Hardware might wrap after we test in userland. * hw_seq seq * | | * ------------------------------------------------------------ * seq - hw_seq = 5. If we call IRQ_WAIT, it will likely see seq >= * hw_seq and wait. However, suppose hw_seq wraps before we make it * into the kernel. The kernel sees hw_seq >= seq and waits for 3 * seconds then returns -EBUSY. This is case C). We should catch * this and then return successfully. * * F) Hardware might take a long time on a buffer. * hw_seq seq * | | * ------------------------------------------------------------------- * seq - hw_seq = 5. If we call IRQ_WAIT, if sequence 2 through 5 * take too long, it will return -EBUSY. Batchbuffers in the * gltestperf demo were seen to take up to 7 seconds. We should * catch early -EBUSY return and keep trying. */ do { /* Keep a copy of last_dispatch so that if the wait -EBUSYs * because the hardware didn't catch up in 3 seconds, we can * see if it at least made progress and retry. */ hw_seq = *bufmgr_fake->last_dispatch; /* Catch case C */ if (seq - hw_seq > 0x40000000) return; ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT, &iw, sizeof(iw)); /* Catch case D */ kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch < -0x40000000); /* Catch case E */ if (ret == -EBUSY && (seq - *bufmgr_fake->last_dispatch > 0x40000000)) ret = 0; /* Catch case F: Allow up to 15 seconds chewing on one buffer. */ if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch)) busy_count = 0; else busy_count++; } while (kernel_lied || ret == -EAGAIN || ret == -EINTR || (ret == -EBUSY && busy_count < 5)); if (ret != 0) { drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__, __LINE__, strerror(-ret)); abort(); } clear_fenced(bufmgr_fake, seq); } static int _fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence) { /* Slight problem with wrap-around: */ return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence); } /** * Allocate a memory manager block for the buffer. */ static int alloc_block(drm_intel_bo *bo) { drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr; struct block *block = (struct block *)calloc(sizeof *block, 1); unsigned int align_log2 = ffs(bo_fake->alignment) - 1; unsigned int sz; if (!block) return 1; sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1); block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0); if (!block->mem) { free(block); return 0; } DRMINITLISTHEAD(block); /* Insert at head or at tail??? */ DRMLISTADDTAIL(block, &bufmgr_fake->lru); block->virtual = (uint8_t *) bufmgr_fake->virtual + block->mem->ofs - bufmgr_fake->low_offset; block->bo = bo; bo_fake->block = block; return 1; } /* Release the card storage associated with buf: */ static void free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block, int skip_dirty_copy) { drm_intel_bo_fake *bo_fake; DBG("free block %p %08x %d %d\n", block, block->mem->ofs, block->on_hardware, block->fenced); if (!block) return; bo_fake = (drm_intel_bo_fake *) block->bo; if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)) skip_dirty_copy = 1; if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) { memcpy(bo_fake->backing_store, block->virtual, block->bo->size); bo_fake->card_dirty = 0; bo_fake->dirty = 1; } if (block->on_hardware) { block->bo = NULL; } else if (block->fenced) { block->bo = NULL; } else { DBG(" - free immediately\n"); DRMLISTDEL(block); mmFreeMem(block->mem); free(block); } } static void alloc_backing_store(drm_intel_bo *bo) { drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr; drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; assert(!bo_fake->backing_store); assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))); bo_fake->backing_store = malloc(bo->size); DBG("alloc_backing - buf %d %p %d\n", bo_fake->id, bo_fake->backing_store, bo->size); assert(bo_fake->backing_store); } static void free_backing_store(drm_intel_bo *bo) { drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; if (bo_fake->backing_store) { assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))); free(bo_fake->backing_store); bo_fake->backing_store = NULL; } } static void set_dirty(drm_intel_bo *bo) { drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr; drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; if (bo_fake->flags & BM_NO_BACKING_STORE && bo_fake->invalidate_cb != NULL) bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr); assert(!(bo_fake->flags & BM_PINNED)); DBG("set_dirty - buf %d\n", bo_fake->id); bo_fake->dirty = 1; } static int evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence) { struct block *block, *tmp; DBG("%s\n", __FUNCTION__); DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) { drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) continue; if (block->fence && max_fence && !FENCE_LTE(block->fence, max_fence)) return 0; set_dirty(&bo_fake->bo); bo_fake->block = NULL; free_block(bufmgr_fake, block, 0); return 1; } return 0; } static int evict_mru(drm_intel_bufmgr_fake *bufmgr_fake) { struct block *block, *tmp; DBG("%s\n", __FUNCTION__); DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) { drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo; if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA)) continue; set_dirty(&bo_fake->bo); bo_fake->block = NULL; free_block(bufmgr_fake, block, 0); return 1; } return 0; } /** * Removes all objects from the fenced list older than the given fence. */ static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie) { struct block *block, *tmp; int ret = 0; bufmgr_fake->last_fence = fence_cookie; DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) { assert(block->fenced); if (_fence_test(bufmgr_fake, block->fence)) { block->fenced = 0; if (!block->bo) { DBG("delayed free: offset %x sz %x\n", block->mem->ofs, block->mem->size); DRMLISTDEL(block); mmFreeMem(block->mem); free(block); } else { DBG("return to lru: offset %x sz %x\n", block->mem->ofs, block->mem->size); DRMLISTDEL(block); DRMLISTADDTAIL(block, &bufmgr_fake->lru); } ret = 1; } else { /* Blocks are ordered by fence, so if one fails, all * from here will fail also: */ DBG("fence not passed: offset %x sz %x %d %d \n", block->mem->ofs, block->mem->size, block->fence, bufmgr_fake->last_fence); break; } } DBG("%s: %d\n", __FUNCTION__, ret); return ret; } static void fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence) { struct block *block, *tmp; DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) { DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n", block, block->mem->size, block->mem->ofs, block->bo, fence); block->fence = fence; block->on_hardware = 0; block->fenced = 1; /* Move to tail of pending list here */ DRMLISTDEL(block); DRMLISTADDTAIL(block, &bufmgr_fake->fenced); } assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware)); } static int evict_and_alloc_block(drm_intel_bo *bo) { drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr; drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; assert(bo_fake->block == NULL); /* Search for already free memory: */ if (alloc_block(bo)) return 1; /* If we're not thrashing, allow lru eviction to dig deeper into * recently used textures. We'll probably be thrashing soon: */ if (!bufmgr_fake->thrashing) { while (evict_lru(bufmgr_fake, 0)) if (alloc_block(bo)) return 1; } /* Keep thrashing counter alive? */ if (bufmgr_fake->thrashing) bufmgr_fake->thrashing = 20; /* Wait on any already pending fences - here we are waiting for any * freed memory that has been submitted to hardware and fenced to * become available: */ while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) { uint32_t fence = bufmgr_fake->fenced.next->fence; _fence_wait_internal(bufmgr_fake, fence); if (alloc_block(bo)) return 1; } if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) { while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) { uint32_t fence = bufmgr_fake->fenced.next->fence; _fence_wait_internal(bufmgr_fake, fence); } if (!bufmgr_fake->thrashing) { DBG("thrashing\n"); } bufmgr_fake->thrashing = 20; if (alloc_block(bo)) return 1; } while (evict_mru(bufmgr_fake)) if (alloc_block(bo)) return 1; DBG("%s 0x%x bytes failed\n", __FUNCTION__, bo->size); return 0; } /*********************************************************************** * Public functions */ /** * Wait for hardware idle by emitting a fence and waiting for it. */ static void drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake) { unsigned int cookie; cookie = _fence_emit_internal(bufmgr_fake); _fence_wait_internal(bufmgr_fake, cookie); } /** * Wait for rendering to a buffer to complete. * * It is assumed that the bathcbuffer which performed the rendering included * the necessary flushing. */ static void drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo) { drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr; drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo; if (bo_fake->block == NULL || !bo_fake->block->fenced) return; _fence_wait_internal(bufmgr_fake, bo_fake->block->fence); } static void drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo) { drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr;