1 /* BEGIN CSTYLED */
2
3 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
4 */
5 /*
6 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
7 * Copyright (c) 2009, Intel Corporation.
8 * All Rights Reserved.
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a
11 * copy of this software and associated documentation files (the
12 * "Software"), to deal in the Software without restriction, including
13 * without limitation the rights to use, copy, modify, merge, publish,
14 * distribute, sub license, and/or sell copies of the Software, and to
15 * permit persons to whom the Software is furnished to do so, subject to
16 * the following conditions:
17 *
18 * The above copyright notice and this permission notice (including the
19 * next paragraph) shall be included in all copies or substantial portions
20 * of the Software.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
25 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
26 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
27 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
28 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
29 *
30 */
31
32 /*
33 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
34 * Use is subject to license terms.
35 */
36
37 #include "drmP.h"
38 #include "drm.h"
39 #include "i915_drm.h"
40 #include "i915_drv.h"
41
42
43 #define MAX_NOPID ((u32)~0)
44
45 /**
46 * Interrupts that are always left unmasked.
47 *
48 * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
49 * we leave them always unmasked in IMR and then control enabling them through
50 * PIPESTAT alone.
51 */
52
53 #define I915_INTERRUPT_ENABLE_FIX (I915_ASLE_INTERRUPT | \
54 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
55 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | \
56 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
57
58 /** Interrupts that we mask and unmask at runtime. */
59 #define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT)
60
61 /** These are all of the interrupts used by the driver */
62 #define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX | \
63 I915_INTERRUPT_ENABLE_VAR)
64
65 void
66 igdng_enable_irq(drm_i915_private_t *dev_priv, u32 mask, int gfx_irq)
67 {
68 if (gfx_irq && ((dev_priv->gt_irq_mask_reg & mask) != 0)) {
69 dev_priv->gt_irq_mask_reg &= ~mask;
70 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
71 (void) I915_READ(GTIMR);
72 } else if ((dev_priv->irq_mask_reg & mask) != 0) {
73 dev_priv->irq_mask_reg &= ~mask;
74 I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
75 (void) I915_READ(DEIMR);
76
77 }
78 }
79
80 static inline void
81 igdng_disable_irq(drm_i915_private_t *dev_priv, u32 mask, int gfx_irq)
82 {
83 if (gfx_irq && ((dev_priv->gt_irq_mask_reg & mask) != mask)) {
84 dev_priv->gt_irq_mask_reg |= mask;
85 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
86 (void) I915_READ(GTIMR);
87 } else if ((dev_priv->irq_mask_reg & mask) != mask) {
88 dev_priv->irq_mask_reg |= mask;
89 I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
90 (void) I915_READ(DEIMR);
91 }
92 }
93
94 /* For display hotplug interrupt */
95 void
96 igdng_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
97 {
98 if ((dev_priv->irq_mask_reg & mask) != 0) {
99 dev_priv->irq_mask_reg &= ~mask;
100 I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
101 (void) I915_READ(DEIMR);
102 }
103 }
104
105 #if 0
106 static inline void
107 igdng_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
108 {
109 if ((dev_priv->irq_mask_reg & mask) != mask) {
110 dev_priv->irq_mask_reg |= mask;
111 I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
112 (void) I915_READ(DEIMR);
113 }
114 }
115 #endif
116
117 static inline void
118 i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
119 {
120 if ((dev_priv->irq_mask_reg & mask) != 0) {
121 dev_priv->irq_mask_reg &= ~mask;
122 I915_WRITE(IMR, dev_priv->irq_mask_reg);
123 (void) I915_READ(IMR);
124 }
125 }
126
127 static inline void
128 i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
129 {
130 if ((dev_priv->irq_mask_reg & mask) != mask) {
131 dev_priv->irq_mask_reg |= mask;
132 I915_WRITE(IMR, dev_priv->irq_mask_reg);
133 (void) I915_READ(IMR);
134 }
135 }
136
137 static inline uint32_t
138 i915_pipestat(int pipe)
139 {
140 if (pipe == 0)
141 return PIPEASTAT;
142 if (pipe == 1)
143 return PIPEBSTAT;
144 return 0;
145 }
146
147 void
148 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, uint32_t mask)
149 {
150 if ((dev_priv->pipestat[pipe] & mask) != mask) {
151 u32 reg = i915_pipestat(pipe);
152
153 dev_priv->pipestat[pipe] |= mask;
154 /* Enable the interrupt, clear any pending status */
155 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
156 (void) I915_READ(reg);
157 }
158 }
159
160 void
161 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
162 {
163 if ((dev_priv->pipestat[pipe] & mask) != 0) {
164 u32 reg = i915_pipestat(pipe);
165
166 dev_priv->pipestat[pipe] &= ~mask;
167 I915_WRITE(reg, dev_priv->pipestat[pipe]);
168 (void) I915_READ(reg);
169 }
170 }
171
172 /**
173 * i915_pipe_enabled - check if a pipe is enabled
174 * @dev: DRM device
175 * @pipe: pipe to check
176 *
177 * Reading certain registers when the pipe is disabled can hang the chip.
178 * Use this routine to make sure the PLL is running and the pipe is active
179 * before reading such registers if unsure.
180 */
181 static int
182 i915_pipe_enabled(struct drm_device *dev, int pipe)
183 {
184 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
185 unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;
186
187 if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
188 return 1;
189
190 return 0;
191 }
192
193 u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
194 {
195 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
196 unsigned long high_frame;
197 unsigned long low_frame;
198 u32 high1, high2, low, count;
199
200 high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
201 low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
202
203 if (!i915_pipe_enabled(dev, pipe)) {
204 DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
205 return 0;
206 }
207
208 /*
209 * High & low register fields aren't synchronized, so make sure
210 * we get a low value that's stable across two reads of the high
211 * register.
212 */
213 do {
214 high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
215 PIPE_FRAME_HIGH_SHIFT);
216 low = ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
217 PIPE_FRAME_LOW_SHIFT);
218 high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
219 PIPE_FRAME_HIGH_SHIFT);
220 } while (high1 != high2);
221
222 count = (high1 << 8) | low;
223
224 return count;
225 }
226
227 /**
228 * i915_capture_error_state - capture an error record for later analysis
229 * @dev: drm device
230 *
231 * Should be called when an error is detected (either a hang or an error
232 * interrupt) to capture error state from the time of the error. Fills
233 * out a structure which becomes available in debugfs for user level tools
234 * to pick up.
235 */
236 static void i915_capture_error_state(struct drm_device *dev)
237 {
238 struct drm_i915_private *dev_priv = dev->dev_private;
239 struct drm_i915_error_state *error;
240
241 spin_lock_irqsave(&dev_priv->error_lock, flags);
242 #if 0
243 if (dev_priv->first_error)
244 goto out;
245 #endif
246 error = drm_alloc(sizeof(*error), DRM_MEM_DRIVER);
247 if (!error) {
248 DRM_DEBUG("out ot memory, not capturing error state\n");
249 goto out;
250 }
251
252 error->eir = I915_READ(EIR);
253 error->pgtbl_er = I915_READ(PGTBL_ER);
254 error->pipeastat = I915_READ(PIPEASTAT);
255 error->pipebstat = I915_READ(PIPEBSTAT);
256 error->instpm = I915_READ(INSTPM);
257 if (!IS_I965G(dev)) {
258 error->ipeir = I915_READ(IPEIR);
259 error->ipehr = I915_READ(IPEHR);
260 error->instdone = I915_READ(INSTDONE);
261 error->acthd = I915_READ(ACTHD);
262 } else {
263 error->ipeir = I915_READ(IPEIR_I965);
264 error->ipehr = I915_READ(IPEHR_I965);
265 error->instdone = I915_READ(INSTDONE_I965);
266 error->instps = I915_READ(INSTPS);
267 error->instdone1 = I915_READ(INSTDONE1);
268 error->acthd = I915_READ(ACTHD_I965);
269 }
270
271 (void) uniqtime(&error->time);
272
273 dev_priv->first_error = error;
274
275 DRM_DEBUG("Time: %ld s %ld us\n", error->time.tv_sec,
276 error->time.tv_usec);
277 DRM_DEBUG("EIR: 0x%08x\n", error->eir);
278 DRM_DEBUG(" PGTBL_ER: 0x%08x\n", error->pgtbl_er);
279 DRM_DEBUG(" INSTPM: 0x%08x\n", error->instpm);
280 DRM_DEBUG(" IPEIR: 0x%08x\n", error->ipeir);
281 DRM_DEBUG(" IPEHR: 0x%08x\n", error->ipehr);
282 DRM_DEBUG(" INSTDONE: 0x%08x\n", error->instdone);
283 DRM_DEBUG(" ACTHD: 0x%08x\n", error->acthd);
284 DRM_DEBUG(" DMA_FADD_P: 0x%08x\n", I915_READ(0x2078));
285 if (IS_I965G(dev)) {
286 DRM_DEBUG(" INSTPS: 0x%08x\n", error->instps);
287 DRM_DEBUG(" INSTDONE1: 0x%08x\n", error->instdone1);
288 }
289 drm_free(error, sizeof(*error), DRM_MEM_DRIVER);
290 out:
291 spin_unlock_irqrestore(&dev_priv->error_lock, flags);
292 }
293
294 /**
295 * i915_handle_error - handle an error interrupt
296 * @dev: drm device
297 *
298 * Do some basic checking of regsiter state at error interrupt time and
299 * dump it to the syslog. Also call i915_capture_error_state() to make
300 * sure we get a record and make it available in debugfs. Fire a uevent
301 * so userspace knows something bad happened (should trigger collection
302 * of a ring dump etc.).
303 */
304 void i915_handle_error(struct drm_device *dev)
305 {
306 struct drm_i915_private *dev_priv = dev->dev_private;
307 u32 eir = I915_READ(EIR);
308 u32 pipea_stats = I915_READ(PIPEASTAT);
309 u32 pipeb_stats = I915_READ(PIPEBSTAT);
310
311 i915_capture_error_state(dev);
312
313 DRM_DEBUG("render error detected, EIR: 0x%08x\n",
314 eir);
315
316 if (IS_G4X(dev)) {
317 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
318 u32 ipeir = I915_READ(IPEIR_I965);
319
320 DRM_DEBUG(" IPEIR: 0x%08x\n",
321 I915_READ(IPEIR_I965));
322 DRM_DEBUG(" IPEHR: 0x%08x\n",
323 I915_READ(IPEHR_I965));
324 DRM_DEBUG(" INSTDONE: 0x%08x\n",
325 I915_READ(INSTDONE_I965));
326 DRM_DEBUG(" INSTPS: 0x%08x\n",
327 I915_READ(INSTPS));
328 DRM_DEBUG(" INSTDONE1: 0x%08x\n",
329 I915_READ(INSTDONE1));
330 DRM_DEBUG(" ACTHD: 0x%08x\n",
331 I915_READ(ACTHD_I965));
332 I915_WRITE(IPEIR_I965, ipeir);
333 (void)I915_READ(IPEIR_I965);
334 }
335 if (eir & GM45_ERROR_PAGE_TABLE) {
336 u32 pgtbl_err = I915_READ(PGTBL_ER);
337 DRM_DEBUG("page table error\n");
338 DRM_DEBUG(" PGTBL_ER: 0x%08x\n",
339 pgtbl_err);
340 I915_WRITE(PGTBL_ER, pgtbl_err);
341 (void)I915_READ(PGTBL_ER);
342 }
343 }
344
345 if (IS_I9XX(dev)) {
346 if (eir & I915_ERROR_PAGE_TABLE) {
347 u32 pgtbl_err = I915_READ(PGTBL_ER);
348 DRM_DEBUG("page table error\n");
349 DRM_DEBUG("PGTBL_ER: 0x%08x\n",
350 pgtbl_err);
351 I915_WRITE(PGTBL_ER, pgtbl_err);
352 (void)I915_READ(PGTBL_ER);
353 }
354 }
355
356 if (eir & I915_ERROR_MEMORY_REFRESH) {
357 DRM_DEBUG("memory refresh error\n");
358 DRM_DEBUG("PIPEASTAT: 0x%08x\n",
359 pipea_stats);
360 DRM_DEBUG("PIPEBSTAT: 0x%08x\n",
361 pipeb_stats);
362 /* pipestat has already been acked */
363 }
364 if (eir & I915_ERROR_INSTRUCTION) {
365 DRM_DEBUG("instruction error\n");
366 DRM_DEBUG(" INSTPM: 0x%08x\n",
367 I915_READ(INSTPM));
368 if (!IS_I965G(dev)) {
369 u32 ipeir = I915_READ(IPEIR);
370
371 DRM_DEBUG(" IPEIR: 0x%08x\n",
372 I915_READ(IPEIR));
373 DRM_DEBUG(" IPEHR: 0x%08x\n",
374 I915_READ(IPEHR));
375 DRM_DEBUG(" INSTDONE: 0x%08x\n",
376 I915_READ(INSTDONE));
377 DRM_DEBUG(" ACTHD: 0x%08x\n",
378 I915_READ(ACTHD));
379 I915_WRITE(IPEIR, ipeir);
380 (void)I915_READ(IPEIR);
381 } else {
382 u32 ipeir = I915_READ(IPEIR_I965);
383
384 DRM_DEBUG(" IPEIR: 0x%08x\n",
385 I915_READ(IPEIR_I965));
386 DRM_DEBUG(" IPEHR: 0x%08x\n",
387 I915_READ(IPEHR_I965));
388 DRM_DEBUG(" INSTDONE: 0x%08x\n",
389 I915_READ(INSTDONE_I965));
390 DRM_DEBUG(" INSTPS: 0x%08x\n",
391 I915_READ(INSTPS));
392 DRM_DEBUG(" INSTDONE1: 0x%08x\n",
393 I915_READ(INSTDONE1));
394 DRM_DEBUG(" ACTHD: 0x%08x\n",
395 I915_READ(ACTHD_I965));
396 I915_WRITE(IPEIR_I965, ipeir);
397 (void)I915_READ(IPEIR_I965);
398 }
399 }
400
401 I915_WRITE(EIR, eir);
402 (void)I915_READ(EIR);
403 eir = I915_READ(EIR);
404 if (eir) {
405 /*
406 * some errors might have become stuck,
407 * mask them.
408 */
409 DRM_DEBUG("EIR stuck: 0x%08x, masking\n", eir);
410 I915_WRITE(EMR, I915_READ(EMR) | eir);
411 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
412 }
413
414 }
415
416 u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
417 {
418 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
419 int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45;
420
421 if (!i915_pipe_enabled(dev, pipe)) {
422 DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
423 return 0;
424 }
425
426 return I915_READ(reg);
427 }
428
429 irqreturn_t igdng_irq_handler(struct drm_device *dev)
430 {
431 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
432 int ret = IRQ_NONE;
433 u32 de_iir, gt_iir, de_ier;
434 u32 new_de_iir, new_gt_iir;
435 int vblank = 0;
436
437 /* disable master interrupt before clearing iir */
438 de_ier = I915_READ(DEIER);
439 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
440 (void)I915_READ(DEIER);
441
442 de_iir = I915_READ(DEIIR);
443 gt_iir = I915_READ(GTIIR);
444
445 for (;;) {
446 if (de_iir == 0 && gt_iir == 0)
447 break;
448
449 ret = IRQ_HANDLED;
450
451 I915_WRITE(DEIIR, de_iir);
452 new_de_iir = I915_READ(DEIIR);
453 I915_WRITE(GTIIR, gt_iir);
454 new_gt_iir = I915_READ(GTIIR);
455
456 if (dev_priv->sarea_priv) {
457 dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
458
459 }
460
461 if (gt_iir & GT_USER_INTERRUPT) {
462 dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
463 DRM_WAKEUP(&dev_priv->irq_queue);
464 }
465 if (de_iir & DE_PIPEA_VBLANK) {
466 vblank++;
467 drm_handle_vblank(dev, 0);
468 }
469
470 if (de_iir & DE_PIPEB_VBLANK) {
471 vblank++;
472 drm_handle_vblank(dev, 1);
473 }
474
475 de_iir = new_de_iir;
476 gt_iir = new_gt_iir;
477 }
478
479 I915_WRITE(DEIER, de_ier);
480 (void)I915_READ(DEIER);
481
482 return ret;
483 }
484
485 irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
486 {
487 drm_device_t *dev = (drm_device_t *) (void *) arg;
488 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
489 u32 iir;
490 u32 pipea_stats = 0, pipeb_stats = 0;
491 int vblank = 0;
492
493 if (IS_IGDNG(dev))
494 return igdng_irq_handler(dev);
495
496 iir = I915_READ(IIR);
497
498 if (iir == 0) {
499 return IRQ_NONE;
500 }
501 start:
502
503 if (dev_priv->sarea_priv) {
504 if (dev_priv->hw_status_page)
505 dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
506 }
507
508 I915_WRITE(IIR, iir);
509
510 (void) I915_READ(IIR); /* Flush posted writes */
511
512
513 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
514 i915_handle_error(dev);
515
516 if (iir & I915_USER_INTERRUPT) {
517 dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
518 DRM_WAKEUP(&dev_priv->irq_queue);
519 }
520
521 if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
522 pipea_stats = I915_READ(PIPEASTAT);
523
524 /* The vblank interrupt gets enabled even if we didn't ask for
525 it, so make sure it's shut down again */
526 if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A))
527 pipea_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
528 PIPE_VBLANK_INTERRUPT_ENABLE);
529 else if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
530 PIPE_VBLANK_INTERRUPT_STATUS))
531 {
532 vblank++;
533 drm_handle_vblank(dev, 0);
534 }
535
536 I915_WRITE(PIPEASTAT, pipea_stats);
537 }
538 if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
539 pipeb_stats = I915_READ(PIPEBSTAT);
540
541 /* The vblank interrupt gets enabled even if we didn't ask for
542 it, so make sure it's shut down again */
543 if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B))
544 pipeb_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
545 PIPE_VBLANK_INTERRUPT_ENABLE);
546 else if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
547 PIPE_VBLANK_INTERRUPT_STATUS))
548 {
549 vblank++;
550 drm_handle_vblank(dev, 1);
551 }
552
553 I915_WRITE(PIPEBSTAT, pipeb_stats);
554 }
555 return IRQ_HANDLED;
556
557 }
558
559 int i915_emit_irq(drm_device_t * dev)
560 {
561
562 drm_i915_private_t *dev_priv = dev->dev_private;
563 RING_LOCALS;
564
565 i915_kernel_lost_context(dev);
566
567 dev_priv->counter++;
568 if (dev_priv->counter > 0x7FFFFFFFUL)
569 dev_priv->counter = 1;
570 if (dev_priv->sarea_priv)
571 dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
572
573 #if defined(__i386)
574 if (IS_GM45(dev)) {
575 BEGIN_LP_RING(3);
576 OUT_RING(MI_STORE_DWORD_INDEX);
577 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
578 OUT_RING(dev_priv->counter);
579 ADVANCE_LP_RING();
580
581 (void) READ_BREADCRUMB(dev_priv);
582 BEGIN_LP_RING(2);
583 OUT_RING(0);
584 OUT_RING(MI_USER_INTERRUPT);
585 ADVANCE_LP_RING();
586 } else {
587 #endif /* __i386 */
588 BEGIN_LP_RING(4);
589 OUT_RING(MI_STORE_DWORD_INDEX);
590 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
591 OUT_RING(dev_priv->counter);
592 OUT_RING(MI_USER_INTERRUPT);
593 ADVANCE_LP_RING();
594 #if defined(__i386)
595 }
596 #endif /* __i386 */
597
598 #if defined(__i386)
599 if (IS_I965GM(dev) || IS_IGDNG(dev) || IS_GM45(dev))
600 #else
601 if (IS_I965GM(dev) || IS_IGDNG(dev))
602 #endif /* __i386 */
603 {
604 (void) READ_BREADCRUMB(dev_priv);
605 BEGIN_LP_RING(2);
606 OUT_RING(0);
607 OUT_RING(0);
608 ADVANCE_LP_RING();
609 (void) READ_BREADCRUMB(dev_priv);
610 }
611
612 return dev_priv->counter;
613 }
614
615 void i915_user_irq_on(struct drm_device *dev)
616 {
617 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
618 spin_lock(&dev_priv->user_irq_lock);
619 if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)){
620 if (IS_IGDNG(dev))
621 igdng_enable_irq(dev_priv, GT_USER_INTERRUPT, 1);
622 else
623 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
624 }
625 spin_unlock(&dev_priv->user_irq_lock);
626
627 }
628
629 void i915_user_irq_off(struct drm_device *dev)
630 {
631 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
632 spin_lock(&dev_priv->user_irq_lock);
633 if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
634 if (IS_IGDNG(dev))
635 igdng_disable_irq(dev_priv, GT_USER_INTERRUPT, 1);
636 else
637 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
638 }
639 spin_unlock(&dev_priv->user_irq_lock);
640 }
641
642
643 static int i915_wait_irq(drm_device_t * dev, int irq_nr)
644 {
645 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
646 int ret = 0;
647 int wait_time = 0;
648
649 if (!dev_priv) {
650 DRM_ERROR("called with no initialization\n");
651 return -EINVAL;
652 }
653
654 waitmore:
655 wait_time++;
656 if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
657 if (dev_priv->sarea_priv) {
658 dev_priv->sarea_priv->last_dispatch =
659 READ_BREADCRUMB(dev_priv);
660 }
661 return 0;
662 }
663 DRM_DEBUG("i915_wait_irq: irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv));
664 i915_user_irq_on(dev);
665 DRM_WAIT_ON(ret, &dev_priv->irq_queue, 3 * DRM_HZ,
666 READ_BREADCRUMB(dev_priv) >= irq_nr);
667 i915_user_irq_off(dev);
668
669 if (ret == EBUSY) {
670 if (wait_time > 5) {
671 DRM_DEBUG("%d: EBUSY -- rec: %d emitted: %d\n",
672 ret,
673 READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
674 return ret;
675 }
676 goto waitmore;
677 }
678
679 if (dev_priv->sarea_priv)
680 dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
681
682 if (ret == EINTR) {
683 if (wait_time > 5) {
684 DRM_DEBUG("EINTR wait %d now %d", dev_priv->counter, READ_BREADCRUMB(dev_priv));
685 return ret;
686 }
687 goto waitmore;
688 }
689
690 return ret;
691 }
692
693
694 /* Needs the lock as it touches the ring.
695 */
696 /*ARGSUSED*/
697 int i915_irq_emit(DRM_IOCTL_ARGS)
698 {
699 DRM_DEVICE;
700 drm_i915_private_t *dev_priv = dev->dev_private;
701 drm_i915_irq_emit_t emit;
702 int result;
703
704 LOCK_TEST_WITH_RETURN(dev, fpriv);
705
706 if (!dev_priv) {
707 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
708 return (EINVAL);
709 }
710
711
712 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
713 drm_i915_irq_emit32_t irq_emit32;
714
715 DRM_COPYFROM_WITH_RETURN(&irq_emit32,
716 (drm_i915_irq_emit32_t __user *) data,
717 sizeof (drm_i915_irq_emit32_t));
718 emit.irq_seq = (int __user *)(uintptr_t)irq_emit32.irq_seq;
719 } else
720 DRM_COPYFROM_WITH_RETURN(&emit,
721 (drm_i915_irq_emit_t __user *) data, sizeof(emit));
722
723 spin_lock(&dev->struct_mutex);
724 result = i915_emit_irq(dev);
725 spin_unlock(&dev->struct_mutex);
726
727 if (DRM_COPY_TO_USER(emit.irq_seq, &result, sizeof(int))) {
728 DRM_ERROR("copy_to_user\n");
729 return (EFAULT);
730 }
731
732 return 0;
733 }
734
735 /* Doesn't need the hardware lock.
736 */
737 /*ARGSUSED*/
738 int i915_irq_wait(DRM_IOCTL_ARGS)
739 {
740 DRM_DEVICE;
741 drm_i915_private_t *dev_priv = dev->dev_private;
742 drm_i915_irq_wait_t irqwait;
743
744 if (!dev_priv) {
745 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
746 return (EINVAL);
747 }
748
749 DRM_COPYFROM_WITH_RETURN(&irqwait,
750 (drm_i915_irq_wait_t __user *) data, sizeof(irqwait));
751
752 return i915_wait_irq(dev, irqwait.irq_seq);
753 }
754
755 static void igdng_enable_vblank(struct drm_device *dev, int pipe)
756 {
757 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
758 u32 vblank;
759
760 if (pipe == 0)
761 vblank = DE_PIPEA_VBLANK;
762 else
763 vblank = DE_PIPEB_VBLANK;
764
765 if ((dev_priv->de_irq_enable_reg & vblank) == 0) {
766 igdng_enable_irq(dev_priv, vblank, 0);
767 dev_priv->de_irq_enable_reg |= vblank;
768 I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
769 (void) I915_READ(DEIER);
770 }
771 }
772
773 static void igdng_disable_vblank(struct drm_device *dev, int pipe)
774 {
775 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
776 u32 vblank;
777
778 if (pipe == 0)
779 vblank = DE_PIPEA_VBLANK;
780 else
781 vblank = DE_PIPEB_VBLANK;
782
783 if ((dev_priv->de_irq_enable_reg & vblank) != 0) {
784 igdng_disable_irq(dev_priv, vblank, 0);
785 dev_priv->de_irq_enable_reg &= ~vblank;
786 I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
787 (void) I915_READ(DEIER);
788 }
789 }
790
791 int i915_enable_vblank(struct drm_device *dev, int pipe)
792 {
793 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
794 int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
795 u32 pipeconf;
796
797 pipeconf = I915_READ(pipeconf_reg);
798 if (!(pipeconf & PIPEACONF_ENABLE))
799 return -EINVAL;
800
801 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
802 if (IS_IGDNG(dev))
803 igdng_enable_vblank(dev, pipe);
804 else if (IS_I965G(dev))
805 i915_enable_pipestat(dev_priv, pipe,
806 PIPE_START_VBLANK_INTERRUPT_ENABLE);
807 else
808 i915_enable_pipestat(dev_priv, pipe,
809 PIPE_VBLANK_INTERRUPT_ENABLE);
810 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
811
812 return 0;
813 }
814
815 void i915_disable_vblank(struct drm_device *dev, int pipe)
816 {
817 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
818
819 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
820 if (IS_IGDNG(dev))
821 igdng_disable_vblank(dev, pipe);
822 else
823 i915_disable_pipestat(dev_priv, pipe,
824 PIPE_VBLANK_INTERRUPT_ENABLE |
825 PIPE_START_VBLANK_INTERRUPT_ENABLE);
826 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
827 }
828
829 /* Set the vblank monitor pipe
830 */
831 /*ARGSUSED*/
832 int i915_vblank_pipe_set(DRM_IOCTL_ARGS)
833 {
834 DRM_DEVICE;
835 drm_i915_private_t *dev_priv = dev->dev_private;
836
837 if (!dev_priv) {
838 DRM_ERROR("called with no initialization\n");
839 return (-EINVAL);
840 }
841
842 return (0);
843 }
844
845 /*ARGSUSED*/
846 int i915_vblank_pipe_get(DRM_IOCTL_ARGS)
847 {
848 DRM_DEVICE;
849 drm_i915_private_t *dev_priv = dev->dev_private;
850 drm_i915_vblank_pipe_t pipe;
851
852 if (!dev_priv) {
853 DRM_ERROR("called with no initialization\n");
854 return -EINVAL;
855 }
856
857 DRM_COPYFROM_WITH_RETURN(&pipe, (drm_i915_vblank_pipe_t __user *)data, sizeof (pipe));
858
859 pipe.pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
860
861 return 0;
862 }
863
864 /**
865 * Schedule buffer swap at given vertical blank.
866 */
867 /*ARGSUSED*/
868 int i915_vblank_swap(DRM_IOCTL_ARGS)
869 {
870 /* The delayed swap mechanism was fundamentally racy, and has been
871 * removed. The model was that the client requested a delayed flip/swap
872 * from the kernel, then waited for vblank before continuing to perform
873 * rendering. The problem was that the kernel might wake the client
874 * up before it dispatched the vblank swap (since the lock has to be
875 * held while touching the ringbuffer), in which case the client would
876 * clear and start the next frame before the swap occurred, and
877 * flicker would occur in addition to likely missing the vblank.
878 *
879 * In the absence of this ioctl, userland falls back to a correct path
880 * of waiting for a vblank, then dispatching the swap on its own.
881 * Context switching to userland and back is plenty fast enough for
882 * meeting the requirements of vblank swapping.
883 */
884 return -EINVAL;
885
886 }
887
888 /* drm_dma.h hooks
889 */
890
891 static void igdng_irq_preinstall(struct drm_device *dev)
892 {
893 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
894
895 I915_WRITE(HWSTAM, 0xeffe);
896
897 /* XXX hotplug from PCH */
898
899 I915_WRITE(DEIMR, 0xffffffff);
900 I915_WRITE(DEIER, 0x0);
901 (void) I915_READ(DEIER);
902
903 /* and GT */
904 I915_WRITE(GTIMR, 0xffffffff);
905 I915_WRITE(GTIER, 0x0);
906 (void) I915_READ(GTIER);
907 }
908
909 static int igdng_irq_postinstall(struct drm_device *dev)
910 {
911 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
912 /* enable kind of interrupts always enabled */
913 u32 display_mask = DE_MASTER_IRQ_CONTROL /*| DE_PCH_EVENT */;
914 u32 render_mask = GT_USER_INTERRUPT;
915
916 dev_priv->irq_mask_reg = ~display_mask;
917 dev_priv->de_irq_enable_reg = display_mask;
918
919 /* should always can generate irq */
920 I915_WRITE(DEIIR, I915_READ(DEIIR));
921 (void) I915_READ(DEIIR);
922 I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
923 I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
924 (void) I915_READ(DEIER);
925
926 /* user interrupt should be enabled, but masked initial */
927 dev_priv->gt_irq_mask_reg = 0xffffffff;
928 dev_priv->gt_irq_enable_reg = render_mask;
929
930 I915_WRITE(GTIIR, I915_READ(GTIIR));
931 (void) I915_READ(GTIIR);
932 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
933 I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg);
934 (void) I915_READ(GTIER);
935
936 return 0;
937 }
938
939 static void igdng_irq_uninstall(struct drm_device *dev)
940 {
941 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
942 I915_WRITE(HWSTAM, 0xffffffff);
943
944 I915_WRITE(DEIMR, 0xffffffff);
945 I915_WRITE(DEIER, 0x0);
946 I915_WRITE(DEIIR, I915_READ(DEIIR));
947
948 I915_WRITE(GTIMR, 0xffffffff);
949 I915_WRITE(GTIER, 0x0);
950 I915_WRITE(GTIIR, I915_READ(GTIIR));
951 }
952
953 int i915_driver_irq_preinstall(drm_device_t * dev)
954 {
955 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
956
957 if (!dev_priv->mmio_map)
958 return -EINVAL;
959
960 if (IS_IGDNG(dev)) {
961 igdng_irq_preinstall(dev);
962 return 0;
963 }
964
965 I915_WRITE16(HWSTAM, 0xeffe);
966 I915_WRITE(PIPEASTAT, 0);
967 I915_WRITE(PIPEBSTAT, 0);
968 I915_WRITE(IMR, 0xffffffff);
969 I915_WRITE16(IER, 0x0);
970 (void) I915_READ(IER);
971
972 return 0;
973 }
974
975 void i915_driver_irq_postinstall(drm_device_t * dev)
976 {
977 int error_mask;
978 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
979
980 dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
981
982 if (IS_IGDNG(dev)) {
983 (void) igdng_irq_postinstall(dev);
984 DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));
985 return;
986 }
987
988 /* Unmask the interrupts that we always want on. */
989 dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;
990
991 dev_priv->pipestat[0] = 0;
992 dev_priv->pipestat[1] = 0;
993
994 /*
995 * Enable some error detection, note the instruction error mask
996 * bit is reserved, so we leave it masked.
997 */
998 if (IS_G4X(dev)) {
999 error_mask = ~(GM45_ERROR_PAGE_TABLE |
1000 GM45_ERROR_MEM_PRIV |
1001 GM45_ERROR_CP_PRIV |
1002 I915_ERROR_MEMORY_REFRESH);
1003 } else {
1004 error_mask = ~(I915_ERROR_PAGE_TABLE |
1005 I915_ERROR_MEMORY_REFRESH);
1006 }
1007 I915_WRITE(EMR, error_mask);
1008
1009 /* Disable pipe interrupt enables, clear pending pipe status */
1010 I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
1011 I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
1012 (void) I915_READ(PIPEASTAT);
1013 (void) I915_READ(PIPEBSTAT);
1014 /* Clear pending interrupt status */
1015 I915_WRITE(IIR, I915_READ(IIR));
1016
1017 (void) I915_READ(IIR);
1018 I915_WRITE(IMR, dev_priv->irq_mask_reg);
1019 I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
1020 (void) I915_READ(IER);
1021
1022 DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));
1023
1024 return;
1025 }
1026
1027 void i915_driver_irq_uninstall(drm_device_t * dev)
1028 {
1029 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1030 if ((!dev_priv) || (dev->irq_enabled == 0))
1031 return;
1032
1033 dev_priv->vblank_pipe = 0;
1034
1035 if (IS_IGDNG(dev)) {
1036 igdng_irq_uninstall(dev);
1037 DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
1038 return;
1039 }
1040
1041 I915_WRITE(HWSTAM, 0xffffffff);
1042 I915_WRITE(PIPEASTAT, 0);
1043 I915_WRITE(PIPEBSTAT, 0);
1044 I915_WRITE(IMR, 0xffffffff);
1045 I915_WRITE(IER, 0x0);
1046
1047 I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
1048 I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
1049 I915_WRITE(IIR, I915_READ(IIR));
1050
1051 DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
1052 }