1 /* BEGIN CSTYLED */
   2 
   3 /* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
   4  */
   5 /*
   6  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   7  * Copyright (c) 2009, Intel Corporation.
   8  * All Rights Reserved.
   9  *
  10  * Permission is hereby granted, free of charge, to any person obtaining a
  11  * copy of this software and associated documentation files (the
  12  * "Software"), to deal in the Software without restriction, including
  13  * without limitation the rights to use, copy, modify, merge, publish,
  14  * distribute, sub license, and/or sell copies of the Software, and to
  15  * permit persons to whom the Software is furnished to do so, subject to
  16  * the following conditions:
  17  *
  18  * The above copyright notice and this permission notice (including the
  19  * next paragraph) shall be included in all copies or substantial portions
  20  * of the Software.
  21  *
  22  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  23  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  24  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  25  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  26  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  27  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  28  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  29  *
  30  */
  31 
  32 /*
  33  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  34  * Use is subject to license terms.
  35  */
  36 
  37 #include "drmP.h"
  38 #include "drm.h"
  39 #include "i915_drm.h"
  40 #include "i915_drv.h"
  41 
  42 
  43 #define MAX_NOPID ((u32)~0)
  44 
  45 /**
  46  * Interrupts that are always left unmasked.
  47  *
  48  * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
  49  * we leave them always unmasked in IMR and then control enabling them through
  50  * PIPESTAT alone.
  51  */
  52 
  53 #define I915_INTERRUPT_ENABLE_FIX (I915_ASLE_INTERRUPT |                 \
  54                                    I915_DISPLAY_PIPE_A_EVENT_INTERRUPT | \
  55                                    I915_DISPLAY_PIPE_B_EVENT_INTERRUPT | \
  56                                    I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
  57 
  58 /** Interrupts that we mask and unmask at runtime. */
  59 #define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT)
  60 
  61 /** These are all of the interrupts used by the driver */
  62 #define I915_INTERRUPT_ENABLE_MASK (I915_INTERRUPT_ENABLE_FIX | \
  63                                     I915_INTERRUPT_ENABLE_VAR)
  64 
  65 void
  66 igdng_enable_irq(drm_i915_private_t *dev_priv, u32 mask, int gfx_irq)
  67 {
  68         if (gfx_irq && ((dev_priv->gt_irq_mask_reg & mask) != 0)) {
  69                 dev_priv->gt_irq_mask_reg &= ~mask;
  70                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
  71                 (void) I915_READ(GTIMR);
  72         } else if ((dev_priv->irq_mask_reg & mask) != 0) {
  73                 dev_priv->irq_mask_reg &= ~mask;
  74                 I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
  75                 (void) I915_READ(DEIMR);
  76 
  77         }
  78 }
  79 
  80 static inline void
  81 igdng_disable_irq(drm_i915_private_t *dev_priv, u32 mask, int gfx_irq)
  82 {
  83         if (gfx_irq && ((dev_priv->gt_irq_mask_reg & mask) != mask)) {
  84                 dev_priv->gt_irq_mask_reg |= mask;
  85                 I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
  86                 (void) I915_READ(GTIMR);
  87         } else if ((dev_priv->irq_mask_reg & mask) != mask) {
  88                 dev_priv->irq_mask_reg |= mask;
  89                 I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
  90                 (void) I915_READ(DEIMR);
  91         }
  92 }
  93 
  94 /* For display hotplug interrupt */
  95 void
  96 igdng_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
  97 {
  98        if ((dev_priv->irq_mask_reg & mask) != 0) {
  99                dev_priv->irq_mask_reg &= ~mask;
 100                I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
 101                (void) I915_READ(DEIMR);
 102        }
 103 }
 104 
 105 #if 0
 106 static inline void
 107 igdng_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 108 {
 109        if ((dev_priv->irq_mask_reg & mask) != mask) {
 110                dev_priv->irq_mask_reg |= mask;
 111                I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
 112                (void) I915_READ(DEIMR);
 113        }
 114 }
 115 #endif
 116 
 117 static inline void
 118 i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
 119 {
 120         if ((dev_priv->irq_mask_reg & mask) != 0) {
 121                 dev_priv->irq_mask_reg &= ~mask;
 122                 I915_WRITE(IMR, dev_priv->irq_mask_reg);
 123                 (void) I915_READ(IMR);
 124         }
 125 }
 126 
 127 static inline void
 128 i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
 129 {
 130         if ((dev_priv->irq_mask_reg & mask) != mask) {
 131                 dev_priv->irq_mask_reg |= mask;
 132                 I915_WRITE(IMR, dev_priv->irq_mask_reg);
 133                 (void) I915_READ(IMR);
 134         }
 135 }
 136 
 137 static inline uint32_t
 138 i915_pipestat(int pipe)
 139 {
 140         if (pipe == 0)
 141                 return PIPEASTAT;
 142         if (pipe == 1)
 143                 return PIPEBSTAT;
 144         return 0;
 145 }
 146 
 147 void
 148 i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, uint32_t mask)
 149 {
 150         if ((dev_priv->pipestat[pipe] & mask) != mask) {
 151                 u32 reg = i915_pipestat(pipe);
 152 
 153                 dev_priv->pipestat[pipe] |= mask;
 154                 /* Enable the interrupt, clear any pending status */
 155                 I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16));
 156                 (void) I915_READ(reg);
 157         }
 158 }
 159 
 160 void
 161 i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask)
 162 {
 163         if ((dev_priv->pipestat[pipe] & mask) != 0) {
 164                 u32 reg = i915_pipestat(pipe);
 165 
 166                 dev_priv->pipestat[pipe] &= ~mask;
 167                 I915_WRITE(reg, dev_priv->pipestat[pipe]);
 168                 (void) I915_READ(reg);
 169         }
 170 }
 171 
 172 /**
 173  * i915_pipe_enabled - check if a pipe is enabled
 174  * @dev: DRM device
 175  * @pipe: pipe to check
 176  *
 177  * Reading certain registers when the pipe is disabled can hang the chip.
 178  * Use this routine to make sure the PLL is running and the pipe is active
 179  * before reading such registers if unsure.
 180  */
 181 static int
 182 i915_pipe_enabled(struct drm_device *dev, int pipe)
 183 {
 184         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 185         unsigned long pipeconf = pipe ? PIPEBCONF : PIPEACONF;
 186 
 187         if (I915_READ(pipeconf) & PIPEACONF_ENABLE)
 188                 return 1;
 189 
 190         return 0;
 191 }
 192 
 193 u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 194 {
 195         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 196         unsigned long high_frame;
 197         unsigned long low_frame;
 198         u32 high1, high2, low, count;
 199 
 200         high_frame = pipe ? PIPEBFRAMEHIGH : PIPEAFRAMEHIGH;
 201         low_frame = pipe ? PIPEBFRAMEPIXEL : PIPEAFRAMEPIXEL;
 202 
 203         if (!i915_pipe_enabled(dev, pipe)) {
 204             DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
 205             return 0;
 206         }
 207 
 208         /*
 209          * High & low register fields aren't synchronized, so make sure
 210          * we get a low value that's stable across two reads of the high
 211          * register.
 212          */
 213         do {
 214                 high1 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
 215                          PIPE_FRAME_HIGH_SHIFT);
 216                 low =  ((I915_READ(low_frame) & PIPE_FRAME_LOW_MASK) >>
 217                         PIPE_FRAME_LOW_SHIFT);
 218                 high2 = ((I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK) >>
 219                          PIPE_FRAME_HIGH_SHIFT);
 220         } while (high1 != high2);
 221 
 222         count = (high1 << 8) | low;
 223 
 224         return count;
 225 }
 226 
 227 /**
 228  * i915_capture_error_state - capture an error record for later analysis
 229  * @dev: drm device
 230  *
 231  * Should be called when an error is detected (either a hang or an error
 232  * interrupt) to capture error state from the time of the error.  Fills
 233  * out a structure which becomes available in debugfs for user level tools
 234  * to pick up.
 235  */
 236 static void i915_capture_error_state(struct drm_device *dev)
 237 {
 238         struct drm_i915_private *dev_priv = dev->dev_private;
 239         struct drm_i915_error_state *error;
 240 
 241         spin_lock_irqsave(&dev_priv->error_lock, flags);
 242 #if 0
 243         if (dev_priv->first_error)
 244                 goto out;
 245 #endif
 246         error = drm_alloc(sizeof(*error), DRM_MEM_DRIVER);
 247         if (!error) {
 248                 DRM_DEBUG("out ot memory, not capturing error state\n");
 249                 goto out;
 250         }
 251 
 252         error->eir = I915_READ(EIR);
 253         error->pgtbl_er = I915_READ(PGTBL_ER);
 254         error->pipeastat = I915_READ(PIPEASTAT);
 255         error->pipebstat = I915_READ(PIPEBSTAT);
 256         error->instpm = I915_READ(INSTPM);
 257         if (!IS_I965G(dev)) {
 258                 error->ipeir = I915_READ(IPEIR);
 259                 error->ipehr = I915_READ(IPEHR);
 260                 error->instdone = I915_READ(INSTDONE);
 261                 error->acthd = I915_READ(ACTHD);
 262         } else {
 263                 error->ipeir = I915_READ(IPEIR_I965);
 264                 error->ipehr = I915_READ(IPEHR_I965);
 265                 error->instdone = I915_READ(INSTDONE_I965);
 266                 error->instps = I915_READ(INSTPS);
 267                 error->instdone1 = I915_READ(INSTDONE1);
 268                 error->acthd = I915_READ(ACTHD_I965);
 269         }
 270 
 271         (void) uniqtime(&error->time);
 272 
 273         dev_priv->first_error = error;
 274         
 275         DRM_DEBUG("Time: %ld s %ld us\n", error->time.tv_sec,
 276                    error->time.tv_usec);
 277         DRM_DEBUG("EIR: 0x%08x\n", error->eir);
 278         DRM_DEBUG("  PGTBL_ER: 0x%08x\n", error->pgtbl_er);
 279         DRM_DEBUG("  INSTPM: 0x%08x\n", error->instpm);
 280         DRM_DEBUG("  IPEIR: 0x%08x\n", error->ipeir);
 281         DRM_DEBUG("  IPEHR: 0x%08x\n", error->ipehr);
 282         DRM_DEBUG("  INSTDONE: 0x%08x\n", error->instdone);
 283         DRM_DEBUG("  ACTHD: 0x%08x\n", error->acthd);
 284         DRM_DEBUG("  DMA_FADD_P: 0x%08x\n", I915_READ(0x2078));
 285         if (IS_I965G(dev)) {
 286                 DRM_DEBUG("  INSTPS: 0x%08x\n", error->instps);
 287                 DRM_DEBUG("  INSTDONE1: 0x%08x\n", error->instdone1);
 288         }
 289         drm_free(error, sizeof(*error), DRM_MEM_DRIVER);
 290 out:
 291         spin_unlock_irqrestore(&dev_priv->error_lock, flags);
 292 }
 293 
 294 /**
 295  * i915_handle_error - handle an error interrupt
 296  * @dev: drm device
 297  *
 298  * Do some basic checking of regsiter state at error interrupt time and
 299  * dump it to the syslog.  Also call i915_capture_error_state() to make
 300  * sure we get a record and make it available in debugfs.  Fire a uevent
 301  * so userspace knows something bad happened (should trigger collection
 302  * of a ring dump etc.).
 303  */
 304 void i915_handle_error(struct drm_device *dev)
 305 {
 306         struct drm_i915_private *dev_priv = dev->dev_private;
 307         u32 eir = I915_READ(EIR);
 308         u32 pipea_stats = I915_READ(PIPEASTAT);
 309         u32 pipeb_stats = I915_READ(PIPEBSTAT);
 310 
 311         i915_capture_error_state(dev);
 312 
 313         DRM_DEBUG("render error detected, EIR: 0x%08x\n",
 314                eir);
 315 
 316         if (IS_G4X(dev)) {
 317                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
 318                         u32 ipeir = I915_READ(IPEIR_I965);
 319 
 320                         DRM_DEBUG("  IPEIR: 0x%08x\n",
 321                                I915_READ(IPEIR_I965));
 322                         DRM_DEBUG("  IPEHR: 0x%08x\n",
 323                                I915_READ(IPEHR_I965));
 324                         DRM_DEBUG("  INSTDONE: 0x%08x\n",
 325                                I915_READ(INSTDONE_I965));
 326                         DRM_DEBUG("  INSTPS: 0x%08x\n",
 327                                I915_READ(INSTPS));
 328                         DRM_DEBUG("  INSTDONE1: 0x%08x\n",
 329                                I915_READ(INSTDONE1));
 330                         DRM_DEBUG("  ACTHD: 0x%08x\n",
 331                                I915_READ(ACTHD_I965));
 332                         I915_WRITE(IPEIR_I965, ipeir);
 333                         (void)I915_READ(IPEIR_I965);
 334                 }
 335                 if (eir & GM45_ERROR_PAGE_TABLE) {
 336                         u32 pgtbl_err = I915_READ(PGTBL_ER);
 337                         DRM_DEBUG("page table error\n");
 338                         DRM_DEBUG("  PGTBL_ER: 0x%08x\n",
 339                                pgtbl_err);
 340                         I915_WRITE(PGTBL_ER, pgtbl_err);
 341                         (void)I915_READ(PGTBL_ER);
 342                 }
 343         }
 344 
 345         if (IS_I9XX(dev)) {
 346                 if (eir & I915_ERROR_PAGE_TABLE) {
 347                         u32 pgtbl_err = I915_READ(PGTBL_ER);
 348                         DRM_DEBUG("page table error\n");
 349                         DRM_DEBUG("PGTBL_ER: 0x%08x\n",
 350                                pgtbl_err);
 351                         I915_WRITE(PGTBL_ER, pgtbl_err);
 352                         (void)I915_READ(PGTBL_ER);
 353                 }
 354         }
 355 
 356         if (eir & I915_ERROR_MEMORY_REFRESH) {
 357                 DRM_DEBUG("memory refresh error\n");
 358                 DRM_DEBUG("PIPEASTAT: 0x%08x\n",
 359                        pipea_stats);
 360                 DRM_DEBUG("PIPEBSTAT: 0x%08x\n",
 361                        pipeb_stats);
 362                 /* pipestat has already been acked */
 363         }
 364         if (eir & I915_ERROR_INSTRUCTION) {
 365                 DRM_DEBUG("instruction error\n");
 366                 DRM_DEBUG("  INSTPM: 0x%08x\n",
 367                        I915_READ(INSTPM));
 368                 if (!IS_I965G(dev)) {
 369                         u32 ipeir = I915_READ(IPEIR);
 370 
 371                         DRM_DEBUG("  IPEIR: 0x%08x\n",
 372                                I915_READ(IPEIR));
 373                         DRM_DEBUG("  IPEHR: 0x%08x\n",
 374                                I915_READ(IPEHR));
 375                         DRM_DEBUG("  INSTDONE: 0x%08x\n",
 376                                I915_READ(INSTDONE));
 377                         DRM_DEBUG("  ACTHD: 0x%08x\n",
 378                                I915_READ(ACTHD));
 379                         I915_WRITE(IPEIR, ipeir);
 380                         (void)I915_READ(IPEIR);
 381                 } else {
 382                         u32 ipeir = I915_READ(IPEIR_I965);
 383 
 384                         DRM_DEBUG("  IPEIR: 0x%08x\n",
 385                                I915_READ(IPEIR_I965));
 386                         DRM_DEBUG("  IPEHR: 0x%08x\n",
 387                                I915_READ(IPEHR_I965));
 388                         DRM_DEBUG("  INSTDONE: 0x%08x\n",
 389                                I915_READ(INSTDONE_I965));
 390                         DRM_DEBUG("  INSTPS: 0x%08x\n",
 391                                I915_READ(INSTPS));
 392                         DRM_DEBUG("  INSTDONE1: 0x%08x\n",
 393                                I915_READ(INSTDONE1));
 394                         DRM_DEBUG("  ACTHD: 0x%08x\n",
 395                                I915_READ(ACTHD_I965));
 396                         I915_WRITE(IPEIR_I965, ipeir);
 397                         (void)I915_READ(IPEIR_I965);
 398                 }
 399         }
 400 
 401         I915_WRITE(EIR, eir);
 402         (void)I915_READ(EIR);
 403         eir = I915_READ(EIR);
 404         if (eir) {
 405                 /*
 406                  * some errors might have become stuck,
 407                  * mask them.
 408                  */
 409                 DRM_DEBUG("EIR stuck: 0x%08x, masking\n", eir);
 410                 I915_WRITE(EMR, I915_READ(EMR) | eir);
 411                 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
 412         }
 413 
 414 }
 415 
 416 u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 417 {
 418        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 419        int reg = pipe ? PIPEB_FRMCOUNT_GM45 : PIPEA_FRMCOUNT_GM45;
 420 
 421        if (!i915_pipe_enabled(dev, pipe)) {
 422                 DRM_ERROR("trying to get vblank count for disabled pipe %d\n", pipe);
 423                return 0;
 424        }
 425 
 426        return I915_READ(reg);
 427 }
 428 
 429 irqreturn_t igdng_irq_handler(struct drm_device *dev)
 430 {
 431        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 432        int ret = IRQ_NONE;
 433        u32 de_iir, gt_iir, de_ier;
 434        u32 new_de_iir, new_gt_iir;
 435        int vblank = 0;
 436 
 437         /* disable master interrupt before clearing iir  */
 438         de_ier = I915_READ(DEIER);
 439         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
 440         (void)I915_READ(DEIER);
 441 
 442        de_iir = I915_READ(DEIIR);
 443        gt_iir = I915_READ(GTIIR);
 444 
 445        for (;;) {
 446                if (de_iir == 0 && gt_iir == 0)
 447                        break;
 448 
 449                ret = IRQ_HANDLED;
 450 
 451                I915_WRITE(DEIIR, de_iir);
 452                new_de_iir = I915_READ(DEIIR);
 453                I915_WRITE(GTIIR, gt_iir);
 454                new_gt_iir = I915_READ(GTIIR);
 455 
 456         if (dev_priv->sarea_priv) {
 457             dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 458 
 459         }
 460 
 461                if (gt_iir & GT_USER_INTERRUPT) {
 462                        dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
 463                        DRM_WAKEUP(&dev_priv->irq_queue);
 464                }
 465                if (de_iir & DE_PIPEA_VBLANK) {
 466                        vblank++;
 467                        drm_handle_vblank(dev, 0);
 468                }
 469 
 470                if (de_iir & DE_PIPEB_VBLANK) {
 471                        vblank++;
 472                        drm_handle_vblank(dev, 1);
 473                }
 474 
 475                de_iir = new_de_iir;
 476                gt_iir = new_gt_iir;
 477        }
 478 
 479         I915_WRITE(DEIER, de_ier);
 480         (void)I915_READ(DEIER);
 481 
 482        return ret;
 483 }
 484 
 485 irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS)
 486 {
 487         drm_device_t *dev = (drm_device_t *) (void *) arg;
 488         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 489         u32 iir;
 490         u32 pipea_stats = 0, pipeb_stats = 0;
 491         int vblank = 0;
 492 
 493         if (IS_IGDNG(dev))
 494                 return igdng_irq_handler(dev);
 495 
 496         iir = I915_READ(IIR);
 497 
 498         if (iir == 0) {
 499                 return IRQ_NONE;
 500         }
 501 start:
 502 
 503         if (dev_priv->sarea_priv) {
 504                 if (dev_priv->hw_status_page)
 505                         dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 506         }
 507 
 508         I915_WRITE(IIR, iir);
 509 
 510         (void) I915_READ(IIR); /* Flush posted writes */
 511 
 512 
 513         if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
 514                 i915_handle_error(dev);
 515 
 516         if (iir & I915_USER_INTERRUPT) {
 517                 dev_priv->mm.irq_gem_seqno = i915_get_gem_seqno(dev);
 518                 DRM_WAKEUP(&dev_priv->irq_queue);
 519         }
 520 
 521         if (iir & I915_DISPLAY_PIPE_A_EVENT_INTERRUPT) {
 522                 pipea_stats = I915_READ(PIPEASTAT);
 523 
 524                 /* The vblank interrupt gets enabled even if we didn't ask for
 525                    it, so make sure it's shut down again */
 526                 if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_A))
 527                         pipea_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
 528                                          PIPE_VBLANK_INTERRUPT_ENABLE);
 529                 else if (pipea_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
 530                                         PIPE_VBLANK_INTERRUPT_STATUS))
 531                 {
 532                         vblank++;
 533                         drm_handle_vblank(dev, 0);
 534                 }
 535 
 536                 I915_WRITE(PIPEASTAT, pipea_stats);
 537         }
 538         if (iir & I915_DISPLAY_PIPE_B_EVENT_INTERRUPT) {
 539                 pipeb_stats = I915_READ(PIPEBSTAT);
 540 
 541                 /* The vblank interrupt gets enabled even if we didn't ask for
 542                    it, so make sure it's shut down again */
 543                 if (!(dev_priv->vblank_pipe & DRM_I915_VBLANK_PIPE_B))
 544                         pipeb_stats &= ~(PIPE_START_VBLANK_INTERRUPT_ENABLE |
 545                                          PIPE_VBLANK_INTERRUPT_ENABLE);
 546                 else if (pipeb_stats & (PIPE_START_VBLANK_INTERRUPT_STATUS|
 547                                         PIPE_VBLANK_INTERRUPT_STATUS))
 548                 {
 549                         vblank++;
 550                         drm_handle_vblank(dev, 1);
 551                 }
 552 
 553                 I915_WRITE(PIPEBSTAT, pipeb_stats);
 554         }
 555        return IRQ_HANDLED;
 556 
 557 }
 558 
 559 int i915_emit_irq(drm_device_t * dev)
 560 {
 561 
 562         drm_i915_private_t *dev_priv = dev->dev_private;
 563         RING_LOCALS;
 564 
 565         i915_kernel_lost_context(dev);
 566         
 567         dev_priv->counter++;
 568         if (dev_priv->counter > 0x7FFFFFFFUL)
 569                 dev_priv->counter = 1;
 570         if (dev_priv->sarea_priv)
 571                 dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
 572 
 573 #if defined(__i386)
 574         if (IS_GM45(dev)) {
 575                 BEGIN_LP_RING(3);
 576                 OUT_RING(MI_STORE_DWORD_INDEX);
 577                 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 578                 OUT_RING(dev_priv->counter);
 579                 ADVANCE_LP_RING();
 580 
 581                 (void) READ_BREADCRUMB(dev_priv);
 582                 BEGIN_LP_RING(2);
 583                 OUT_RING(0);
 584                 OUT_RING(MI_USER_INTERRUPT);
 585                 ADVANCE_LP_RING();
 586         } else {
 587 #endif  /* __i386 */
 588         BEGIN_LP_RING(4);
 589         OUT_RING(MI_STORE_DWORD_INDEX);
 590         OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 591         OUT_RING(dev_priv->counter);
 592         OUT_RING(MI_USER_INTERRUPT);
 593         ADVANCE_LP_RING();
 594 #if defined(__i386)
 595         }
 596 #endif  /* __i386 */
 597 
 598 #if defined(__i386)
 599         if (IS_I965GM(dev) || IS_IGDNG(dev) || IS_GM45(dev))
 600 #else
 601         if (IS_I965GM(dev) || IS_IGDNG(dev))
 602 #endif  /* __i386 */
 603         {
 604                 (void) READ_BREADCRUMB(dev_priv);
 605                 BEGIN_LP_RING(2);
 606                 OUT_RING(0);
 607                 OUT_RING(0);
 608                 ADVANCE_LP_RING();
 609                 (void) READ_BREADCRUMB(dev_priv);
 610         }
 611 
 612         return dev_priv->counter;
 613 }
 614 
 615 void i915_user_irq_on(struct drm_device *dev)
 616 {
 617         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 618         spin_lock(&dev_priv->user_irq_lock);
 619         if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)){
 620                if (IS_IGDNG(dev))
 621                        igdng_enable_irq(dev_priv, GT_USER_INTERRUPT, 1);
 622                else
 623                        i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 624         }
 625         spin_unlock(&dev_priv->user_irq_lock);
 626 
 627 }
 628                 
 629 void i915_user_irq_off(struct drm_device *dev)
 630 {
 631         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 632         spin_lock(&dev_priv->user_irq_lock);
 633         if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
 634                if (IS_IGDNG(dev))
 635                        igdng_disable_irq(dev_priv, GT_USER_INTERRUPT, 1);
 636                else
 637                        i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 638         }
 639         spin_unlock(&dev_priv->user_irq_lock);
 640 }
 641 
 642 
 643 static int i915_wait_irq(drm_device_t * dev, int irq_nr)
 644 {
 645         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 646         int ret = 0;
 647         int wait_time = 0;
 648 
 649         if (!dev_priv) {
 650                 DRM_ERROR("called with no initialization\n");
 651                 return -EINVAL;
 652         }
 653 
 654 waitmore:
 655         wait_time++;
 656         if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
 657                 if (dev_priv->sarea_priv) {
 658                         dev_priv->sarea_priv->last_dispatch =
 659                                 READ_BREADCRUMB(dev_priv);
 660                 }
 661                 return 0;
 662         }
 663         DRM_DEBUG("i915_wait_irq: irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv));
 664         i915_user_irq_on(dev);
 665         DRM_WAIT_ON(ret, &dev_priv->irq_queue, 3 * DRM_HZ,
 666                     READ_BREADCRUMB(dev_priv) >= irq_nr);
 667         i915_user_irq_off(dev);
 668 
 669         if (ret == EBUSY) {
 670                 if (wait_time > 5) {
 671                 DRM_DEBUG("%d: EBUSY -- rec: %d emitted: %d\n",
 672                           ret,
 673                           READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
 674                         return ret;
 675                 }
 676                 goto waitmore;
 677         }
 678 
 679         if (dev_priv->sarea_priv)
 680                 dev_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 681 
 682         if (ret == EINTR) {
 683                 if (wait_time > 5) {
 684                         DRM_DEBUG("EINTR wait %d now %d", dev_priv->counter, READ_BREADCRUMB(dev_priv));
 685                         return ret;
 686                 }
 687                 goto waitmore;
 688         }
 689 
 690         return ret;
 691 }
 692 
 693 
 694 /* Needs the lock as it touches the ring.
 695  */
 696 /*ARGSUSED*/
 697 int i915_irq_emit(DRM_IOCTL_ARGS)
 698 {
 699         DRM_DEVICE;
 700         drm_i915_private_t *dev_priv = dev->dev_private;
 701         drm_i915_irq_emit_t emit;
 702         int result;
 703 
 704         LOCK_TEST_WITH_RETURN(dev, fpriv);
 705 
 706         if (!dev_priv) {
 707                 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
 708                 return (EINVAL);
 709         }
 710 
 711 
 712         if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
 713                 drm_i915_irq_emit32_t irq_emit32;
 714 
 715                 DRM_COPYFROM_WITH_RETURN(&irq_emit32,
 716                         (drm_i915_irq_emit32_t __user *) data,
 717                         sizeof (drm_i915_irq_emit32_t));
 718                 emit.irq_seq = (int __user *)(uintptr_t)irq_emit32.irq_seq;
 719         } else
 720                 DRM_COPYFROM_WITH_RETURN(&emit,
 721                     (drm_i915_irq_emit_t __user *) data, sizeof(emit));
 722 
 723         spin_lock(&dev->struct_mutex);
 724         result = i915_emit_irq(dev);
 725         spin_unlock(&dev->struct_mutex);
 726 
 727         if (DRM_COPY_TO_USER(emit.irq_seq, &result, sizeof(int))) {
 728                 DRM_ERROR("copy_to_user\n");
 729                 return (EFAULT);
 730         }
 731 
 732         return 0;
 733 }
 734 
 735 /* Doesn't need the hardware lock.
 736  */
 737 /*ARGSUSED*/
 738 int i915_irq_wait(DRM_IOCTL_ARGS)
 739 {
 740         DRM_DEVICE;
 741         drm_i915_private_t *dev_priv = dev->dev_private;
 742         drm_i915_irq_wait_t irqwait;
 743 
 744         if (!dev_priv) {
 745                 DRM_ERROR("%s called with no initialization\n", __FUNCTION__);
 746                 return (EINVAL);
 747         }
 748 
 749         DRM_COPYFROM_WITH_RETURN(&irqwait,
 750             (drm_i915_irq_wait_t __user *) data, sizeof(irqwait));
 751 
 752         return i915_wait_irq(dev, irqwait.irq_seq);
 753 }
 754 
 755 static void igdng_enable_vblank(struct drm_device *dev, int pipe)
 756 {
 757         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 758         u32 vblank;
 759 
 760         if (pipe == 0)
 761                 vblank = DE_PIPEA_VBLANK;
 762         else
 763                 vblank = DE_PIPEB_VBLANK;
 764 
 765         if ((dev_priv->de_irq_enable_reg & vblank) == 0) {
 766                 igdng_enable_irq(dev_priv, vblank, 0);
 767                 dev_priv->de_irq_enable_reg |= vblank;
 768                 I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
 769                 (void) I915_READ(DEIER);
 770         }
 771 }
 772 
 773 static void igdng_disable_vblank(struct drm_device *dev, int pipe)
 774 {
 775         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 776         u32 vblank;
 777 
 778         if (pipe == 0)
 779                 vblank = DE_PIPEA_VBLANK;
 780         else
 781                 vblank = DE_PIPEB_VBLANK;
 782 
 783         if ((dev_priv->de_irq_enable_reg & vblank) != 0) {
 784                 igdng_disable_irq(dev_priv, vblank, 0);
 785                 dev_priv->de_irq_enable_reg &= ~vblank;
 786                 I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
 787                 (void) I915_READ(DEIER);
 788         }
 789 }
 790 
 791 int i915_enable_vblank(struct drm_device *dev, int pipe)
 792 {
 793         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 794         int pipeconf_reg = (pipe == 0) ? PIPEACONF : PIPEBCONF;
 795         u32 pipeconf;
 796 
 797         pipeconf = I915_READ(pipeconf_reg);
 798         if (!(pipeconf & PIPEACONF_ENABLE))
 799                 return -EINVAL;
 800 
 801         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 802         if (IS_IGDNG(dev))
 803                 igdng_enable_vblank(dev, pipe);
 804         else if (IS_I965G(dev))
 805                 i915_enable_pipestat(dev_priv, pipe,
 806                                      PIPE_START_VBLANK_INTERRUPT_ENABLE);
 807         else
 808                 i915_enable_pipestat(dev_priv, pipe,
 809                                      PIPE_VBLANK_INTERRUPT_ENABLE);
 810         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 811 
 812         return 0;
 813 }
 814 
 815 void i915_disable_vblank(struct drm_device *dev, int pipe)
 816 {
 817         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 818 
 819         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 820         if (IS_IGDNG(dev))
 821                 igdng_disable_vblank(dev, pipe);
 822         else
 823         i915_disable_pipestat(dev_priv, pipe,
 824                               PIPE_VBLANK_INTERRUPT_ENABLE |
 825                               PIPE_START_VBLANK_INTERRUPT_ENABLE);
 826         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 827 }
 828 
 829 /* Set the vblank monitor pipe
 830  */
 831 /*ARGSUSED*/
 832 int i915_vblank_pipe_set(DRM_IOCTL_ARGS)
 833 {
 834         DRM_DEVICE;
 835         drm_i915_private_t *dev_priv = dev->dev_private;
 836 
 837         if (!dev_priv) {
 838                 DRM_ERROR("called with no initialization\n");
 839                 return (-EINVAL);
 840         }
 841 
 842         return (0);
 843 }
 844 
 845 /*ARGSUSED*/
 846 int i915_vblank_pipe_get(DRM_IOCTL_ARGS)
 847 {
 848         DRM_DEVICE;
 849         drm_i915_private_t *dev_priv = dev->dev_private;
 850         drm_i915_vblank_pipe_t pipe;
 851 
 852         if (!dev_priv) {
 853                 DRM_ERROR("called with no initialization\n");
 854                 return -EINVAL;
 855         }
 856 
 857         DRM_COPYFROM_WITH_RETURN(&pipe, (drm_i915_vblank_pipe_t __user *)data, sizeof (pipe));
 858 
 859         pipe.pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
 860 
 861         return 0;
 862 }
 863 
 864 /**
 865  * Schedule buffer swap at given vertical blank.
 866  */
 867 /*ARGSUSED*/
 868 int i915_vblank_swap(DRM_IOCTL_ARGS)
 869 {
 870         /* The delayed swap mechanism was fundamentally racy, and has been
 871         * removed.  The model was that the client requested a delayed flip/swap
 872         * from the kernel, then waited for vblank before continuing to perform
 873         * rendering.  The problem was that the kernel might wake the client
 874         * up before it dispatched the vblank swap (since the lock has to be
 875         * held while touching the ringbuffer), in which case the client would
 876         * clear and start the next frame before the swap occurred, and
 877         * flicker would occur in addition to likely missing the vblank.
 878         *
 879         * In the absence of this ioctl, userland falls back to a correct path
 880         * of waiting for a vblank, then dispatching the swap on its own.
 881         * Context switching to userland and back is plenty fast enough for
 882         * meeting the requirements of vblank swapping.
 883         */
 884         return -EINVAL;
 885 
 886 }
 887 
 888 /* drm_dma.h hooks
 889 */
 890 
 891 static void igdng_irq_preinstall(struct drm_device *dev)
 892 {
 893        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 894 
 895        I915_WRITE(HWSTAM, 0xeffe);
 896 
 897       /* XXX hotplug from PCH */
 898 
 899        I915_WRITE(DEIMR, 0xffffffff);
 900        I915_WRITE(DEIER, 0x0);
 901        (void) I915_READ(DEIER);
 902 
 903        /* and GT */
 904        I915_WRITE(GTIMR, 0xffffffff);
 905        I915_WRITE(GTIER, 0x0);
 906        (void) I915_READ(GTIER);
 907 }
 908 
 909 static int igdng_irq_postinstall(struct drm_device *dev)
 910 {
 911        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 912        /* enable kind of interrupts always enabled */
 913        u32 display_mask = DE_MASTER_IRQ_CONTROL /*| DE_PCH_EVENT */;
 914        u32 render_mask = GT_USER_INTERRUPT;
 915 
 916        dev_priv->irq_mask_reg = ~display_mask;
 917        dev_priv->de_irq_enable_reg = display_mask;
 918 
 919        /* should always can generate irq */
 920        I915_WRITE(DEIIR, I915_READ(DEIIR));
 921        (void) I915_READ(DEIIR);
 922        I915_WRITE(DEIMR, dev_priv->irq_mask_reg);
 923        I915_WRITE(DEIER, dev_priv->de_irq_enable_reg);
 924        (void) I915_READ(DEIER);
 925 
 926        /* user interrupt should be enabled, but masked initial */
 927        dev_priv->gt_irq_mask_reg = 0xffffffff;
 928        dev_priv->gt_irq_enable_reg = render_mask;
 929 
 930        I915_WRITE(GTIIR, I915_READ(GTIIR));
 931        (void) I915_READ(GTIIR);
 932        I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg);
 933        I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg);
 934        (void) I915_READ(GTIER);
 935 
 936        return 0;
 937 }
 938 
 939 static void igdng_irq_uninstall(struct drm_device *dev)
 940 {
 941        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 942        I915_WRITE(HWSTAM, 0xffffffff);
 943 
 944        I915_WRITE(DEIMR, 0xffffffff);
 945        I915_WRITE(DEIER, 0x0);
 946        I915_WRITE(DEIIR, I915_READ(DEIIR));
 947 
 948        I915_WRITE(GTIMR, 0xffffffff);
 949        I915_WRITE(GTIER, 0x0);
 950        I915_WRITE(GTIIR, I915_READ(GTIIR));
 951 }
 952 
 953 int i915_driver_irq_preinstall(drm_device_t * dev)
 954 {
 955         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 956 
 957         if (!dev_priv->mmio_map)
 958                 return -EINVAL;
 959 
 960         if (IS_IGDNG(dev)) {
 961                igdng_irq_preinstall(dev);
 962                return 0;
 963         }
 964 
 965         I915_WRITE16(HWSTAM, 0xeffe);
 966         I915_WRITE(PIPEASTAT, 0);
 967         I915_WRITE(PIPEBSTAT, 0);
 968         I915_WRITE(IMR, 0xffffffff);
 969         I915_WRITE16(IER, 0x0);
 970         (void) I915_READ(IER);
 971 
 972         return 0;
 973 }
 974 
 975 void i915_driver_irq_postinstall(drm_device_t * dev)
 976 {
 977         int error_mask;
 978         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 979 
 980         dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
 981 
 982         if (IS_IGDNG(dev)) {
 983                 (void) igdng_irq_postinstall(dev);
 984                 DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));
 985                 return;
 986         }
 987 
 988         /* Unmask the interrupts that we always want on. */
 989         dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX;
 990 
 991         dev_priv->pipestat[0] = 0;
 992         dev_priv->pipestat[1] = 0;
 993 
 994         /*
 995          * Enable some error detection, note the instruction error mask
 996          * bit is reserved, so we leave it masked.
 997          */
 998         if (IS_G4X(dev)) {
 999                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
1000                                GM45_ERROR_MEM_PRIV |
1001                                GM45_ERROR_CP_PRIV |
1002                                I915_ERROR_MEMORY_REFRESH);
1003         } else {
1004                 error_mask = ~(I915_ERROR_PAGE_TABLE |
1005                                I915_ERROR_MEMORY_REFRESH);
1006         }
1007         I915_WRITE(EMR, error_mask);
1008 
1009         /* Disable pipe interrupt enables, clear pending pipe status */
1010         I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
1011         I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
1012         (void) I915_READ(PIPEASTAT);
1013         (void) I915_READ(PIPEBSTAT);
1014         /* Clear pending interrupt status */
1015         I915_WRITE(IIR, I915_READ(IIR));
1016 
1017         (void) I915_READ(IIR);
1018         I915_WRITE(IMR, dev_priv->irq_mask_reg);
1019         I915_WRITE(IER, I915_INTERRUPT_ENABLE_MASK);
1020         (void) I915_READ(IER);
1021 
1022         DRM_INIT_WAITQUEUE(&dev_priv->irq_queue, DRM_INTR_PRI(dev));
1023 
1024         return;
1025 }
1026 
1027 void i915_driver_irq_uninstall(drm_device_t * dev)
1028 {
1029         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
1030         if ((!dev_priv) || (dev->irq_enabled == 0))
1031                 return;
1032 
1033         dev_priv->vblank_pipe = 0;
1034 
1035         if (IS_IGDNG(dev)) {
1036                 igdng_irq_uninstall(dev);
1037                 DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
1038                 return;
1039         }
1040 
1041         I915_WRITE(HWSTAM, 0xffffffff);
1042         I915_WRITE(PIPEASTAT, 0);
1043         I915_WRITE(PIPEBSTAT, 0);
1044         I915_WRITE(IMR, 0xffffffff);
1045         I915_WRITE(IER, 0x0);
1046 
1047         I915_WRITE(PIPEASTAT, I915_READ(PIPEASTAT) & 0x8000ffff);
1048         I915_WRITE(PIPEBSTAT, I915_READ(PIPEBSTAT) & 0x8000ffff);
1049         I915_WRITE(IIR, I915_READ(IIR));
1050 
1051         DRM_FINI_WAITQUEUE(&dev_priv->irq_queue);
1052 }