drivers/gpu/drm/i915/i915_dma.c

   1 /* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
   2  */
   3 /*
   4  * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
   5  * All Rights Reserved.
   6  *
   7  * Permission is hereby granted, free of charge, to any person obtaining a
   8  * copy of this software and associated documentation files (the
   9  * "Software"), to deal in the Software without restriction, including
  10  * without limitation the rights to use, copy, modify, merge, publish,
  11  * distribute, sub license, and/or sell copies of the Software, and to
  12  * permit persons to whom the Software is furnished to do so, subject to
  13  * the following conditions:
  14  *
  15  * The above copyright notice and this permission notice (including the
  16  * next paragraph) shall be included in all copies or substantial portions
  17  * of the Software.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  20  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  22  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  23  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  24  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  25  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  26  *
  27  */
  28
  29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  30
  31 #include <drm/drmP.h>
  32 #include <drm/drm_crtc_helper.h>
  33 #include <drm/drm_fb_helper.h>
  34 #include "intel_drv.h"
  35 #include <drm/i915_drm.h>
  36 #include "i915_drv.h"
  37 #include "i915_trace.h"
  38 #include <linux/pci.h>
  39 #include <linux/console.h>
  40 #include <linux/vt.h>
  41 #include <linux/vgaarb.h>
  42 #include <linux/acpi.h>
  43 #include <linux/pnp.h>
  44 #include <linux/vga_switcheroo.h>
  45 #include <linux/slab.h>
  46 #include <acpi/video.h>
  47 #include <linux/pm.h>
  48 #include <linux/pm_runtime.h>
  49 #include <linux/oom.h>
  50
  51 #define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS])
  52
  53 #define BEGIN_LP_RING(n) \
  54         intel_ring_begin(LP_RING(dev_priv), (n))
  55
  56 #define OUT_RING(x) \
  57         intel_ring_emit(LP_RING(dev_priv), x)
  58
  59 #define ADVANCE_LP_RING() \
  60         __intel_ring_advance(LP_RING(dev_priv))
  61
  62 /**
  63  * Lock test for when it's just for synchronization of ring access.
  64  *
  65  * In that case, we don't need to do it when GEM is initialized as nobody else
  66  * has access to the ring.
  67  */
  68 #define RING_LOCK_TEST_WITH_RETURN(dev, file) do {                      \
  69         if (LP_RING(dev->dev_private)->buffer->obj == NULL)                     \
  70                 LOCK_TEST_WITH_RETURN(dev, file);                       \
  71 } while (0)
  72
  73 static inline u32
  74 intel_read_legacy_status_page(struct drm_i915_private *dev_priv, int reg)
  75 {
  76         if (I915_NEED_GFX_HWS(dev_priv->dev))
  77                 return ioread32(dev_priv->dri1.gfx_hws_cpu_addr + reg);
  78         else
  79                 return intel_read_status_page(LP_RING(dev_priv), reg);
  80 }
  81
  82 #define READ_HWSP(dev_priv, reg) intel_read_legacy_status_page(dev_priv, reg)
  83 #define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
  84 #define I915_BREADCRUMB_INDEX           0x21
  85
  86 void i915_update_dri1_breadcrumb(struct drm_device *dev)
  87 {
  88         struct drm_i915_private *dev_priv = dev->dev_private;
  89         struct drm_i915_master_private *master_priv;
  90
  91         /*
  92          * The dri breadcrumb update races against the drm master disappearing.
  93          * Instead of trying to fix this (this is by far not the only ums issue)
  94          * just don't do the update in kms mode.
  95          */
  96         if (drm_core_check_feature(dev, DRIVER_MODESET))
  97                 return;
  98
  99         if (dev->primary->master) {
 100                 master_priv = dev->primary->master->driver_priv;
 101                 if (master_priv->sarea_priv)
 102                         master_priv->sarea_priv->last_dispatch =
 103                                 READ_BREADCRUMB(dev_priv);
 104         }
 105 }
 106
 107 static void i915_write_hws_pga(struct drm_device *dev)
 108 {
 109         struct drm_i915_private *dev_priv = dev->dev_private;
 110         u32 addr;
 111
 112         addr = dev_priv->status_page_dmah->busaddr;
 113         if (INTEL_INFO(dev)->gen >= 4)
 114                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
 115         I915_WRITE(HWS_PGA, addr);
 116 }
 117
 118 /**
 119  * Frees the hardware status page, whether it's a physical address or a virtual
 120  * address set up by the X Server.
 121  */
 122 static void i915_free_hws(struct drm_device *dev)
 123 {
 124         struct drm_i915_private *dev_priv = dev->dev_private;
 125         struct intel_engine_cs *ring = LP_RING(dev_priv);
 126
 127         if (dev_priv->status_page_dmah) {
 128                 drm_pci_free(dev, dev_priv->status_page_dmah);
 129                 dev_priv->status_page_dmah = NULL;
 130         }
 131
 132         if (ring->status_page.gfx_addr) {
 133                 ring->status_page.gfx_addr = 0;
 134                 iounmap(dev_priv->dri1.gfx_hws_cpu_addr);
 135         }
 136
 137         /* Need to rewrite hardware status page */
 138         I915_WRITE(HWS_PGA, 0x1ffff000);
 139 }
 140
 141 void i915_kernel_lost_context(struct drm_device *dev)
 142 {
 143         struct drm_i915_private *dev_priv = dev->dev_private;
 144         struct drm_i915_master_private *master_priv;
 145         struct intel_engine_cs *ring = LP_RING(dev_priv);
 146         struct intel_ringbuffer *ringbuf = ring->buffer;
 147
 148         /*
 149          * We should never lose context on the ring with modesetting
 150          * as we don't expose it to userspace
 151          */
 152         if (drm_core_check_feature(dev, DRIVER_MODESET))
 153                 return;
 154
 155         ringbuf->head = I915_READ_HEAD(ring) & HEAD_ADDR;
 156         ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
 157         ringbuf->space = ringbuf->head - (ringbuf->tail + I915_RING_FREE_SPACE);
 158         if (ringbuf->space < 0)
 159                 ringbuf->space += ringbuf->size;
 160
 161         if (!dev->primary->master)
 162                 return;
 163
 164         master_priv = dev->primary->master->driver_priv;
 165         if (ringbuf->head == ringbuf->tail && master_priv->sarea_priv)
 166                 master_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY;
 167 }
 168
 169 static int i915_dma_cleanup(struct drm_device *dev)
 170 {
 171         struct drm_i915_private *dev_priv = dev->dev_private;
 172         int i;
 173
 174         /* Make sure interrupts are disabled here because the uninstall ioctl
 175          * may not have been called from userspace and after dev_private
 176          * is freed, it's too late.
 177          */
 178         if (dev->irq_enabled)
 179                 drm_irq_uninstall(dev);
 180
 181         mutex_lock(&dev->struct_mutex);
 182         for (i = 0; i < I915_NUM_RINGS; i++)
 183                 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
 184         mutex_unlock(&dev->struct_mutex);
 185
 186         /* Clear the HWS virtual address at teardown */
 187         if (I915_NEED_GFX_HWS(dev))
 188                 i915_free_hws(dev);
 189
 190         return 0;
 191 }
 192
 193 static int i915_initialize(struct drm_device *dev, drm_i915_init_t *init)
 194 {
 195         struct drm_i915_private *dev_priv = dev->dev_private;
 196         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 197         int ret;
 198
 199         master_priv->sarea = drm_getsarea(dev);
 200         if (master_priv->sarea) {
 201                 master_priv->sarea_priv = (drm_i915_sarea_t *)
 202                         ((u8 *)master_priv->sarea->handle + init->sarea_priv_offset);
 203         } else {
 204                 DRM_DEBUG_DRIVER("sarea not found assuming DRI2 userspace\n");
 205         }
 206
 207         if (init->ring_size != 0) {
 208                 if (LP_RING(dev_priv)->buffer->obj != NULL) {
 209                         i915_dma_cleanup(dev);
 210                         DRM_ERROR("Client tried to initialize ringbuffer in "
 211                                   "GEM mode\n");
 212                         return -EINVAL;
 213                 }
 214
 215                 ret = intel_render_ring_init_dri(dev,
 216                                                  init->ring_start,
 217                                                  init->ring_size);
 218                 if (ret) {
 219                         i915_dma_cleanup(dev);
 220                         return ret;
 221                 }
 222         }
 223
 224         dev_priv->dri1.cpp = init->cpp;
 225         dev_priv->dri1.back_offset = init->back_offset;
 226         dev_priv->dri1.front_offset = init->front_offset;
 227         dev_priv->dri1.current_page = 0;
 228         if (master_priv->sarea_priv)
 229                 master_priv->sarea_priv->pf_current_page = 0;
 230
 231         /* Allow hardware batchbuffers unless told otherwise.
 232          */
 233         dev_priv->dri1.allow_batchbuffer = 1;
 234
 235         return 0;
 236 }
 237
 238 static int i915_dma_resume(struct drm_device *dev)
 239 {
 240         struct drm_i915_private *dev_priv = dev->dev_private;
 241         struct intel_engine_cs *ring = LP_RING(dev_priv);
 242
 243         DRM_DEBUG_DRIVER("%s\n", __func__);
 244
 245         if (ring->buffer->virtual_start == NULL) {
 246                 DRM_ERROR("can not ioremap virtual address for"
 247                           " ring buffer\n");
 248                 return -ENOMEM;
 249         }
 250
 251         /* Program Hardware Status Page */
 252         if (!ring->status_page.page_addr) {
 253                 DRM_ERROR("Can not find hardware status page\n");
 254                 return -EINVAL;
 255         }
 256         DRM_DEBUG_DRIVER("hw status page @ %p\n",
 257                                 ring->status_page.page_addr);
 258         if (ring->status_page.gfx_addr != 0)
 259                 intel_ring_setup_status_page(ring);
 260         else
 261                 i915_write_hws_pga(dev);
 262
 263         DRM_DEBUG_DRIVER("Enabled hardware status page\n");
 264
 265         return 0;
 266 }
 267
 268 static int i915_dma_init(struct drm_device *dev, void *data,
 269                          struct drm_file *file_priv)
 270 {
 271         drm_i915_init_t *init = data;
 272         int retcode = 0;
 273
 274         if (drm_core_check_feature(dev, DRIVER_MODESET))
 275                 return -ENODEV;
 276
 277         switch (init->func) {
 278         case I915_INIT_DMA:
 279                 retcode = i915_initialize(dev, init);
 280                 break;
 281         case I915_CLEANUP_DMA:
 282                 retcode = i915_dma_cleanup(dev);
 283                 break;
 284         case I915_RESUME_DMA:
 285                 retcode = i915_dma_resume(dev);
 286                 break;
 287         default:
 288                 retcode = -EINVAL;
 289                 break;
 290         }
 291
 292         return retcode;
 293 }
 294
 295 /* Implement basically the same security restrictions as hardware does
 296  * for MI_BATCH_NON_SECURE.  These can be made stricter at any time.
 297  *
 298  * Most of the calculations below involve calculating the size of a
 299  * particular instruction.  It's important to get the size right as
 300  * that tells us where the next instruction to check is.  Any illegal
 301  * instruction detected will be given a size of zero, which is a
 302  * signal to abort the rest of the buffer.
 303  */
 304 static int validate_cmd(int cmd)
 305 {
 306         switch (((cmd >> 29) & 0x7)) {
 307         case 0x0:
 308                 switch ((cmd >> 23) & 0x3f) {
 309                 case 0x0:
 310                         return 1;       /* MI_NOOP */
 311                 case 0x4:
 312                         return 1;       /* MI_FLUSH */
 313                 default:
 314                         return 0;       /* disallow everything else */
 315                 }
 316                 break;
 317         case 0x1:
 318                 return 0;       /* reserved */
 319         case 0x2:
 320                 return (cmd & 0xff) + 2;        /* 2d commands */
 321         case 0x3:
 322                 if (((cmd >> 24) & 0x1f) <= 0x18)
 323                         return 1;
 324
 325                 switch ((cmd >> 24) & 0x1f) {
 326                 case 0x1c:
 327                         return 1;
 328                 case 0x1d:
 329                         switch ((cmd >> 16) & 0xff) {
 330                         case 0x3:
 331                                 return (cmd & 0x1f) + 2;
 332                         case 0x4:
 333                                 return (cmd & 0xf) + 2;
 334                         default:
 335                                 return (cmd & 0xffff) + 2;
 336                         }
 337                 case 0x1e:
 338                         if (cmd & (1 << 23))
 339                                 return (cmd & 0xffff) + 1;
 340                         else
 341                                 return 1;
 342                 case 0x1f:
 343                         if ((cmd & (1 << 23)) == 0)     /* inline vertices */
 344                                 return (cmd & 0x1ffff) + 2;
 345                         else if (cmd & (1 << 17))       /* indirect random */
 346                                 if ((cmd & 0xffff) == 0)
 347                                         return 0;       /* unknown length, too hard */
 348                                 else
 349                                         return (((cmd & 0xffff) + 1) / 2) + 1;
 350                         else
 351                                 return 2;       /* indirect sequential */
 352                 default:
 353                         return 0;
 354                 }
 355         default:
 356                 return 0;
 357         }
 358
 359         return 0;
 360 }
 361
 362 static int i915_emit_cmds(struct drm_device *dev, int *buffer, int dwords)
 363 {
 364         struct drm_i915_private *dev_priv = dev->dev_private;
 365         int i, ret;
 366
 367         if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->buffer->size - 8)
 368                 return -EINVAL;
 369
 370         for (i = 0; i < dwords;) {
 371                 int sz = validate_cmd(buffer[i]);
 372
 373                 if (sz == 0 || i + sz > dwords)
 374                         return -EINVAL;
 375                 i += sz;
 376         }
 377
 378         ret = BEGIN_LP_RING((dwords+1)&~1);
 379         if (ret)
 380                 return ret;
 381
 382         for (i = 0; i < dwords; i++)
 383                 OUT_RING(buffer[i]);
 384         if (dwords & 1)
 385                 OUT_RING(0);
 386
 387         ADVANCE_LP_RING();
 388
 389         return 0;
 390 }
 391
 392 int
 393 i915_emit_box(struct drm_device *dev,
 394               struct drm_clip_rect *box,
 395               int DR1, int DR4)
 396 {
 397         struct drm_i915_private *dev_priv = dev->dev_private;
 398         int ret;
 399
 400         if (box->y2 <= box->y1 || box->x2 <= box->x1 ||
 401             box->y2 <= 0 || box->x2 <= 0) {
 402                 DRM_ERROR("Bad box %d,%d..%d,%d\n",
 403                           box->x1, box->y1, box->x2, box->y2);
 404                 return -EINVAL;
 405         }
 406
 407         if (INTEL_INFO(dev)->gen >= 4) {
 408                 ret = BEGIN_LP_RING(4);
 409                 if (ret)
 410                         return ret;
 411
 412                 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
 413                 OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
 414                 OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
 415                 OUT_RING(DR4);
 416         } else {
 417                 ret = BEGIN_LP_RING(6);
 418                 if (ret)
 419                         return ret;
 420
 421                 OUT_RING(GFX_OP_DRAWRECT_INFO);
 422                 OUT_RING(DR1);
 423                 OUT_RING((box->x1 & 0xffff) | (box->y1 << 16));
 424                 OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16));
 425                 OUT_RING(DR4);
 426                 OUT_RING(0);
 427         }
 428         ADVANCE_LP_RING();
 429
 430         return 0;
 431 }
 432
 433 /* XXX: Emitting the counter should really be moved to part of the IRQ
 434  * emit. For now, do it in both places:
 435  */
 436
 437 static void i915_emit_breadcrumb(struct drm_device *dev)
 438 {
 439         struct drm_i915_private *dev_priv = dev->dev_private;
 440         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 441
 442         dev_priv->dri1.counter++;
 443         if (dev_priv->dri1.counter > 0x7FFFFFFFUL)
 444                 dev_priv->dri1.counter = 0;
 445         if (master_priv->sarea_priv)
 446                 master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter;
 447
 448         if (BEGIN_LP_RING(4) == 0) {
 449                 OUT_RING(MI_STORE_DWORD_INDEX);
 450                 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 451                 OUT_RING(dev_priv->dri1.counter);
 452                 OUT_RING(0);
 453                 ADVANCE_LP_RING();
 454         }
 455 }
 456
 457 static int i915_dispatch_cmdbuffer(struct drm_device *dev,
 458                                    drm_i915_cmdbuffer_t *cmd,
 459                                    struct drm_clip_rect *cliprects,
 460                                    void *cmdbuf)
 461 {
 462         int nbox = cmd->num_cliprects;
 463         int i = 0, count, ret;
 464
 465         if (cmd->sz & 0x3) {
 466                 DRM_ERROR("alignment");
 467                 return -EINVAL;
 468         }
 469
 470         i915_kernel_lost_context(dev);
 471
 472         count = nbox ? nbox : 1;
 473
 474         for (i = 0; i < count; i++) {
 475                 if (i < nbox) {
 476                         ret = i915_emit_box(dev, &cliprects[i],
 477                                             cmd->DR1, cmd->DR4);
 478                         if (ret)
 479                                 return ret;
 480                 }
 481
 482                 ret = i915_emit_cmds(dev, cmdbuf, cmd->sz / 4);
 483                 if (ret)
 484                         return ret;
 485         }
 486
 487         i915_emit_breadcrumb(dev);
 488         return 0;
 489 }
 490
 491 static int i915_dispatch_batchbuffer(struct drm_device *dev,
 492                                      drm_i915_batchbuffer_t *batch,
 493                                      struct drm_clip_rect *cliprects)
 494 {
 495         struct drm_i915_private *dev_priv = dev->dev_private;
 496         int nbox = batch->num_cliprects;
 497         int i, count, ret;
 498
 499         if ((batch->start | batch->used) & 0x7) {
 500                 DRM_ERROR("alignment");
 501                 return -EINVAL;
 502         }
 503
 504         i915_kernel_lost_context(dev);
 505
 506         count = nbox ? nbox : 1;
 507         for (i = 0; i < count; i++) {
 508                 if (i < nbox) {
 509                         ret = i915_emit_box(dev, &cliprects[i],
 510                                             batch->DR1, batch->DR4);
 511                         if (ret)
 512                                 return ret;
 513                 }
 514
 515                 if (!IS_I830(dev) && !IS_845G(dev)) {
 516                         ret = BEGIN_LP_RING(2);
 517                         if (ret)
 518                                 return ret;
 519
 520                         if (INTEL_INFO(dev)->gen >= 4) {
 521                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
 522                                 OUT_RING(batch->start);
 523                         } else {
 524                                 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
 525                                 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
 526                         }
 527                 } else {
 528                         ret = BEGIN_LP_RING(4);
 529                         if (ret)
 530                                 return ret;
 531
 532                         OUT_RING(MI_BATCH_BUFFER);
 533                         OUT_RING(batch->start | MI_BATCH_NON_SECURE);
 534                         OUT_RING(batch->start + batch->used - 4);
 535                         OUT_RING(0);
 536                 }
 537                 ADVANCE_LP_RING();
 538         }
 539
 540
 541         if (IS_G4X(dev) || IS_GEN5(dev)) {
 542                 if (BEGIN_LP_RING(2) == 0) {
 543                         OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP);
 544                         OUT_RING(MI_NOOP);
 545                         ADVANCE_LP_RING();
 546                 }
 547         }
 548
 549         i915_emit_breadcrumb(dev);
 550         return 0;
 551 }
 552
 553 static int i915_dispatch_flip(struct drm_device *dev)
 554 {
 555         struct drm_i915_private *dev_priv = dev->dev_private;
 556         struct drm_i915_master_private *master_priv =
 557                 dev->primary->master->driver_priv;
 558         int ret;
 559
 560         if (!master_priv->sarea_priv)
 561                 return -EINVAL;
 562
 563         DRM_DEBUG_DRIVER("%s: page=%d pfCurrentPage=%d\n",
 564                           __func__,
 565                          dev_priv->dri1.current_page,
 566                          master_priv->sarea_priv->pf_current_page);
 567
 568         i915_kernel_lost_context(dev);
 569
 570         ret = BEGIN_LP_RING(10);
 571         if (ret)
 572                 return ret;
 573
 574         OUT_RING(MI_FLUSH | MI_READ_FLUSH);
 575         OUT_RING(0);
 576
 577         OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
 578         OUT_RING(0);
 579         if (dev_priv->dri1.current_page == 0) {
 580                 OUT_RING(dev_priv->dri1.back_offset);
 581                 dev_priv->dri1.current_page = 1;
 582         } else {
 583                 OUT_RING(dev_priv->dri1.front_offset);
 584                 dev_priv->dri1.current_page = 0;
 585         }
 586         OUT_RING(0);
 587
 588         OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
 589         OUT_RING(0);
 590
 591         ADVANCE_LP_RING();
 592
 593         master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter++;
 594
 595         if (BEGIN_LP_RING(4) == 0) {
 596                 OUT_RING(MI_STORE_DWORD_INDEX);
 597                 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 598                 OUT_RING(dev_priv->dri1.counter);
 599                 OUT_RING(0);
 600                 ADVANCE_LP_RING();
 601         }
 602
 603         master_priv->sarea_priv->pf_current_page = dev_priv->dri1.current_page;
 604         return 0;
 605 }
 606
 607 static int i915_quiescent(struct drm_device *dev)
 608 {
 609         i915_kernel_lost_context(dev);
 610         return intel_ring_idle(LP_RING(dev->dev_private));
 611 }
 612
 613 static int i915_flush_ioctl(struct drm_device *dev, void *data,
 614                             struct drm_file *file_priv)
 615 {
 616         int ret;
 617
 618         if (drm_core_check_feature(dev, DRIVER_MODESET))
 619                 return -ENODEV;
 620
 621         RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 622
 623         mutex_lock(&dev->struct_mutex);
 624         ret = i915_quiescent(dev);
 625         mutex_unlock(&dev->struct_mutex);
 626
 627         return ret;
 628 }
 629
 630 static int i915_batchbuffer(struct drm_device *dev, void *data,
 631                             struct drm_file *file_priv)
 632 {
 633         struct drm_i915_private *dev_priv = dev->dev_private;
 634         struct drm_i915_master_private *master_priv;
 635         drm_i915_sarea_t *sarea_priv;
 636         drm_i915_batchbuffer_t *batch = data;
 637         int ret;
 638         struct drm_clip_rect *cliprects = NULL;
 639
 640         if (drm_core_check_feature(dev, DRIVER_MODESET))
 641                 return -ENODEV;
 642
 643         master_priv = dev->primary->master->driver_priv;
 644         sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv;
 645
 646         if (!dev_priv->dri1.allow_batchbuffer) {
 647                 DRM_ERROR("Batchbuffer ioctl disabled\n");
 648                 return -EINVAL;
 649         }
 650
 651         DRM_DEBUG_DRIVER("i915 batchbuffer, start %x used %d cliprects %d\n",
 652                         batch->start, batch->used, batch->num_cliprects);
 653
 654         RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 655
 656         if (batch->num_cliprects < 0)
 657                 return -EINVAL;
 658
 659         if (batch->num_cliprects) {
 660                 cliprects = kcalloc(batch->num_cliprects,
 661                                     sizeof(*cliprects),
 662                                     GFP_KERNEL);
 663                 if (cliprects == NULL)
 664                         return -ENOMEM;
 665
 666                 ret = copy_from_user(cliprects, batch->cliprects,
 667                                      batch->num_cliprects *
 668                                      sizeof(struct drm_clip_rect));
 669                 if (ret != 0) {
 670                         ret = -EFAULT;
 671                         goto fail_free;
 672                 }
 673         }
 674
 675         mutex_lock(&dev->struct_mutex);
 676         ret = i915_dispatch_batchbuffer(dev, batch, cliprects);
 677         mutex_unlock(&dev->struct_mutex);
 678
 679         if (sarea_priv)
 680                 sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 681
 682 fail_free:
 683         kfree(cliprects);
 684
 685         return ret;
 686 }
 687
 688 static int i915_cmdbuffer(struct drm_device *dev, void *data,
 689                           struct drm_file *file_priv)
 690 {
 691         struct drm_i915_private *dev_priv = dev->dev_private;
 692         struct drm_i915_master_private *master_priv;
 693         drm_i915_sarea_t *sarea_priv;
 694         drm_i915_cmdbuffer_t *cmdbuf = data;
 695         struct drm_clip_rect *cliprects = NULL;
 696         void *batch_data;
 697         int ret;
 698
 699         DRM_DEBUG_DRIVER("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
 700                         cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects);
 701
 702         if (drm_core_check_feature(dev, DRIVER_MODESET))
 703                 return -ENODEV;
 704
 705         master_priv = dev->primary->master->driver_priv;
 706         sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv;
 707
 708         RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 709
 710         if (cmdbuf->num_cliprects < 0)
 711                 return -EINVAL;
 712
 713         batch_data = kmalloc(cmdbuf->sz, GFP_KERNEL);
 714         if (batch_data == NULL)
 715                 return -ENOMEM;
 716
 717         ret = copy_from_user(batch_data, cmdbuf->buf, cmdbuf->sz);
 718         if (ret != 0) {
 719                 ret = -EFAULT;
 720                 goto fail_batch_free;
 721         }
 722
 723         if (cmdbuf->num_cliprects) {
 724                 cliprects = kcalloc(cmdbuf->num_cliprects,
 725                                     sizeof(*cliprects), GFP_KERNEL);
 726                 if (cliprects == NULL) {
 727                         ret = -ENOMEM;
 728                         goto fail_batch_free;
 729                 }
 730
 731                 ret = copy_from_user(cliprects, cmdbuf->cliprects,
 732                                      cmdbuf->num_cliprects *
 733                                      sizeof(struct drm_clip_rect));
 734                 if (ret != 0) {
 735                         ret = -EFAULT;
 736                         goto fail_clip_free;
 737                 }
 738         }
 739
 740         mutex_lock(&dev->struct_mutex);
 741         ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data);
 742         mutex_unlock(&dev->struct_mutex);
 743         if (ret) {
 744                 DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
 745                 goto fail_clip_free;
 746         }
 747
 748         if (sarea_priv)
 749                 sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 750
 751 fail_clip_free:
 752         kfree(cliprects);
 753 fail_batch_free:
 754         kfree(batch_data);
 755
 756         return ret;
 757 }
 758
 759 static int i915_emit_irq(struct drm_device *dev)
 760 {
 761         struct drm_i915_private *dev_priv = dev->dev_private;
 762         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 763
 764         i915_kernel_lost_context(dev);
 765
 766         DRM_DEBUG_DRIVER("\n");
 767
 768         dev_priv->dri1.counter++;
 769         if (dev_priv->dri1.counter > 0x7FFFFFFFUL)
 770                 dev_priv->dri1.counter = 1;
 771         if (master_priv->sarea_priv)
 772                 master_priv->sarea_priv->last_enqueue = dev_priv->dri1.counter;
 773
 774         if (BEGIN_LP_RING(4) == 0) {
 775                 OUT_RING(MI_STORE_DWORD_INDEX);
 776                 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 777                 OUT_RING(dev_priv->dri1.counter);
 778                 OUT_RING(MI_USER_INTERRUPT);
 779                 ADVANCE_LP_RING();
 780         }
 781
 782         return dev_priv->dri1.counter;
 783 }
 784
 785 static int i915_wait_irq(struct drm_device *dev, int irq_nr)
 786 {
 787         struct drm_i915_private *dev_priv = dev->dev_private;
 788         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 789         int ret = 0;
 790         struct intel_engine_cs *ring = LP_RING(dev_priv);
 791
 792         DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr,
 793                   READ_BREADCRUMB(dev_priv));
 794
 795         if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
 796                 if (master_priv->sarea_priv)
 797                         master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
 798                 return 0;
 799         }
 800
 801         if (master_priv->sarea_priv)
 802                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 803
 804         if (ring->irq_get(ring)) {
 805                 DRM_WAIT_ON(ret, ring->irq_queue, 3 * HZ,
 806                             READ_BREADCRUMB(dev_priv) >= irq_nr);
 807                 ring->irq_put(ring);
 808         } else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000))
 809                 ret = -EBUSY;
 810
 811         if (ret == -EBUSY) {
 812                 DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
 813                           READ_BREADCRUMB(dev_priv), (int)dev_priv->dri1.counter);
 814         }
 815
 816         return ret;
 817 }
 818
 819 /* Needs the lock as it touches the ring.
 820  */
 821 static int i915_irq_emit(struct drm_device *dev, void *data,
 822                          struct drm_file *file_priv)
 823 {
 824         struct drm_i915_private *dev_priv = dev->dev_private;
 825         drm_i915_irq_emit_t *emit = data;
 826         int result;
 827
 828         if (drm_core_check_feature(dev, DRIVER_MODESET))
 829                 return -ENODEV;
 830
 831         if (!dev_priv || !LP_RING(dev_priv)->buffer->virtual_start) {
 832                 DRM_ERROR("called with no initialization\n");
 833                 return -EINVAL;
 834         }
 835
 836         RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 837
 838         mutex_lock(&dev->struct_mutex);
 839         result = i915_emit_irq(dev);
 840         mutex_unlock(&dev->struct_mutex);
 841
 842         if (copy_to_user(emit->irq_seq, &result, sizeof(int))) {
 843                 DRM_ERROR("copy_to_user\n");
 844                 return -EFAULT;
 845         }
 846
 847         return 0;
 848 }
 849
 850 /* Doesn't need the hardware lock.
 851  */
 852 static int i915_irq_wait(struct drm_device *dev, void *data,
 853                          struct drm_file *file_priv)
 854 {
 855         struct drm_i915_private *dev_priv = dev->dev_private;
 856         drm_i915_irq_wait_t *irqwait = data;
 857
 858         if (drm_core_check_feature(dev, DRIVER_MODESET))
 859                 return -ENODEV;
 860
 861         if (!dev_priv) {
 862                 DRM_ERROR("called with no initialization\n");
 863                 return -EINVAL;
 864         }
 865
 866         return i915_wait_irq(dev, irqwait->irq_seq);
 867 }
 868
 869 static int i915_vblank_pipe_get(struct drm_device *dev, void *data,
 870                          struct drm_file *file_priv)
 871 {
 872         struct drm_i915_private *dev_priv = dev->dev_private;
 873         drm_i915_vblank_pipe_t *pipe = data;
 874
 875         if (drm_core_check_feature(dev, DRIVER_MODESET))
 876                 return -ENODEV;
 877
 878         if (!dev_priv) {
 879                 DRM_ERROR("called with no initialization\n");
 880                 return -EINVAL;
 881         }
 882
 883         pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
 884
 885         return 0;
 886 }
 887
 888 /**
 889  * Schedule buffer swap at given vertical blank.
 890  */
 891 static int i915_vblank_swap(struct drm_device *dev, void *data,
 892                      struct drm_file *file_priv)
 893 {
 894         /* The delayed swap mechanism was fundamentally racy, and has been
 895          * removed.  The model was that the client requested a delayed flip/swap
 896          * from the kernel, then waited for vblank before continuing to perform
 897          * rendering.  The problem was that the kernel might wake the client
 898          * up before it dispatched the vblank swap (since the lock has to be
 899          * held while touching the ringbuffer), in which case the client would
 900          * clear and start the next frame before the swap occurred, and
 901          * flicker would occur in addition to likely missing the vblank.
 902          *
 903          * In the absence of this ioctl, userland falls back to a correct path
 904          * of waiting for a vblank, then dispatching the swap on its own.
 905          * Context switching to userland and back is plenty fast enough for
 906          * meeting the requirements of vblank swapping.
 907          */
 908         return -EINVAL;
 909 }
 910
 911 static int i915_flip_bufs(struct drm_device *dev, void *data,
 912                           struct drm_file *file_priv)
 913 {
 914         int ret;
 915
 916         if (drm_core_check_feature(dev, DRIVER_MODESET))
 917                 return -ENODEV;
 918
 919         DRM_DEBUG_DRIVER("%s\n", __func__);
 920
 921         RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 922
 923         mutex_lock(&dev->struct_mutex);
 924         ret = i915_dispatch_flip(dev);
 925         mutex_unlock(&dev->struct_mutex);
 926
 927         return ret;
 928 }
 929
 930 static int i915_getparam(struct drm_device *dev, void *data,
 931                          struct drm_file *file_priv)
 932 {
 933         struct drm_i915_private *dev_priv = dev->dev_private;
 934         drm_i915_getparam_t *param = data;
 935         int value;
 936
 937         if (!dev_priv) {
 938                 DRM_ERROR("called with no initialization\n");
 939                 return -EINVAL;
 940         }
 941
 942         switch (param->param) {
 943         case I915_PARAM_IRQ_ACTIVE:
 944                 value = dev->pdev->irq ? 1 : 0;
 945                 break;
 946         case I915_PARAM_ALLOW_BATCHBUFFER:
 947                 value = dev_priv->dri1.allow_batchbuffer ? 1 : 0;
 948                 break;
 949         case I915_PARAM_LAST_DISPATCH:
 950                 value = READ_BREADCRUMB(dev_priv);
 951                 break;
 952         case I915_PARAM_CHIPSET_ID:
 953                 value = dev->pdev->device;
 954                 break;
 955         case I915_PARAM_HAS_GEM:
 956                 value = 1;
 957                 break;
 958         case I915_PARAM_NUM_FENCES_AVAIL:
 959                 value = dev_priv->num_fence_regs - dev_priv->fence_reg_start;
 960                 break;
 961         case I915_PARAM_HAS_OVERLAY:
 962                 value = dev_priv->overlay ? 1 : 0;
 963                 break;
 964         case I915_PARAM_HAS_PAGEFLIPPING:
 965                 value = 1;
 966                 break;
 967         case I915_PARAM_HAS_EXECBUF2:
 968                 /* depends on GEM */
 969                 value = 1;
 970                 break;
 971         case I915_PARAM_HAS_BSD:
 972                 value = intel_ring_initialized(&dev_priv->ring[VCS]);
 973                 break;
 974         case I915_PARAM_HAS_BLT:
 975                 value = intel_ring_initialized(&dev_priv->ring[BCS]);
 976                 break;
 977         case I915_PARAM_HAS_VEBOX:
 978                 value = intel_ring_initialized(&dev_priv->ring[VECS]);
 979                 break;
 980         case I915_PARAM_HAS_RELAXED_FENCING:
 981                 value = 1;
 982                 break;
 983         case I915_PARAM_HAS_COHERENT_RINGS:
 984                 value = 1;
 985                 break;
 986         case I915_PARAM_HAS_EXEC_CONSTANTS:
 987                 value = INTEL_INFO(dev)->gen >= 4;
 988                 break;
 989         case I915_PARAM_HAS_RELAXED_DELTA:
 990                 value = 1;
 991                 break;
 992         case I915_PARAM_HAS_GEN7_SOL_RESET:
 993                 value = 1;
 994                 break;
 995         case I915_PARAM_HAS_LLC:
 996                 value = HAS_LLC(dev);
 997                 break;
 998         case I915_PARAM_HAS_WT:
 999                 value = HAS_WT(dev);
1000                 break;
1001         case I915_PARAM_HAS_ALIASING_PPGTT:
1002                 value = dev_priv->mm.aliasing_ppgtt || USES_FULL_PPGTT(dev);
1003                 break;
1004         case I915_PARAM_HAS_WAIT_TIMEOUT:
1005                 value = 1;
1006                 break;
1007         case I915_PARAM_HAS_SEMAPHORES:
1008                 value = i915_semaphore_is_enabled(dev);
1009                 break;
1010         case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
1011                 value = 1;
1012                 break;
1013         case I915_PARAM_HAS_SECURE_BATCHES:
1014                 value = capable(CAP_SYS_ADMIN);
1015                 break;
1016         case I915_PARAM_HAS_PINNED_BATCHES:
1017                 value = 1;
1018                 break;
1019         case I915_PARAM_HAS_EXEC_NO_RELOC:
1020                 value = 1;
1021                 break;
1022         case I915_PARAM_HAS_EXEC_HANDLE_LUT:
1023                 value = 1;
1024                 break;
1025         case I915_PARAM_CMD_PARSER_VERSION:
1026                 value = i915_cmd_parser_get_version();
1027                 break;
1028         default:
1029                 DRM_DEBUG("Unknown parameter %d\n", param->param);
1030                 return -EINVAL;
1031         }
1032
1033         if (copy_to_user(param->value, &value, sizeof(int))) {
1034                 DRM_ERROR("copy_to_user failed\n");
1035                 return -EFAULT;
1036         }
1037
1038         return 0;
1039 }
1040
1041 static int i915_setparam(struct drm_device *dev, void *data,
1042                          struct drm_file *file_priv)
1043 {
1044         struct drm_i915_private *dev_priv = dev->dev_private;
1045         drm_i915_setparam_t *param = data;
1046
1047         if (!dev_priv) {
1048                 DRM_ERROR("called with no initialization\n");
1049                 return -EINVAL;
1050         }
1051
1052         switch (param->param) {
1053         case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
1054                 break;
1055         case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
1056                 break;
1057         case I915_SETPARAM_ALLOW_BATCHBUFFER:
1058                 dev_priv->dri1.allow_batchbuffer = param->value ? 1 : 0;
1059                 break;
1060         case I915_SETPARAM_NUM_USED_FENCES:
1061                 if (param->value > dev_priv->num_fence_regs ||
1062                     param->value < 0)
1063                         return -EINVAL;
1064                 /* Userspace can use first N regs */
1065                 dev_priv->fence_reg_start = param->value;
1066                 break;
1067         default:
1068                 DRM_DEBUG_DRIVER("unknown parameter %d\n",
1069                                         param->param);
1070                 return -EINVAL;
1071         }
1072
1073         return 0;
1074 }
1075
1076 static int i915_set_status_page(struct drm_device *dev, void *data,
1077                                 struct drm_file *file_priv)
1078 {
1079         struct drm_i915_private *dev_priv = dev->dev_private;
1080         drm_i915_hws_addr_t *hws = data;
1081         struct intel_engine_cs *ring;
1082
1083         if (drm_core_check_feature(dev, DRIVER_MODESET))
1084                 return -ENODEV;
1085
1086         if (!I915_NEED_GFX_HWS(dev))
1087                 return -EINVAL;
1088
1089         if (!dev_priv) {
1090                 DRM_ERROR("called with no initialization\n");
1091                 return -EINVAL;
1092         }
1093
1094         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1095                 WARN(1, "tried to set status page when mode setting active\n");
1096                 return 0;
1097         }
1098
1099         DRM_DEBUG_DRIVER("set status page addr 0x%08x\n", (u32)hws->addr);
1100
1101         ring = LP_RING(dev_priv);
1102         ring->status_page.gfx_addr = hws->addr & (0x1ffff<<12);
1103
1104         dev_priv->dri1.gfx_hws_cpu_addr =
1105                 ioremap_wc(dev_priv->gtt.mappable_base + hws->addr, 4096);
1106         if (dev_priv->dri1.gfx_hws_cpu_addr == NULL) {
1107                 i915_dma_cleanup(dev);
1108                 ring->status_page.gfx_addr = 0;
1109                 DRM_ERROR("can not ioremap virtual address for"
1110                                 " G33 hw status page\n");
1111                 return -ENOMEM;
1112         }
1113
1114         memset_io(dev_priv->dri1.gfx_hws_cpu_addr, 0, PAGE_SIZE);
1115         I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
1116
1117         DRM_DEBUG_DRIVER("load hws HWS_PGA with gfx mem 0x%x\n",
1118                          ring->status_page.gfx_addr);
1119         DRM_DEBUG_DRIVER("load hws at %p\n",
1120                          ring->status_page.page_addr);
1121         return 0;
1122 }
1123
1124 static int i915_get_bridge_dev(struct drm_device *dev)
1125 {
1126         struct drm_i915_private *dev_priv = dev->dev_private;
1127
1128         dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
1129         if (!dev_priv->bridge_dev) {
1130                 DRM_ERROR("bridge device not found\n");
1131                 return -1;
1132         }
1133         return 0;
1134 }
1135
1136 #define MCHBAR_I915 0x44
1137 #define MCHBAR_I965 0x48
1138 #define MCHBAR_SIZE (4*4096)
1139
1140 #define DEVEN_REG 0x54
1141 #define   DEVEN_MCHBAR_EN (1 << 28)
1142
1143 /* Allocate space for the MCH regs if needed, return nonzero on error */
1144 static int
1145 intel_alloc_mchbar_resource(struct drm_device *dev)
1146 {
1147         struct drm_i915_private *dev_priv = dev->dev_private;
1148         int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
1149         u32 temp_lo, temp_hi = 0;
1150         u64 mchbar_addr;
1151         int ret;
1152
1153         if (INTEL_INFO(dev)->gen >= 4)
1154                 pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
1155         pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
1156         mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
1157
1158         /* If ACPI doesn't have it, assume we need to allocate it ourselves */
1159 #ifdef CONFIG_PNP
1160         if (mchbar_addr &&
1161             pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
1162                 return 0;
1163 #endif
1164
1165         /* Get some space for it */
1166         dev_priv->mch_res.name = "i915 MCHBAR";
1167         dev_priv->mch_res.flags = IORESOURCE_MEM;
1168         ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
1169                                      &dev_priv->mch_res,
1170                                      MCHBAR_SIZE, MCHBAR_SIZE,
1171                                      PCIBIOS_MIN_MEM,
1172                                      0, pcibios_align_resource,
1173                                      dev_priv->bridge_dev);
1174         if (ret) {
1175                 DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
1176                 dev_priv->mch_res.start = 0;
1177                 return ret;
1178         }
1179
1180         if (INTEL_INFO(dev)->gen >= 4)
1181                 pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
1182                                        upper_32_bits(dev_priv->mch_res.start));
1183
1184         pci_write_config_dword(dev_priv->bridge_dev, reg,
1185                                lower_32_bits(dev_priv->mch_res.start));
1186         return 0;
1187 }
1188
1189 /* Setup MCHBAR if possible, return true if we should disable it again */
1190 static void
1191 intel_setup_mchbar(struct drm_device *dev)
1192 {
1193         struct drm_i915_private *dev_priv = dev->dev_private;
1194         int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
1195         u32 temp;
1196         bool enabled;
1197
1198         if (IS_VALLEYVIEW(dev))
1199                 return;
1200
1201         dev_priv->mchbar_need_disable = false;
1202
1203         if (IS_I915G(dev) || IS_I915GM(dev)) {
1204                 pci_read_config_dword(dev_priv->bridge_dev, DEVEN_REG, &temp);
1205                 enabled = !!(temp & DEVEN_MCHBAR_EN);
1206         } else {
1207                 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
1208                 enabled = temp & 1;
1209         }
1210
1211         /* If it's already enabled, don't have to do anything */
1212         if (enabled)
1213                 return;
1214
1215         if (intel_alloc_mchbar_resource(dev))
1216                 return;
1217
1218         dev_priv->mchbar_need_disable = true;
1219
1220         /* Space is allocated or reserved, so enable it. */
1221         if (IS_I915G(dev) || IS_I915GM(dev)) {
1222                 pci_write_config_dword(dev_priv->bridge_dev, DEVEN_REG,
1223                                        temp | DEVEN_MCHBAR_EN);
1224         } else {
1225                 pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
1226                 pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
1227         }
1228 }
1229
1230 static void
1231 intel_teardown_mchbar(struct drm_device *dev)
1232 {
1233         struct drm_i915_private *dev_priv = dev->dev_private;
1234         int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
1235         u32 temp;
1236
1237         if (dev_priv->mchbar_need_disable) {
1238                 if (IS_I915G(dev) || IS_I915GM(dev)) {
1239                         pci_read_config_dword(dev_priv->bridge_dev, DEVEN_REG, &temp);
1240                         temp &= ~DEVEN_MCHBAR_EN;
1241                         pci_write_config_dword(dev_priv->bridge_dev, DEVEN_REG, temp);
1242                 } else {
1243                         pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
1244                         temp &= ~1;
1245                         pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp);
1246                 }
1247         }
1248
1249         if (dev_priv->mch_res.start)
1250                 release_resource(&dev_priv->mch_res);
1251 }
1252
1253 /* true = enable decode, false = disable decoder */
1254 static unsigned int i915_vga_set_decode(void *cookie, bool state)
1255 {
1256         struct drm_device *dev = cookie;
1257
1258         intel_modeset_vga_set_state(dev, state);
1259         if (state)
1260                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1261                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1262         else
1263                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1264 }
1265
1266 static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1267 {
1268         struct drm_device *dev = pci_get_drvdata(pdev);
1269         pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
1270
1271         if (state == VGA_SWITCHEROO_ON) {
1272                 pr_info("switched on\n");
1273                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1274                 /* i915 resume handler doesn't set to D0 */
1275                 pci_set_power_state(dev->pdev, PCI_D0);
1276                 i915_resume(dev);
1277                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1278         } else {
1279                 pr_err("switched off\n");
1280                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1281                 i915_suspend(dev, pmm);
1282                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1283         }
1284 }
1285
1286 static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
1287 {
1288         struct drm_device *dev = pci_get_drvdata(pdev);
1289
1290         /*
1291          * FIXME: open_count is protected by drm_global_mutex but that would lead to
1292          * locking inversion with the driver load path. And the access here is
1293          * completely racy anyway. So don't bother with locking for now.
1294          */
1295         return dev->open_count == 0;
1296 }
1297
1298 static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
1299         .set_gpu_state = i915_switcheroo_set_state,
1300         .reprobe = NULL,
1301         .can_switch = i915_switcheroo_can_switch,
1302 };
1303
1304 static int i915_load_modeset_init(struct drm_device *dev)
1305 {
1306         struct drm_i915_private *dev_priv = dev->dev_private;
1307         int ret;
1308
1309         ret = intel_parse_bios(dev);
1310         if (ret)
1311                 DRM_INFO("failed to find VBIOS tables\n");
1312
1313         /* If we have > 1 VGA cards, then we need to arbitrate access
1314          * to the common VGA resources.
1315          *
1316          * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
1317          * then we do not take part in VGA arbitration and the
1318          * vga_client_register() fails with -ENODEV.
1319          */
1320         ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
1321         if (ret && ret != -ENODEV)
1322                 goto out;
1323
1324         intel_register_dsm_handler();
1325
1326         ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false);
1327         if (ret)
1328                 goto cleanup_vga_client;
1329
1330         /* Initialise stolen first so that we may reserve preallocated
1331          * objects for the BIOS to KMS transition.
1332          */
1333         ret = i915_gem_init_stolen(dev);
1334         if (ret)
1335                 goto cleanup_vga_switcheroo;
1336
1337         intel_power_domains_init_hw(dev_priv);
1338
1339         /*
1340          * We enable some interrupt sources in our postinstall hooks, so mark
1341          * interrupts as enabled _before_ actually enabling them to avoid
1342          * special cases in our ordering checks.
1343          */
1344         dev_priv->pm._irqs_disabled = false;
1345
1346         ret = drm_irq_install(dev, dev->pdev->irq);
1347         if (ret)
1348                 goto cleanup_gem_stolen;
1349
1350         /* Important: The output setup functions called by modeset_init need
1351          * working irqs for e.g. gmbus and dp aux transfers. */
1352         intel_modeset_init(dev);
1353
1354         ret = i915_gem_init(dev);
1355         if (ret)
1356                 goto cleanup_irq;
1357
1358         INIT_WORK(&dev_priv->console_resume_work, intel_console_resume);
1359
1360         intel_modeset_gem_init(dev);
1361
1362         /* Always safe in the mode setting case. */
1363         /* FIXME: do pre/post-mode set stuff in core KMS code */
1364         dev->vblank_disable_allowed = true;
1365         if (INTEL_INFO(dev)->num_pipes == 0)
1366                 return 0;
1367
1368         ret = intel_fbdev_init(dev);
1369         if (ret)
1370                 goto cleanup_gem;
1371
1372         /* Only enable hotplug handling once the fbdev is fully set up. */
1373         intel_hpd_init(dev);
1374
1375         /*
1376          * Some ports require correctly set-up hpd registers for detection to
1377          * work properly (leading to ghost connected connector status), e.g. VGA
1378          * on gm45.  Hence we can only set up the initial fbdev config after hpd
1379          * irqs are fully enabled. Now we should scan for the initial config
1380          * only once hotplug handling is enabled, but due to screwed-up locking
1381          * around kms/fbdev init we can't protect the fdbev initial config
1382          * scanning against hotplug events. Hence do this first and ignore the
1383          * tiny window where we will loose hotplug notifactions.
1384          */
1385         intel_fbdev_initial_config(dev);
1386
1387         drm_kms_helper_poll_init(dev);
1388
1389         return 0;
1390
1391 cleanup_gem:
1392         mutex_lock(&dev->struct_mutex);
1393         i915_gem_cleanup_ringbuffer(dev);
1394         i915_gem_context_fini(dev);
1395         mutex_unlock(&dev->struct_mutex);
1396         WARN_ON(dev_priv->mm.aliasing_ppgtt);
1397 cleanup_irq:
1398         drm_irq_uninstall(dev);
1399 cleanup_gem_stolen:
1400         i915_gem_cleanup_stolen(dev);
1401 cleanup_vga_switcheroo:
1402         vga_switcheroo_unregister_client(dev->pdev);
1403 cleanup_vga_client:
1404         vga_client_register(dev->pdev, NULL, NULL, NULL);
1405 out:
1406         return ret;
1407 }
1408
1409 int i915_master_create(struct drm_device *dev, struct drm_master *master)
1410 {
1411         struct drm_i915_master_private *master_priv;
1412
1413         master_priv = kzalloc(sizeof(*master_priv), GFP_KERNEL);
1414         if (!master_priv)
1415                 return -ENOMEM;
1416
1417         master->driver_priv = master_priv;
1418         return 0;
1419 }
1420
1421 void i915_master_destroy(struct drm_device *dev, struct drm_master *master)
1422 {
1423         struct drm_i915_master_private *master_priv = master->driver_priv;
1424
1425         if (!master_priv)
1426                 return;
1427
1428         kfree(master_priv);
1429
1430         master->driver_priv = NULL;
1431 }
1432
1433 #if IS_ENABLED(CONFIG_FB)
1434 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
1435 {
1436         struct apertures_struct *ap;
1437         struct pci_dev *pdev = dev_priv->dev->pdev;
1438         bool primary;
1439         int ret;
1440
1441         ap = alloc_apertures(1);
1442         if (!ap)
1443                 return -ENOMEM;
1444
1445         ap->ranges[0].base = dev_priv->gtt.mappable_base;
1446         ap->ranges[0].size = dev_priv->gtt.mappable_end;
1447
1448         primary =
1449                 pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
1450
1451         ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
1452
1453         kfree(ap);
1454
1455         return ret;
1456 }
1457 #else
1458 static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
1459 {
1460         return 0;
1461 }
1462 #endif
1463
1464 #if !defined(CONFIG_VGA_CONSOLE)
1465 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
1466 {
1467         return 0;
1468 }
1469 #elif !defined(CONFIG_DUMMY_CONSOLE)
1470 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
1471 {
1472         return -ENODEV;
1473 }
1474 #else
1475 static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
1476 {
1477         int ret = 0;
1478
1479         DRM_INFO("Replacing VGA console driver\n");
1480
1481         console_lock();
1482         if (con_is_bound(&vga_con))
1483                 ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
1484         if (ret == 0) {
1485                 ret = do_unregister_con_driver(&vga_con);
1486
1487                 /* Ignore "already unregistered". */
1488                 if (ret == -ENODEV)
1489                         ret = 0;
1490         }
1491         console_unlock();
1492
1493         return ret;
1494 }
1495 #endif
1496
1497 static void i915_dump_device_info(struct drm_i915_private *dev_priv)
1498 {
1499         const struct intel_device_info *info = &dev_priv->info;
1500
1501 #define PRINT_S(name) "%s"
1502 #define SEP_EMPTY
1503 #define PRINT_FLAG(name) info->name ? #name "," : ""
1504 #define SEP_COMMA ,
1505         DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags="
1506                          DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
1507                          info->gen,
1508                          dev_priv->dev->pdev->device,
1509                          dev_priv->dev->pdev->revision,
1510                          DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
1511 #undef PRINT_S
1512 #undef SEP_EMPTY
1513 #undef PRINT_FLAG
1514 #undef SEP_COMMA
1515 }
1516
1517 /*
1518  * Determine various intel_device_info fields at runtime.
1519  *
1520  * Use it when either:
1521  *   - it's judged too laborious to fill n static structures with the limit
1522  *     when a simple if statement does the job,
1523  *   - run-time checks (eg read fuse/strap registers) are needed.
1524  *
1525  * This function needs to be called:
1526  *   - after the MMIO has been setup as we are reading registers,
1527  *   - after the PCH has been detected,
1528  *   - before the first usage of the fields it can tweak.
1529  */
1530 static void intel_device_info_runtime_init(struct drm_device *dev)
1531 {
1532         struct drm_i915_private *dev_priv = dev->dev_private;
1533         struct intel_device_info *info;
1534         enum pipe pipe;
1535
1536         info = (struct intel_device_info *)&dev_priv->info;
1537
1538         if (IS_VALLEYVIEW(dev))
1539                 for_each_pipe(pipe)
1540                         info->num_sprites[pipe] = 2;
1541         else
1542                 for_each_pipe(pipe)
1543                         info->num_sprites[pipe] = 1;
1544
1545         if (i915.disable_display) {
1546                 DRM_INFO("Display disabled (module parameter)\n");
1547                 info->num_pipes = 0;
1548         } else if (info->num_pipes > 0 &&
1549                    (INTEL_INFO(dev)->gen == 7 || INTEL_INFO(dev)->gen == 8) &&
1550                    !IS_VALLEYVIEW(dev)) {
1551                 u32 fuse_strap = I915_READ(FUSE_STRAP);
1552                 u32 sfuse_strap = I915_READ(SFUSE_STRAP);
1553
1554                 /*
1555                  * SFUSE_STRAP is supposed to have a bit signalling the display
1556                  * is fused off. Unfortunately it seems that, at least in
1557                  * certain cases, fused off display means that PCH display
1558                  * reads don't land anywhere. In that case, we read 0s.
1559                  *
1560                  * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK
1561                  * should be set when taking over after the firmware.
1562                  */
1563                 if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE ||
1564                     sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED ||
1565                     (dev_priv->pch_type == PCH_CPT &&
1566                      !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
1567                         DRM_INFO("Display fused off, disabling\n");
1568                         info->num_pipes = 0;
1569                 }
1570         }
1571 }
1572
1573 /**
1574  * i915_driver_load - setup chip and create an initial config
1575  * @dev: DRM device
1576  * @flags: startup flags
1577  *
1578  * The driver load routine has to do several things:
1579  *   - drive output discovery via intel_modeset_init()
1580  *   - initialize the memory manager
1581  *   - allocate initial config memory
1582  *   - setup the DRM framebuffer with the allocated memory
1583  */
1584 int i915_driver_load(struct drm_device *dev, unsigned long flags)
1585 {
1586         struct drm_i915_private *dev_priv;
1587         struct intel_device_info *info, *device_info;
1588         int ret = 0, mmio_bar, mmio_size;
1589         uint32_t aperture_size;
1590
1591         info = (struct intel_device_info *) flags;
1592
1593         /* Refuse to load on gen6+ without kms enabled. */
1594         if (info->gen >= 6 && !drm_core_check_feature(dev, DRIVER_MODESET)) {
1595                 DRM_INFO("Your hardware requires kernel modesetting (KMS)\n");
1596                 DRM_INFO("See CONFIG_DRM_I915_KMS, nomodeset, and i915.modeset parameters\n");
1597                 return -ENODEV;
1598         }
1599
1600         /* UMS needs agp support. */
1601         if (!drm_core_check_feature(dev, DRIVER_MODESET) && !dev->agp)
1602                 return -EINVAL;
1603
1604         dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
1605         if (dev_priv == NULL)
1606                 return -ENOMEM;
1607
1608         dev->dev_private = dev_priv;
1609         dev_priv->dev = dev;
1610
1611         /* copy initial configuration to dev_priv->info */
1612         device_info = (struct intel_device_info *)&dev_priv->info;
1613         *device_info = *info;
1614
1615         spin_lock_init(&dev_priv->irq_lock);
1616         spin_lock_init(&dev_priv->gpu_error.lock);
1617         spin_lock_init(&dev_priv->backlight_lock);
1618         spin_lock_init(&dev_priv->uncore.lock);
1619         spin_lock_init(&dev_priv->mm.object_stat_lock);
1620         spin_lock_init(&dev_priv->mmio_flip_lock);
1621         mutex_init(&dev_priv->dpio_lock);
1622         mutex_init(&dev_priv->modeset_restore_lock);
1623
1624         intel_pm_setup(dev);
1625
1626         intel_display_crc_init(dev);
1627
1628         i915_dump_device_info(dev_priv);
1629
1630         /* Not all pre-production machines fall into this category, only the
1631          * very first ones. Almost everything should work, except for maybe
1632          * suspend/resume. And we don't implement workarounds that affect only
1633          * pre-production machines. */
1634         if (IS_HSW_EARLY_SDV(dev))
1635                 DRM_INFO("This is an early pre-production Haswell machine. "
1636                          "It may not be fully functional.\n");
1637
1638         if (i915_get_bridge_dev(dev)) {
1639                 ret = -EIO;
1640                 goto free_priv;
1641         }
1642
1643         mmio_bar = IS_GEN2(dev) ? 1 : 0;
1644         /* Before gen4, the registers and the GTT are behind different BARs.
1645          * However, from gen4 onwards, the registers and the GTT are shared
1646          * in the same BAR, so we want to restrict this ioremap from
1647          * clobbering the GTT which we want ioremap_wc instead. Fortunately,
1648          * the register BAR remains the same size for all the earlier
1649          * generations up to Ironlake.
1650          */
1651         if (info->gen < 5)
1652                 mmio_size = 512*1024;
1653         else
1654                 mmio_size = 2*1024*1024;
1655
1656         dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
1657         if (!dev_priv->regs) {
1658                 DRM_ERROR("failed to map registers\n");
1659                 ret = -EIO;
1660                 goto put_bridge;
1661         }
1662
1663         /* This must be called before any calls to HAS_PCH_* */
1664         intel_detect_pch(dev);
1665
1666         intel_uncore_init(dev);
1667
1668         ret = i915_gem_gtt_init(dev);
1669         if (ret)
1670                 goto out_regs;
1671
1672         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1673                 ret = i915_kick_out_vgacon(dev_priv);
1674                 if (ret) {
1675                         DRM_ERROR("failed to remove conflicting VGA console\n");
1676                         goto out_gtt;
1677                 }
1678
1679                 ret = i915_kick_out_firmware_fb(dev_priv);
1680                 if (ret) {
1681                         DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
1682                         goto out_gtt;
1683                 }
1684         }
1685
1686         pci_set_master(dev->pdev);
1687
1688         /* overlay on gen2 is broken and can't address above 1G */
1689         if (IS_GEN2(dev))
1690                 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
1691
1692         /* 965GM sometimes incorrectly writes to hardware status page (HWS)
1693          * using 32bit addressing, overwriting memory if HWS is located
1694          * above 4GB.
1695          *
1696          * The documentation also mentions an issue with undefined
1697          * behaviour if any general state is accessed within a page above 4GB,
1698          * which also needs to be handled carefully.
1699          */
1700         if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
1701                 dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
1702
1703         aperture_size = dev_priv->gtt.mappable_end;
1704
1705         dev_priv->gtt.mappable =
1706                 io_mapping_create_wc(dev_priv->gtt.mappable_base,
1707                                      aperture_size);
1708         if (dev_priv->gtt.mappable == NULL) {
1709                 ret = -EIO;
1710                 goto out_gtt;
1711         }
1712
1713         dev_priv->gtt.mtrr = arch_phys_wc_add(dev_priv->gtt.mappable_base,
1714                                               aperture_size);
1715
1716         /* The i915 workqueue is primarily used for batched retirement of
1717          * requests (and thus managing bo) once the task has been completed
1718          * by the GPU. i915_gem_retire_requests() is called directly when we
1719          * need high-priority retirement, such as waiting for an explicit
1720          * bo.
1721          *
1722          * It is also used for periodic low-priority events, such as
1723          * idle-timers and recording error state.
1724          *
1725          * All tasks on the workqueue are expected to acquire the dev mutex
1726          * so there is no point in running more than one instance of the
1727          * workqueue at any time.  Use an ordered one.
1728          */
1729         dev_priv->wq = alloc_ordered_workqueue("i915", 0);
1730         if (dev_priv->wq == NULL) {
1731                 DRM_ERROR("Failed to create our workqueue.\n");
1732                 ret = -ENOMEM;
1733                 goto out_mtrrfree;
1734         }
1735
1736         dev_priv->dp_wq = alloc_ordered_workqueue("i915-dp", 0);
1737         if (dev_priv->dp_wq == NULL) {
1738                 DRM_ERROR("Failed to create our dp workqueue.\n");
1739                 ret = -ENOMEM;
1740                 goto out_freewq;
1741         }
1742
1743         intel_irq_init(dev);
1744         intel_uncore_sanitize(dev);
1745
1746         /* Try to make sure MCHBAR is enabled before poking at it */
1747         intel_setup_mchbar(dev);
1748         intel_setup_gmbus(dev);
1749         intel_opregion_setup(dev);
1750
1751         intel_setup_bios(dev);
1752
1753         i915_gem_load(dev);
1754
1755         /* On the 945G/GM, the chipset reports the MSI capability on the
1756          * integrated graphics even though the support isn't actually there
1757          * according to the published specs.  It doesn't appear to function
1758          * correctly in testing on 945G.
1759          * This may be a side effect of MSI having been made available for PEG
1760          * and the registers being closely associated.
1761          *
1762          * According to chipset errata, on the 965GM, MSI interrupts may
1763          * be lost or delayed, but we use them anyways to avoid
1764          * stuck interrupts on some machines.
1765          */
1766         if (!IS_I945G(dev) && !IS_I945GM(dev))
1767                 pci_enable_msi(dev->pdev);
1768
1769         intel_device_info_runtime_init(dev);
1770
1771         if (INTEL_INFO(dev)->num_pipes) {
1772                 ret = drm_vblank_init(dev, INTEL_INFO(dev)->num_pipes);
1773                 if (ret)
1774                         goto out_gem_unload;
1775         }
1776
1777         intel_power_domains_init(dev_priv);
1778
1779         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1780                 ret = i915_load_modeset_init(dev);
1781                 if (ret < 0) {
1782                         DRM_ERROR("failed to init modeset\n");
1783                         goto out_power_well;
1784                 }
1785         } else {
1786                 /* Start out suspended in ums mode. */
1787                 dev_priv->ums.mm_suspended = 1;
1788         }
1789
1790         i915_setup_sysfs(dev);
1791
1792         if (INTEL_INFO(dev)->num_pipes) {
1793                 /* Must be done after probing outputs */
1794                 intel_opregion_init(dev);
1795                 acpi_video_register();
1796         }
1797
1798         if (IS_GEN5(dev))
1799                 intel_gpu_ips_init(dev_priv);
1800
1801         intel_init_runtime_pm(dev_priv);
1802
1803         return 0;
1804
1805 out_power_well:
1806         intel_power_domains_remove(dev_priv);
1807         drm_vblank_cleanup(dev);
1808 out_gem_unload:
1809         WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
1810         unregister_shrinker(&dev_priv->mm.shrinker);
1811
1812         if (dev->pdev->msi_enabled)
1813                 pci_disable_msi(dev->pdev);
1814
1815         intel_teardown_gmbus(dev);
1816         intel_teardown_mchbar(dev);
1817         pm_qos_remove_request(&dev_priv->pm_qos);
1818         destroy_workqueue(dev_priv->dp_wq);
1819 out_freewq:
1820         destroy_workqueue(dev_priv->wq);
1821 out_mtrrfree:
1822         arch_phys_wc_del(dev_priv->gtt.mtrr);
1823         io_mapping_free(dev_priv->gtt.mappable);
1824 out_gtt:
1825         dev_priv->gtt.base.cleanup(&dev_priv->gtt.base);
1826 out_regs:
1827         intel_uncore_fini(dev);
1828         pci_iounmap(dev->pdev, dev_priv->regs);
1829 put_bridge:
1830         pci_dev_put(dev_priv->bridge_dev);
1831 free_priv:
1832         if (dev_priv->slab)
1833                 kmem_cache_destroy(dev_priv->slab);
1834         kfree(dev_priv);
1835         return ret;
1836 }
1837
1838 int i915_driver_unload(struct drm_device *dev)
1839 {
1840         struct drm_i915_private *dev_priv = dev->dev_private;
1841         int ret;
1842
1843         ret = i915_gem_suspend(dev);
1844         if (ret) {
1845                 DRM_ERROR("failed to idle hardware: %d\n", ret);
1846                 return ret;
1847         }
1848
1849         intel_fini_runtime_pm(dev_priv);
1850
1851         intel_gpu_ips_teardown();
1852
1853         /* The i915.ko module is still not prepared to be loaded when
1854          * the power well is not enabled, so just enable it in case
1855          * we're going to unload/reload. */
1856         intel_display_set_init_power(dev_priv, true);
1857         intel_power_domains_remove(dev_priv);
1858
1859         i915_teardown_sysfs(dev);
1860
1861         WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
1862         unregister_shrinker(&dev_priv->mm.shrinker);
1863
1864         io_mapping_free(dev_priv->gtt.mappable);
1865         arch_phys_wc_del(dev_priv->gtt.mtrr);
1866
1867         acpi_video_unregister();
1868
1869         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1870                 intel_fbdev_fini(dev);
1871                 intel_modeset_cleanup(dev);
1872                 cancel_work_sync(&dev_priv->console_resume_work);
1873
1874                 /*
1875                  * free the memory space allocated for the child device
1876                  * config parsed from VBT
1877                  */
1878                 if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
1879                         kfree(dev_priv->vbt.child_dev);
1880                         dev_priv->vbt.child_dev = NULL;
1881                         dev_priv->vbt.child_dev_num = 0;
1882                 }
1883
1884                 vga_switcheroo_unregister_client(dev->pdev);
1885                 vga_client_register(dev->pdev, NULL, NULL, NULL);
1886         }
1887
1888         /* Free error state after interrupts are fully disabled. */
1889         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
1890         cancel_work_sync(&dev_priv->gpu_error.work);
1891         i915_destroy_error_state(dev);
1892
1893         if (dev->pdev->msi_enabled)
1894                 pci_disable_msi(dev->pdev);
1895
1896         intel_opregion_fini(dev);
1897
1898         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1899                 /* Flush any outstanding unpin_work. */
1900                 flush_workqueue(dev_priv->wq);
1901
1902                 mutex_lock(&dev->struct_mutex);
1903                 i915_gem_cleanup_ringbuffer(dev);
1904                 i915_gem_context_fini(dev);
1905                 WARN_ON(dev_priv->mm.aliasing_ppgtt);
1906                 mutex_unlock(&dev->struct_mutex);
1907                 i915_gem_cleanup_stolen(dev);
1908
1909                 if (!I915_NEED_GFX_HWS(dev))
1910                         i915_free_hws(dev);
1911         }
1912
1913         WARN_ON(!list_empty(&dev_priv->vm_list));
1914
1915         drm_vblank_cleanup(dev);
1916
1917         intel_teardown_gmbus(dev);
1918         intel_teardown_mchbar(dev);
1919
1920         destroy_workqueue(dev_priv->dp_wq);
1921         destroy_workqueue(dev_priv->wq);
1922         pm_qos_remove_request(&dev_priv->pm_qos);
1923
1924         dev_priv->gtt.base.cleanup(&dev_priv->gtt.base);
1925
1926         intel_uncore_fini(dev);
1927         if (dev_priv->regs != NULL)
1928                 pci_iounmap(dev->pdev, dev_priv->regs);
1929
1930         if (dev_priv->slab)
1931                 kmem_cache_destroy(dev_priv->slab);
1932
1933         pci_dev_put(dev_priv->bridge_dev);
1934         kfree(dev_priv);
1935
1936         return 0;
1937 }
1938
1939 int i915_driver_open(struct drm_device *dev, struct drm_file *file)
1940 {
1941         int ret;
1942
1943         ret = i915_gem_open(dev, file);
1944         if (ret)
1945                 return ret;
1946
1947         return 0;
1948 }
1949
1950 /**
1951  * i915_driver_lastclose - clean up after all DRM clients have exited
1952  * @dev: DRM device
1953  *
1954  * Take care of cleaning up after all DRM clients have exited.  In the
1955  * mode setting case, we want to restore the kernel's initial mode (just
1956  * in case the last client left us in a bad state).
1957  *
1958  * Additionally, in the non-mode setting case, we'll tear down the GTT
1959  * and DMA structures, since the kernel won't be using them, and clea
1960  * up any GEM state.
1961  */
1962 void i915_driver_lastclose(struct drm_device *dev)
1963 {
1964         struct drm_i915_private *dev_priv = dev->dev_private;
1965
1966         /* On gen6+ we refuse to init without kms enabled, but then the drm core
1967          * goes right around and calls lastclose. Check for this and don't clean
1968          * up anything. */
1969         if (!dev_priv)
1970                 return;
1971
1972         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1973                 intel_fbdev_restore_mode(dev);
1974                 vga_switcheroo_process_delayed_switch();
1975                 return;
1976         }
1977
1978         i915_gem_lastclose(dev);
1979
1980         i915_dma_cleanup(dev);
1981 }
1982
1983 void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
1984 {
1985         mutex_lock(&dev->struct_mutex);
1986         i915_gem_context_close(dev, file);
1987         i915_gem_release(dev, file);
1988         mutex_unlock(&dev->struct_mutex);
1989 }
1990
1991 void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
1992 {
1993         struct drm_i915_file_private *file_priv = file->driver_priv;
1994
1995         if (file_priv && file_priv->bsd_ring)
1996                 file_priv->bsd_ring = NULL;
1997         kfree(file_priv);
1998 }
1999
2000 const struct drm_ioctl_desc i915_ioctls[] = {
2001         DRM_IOCTL_DEF_DRV(I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2002         DRM_IOCTL_DEF_DRV(I915_FLUSH, i915_flush_ioctl, DRM_AUTH),
2003         DRM_IOCTL_DEF_DRV(I915_FLIP, i915_flip_bufs, DRM_AUTH),
2004         DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, i915_batchbuffer, DRM_AUTH),
2005         DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, i915_irq_emit, DRM_AUTH),
2006         DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, i915_irq_wait, DRM_AUTH),
2007         DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
2008         DRM_IOCTL_DEF_DRV(I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2009         DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
2010         DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
2011         DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2012         DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH),
2013         DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2014         DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2015         DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  i915_vblank_pipe_get, DRM_AUTH),
2016         DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH),
2017         DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
2018         DRM_IOCTL_DEF_DRV(I915_GEM_INIT, i915_gem_init_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
2019         DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH|DRM_UNLOCKED),
2020         DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
2021         DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
2022         DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY|DRM_UNLOCKED),
2023         DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
2024         DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2025         DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2026         DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
2027         DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
2028         DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY|DRM_UNLOCKED),
2029         DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2030         DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2031         DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2032         DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2033         DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2034         DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2035         DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2036         DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2037         DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2038         DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2039         DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, DRM_UNLOCKED),
2040         DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2041         DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
2042         DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
2043         DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
2044         DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, intel_sprite_get_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
2045         DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
2046         DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2047         DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2048         DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2049         DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_get_reset_stats_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2050         DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW),
2051 };
2052
2053 int i915_max_ioctl = ARRAY_SIZE(i915_ioctls);
2054
2055 /*
2056  * This is really ugly: Because old userspace abused the linux agp interface to
2057  * manage the gtt, we need to claim that all intel devices are agp.  For
2058  * otherwise the drm core refuses to initialize the agp support code.
2059  */
2060 int i915_driver_device_is_agp(struct drm_device *dev)
2061 {
2062         return 1;
2063 }