drivers/misc/habanalabs/device.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #define pr_fmt(fmt)                     "habanalabs: " fmt
   9
  10 #include "habanalabs.h"
  11
  12 #include <linux/pci.h>
  13 #include <linux/sched/signal.h>
  14 #include <linux/hwmon.h>
  15 #include <uapi/misc/habanalabs.h>
  16
  17 #define HL_PLDM_PENDING_RESET_PER_SEC   (HL_PENDING_RESET_PER_SEC * 10)
  18
  19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
  20 {
  21         if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
  22                 return true;
  23         else
  24                 return false;
  25 }
  26
  27 enum hl_device_status hl_device_status(struct hl_device *hdev)
  28 {
  29         enum hl_device_status status;
  30
  31         if (hdev->disabled)
  32                 status = HL_DEVICE_STATUS_MALFUNCTION;
  33         else if (atomic_read(&hdev->in_reset))
  34                 status = HL_DEVICE_STATUS_IN_RESET;
  35         else
  36                 status = HL_DEVICE_STATUS_OPERATIONAL;
  37
  38         return status;
  39 };
  40
  41 static void hpriv_release(struct kref *ref)
  42 {
  43         struct hl_fpriv *hpriv;
  44         struct hl_device *hdev;
  45
  46         hpriv = container_of(ref, struct hl_fpriv, refcount);
  47
  48         hdev = hpriv->hdev;
  49
  50         put_pid(hpriv->taskpid);
  51
  52         hl_debugfs_remove_file(hpriv);
  53
  54         mutex_destroy(&hpriv->restore_phase_mutex);
  55
  56         kfree(hpriv);
  57
  58         /* Now the FD is really closed */
  59         atomic_dec(&hdev->fd_open_cnt);
  60
  61         /* This allows a new user context to open the device */
  62         hdev->user_ctx = NULL;
  63 }
  64
  65 void hl_hpriv_get(struct hl_fpriv *hpriv)
  66 {
  67         kref_get(&hpriv->refcount);
  68 }
  69
  70 void hl_hpriv_put(struct hl_fpriv *hpriv)
  71 {
  72         kref_put(&hpriv->refcount, hpriv_release);
  73 }
  74
  75 /*
  76  * hl_device_release - release function for habanalabs device
  77  *
  78  * @inode: pointer to inode structure
  79  * @filp: pointer to file structure
  80  *
  81  * Called when process closes an habanalabs device
  82  */
  83 static int hl_device_release(struct inode *inode, struct file *filp)
  84 {
  85         struct hl_fpriv *hpriv = filp->private_data;
  86
  87         hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
  88         hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
  89
  90         filp->private_data = NULL;
  91
  92         hl_hpriv_put(hpriv);
  93
  94         return 0;
  95 }
  96
  97 /*
  98  * hl_mmap - mmap function for habanalabs device
  99  *
 100  * @*filp: pointer to file structure
 101  * @*vma: pointer to vm_area_struct of the process
 102  *
 103  * Called when process does an mmap on habanalabs device. Call the device's mmap
 104  * function at the end of the common code.
 105  */
 106 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
 107 {
 108         struct hl_fpriv *hpriv = filp->private_data;
 109
 110         if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) {
 111                 vma->vm_pgoff ^= HL_MMAP_CB_MASK;
 112                 return hl_cb_mmap(hpriv, vma);
 113         }
 114
 115         return -EINVAL;
 116 }
 117
 118 static const struct file_operations hl_ops = {
 119         .owner = THIS_MODULE,
 120         .open = hl_device_open,
 121         .release = hl_device_release,
 122         .mmap = hl_mmap,
 123         .unlocked_ioctl = hl_ioctl,
 124         .compat_ioctl = hl_ioctl
 125 };
 126
 127 /*
 128  * device_setup_cdev - setup cdev and device for habanalabs device
 129  *
 130  * @hdev: pointer to habanalabs device structure
 131  * @hclass: pointer to the class object of the device
 132  * @minor: minor number of the specific device
 133  * @fpos : file operations to install for this device
 134  *
 135  * Create a cdev and a Linux device for habanalabs's device. Need to be
 136  * called at the end of the habanalabs device initialization process,
 137  * because this function exposes the device to the user
 138  */
 139 static int device_setup_cdev(struct hl_device *hdev, struct class *hclass,
 140                                 int minor, const struct file_operations *fops)
 141 {
 142         int err, devno = MKDEV(hdev->major, minor);
 143         struct cdev *hdev_cdev = &hdev->cdev;
 144         char *name;
 145
 146         name = kasprintf(GFP_KERNEL, "hl%d", hdev->id);
 147         if (!name)
 148                 return -ENOMEM;
 149
 150         cdev_init(hdev_cdev, fops);
 151         hdev_cdev->owner = THIS_MODULE;
 152         err = cdev_add(hdev_cdev, devno, 1);
 153         if (err) {
 154                 pr_err("Failed to add char device %s\n", name);
 155                 goto err_cdev_add;
 156         }
 157
 158         hdev->dev = device_create(hclass, NULL, devno, NULL, "%s", name);
 159         if (IS_ERR(hdev->dev)) {
 160                 pr_err("Failed to create device %s\n", name);
 161                 err = PTR_ERR(hdev->dev);
 162                 goto err_device_create;
 163         }
 164
 165         dev_set_drvdata(hdev->dev, hdev);
 166
 167         kfree(name);
 168
 169         return 0;
 170
 171 err_device_create:
 172         cdev_del(hdev_cdev);
 173 err_cdev_add:
 174         kfree(name);
 175         return err;
 176 }
 177
 178 /*
 179  * device_early_init - do some early initialization for the habanalabs device
 180  *
 181  * @hdev: pointer to habanalabs device structure
 182  *
 183  * Install the relevant function pointers and call the early_init function,
 184  * if such a function exists
 185  */
 186 static int device_early_init(struct hl_device *hdev)
 187 {
 188         int rc;
 189
 190         switch (hdev->asic_type) {
 191         case ASIC_GOYA:
 192                 goya_set_asic_funcs(hdev);
 193                 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
 194                 break;
 195         default:
 196                 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 197                         hdev->asic_type);
 198                 return -EINVAL;
 199         }
 200
 201         rc = hdev->asic_funcs->early_init(hdev);
 202         if (rc)
 203                 return rc;
 204
 205         rc = hl_asid_init(hdev);
 206         if (rc)
 207                 goto early_fini;
 208
 209         hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
 210         if (hdev->cq_wq == NULL) {
 211                 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
 212                 rc = -ENOMEM;
 213                 goto asid_fini;
 214         }
 215
 216         hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
 217         if (hdev->eq_wq == NULL) {
 218                 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
 219                 rc = -ENOMEM;
 220                 goto free_cq_wq;
 221         }
 222
 223         hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
 224                                         GFP_KERNEL);
 225         if (!hdev->hl_chip_info) {
 226                 rc = -ENOMEM;
 227                 goto free_eq_wq;
 228         }
 229
 230         hl_cb_mgr_init(&hdev->kernel_cb_mgr);
 231
 232         mutex_init(&hdev->fd_open_cnt_lock);
 233         mutex_init(&hdev->send_cpu_message_lock);
 234         mutex_init(&hdev->debug_lock);
 235         mutex_init(&hdev->mmu_cache_lock);
 236         INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
 237         spin_lock_init(&hdev->hw_queues_mirror_lock);
 238         atomic_set(&hdev->in_reset, 0);
 239         atomic_set(&hdev->fd_open_cnt, 0);
 240         atomic_set(&hdev->cs_active_cnt, 0);
 241
 242         return 0;
 243
 244 free_eq_wq:
 245         destroy_workqueue(hdev->eq_wq);
 246 free_cq_wq:
 247         destroy_workqueue(hdev->cq_wq);
 248 asid_fini:
 249         hl_asid_fini(hdev);
 250 early_fini:
 251         if (hdev->asic_funcs->early_fini)
 252                 hdev->asic_funcs->early_fini(hdev);
 253
 254         return rc;
 255 }
 256
 257 /*
 258  * device_early_fini - finalize all that was done in device_early_init
 259  *
 260  * @hdev: pointer to habanalabs device structure
 261  *
 262  */
 263 static void device_early_fini(struct hl_device *hdev)
 264 {
 265         mutex_destroy(&hdev->mmu_cache_lock);
 266         mutex_destroy(&hdev->debug_lock);
 267         mutex_destroy(&hdev->send_cpu_message_lock);
 268
 269         hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
 270
 271         kfree(hdev->hl_chip_info);
 272
 273         destroy_workqueue(hdev->eq_wq);
 274         destroy_workqueue(hdev->cq_wq);
 275
 276         hl_asid_fini(hdev);
 277
 278         if (hdev->asic_funcs->early_fini)
 279                 hdev->asic_funcs->early_fini(hdev);
 280
 281         mutex_destroy(&hdev->fd_open_cnt_lock);
 282 }
 283
 284 static void set_freq_to_low_job(struct work_struct *work)
 285 {
 286         struct hl_device *hdev = container_of(work, struct hl_device,
 287                                                 work_freq.work);
 288
 289         if (atomic_read(&hdev->fd_open_cnt) == 0)
 290                 hl_device_set_frequency(hdev, PLL_LOW);
 291
 292         schedule_delayed_work(&hdev->work_freq,
 293                         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
 294 }
 295
 296 static void hl_device_heartbeat(struct work_struct *work)
 297 {
 298         struct hl_device *hdev = container_of(work, struct hl_device,
 299                                                 work_heartbeat.work);
 300
 301         if (hl_device_disabled_or_in_reset(hdev))
 302                 goto reschedule;
 303
 304         if (!hdev->asic_funcs->send_heartbeat(hdev))
 305                 goto reschedule;
 306
 307         dev_err(hdev->dev, "Device heartbeat failed!\n");
 308         hl_device_reset(hdev, true, false);
 309
 310         return;
 311
 312 reschedule:
 313         schedule_delayed_work(&hdev->work_heartbeat,
 314                         usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
 315 }
 316
 317 /*
 318  * device_late_init - do late stuff initialization for the habanalabs device
 319  *
 320  * @hdev: pointer to habanalabs device structure
 321  *
 322  * Do stuff that either needs the device H/W queues to be active or needs
 323  * to happen after all the rest of the initialization is finished
 324  */
 325 static int device_late_init(struct hl_device *hdev)
 326 {
 327         int rc;
 328
 329         if (hdev->asic_funcs->late_init) {
 330                 rc = hdev->asic_funcs->late_init(hdev);
 331                 if (rc) {
 332                         dev_err(hdev->dev,
 333                                 "failed late initialization for the H/W\n");
 334                         return rc;
 335                 }
 336         }
 337
 338         hdev->high_pll = hdev->asic_prop.high_pll;
 339
 340         /* force setting to low frequency */
 341         atomic_set(&hdev->curr_pll_profile, PLL_LOW);
 342
 343         if (hdev->pm_mng_profile == PM_AUTO)
 344                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
 345         else
 346                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
 347
 348         INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
 349         schedule_delayed_work(&hdev->work_freq,
 350         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
 351
 352         if (hdev->heartbeat) {
 353                 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
 354                 schedule_delayed_work(&hdev->work_heartbeat,
 355                                 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
 356         }
 357
 358         hdev->late_init_done = true;
 359
 360         return 0;
 361 }
 362
 363 /*
 364  * device_late_fini - finalize all that was done in device_late_init
 365  *
 366  * @hdev: pointer to habanalabs device structure
 367  *
 368  */
 369 static void device_late_fini(struct hl_device *hdev)
 370 {
 371         if (!hdev->late_init_done)
 372                 return;
 373
 374         cancel_delayed_work_sync(&hdev->work_freq);
 375         if (hdev->heartbeat)
 376                 cancel_delayed_work_sync(&hdev->work_heartbeat);
 377
 378         if (hdev->asic_funcs->late_fini)
 379                 hdev->asic_funcs->late_fini(hdev);
 380
 381         hdev->late_init_done = false;
 382 }
 383
 384 /*
 385  * hl_device_set_frequency - set the frequency of the device
 386  *
 387  * @hdev: pointer to habanalabs device structure
 388  * @freq: the new frequency value
 389  *
 390  * Change the frequency if needed.
 391  * We allose to set PLL to low only if there is no user process
 392  * Returns 0 if no change was done, otherwise returns 1;
 393  */
 394 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
 395 {
 396         enum hl_pll_frequency old_freq =
 397                         (freq == PLL_HIGH) ? PLL_LOW : PLL_HIGH;
 398         int ret;
 399
 400         if (hdev->pm_mng_profile == PM_MANUAL)
 401                 return 0;
 402
 403         ret = atomic_cmpxchg(&hdev->curr_pll_profile, old_freq, freq);
 404         if (ret == freq)
 405                 return 0;
 406
 407         /*
 408          * in case we want to lower frequency, check if device is not
 409          * opened. We must have a check here to workaround race condition with
 410          * hl_device_open
 411          */
 412         if ((freq == PLL_LOW) && (atomic_read(&hdev->fd_open_cnt) > 0)) {
 413                 atomic_set(&hdev->curr_pll_profile, PLL_HIGH);
 414                 return 0;
 415         }
 416
 417         dev_dbg(hdev->dev, "Changing device frequency to %s\n",
 418                 freq == PLL_HIGH ? "high" : "low");
 419
 420         hdev->asic_funcs->set_pll_profile(hdev, freq);
 421
 422         return 1;
 423 }
 424
 425 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
 426 {
 427         int rc = 0;
 428
 429         mutex_lock(&hdev->debug_lock);
 430
 431         if (!enable) {
 432                 if (!hdev->in_debug) {
 433                         dev_err(hdev->dev,
 434                                 "Failed to disable debug mode because device was not in debug mode\n");
 435                         rc = -EFAULT;
 436                         goto out;
 437                 }
 438
 439                 hdev->asic_funcs->halt_coresight(hdev);
 440                 hdev->in_debug = 0;
 441
 442                 goto out;
 443         }
 444
 445         if (hdev->in_debug) {
 446                 dev_err(hdev->dev,
 447                         "Failed to enable debug mode because device is already in debug mode\n");
 448                 rc = -EFAULT;
 449                 goto out;
 450         }
 451
 452         mutex_lock(&hdev->fd_open_cnt_lock);
 453
 454         if (atomic_read(&hdev->fd_open_cnt) > 1) {
 455                 dev_err(hdev->dev,
 456                         "Failed to enable debug mode. More then a single user is using the device\n");
 457                 rc = -EPERM;
 458                 goto unlock_fd_open_lock;
 459         }
 460
 461         hdev->in_debug = 1;
 462
 463 unlock_fd_open_lock:
 464         mutex_unlock(&hdev->fd_open_cnt_lock);
 465 out:
 466         mutex_unlock(&hdev->debug_lock);
 467
 468         return rc;
 469 }
 470
 471 /*
 472  * hl_device_suspend - initiate device suspend
 473  *
 474  * @hdev: pointer to habanalabs device structure
 475  *
 476  * Puts the hw in the suspend state (all asics).
 477  * Returns 0 for success or an error on failure.
 478  * Called at driver suspend.
 479  */
 480 int hl_device_suspend(struct hl_device *hdev)
 481 {
 482         int rc;
 483
 484         pci_save_state(hdev->pdev);
 485
 486         /* Block future CS/VM/JOB completion operations */
 487         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
 488         if (rc) {
 489                 dev_err(hdev->dev, "Can't suspend while in reset\n");
 490                 return -EIO;
 491         }
 492
 493         /* This blocks all other stuff that is not blocked by in_reset */
 494         hdev->disabled = true;
 495
 496         /*
 497          * Flush anyone that is inside the critical section of enqueue
 498          * jobs to the H/W
 499          */
 500         hdev->asic_funcs->hw_queues_lock(hdev);
 501         hdev->asic_funcs->hw_queues_unlock(hdev);
 502
 503         /* Flush processes that are sending message to CPU */
 504         mutex_lock(&hdev->send_cpu_message_lock);
 505         mutex_unlock(&hdev->send_cpu_message_lock);
 506
 507         rc = hdev->asic_funcs->suspend(hdev);
 508         if (rc)
 509                 dev_err(hdev->dev,
 510                         "Failed to disable PCI access of device CPU\n");
 511
 512         /* Shut down the device */
 513         pci_disable_device(hdev->pdev);
 514         pci_set_power_state(hdev->pdev, PCI_D3hot);
 515
 516         return 0;
 517 }
 518
 519 /*
 520  * hl_device_resume - initiate device resume
 521  *
 522  * @hdev: pointer to habanalabs device structure
 523  *
 524  * Bring the hw back to operating state (all asics).
 525  * Returns 0 for success or an error on failure.
 526  * Called at driver resume.
 527  */
 528 int hl_device_resume(struct hl_device *hdev)
 529 {
 530         int rc;
 531
 532         pci_set_power_state(hdev->pdev, PCI_D0);
 533         pci_restore_state(hdev->pdev);
 534         rc = pci_enable_device_mem(hdev->pdev);
 535         if (rc) {
 536                 dev_err(hdev->dev,
 537                         "Failed to enable PCI device in resume\n");
 538                 return rc;
 539         }
 540
 541         pci_set_master(hdev->pdev);
 542
 543         rc = hdev->asic_funcs->resume(hdev);
 544         if (rc) {
 545                 dev_err(hdev->dev, "Failed to resume device after suspend\n");
 546                 goto disable_device;
 547         }
 548
 549
 550         hdev->disabled = false;
 551         atomic_set(&hdev->in_reset, 0);
 552
 553         rc = hl_device_reset(hdev, true, false);
 554         if (rc) {
 555                 dev_err(hdev->dev, "Failed to reset device during resume\n");
 556                 goto disable_device;
 557         }
 558
 559         return 0;
 560
 561 disable_device:
 562         pci_clear_master(hdev->pdev);
 563         pci_disable_device(hdev->pdev);
 564
 565         return rc;
 566 }
 567
 568 static void device_kill_open_processes(struct hl_device *hdev)
 569 {
 570         u16 pending_total, pending_cnt;
 571         struct task_struct *task = NULL;
 572
 573         if (hdev->pldm)
 574                 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
 575         else
 576                 pending_total = HL_PENDING_RESET_PER_SEC;
 577
 578         pending_cnt = pending_total;
 579
 580         /* Flush all processes that are inside hl_open */
 581         mutex_lock(&hdev->fd_open_cnt_lock);
 582
 583         while ((atomic_read(&hdev->fd_open_cnt)) && (pending_cnt)) {
 584
 585                 pending_cnt--;
 586
 587                 dev_info(hdev->dev,
 588                         "Can't HARD reset, waiting for user to close FD\n");
 589                 ssleep(1);
 590         }
 591
 592         if (atomic_read(&hdev->fd_open_cnt)) {
 593                 task = get_pid_task(hdev->user_ctx->hpriv->taskpid,
 594                                         PIDTYPE_PID);
 595                 if (task) {
 596                         dev_info(hdev->dev, "Killing user processes\n");
 597                         send_sig(SIGKILL, task, 1);
 598                         msleep(100);
 599
 600                         put_task_struct(task);
 601                 }
 602         }
 603
 604         /* We killed the open users, but because the driver cleans up after the
 605          * user contexts are closed (e.g. mmu mappings), we need to wait again
 606          * to make sure the cleaning phase is finished before continuing with
 607          * the reset
 608          */
 609
 610         pending_cnt = pending_total;
 611
 612         while ((atomic_read(&hdev->fd_open_cnt)) && (pending_cnt)) {
 613
 614                 pending_cnt--;
 615
 616                 ssleep(1);
 617         }
 618
 619         if (atomic_read(&hdev->fd_open_cnt))
 620                 dev_crit(hdev->dev,
 621                         "Going to hard reset with open user contexts\n");
 622
 623         mutex_unlock(&hdev->fd_open_cnt_lock);
 624
 625 }
 626
 627 static void device_hard_reset_pending(struct work_struct *work)
 628 {
 629         struct hl_device_reset_work *device_reset_work =
 630                 container_of(work, struct hl_device_reset_work, reset_work);
 631         struct hl_device *hdev = device_reset_work->hdev;
 632
 633         device_kill_open_processes(hdev);
 634
 635         hl_device_reset(hdev, true, true);
 636
 637         kfree(device_reset_work);
 638 }
 639
 640 /*
 641  * hl_device_reset - reset the device
 642  *
 643  * @hdev: pointer to habanalabs device structure
 644  * @hard_reset: should we do hard reset to all engines or just reset the
 645  *              compute/dma engines
 646  *
 647  * Block future CS and wait for pending CS to be enqueued
 648  * Call ASIC H/W fini
 649  * Flush all completions
 650  * Re-initialize all internal data structures
 651  * Call ASIC H/W init, late_init
 652  * Test queues
 653  * Enable device
 654  *
 655  * Returns 0 for success or an error on failure.
 656  */
 657 int hl_device_reset(struct hl_device *hdev, bool hard_reset,
 658                         bool from_hard_reset_thread)
 659 {
 660         int i, rc;
 661
 662         if (!hdev->init_done) {
 663                 dev_err(hdev->dev,
 664                         "Can't reset before initialization is done\n");
 665                 return 0;
 666         }
 667
 668         /*
 669          * Prevent concurrency in this function - only one reset should be
 670          * done at any given time. Only need to perform this if we didn't
 671          * get from the dedicated hard reset thread
 672          */
 673         if (!from_hard_reset_thread) {
 674                 /* Block future CS/VM/JOB completion operations */
 675                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
 676                 if (rc)
 677                         return 0;
 678
 679                 /* This also blocks future CS/VM/JOB completion operations */
 680                 hdev->disabled = true;
 681
 682                 /*
 683                  * Flush anyone that is inside the critical section of enqueue
 684                  * jobs to the H/W
 685                  */
 686                 hdev->asic_funcs->hw_queues_lock(hdev);
 687                 hdev->asic_funcs->hw_queues_unlock(hdev);
 688
 689                 dev_err(hdev->dev, "Going to RESET device!\n");
 690         }
 691
 692 again:
 693         if ((hard_reset) && (!from_hard_reset_thread)) {
 694                 struct hl_device_reset_work *device_reset_work;
 695
 696                 hdev->hard_reset_pending = true;
 697
 698                 device_reset_work = kzalloc(sizeof(*device_reset_work),
 699                                                 GFP_ATOMIC);
 700                 if (!device_reset_work) {
 701                         rc = -ENOMEM;
 702                         goto out_err;
 703                 }
 704
 705                 /*
 706                  * Because the reset function can't run from interrupt or
 707                  * from heartbeat work, we need to call the reset function
 708                  * from a dedicated work
 709                  */
 710                 INIT_WORK(&device_reset_work->reset_work,
 711                                 device_hard_reset_pending);
 712                 device_reset_work->hdev = hdev;
 713                 schedule_work(&device_reset_work->reset_work);
 714
 715                 return 0;
 716         }
 717
 718         if (hard_reset) {
 719                 device_late_fini(hdev);
 720
 721                 /*
 722                  * Now that the heartbeat thread is closed, flush processes
 723                  * which are sending messages to CPU
 724                  */
 725                 mutex_lock(&hdev->send_cpu_message_lock);
 726                 mutex_unlock(&hdev->send_cpu_message_lock);
 727         }
 728
 729         /*
 730          * Halt the engines and disable interrupts so we won't get any more
 731          * completions from H/W and we won't have any accesses from the
 732          * H/W to the host machine
 733          */
 734         hdev->asic_funcs->halt_engines(hdev, hard_reset);
 735
 736         /* Go over all the queues, release all CS and their jobs */
 737         hl_cs_rollback_all(hdev);
 738
 739         /* Release kernel context */
 740         if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1))
 741                 hdev->kernel_ctx = NULL;
 742
 743         /* Reset the H/W. It will be in idle state after this returns */
 744         hdev->asic_funcs->hw_fini(hdev, hard_reset);
 745
 746         if (hard_reset) {
 747                 hl_vm_fini(hdev);
 748                 hl_mmu_fini(hdev);
 749                 hl_eq_reset(hdev, &hdev->event_queue);
 750         }
 751
 752         /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
 753         hl_hw_queue_reset(hdev, hard_reset);
 754         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
 755                 hl_cq_reset(hdev, &hdev->completion_queue[i]);
 756
 757         /* Make sure the context switch phase will run again */
 758         if (hdev->user_ctx) {
 759                 atomic_set(&hdev->user_ctx->thread_ctx_switch_token, 1);
 760                 hdev->user_ctx->thread_ctx_switch_wait_token = 0;
 761         }
 762
 763         /* Finished tear-down, starting to re-initialize */
 764
 765         if (hard_reset) {
 766                 hdev->device_cpu_disabled = false;
 767                 hdev->hard_reset_pending = false;
 768
 769                 if (hdev->kernel_ctx) {
 770                         dev_crit(hdev->dev,
 771                                 "kernel ctx was alive during hard reset, something is terribly wrong\n");
 772                         rc = -EBUSY;
 773                         goto out_err;
 774                 }
 775
 776                 rc = hl_mmu_init(hdev);
 777                 if (rc) {
 778                         dev_err(hdev->dev,
 779                                 "Failed to initialize MMU S/W after hard reset\n");
 780                         goto out_err;
 781                 }
 782
 783                 /* Allocate the kernel context */
 784                 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
 785                                                 GFP_KERNEL);
 786                 if (!hdev->kernel_ctx) {
 787                         rc = -ENOMEM;
 788                         goto out_err;
 789                 }
 790
 791                 hdev->user_ctx = NULL;
 792
 793                 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
 794                 if (rc) {
 795                         dev_err(hdev->dev,
 796                                 "failed to init kernel ctx in hard reset\n");
 797                         kfree(hdev->kernel_ctx);
 798                         hdev->kernel_ctx = NULL;
 799                         goto out_err;
 800                 }
 801         }
 802
 803         rc = hdev->asic_funcs->hw_init(hdev);
 804         if (rc) {
 805                 dev_err(hdev->dev,
 806                         "failed to initialize the H/W after reset\n");
 807                 goto out_err;
 808         }
 809
 810         hdev->disabled = false;
 811
 812         /* Check that the communication with the device is working */
 813         rc = hdev->asic_funcs->test_queues(hdev);
 814         if (rc) {
 815                 dev_err(hdev->dev,
 816                         "Failed to detect if device is alive after reset\n");
 817                 goto out_err;
 818         }
 819
 820         if (hard_reset) {
 821                 rc = device_late_init(hdev);
 822                 if (rc) {
 823                         dev_err(hdev->dev,
 824                                 "Failed late init after hard reset\n");
 825                         goto out_err;
 826                 }
 827
 828                 rc = hl_vm_init(hdev);
 829                 if (rc) {
 830                         dev_err(hdev->dev,
 831                                 "Failed to init memory module after hard reset\n");
 832                         goto out_err;
 833                 }
 834
 835                 hl_set_max_power(hdev, hdev->max_power);
 836         } else {
 837                 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
 838                 if (rc) {
 839                         dev_err(hdev->dev,
 840                                 "Failed late init after soft reset\n");
 841                         goto out_err;
 842                 }
 843         }
 844
 845         atomic_set(&hdev->in_reset, 0);
 846
 847         if (hard_reset)
 848                 hdev->hard_reset_cnt++;
 849         else
 850                 hdev->soft_reset_cnt++;
 851
 852         return 0;
 853
 854 out_err:
 855         hdev->disabled = true;
 856
 857         if (hard_reset) {
 858                 dev_err(hdev->dev,
 859                         "Failed to reset! Device is NOT usable\n");
 860                 hdev->hard_reset_cnt++;
 861         } else {
 862                 dev_err(hdev->dev,
 863                         "Failed to do soft-reset, trying hard reset\n");
 864                 hdev->soft_reset_cnt++;
 865                 hard_reset = true;
 866                 goto again;
 867         }
 868
 869         atomic_set(&hdev->in_reset, 0);
 870
 871         return rc;
 872 }
 873
 874 /*
 875  * hl_device_init - main initialization function for habanalabs device
 876  *
 877  * @hdev: pointer to habanalabs device structure
 878  *
 879  * Allocate an id for the device, do early initialization and then call the
 880  * ASIC specific initialization functions. Finally, create the cdev and the
 881  * Linux device to expose it to the user
 882  */
 883 int hl_device_init(struct hl_device *hdev, struct class *hclass)
 884 {
 885         int i, rc, cq_ready_cnt;
 886
 887         /* Create device */
 888         rc = device_setup_cdev(hdev, hclass, hdev->id, &hl_ops);
 889
 890         if (rc)
 891                 goto out_disabled;
 892
 893         /* Initialize ASIC function pointers and perform early init */
 894         rc = device_early_init(hdev);
 895         if (rc)
 896                 goto release_device;
 897
 898         /*
 899          * Start calling ASIC initialization. First S/W then H/W and finally
 900          * late init
 901          */
 902         rc = hdev->asic_funcs->sw_init(hdev);
 903         if (rc)
 904                 goto early_fini;
 905
 906         /*
 907          * Initialize the H/W queues. Must be done before hw_init, because
 908          * there the addresses of the kernel queue are being written to the
 909          * registers of the device
 910          */
 911         rc = hl_hw_queues_create(hdev);
 912         if (rc) {
 913                 dev_err(hdev->dev, "failed to initialize kernel queues\n");
 914                 goto sw_fini;
 915         }
 916
 917         /*
 918          * Initialize the completion queues. Must be done before hw_init,
 919          * because there the addresses of the completion queues are being
 920          * passed as arguments to request_irq
 921          */
 922         hdev->completion_queue =
 923                         kcalloc(hdev->asic_prop.completion_queues_count,
 924                                 sizeof(*hdev->completion_queue), GFP_KERNEL);
 925
 926         if (!hdev->completion_queue) {
 927                 dev_err(hdev->dev, "failed to allocate completion queues\n");
 928                 rc = -ENOMEM;
 929                 goto hw_queues_destroy;
 930         }
 931
 932         for (i = 0, cq_ready_cnt = 0;
 933                         i < hdev->asic_prop.completion_queues_count;
 934                         i++, cq_ready_cnt++) {
 935                 rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
 936                 if (rc) {
 937                         dev_err(hdev->dev,
 938                                 "failed to initialize completion queue\n");
 939                         goto cq_fini;
 940                 }
 941         }
 942
 943         /*
 944          * Initialize the event queue. Must be done before hw_init,
 945          * because there the address of the event queue is being
 946          * passed as argument to request_irq
 947          */
 948         rc = hl_eq_init(hdev, &hdev->event_queue);
 949         if (rc) {
 950                 dev_err(hdev->dev, "failed to initialize event queue\n");
 951                 goto cq_fini;
 952         }
 953
 954         /* MMU S/W must be initialized before kernel context is created */
 955         rc = hl_mmu_init(hdev);
 956         if (rc) {
 957                 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
 958                 goto eq_fini;
 959         }
 960
 961         /* Allocate the kernel context */
 962         hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
 963         if (!hdev->kernel_ctx) {
 964                 rc = -ENOMEM;
 965                 goto mmu_fini;
 966         }
 967
 968         hdev->user_ctx = NULL;
 969
 970         rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
 971         if (rc) {
 972                 dev_err(hdev->dev, "failed to initialize kernel context\n");
 973                 goto free_ctx;
 974         }
 975
 976         rc = hl_cb_pool_init(hdev);
 977         if (rc) {
 978                 dev_err(hdev->dev, "failed to initialize CB pool\n");
 979                 goto release_ctx;
 980         }
 981
 982         rc = hl_sysfs_init(hdev);
 983         if (rc) {
 984                 dev_err(hdev->dev, "failed to initialize sysfs\n");
 985                 goto free_cb_pool;
 986         }
 987
 988         hl_debugfs_add_device(hdev);
 989
 990         if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
 991                 dev_info(hdev->dev,
 992                         "H/W state is dirty, must reset before initializing\n");
 993                 hdev->asic_funcs->hw_fini(hdev, true);
 994         }
 995
 996         rc = hdev->asic_funcs->hw_init(hdev);
 997         if (rc) {
 998                 dev_err(hdev->dev, "failed to initialize the H/W\n");
 999                 rc = 0;
1000                 goto out_disabled;
1001         }
1002
1003         hdev->disabled = false;
1004
1005         /* Check that the communication with the device is working */
1006         rc = hdev->asic_funcs->test_queues(hdev);
1007         if (rc) {
1008                 dev_err(hdev->dev, "Failed to detect if device is alive\n");
1009                 rc = 0;
1010                 goto out_disabled;
1011         }
1012
1013         rc = device_late_init(hdev);
1014         if (rc) {
1015                 dev_err(hdev->dev, "Failed late initialization\n");
1016                 rc = 0;
1017                 goto out_disabled;
1018         }
1019
1020         dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
1021                 hdev->asic_name,
1022                 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
1023
1024         rc = hl_vm_init(hdev);
1025         if (rc) {
1026                 dev_err(hdev->dev, "Failed to initialize memory module\n");
1027                 rc = 0;
1028                 goto out_disabled;
1029         }
1030
1031         /*
1032          * hl_hwmon_init must be called after device_late_init, because only
1033          * there we get the information from the device about which
1034          * hwmon-related sensors the device supports
1035          */
1036         rc = hl_hwmon_init(hdev);
1037         if (rc) {
1038                 dev_err(hdev->dev, "Failed to initialize hwmon\n");
1039                 rc = 0;
1040                 goto out_disabled;
1041         }
1042
1043         dev_notice(hdev->dev,
1044                 "Successfully added device to habanalabs driver\n");
1045
1046         hdev->init_done = true;
1047
1048         return 0;
1049
1050 free_cb_pool:
1051         hl_cb_pool_fini(hdev);
1052 release_ctx:
1053         if (hl_ctx_put(hdev->kernel_ctx) != 1)
1054                 dev_err(hdev->dev,
1055                         "kernel ctx is still alive on initialization failure\n");
1056 free_ctx:
1057         kfree(hdev->kernel_ctx);
1058 mmu_fini:
1059         hl_mmu_fini(hdev);
1060 eq_fini:
1061         hl_eq_fini(hdev, &hdev->event_queue);
1062 cq_fini:
1063         for (i = 0 ; i < cq_ready_cnt ; i++)
1064                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1065         kfree(hdev->completion_queue);
1066 hw_queues_destroy:
1067         hl_hw_queues_destroy(hdev);
1068 sw_fini:
1069         hdev->asic_funcs->sw_fini(hdev);
1070 early_fini:
1071         device_early_fini(hdev);
1072 release_device:
1073         device_destroy(hclass, hdev->dev->devt);
1074         cdev_del(&hdev->cdev);
1075 out_disabled:
1076         hdev->disabled = true;
1077         if (hdev->pdev)
1078                 dev_err(&hdev->pdev->dev,
1079                         "Failed to initialize hl%d. Device is NOT usable !\n",
1080                         hdev->id);
1081         else
1082                 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1083                         hdev->id);
1084
1085         return rc;
1086 }
1087
1088 /*
1089  * hl_device_fini - main tear-down function for habanalabs device
1090  *
1091  * @hdev: pointer to habanalabs device structure
1092  *
1093  * Destroy the device, call ASIC fini functions and release the id
1094  */
1095 void hl_device_fini(struct hl_device *hdev)
1096 {
1097         int i, rc;
1098         ktime_t timeout;
1099
1100         dev_info(hdev->dev, "Removing device\n");
1101
1102         /*
1103          * This function is competing with the reset function, so try to
1104          * take the reset atomic and if we are already in middle of reset,
1105          * wait until reset function is finished. Reset function is designed
1106          * to always finish (could take up to a few seconds in worst case).
1107          */
1108
1109         timeout = ktime_add_us(ktime_get(),
1110                                 HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4);
1111         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1112         while (rc) {
1113                 usleep_range(50, 200);
1114                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1115                 if (ktime_compare(ktime_get(), timeout) > 0) {
1116                         WARN(1, "Failed to remove device because reset function did not finish\n");
1117                         return;
1118                 }
1119         }
1120
1121         /* Mark device as disabled */
1122         hdev->disabled = true;
1123
1124         /*
1125          * Flush anyone that is inside the critical section of enqueue
1126          * jobs to the H/W
1127          */
1128         hdev->asic_funcs->hw_queues_lock(hdev);
1129         hdev->asic_funcs->hw_queues_unlock(hdev);
1130
1131         hdev->hard_reset_pending = true;
1132
1133         device_kill_open_processes(hdev);
1134
1135         hl_hwmon_fini(hdev);
1136
1137         device_late_fini(hdev);
1138
1139         hl_debugfs_remove_device(hdev);
1140
1141         hl_sysfs_fini(hdev);
1142
1143         /*
1144          * Halt the engines and disable interrupts so we won't get any more
1145          * completions from H/W and we won't have any accesses from the
1146          * H/W to the host machine
1147          */
1148         hdev->asic_funcs->halt_engines(hdev, true);
1149
1150         /* Go over all the queues, release all CS and their jobs */
1151         hl_cs_rollback_all(hdev);
1152
1153         hl_cb_pool_fini(hdev);
1154
1155         /* Release kernel context */
1156         if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1157                 dev_err(hdev->dev, "kernel ctx is still alive\n");
1158
1159         /* Reset the H/W. It will be in idle state after this returns */
1160         hdev->asic_funcs->hw_fini(hdev, true);
1161
1162         hl_vm_fini(hdev);
1163
1164         hl_mmu_fini(hdev);
1165
1166         hl_eq_fini(hdev, &hdev->event_queue);
1167
1168         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1169                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1170         kfree(hdev->completion_queue);
1171
1172         hl_hw_queues_destroy(hdev);
1173
1174         /* Call ASIC S/W finalize function */
1175         hdev->asic_funcs->sw_fini(hdev);
1176
1177         device_early_fini(hdev);
1178
1179         /* Hide device from user */
1180         device_destroy(hdev->dev->class, hdev->dev->devt);
1181         cdev_del(&hdev->cdev);
1182
1183         pr_info("removed device successfully\n");
1184 }
1185
1186 /*
1187  * MMIO register access helper functions.
1188  */
1189
1190 /*
1191  * hl_rreg - Read an MMIO register
1192  *
1193  * @hdev: pointer to habanalabs device structure
1194  * @reg: MMIO register offset (in bytes)
1195  *
1196  * Returns the value of the MMIO register we are asked to read
1197  *
1198  */
1199 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1200 {
1201         return readl(hdev->rmmio + reg);
1202 }
1203
1204 /*
1205  * hl_wreg - Write to an MMIO register
1206  *
1207  * @hdev: pointer to habanalabs device structure
1208  * @reg: MMIO register offset (in bytes)
1209  * @val: 32-bit value
1210  *
1211  * Writes the 32-bit value into the MMIO register
1212  *
1213  */
1214 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1215 {
1216         writel(val, hdev->rmmio + reg);
1217 }