drivers/thunderbolt/icm.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Internal Thunderbolt Connection Manager. This is a firmware running on
   4  * the Thunderbolt host controller performing most of the low-level
   5  * handling.
   6  *
   7  * Copyright (C) 2017, Intel Corporation
   8  * Authors: Michael Jamet <michael.jamet@intel.com>
   9  *          Mika Westerberg <mika.westerberg@linux.intel.com>
  10  */
  11
  12 #include <linux/delay.h>
  13 #include <linux/mutex.h>
  14 #include <linux/moduleparam.h>
  15 #include <linux/pci.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/platform_data/x86/apple.h>
  18 #include <linux/sizes.h>
  19 #include <linux/slab.h>
  20 #include <linux/workqueue.h>
  21
  22 #include "ctl.h"
  23 #include "nhi_regs.h"
  24 #include "tb.h"
  25
  26 #define PCIE2CIO_CMD                    0x30
  27 #define PCIE2CIO_CMD_TIMEOUT            BIT(31)
  28 #define PCIE2CIO_CMD_START              BIT(30)
  29 #define PCIE2CIO_CMD_WRITE              BIT(21)
  30 #define PCIE2CIO_CMD_CS_MASK            GENMASK(20, 19)
  31 #define PCIE2CIO_CMD_CS_SHIFT           19
  32 #define PCIE2CIO_CMD_PORT_MASK          GENMASK(18, 13)
  33 #define PCIE2CIO_CMD_PORT_SHIFT         13
  34
  35 #define PCIE2CIO_WRDATA                 0x34
  36 #define PCIE2CIO_RDDATA                 0x38
  37
  38 #define PHY_PORT_CS1                    0x37
  39 #define PHY_PORT_CS1_LINK_DISABLE       BIT(14)
  40 #define PHY_PORT_CS1_LINK_STATE_MASK    GENMASK(29, 26)
  41 #define PHY_PORT_CS1_LINK_STATE_SHIFT   26
  42
  43 #define ICM_TIMEOUT                     5000    /* ms */
  44 #define ICM_RETRIES                     3
  45 #define ICM_APPROVE_TIMEOUT             10000   /* ms */
  46 #define ICM_MAX_LINK                    4
  47
  48 static bool start_icm;
  49 module_param(start_icm, bool, 0444);
  50 MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
  51
  52 /**
  53  * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
  54  * @reply: Reply from ICM firmware is placed here
  55  * @request: Request that is sent to ICM firmware
  56  * @icm: Pointer to ICM private data
  57  */
  58 struct usb4_switch_nvm_auth {
  59         struct icm_usb4_switch_op_response reply;
  60         struct icm_usb4_switch_op request;
  61         struct icm *icm;
  62 };
  63
  64 /**
  65  * struct icm - Internal connection manager private data
  66  * @request_lock: Makes sure only one message is send to ICM at time
  67  * @rescan_work: Work used to rescan the surviving switches after resume
  68  * @upstream_port: Pointer to the PCIe upstream port this host
  69  *                 controller is connected. This is only set for systems
  70  *                 where ICM needs to be started manually
  71  * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
  72  *           (only set when @upstream_port is not %NULL)
  73  * @safe_mode: ICM is in safe mode
  74  * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
  75  * @rpm: Does the controller support runtime PM (RTD3)
  76  * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
  77  * @proto_version: Firmware protocol version
  78  * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
  79  * @veto: Is RTD3 veto in effect
  80  * @is_supported: Checks if we can support ICM on this controller
  81  * @cio_reset: Trigger CIO reset
  82  * @get_mode: Read and return the ICM firmware mode (optional)
  83  * @get_route: Find a route string for given switch
  84  * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
  85  * @driver_ready: Send driver ready message to ICM
  86  * @set_uuid: Set UUID for the root switch (optional)
  87  * @device_connected: Handle device connected ICM message
  88  * @device_disconnected: Handle device disconnected ICM message
  89  * @xdomain_connected: Handle XDomain connected ICM message
  90  * @xdomain_disconnected: Handle XDomain disconnected ICM message
  91  * @rtd3_veto: Handle RTD3 veto notification ICM message
  92  */
  93 struct icm {
  94         struct mutex request_lock;
  95         struct delayed_work rescan_work;
  96         struct pci_dev *upstream_port;
  97         int vnd_cap;
  98         bool safe_mode;
  99         size_t max_boot_acl;
 100         bool rpm;
 101         bool can_upgrade_nvm;
 102         u8 proto_version;
 103         struct usb4_switch_nvm_auth *last_nvm_auth;
 104         bool veto;
 105         bool (*is_supported)(struct tb *tb);
 106         int (*cio_reset)(struct tb *tb);
 107         int (*get_mode)(struct tb *tb);
 108         int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
 109         void (*save_devices)(struct tb *tb);
 110         int (*driver_ready)(struct tb *tb,
 111                             enum tb_security_level *security_level,
 112                             u8 *proto_version, size_t *nboot_acl, bool *rpm);
 113         void (*set_uuid)(struct tb *tb);
 114         void (*device_connected)(struct tb *tb,
 115                                  const struct icm_pkg_header *hdr);
 116         void (*device_disconnected)(struct tb *tb,
 117                                     const struct icm_pkg_header *hdr);
 118         void (*xdomain_connected)(struct tb *tb,
 119                                   const struct icm_pkg_header *hdr);
 120         void (*xdomain_disconnected)(struct tb *tb,
 121                                      const struct icm_pkg_header *hdr);
 122         void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
 123 };
 124
 125 struct icm_notification {
 126         struct work_struct work;
 127         struct icm_pkg_header *pkg;
 128         struct tb *tb;
 129 };
 130
 131 struct ep_name_entry {
 132         u8 len;
 133         u8 type;
 134         u8 data[];
 135 };
 136
 137 #define EP_NAME_INTEL_VSS       0x10
 138
 139 /* Intel Vendor specific structure */
 140 struct intel_vss {
 141         u16 vendor;
 142         u16 model;
 143         u8 mc;
 144         u8 flags;
 145         u16 pci_devid;
 146         u32 nvm_version;
 147 };
 148
 149 #define INTEL_VSS_FLAGS_RTD3    BIT(0)
 150
 151 static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
 152 {
 153         const void *end = ep_name + size;
 154
 155         while (ep_name < end) {
 156                 const struct ep_name_entry *ep = ep_name;
 157
 158                 if (!ep->len)
 159                         break;
 160                 if (ep_name + ep->len > end)
 161                         break;
 162
 163                 if (ep->type == EP_NAME_INTEL_VSS)
 164                         return (const struct intel_vss *)ep->data;
 165
 166                 ep_name += ep->len;
 167         }
 168
 169         return NULL;
 170 }
 171
 172 static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
 173 {
 174         const struct intel_vss *vss;
 175
 176         vss = parse_intel_vss(ep_name, size);
 177         if (vss)
 178                 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
 179
 180         return false;
 181 }
 182
 183 static inline struct tb *icm_to_tb(struct icm *icm)
 184 {
 185         return ((void *)icm - sizeof(struct tb));
 186 }
 187
 188 static inline u8 phy_port_from_route(u64 route, u8 depth)
 189 {
 190         u8 link;
 191
 192         link = depth ? route >> ((depth - 1) * 8) : route;
 193         return tb_phy_port_from_link(link);
 194 }
 195
 196 static inline u8 dual_link_from_link(u8 link)
 197 {
 198         return link ? ((link - 1) ^ 0x01) + 1 : 0;
 199 }
 200
 201 static inline u64 get_route(u32 route_hi, u32 route_lo)
 202 {
 203         return (u64)route_hi << 32 | route_lo;
 204 }
 205
 206 static inline u64 get_parent_route(u64 route)
 207 {
 208         int depth = tb_route_length(route);
 209         return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
 210 }
 211
 212 static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
 213 {
 214         unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
 215         u32 cmd;
 216
 217         do {
 218                 pci_read_config_dword(icm->upstream_port,
 219                                       icm->vnd_cap + PCIE2CIO_CMD, &cmd);
 220                 if (!(cmd & PCIE2CIO_CMD_START)) {
 221                         if (cmd & PCIE2CIO_CMD_TIMEOUT)
 222                                 break;
 223                         return 0;
 224                 }
 225
 226                 msleep(50);
 227         } while (time_before(jiffies, end));
 228
 229         return -ETIMEDOUT;
 230 }
 231
 232 static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
 233                          unsigned int port, unsigned int index, u32 *data)
 234 {
 235         struct pci_dev *pdev = icm->upstream_port;
 236         int ret, vnd_cap = icm->vnd_cap;
 237         u32 cmd;
 238
 239         cmd = index;
 240         cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 241         cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 242         cmd |= PCIE2CIO_CMD_START;
 243         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 244
 245         ret = pci2cio_wait_completion(icm, 5000);
 246         if (ret)
 247                 return ret;
 248
 249         pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
 250         return 0;
 251 }
 252
 253 static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
 254                           unsigned int port, unsigned int index, u32 data)
 255 {
 256         struct pci_dev *pdev = icm->upstream_port;
 257         int vnd_cap = icm->vnd_cap;
 258         u32 cmd;
 259
 260         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
 261
 262         cmd = index;
 263         cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
 264         cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
 265         cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
 266         pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
 267
 268         return pci2cio_wait_completion(icm, 5000);
 269 }
 270
 271 static bool icm_match(const struct tb_cfg_request *req,
 272                       const struct ctl_pkg *pkg)
 273 {
 274         const struct icm_pkg_header *res_hdr = pkg->buffer;
 275         const struct icm_pkg_header *req_hdr = req->request;
 276
 277         if (pkg->frame.eof != req->response_type)
 278                 return false;
 279         if (res_hdr->code != req_hdr->code)
 280                 return false;
 281
 282         return true;
 283 }
 284
 285 static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
 286 {
 287         const struct icm_pkg_header *hdr = pkg->buffer;
 288
 289         if (hdr->packet_id < req->npackets) {
 290                 size_t offset = hdr->packet_id * req->response_size;
 291
 292                 memcpy(req->response + offset, pkg->buffer, req->response_size);
 293         }
 294
 295         return hdr->packet_id == hdr->total_packets - 1;
 296 }
 297
 298 static int icm_request(struct tb *tb, const void *request, size_t request_size,
 299                        void *response, size_t response_size, size_t npackets,
 300                        int retries, unsigned int timeout_msec)
 301 {
 302         struct icm *icm = tb_priv(tb);
 303
 304         do {
 305                 struct tb_cfg_request *req;
 306                 struct tb_cfg_result res;
 307
 308                 req = tb_cfg_request_alloc();
 309                 if (!req)
 310                         return -ENOMEM;
 311
 312                 req->match = icm_match;
 313                 req->copy = icm_copy;
 314                 req->request = request;
 315                 req->request_size = request_size;
 316                 req->request_type = TB_CFG_PKG_ICM_CMD;
 317                 req->response = response;
 318                 req->npackets = npackets;
 319                 req->response_size = response_size;
 320                 req->response_type = TB_CFG_PKG_ICM_RESP;
 321
 322                 mutex_lock(&icm->request_lock);
 323                 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
 324                 mutex_unlock(&icm->request_lock);
 325
 326                 tb_cfg_request_put(req);
 327
 328                 if (res.err != -ETIMEDOUT)
 329                         return res.err == 1 ? -EIO : res.err;
 330
 331                 usleep_range(20, 50);
 332         } while (retries--);
 333
 334         return -ETIMEDOUT;
 335 }
 336
 337 /*
 338  * If rescan is queued to run (we are resuming), postpone it to give the
 339  * firmware some more time to send device connected notifications for next
 340  * devices in the chain.
 341  */
 342 static void icm_postpone_rescan(struct tb *tb)
 343 {
 344         struct icm *icm = tb_priv(tb);
 345
 346         if (delayed_work_pending(&icm->rescan_work))
 347                 mod_delayed_work(tb->wq, &icm->rescan_work,
 348                                  msecs_to_jiffies(500));
 349 }
 350
 351 static void icm_veto_begin(struct tb *tb)
 352 {
 353         struct icm *icm = tb_priv(tb);
 354
 355         if (!icm->veto) {
 356                 icm->veto = true;
 357                 /* Keep the domain powered while veto is in effect */
 358                 pm_runtime_get(&tb->dev);
 359         }
 360 }
 361
 362 static void icm_veto_end(struct tb *tb)
 363 {
 364         struct icm *icm = tb_priv(tb);
 365
 366         if (icm->veto) {
 367                 icm->veto = false;
 368                 /* Allow the domain suspend now */
 369                 pm_runtime_mark_last_busy(&tb->dev);
 370                 pm_runtime_put_autosuspend(&tb->dev);
 371         }
 372 }
 373
 374 static bool icm_firmware_running(const struct tb_nhi *nhi)
 375 {
 376         u32 val;
 377
 378         val = ioread32(nhi->iobase + REG_FW_STS);
 379         return !!(val & REG_FW_STS_ICM_EN);
 380 }
 381
 382 static bool icm_fr_is_supported(struct tb *tb)
 383 {
 384         return !x86_apple_machine;
 385 }
 386
 387 static inline int icm_fr_get_switch_index(u32 port)
 388 {
 389         int index;
 390
 391         if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
 392                 return 0;
 393
 394         index = port >> ICM_PORT_INDEX_SHIFT;
 395         return index != 0xff ? index : 0;
 396 }
 397
 398 static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
 399 {
 400         struct icm_fr_pkg_get_topology_response *switches, *sw;
 401         struct icm_fr_pkg_get_topology request = {
 402                 .hdr = { .code = ICM_GET_TOPOLOGY },
 403         };
 404         size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
 405         int ret, index;
 406         u8 i;
 407
 408         switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
 409         if (!switches)
 410                 return -ENOMEM;
 411
 412         ret = icm_request(tb, &request, sizeof(request), switches,
 413                           sizeof(*switches), npackets, ICM_RETRIES, ICM_TIMEOUT);
 414         if (ret)
 415                 goto err_free;
 416
 417         sw = &switches[0];
 418         index = icm_fr_get_switch_index(sw->ports[link]);
 419         if (!index) {
 420                 ret = -ENODEV;
 421                 goto err_free;
 422         }
 423
 424         sw = &switches[index];
 425         for (i = 1; i < depth; i++) {
 426                 unsigned int j;
 427
 428                 if (!(sw->first_data & ICM_SWITCH_USED)) {
 429                         ret = -ENODEV;
 430                         goto err_free;
 431                 }
 432
 433                 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
 434                         index = icm_fr_get_switch_index(sw->ports[j]);
 435                         if (index > sw->switch_index) {
 436                                 sw = &switches[index];
 437                                 break;
 438                         }
 439                 }
 440         }
 441
 442         *route = get_route(sw->route_hi, sw->route_lo);
 443
 444 err_free:
 445         kfree(switches);
 446         return ret;
 447 }
 448
 449 static void icm_fr_save_devices(struct tb *tb)
 450 {
 451         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
 452 }
 453
 454 static int
 455 icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
 456                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
 457 {
 458         struct icm_fr_pkg_driver_ready_response reply;
 459         struct icm_pkg_driver_ready request = {
 460                 .hdr.code = ICM_DRIVER_READY,
 461         };
 462         int ret;
 463
 464         memset(&reply, 0, sizeof(reply));
 465         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 466                           1, ICM_RETRIES, ICM_TIMEOUT);
 467         if (ret)
 468                 return ret;
 469
 470         if (security_level)
 471                 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
 472
 473         return 0;
 474 }
 475
 476 static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
 477 {
 478         struct icm_fr_pkg_approve_device request;
 479         struct icm_fr_pkg_approve_device reply;
 480         int ret;
 481
 482         memset(&request, 0, sizeof(request));
 483         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 484         request.hdr.code = ICM_APPROVE_DEVICE;
 485         request.connection_id = sw->connection_id;
 486         request.connection_key = sw->connection_key;
 487
 488         memset(&reply, 0, sizeof(reply));
 489         /* Use larger timeout as establishing tunnels can take some time */
 490         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 491                           1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
 492         if (ret)
 493                 return ret;
 494
 495         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 496                 tb_warn(tb, "PCIe tunnel creation failed\n");
 497                 return -EIO;
 498         }
 499
 500         return 0;
 501 }
 502
 503 static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
 504 {
 505         struct icm_fr_pkg_add_device_key request;
 506         struct icm_fr_pkg_add_device_key_response reply;
 507         int ret;
 508
 509         memset(&request, 0, sizeof(request));
 510         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 511         request.hdr.code = ICM_ADD_DEVICE_KEY;
 512         request.connection_id = sw->connection_id;
 513         request.connection_key = sw->connection_key;
 514         memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
 515
 516         memset(&reply, 0, sizeof(reply));
 517         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 518                           1, ICM_RETRIES, ICM_TIMEOUT);
 519         if (ret)
 520                 return ret;
 521
 522         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
 523                 tb_warn(tb, "Adding key to switch failed\n");
 524                 return -EIO;
 525         }
 526
 527         return 0;
 528 }
 529
 530 static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
 531                                        const u8 *challenge, u8 *response)
 532 {
 533         struct icm_fr_pkg_challenge_device request;
 534         struct icm_fr_pkg_challenge_device_response reply;
 535         int ret;
 536
 537         memset(&request, 0, sizeof(request));
 538         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
 539         request.hdr.code = ICM_CHALLENGE_DEVICE;
 540         request.connection_id = sw->connection_id;
 541         request.connection_key = sw->connection_key;
 542         memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
 543
 544         memset(&reply, 0, sizeof(reply));
 545         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 546                           1, ICM_RETRIES, ICM_TIMEOUT);
 547         if (ret)
 548                 return ret;
 549
 550         if (reply.hdr.flags & ICM_FLAGS_ERROR)
 551                 return -EKEYREJECTED;
 552         if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
 553                 return -ENOKEY;
 554
 555         memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
 556
 557         return 0;
 558 }
 559
 560 static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 561                                         int transmit_path, int transmit_ring,
 562                                         int receive_path, int receive_ring)
 563 {
 564         struct icm_fr_pkg_approve_xdomain_response reply;
 565         struct icm_fr_pkg_approve_xdomain request;
 566         int ret;
 567
 568         memset(&request, 0, sizeof(request));
 569         request.hdr.code = ICM_APPROVE_XDOMAIN;
 570         request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
 571         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
 572
 573         request.transmit_path = transmit_path;
 574         request.transmit_ring = transmit_ring;
 575         request.receive_path = receive_path;
 576         request.receive_ring = receive_ring;
 577
 578         memset(&reply, 0, sizeof(reply));
 579         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
 580                           1, ICM_RETRIES, ICM_TIMEOUT);
 581         if (ret)
 582                 return ret;
 583
 584         if (reply.hdr.flags & ICM_FLAGS_ERROR)
 585                 return -EIO;
 586
 587         return 0;
 588 }
 589
 590 static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 591                                            int transmit_path, int transmit_ring,
 592                                            int receive_path, int receive_ring)
 593 {
 594         u8 phy_port;
 595         u8 cmd;
 596
 597         phy_port = tb_phy_port_from_link(xd->link);
 598         if (phy_port == 0)
 599                 cmd = NHI_MAILBOX_DISCONNECT_PA;
 600         else
 601                 cmd = NHI_MAILBOX_DISCONNECT_PB;
 602
 603         nhi_mailbox_cmd(tb->nhi, cmd, 1);
 604         usleep_range(10, 50);
 605         nhi_mailbox_cmd(tb->nhi, cmd, 2);
 606         return 0;
 607 }
 608
 609 static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
 610                                       const uuid_t *uuid)
 611 {
 612         struct tb *tb = parent_sw->tb;
 613         struct tb_switch *sw;
 614
 615         sw = tb_switch_alloc(tb, &parent_sw->dev, route);
 616         if (IS_ERR(sw)) {
 617                 tb_warn(tb, "failed to allocate switch at %llx\n", route);
 618                 return sw;
 619         }
 620
 621         sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
 622         if (!sw->uuid) {
 623                 tb_switch_put(sw);
 624                 return ERR_PTR(-ENOMEM);
 625         }
 626
 627         init_completion(&sw->rpm_complete);
 628         return sw;
 629 }
 630
 631 static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
 632 {
 633         u64 route = tb_route(sw);
 634         int ret;
 635
 636         /* Link the two switches now */
 637         tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 638         tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
 639
 640         ret = tb_switch_add(sw);
 641         if (ret)
 642                 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
 643
 644         return ret;
 645 }
 646
 647 static void update_switch(struct tb_switch *sw, u64 route, u8 connection_id,
 648                           u8 connection_key, u8 link, u8 depth, bool boot)
 649 {
 650         struct tb_switch *parent_sw = tb_switch_parent(sw);
 651
 652         /* Disconnect from parent */
 653         tb_switch_downstream_port(sw)->remote = NULL;
 654         /* Re-connect via updated port */
 655         tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
 656
 657         /* Update with the new addressing information */
 658         sw->config.route_hi = upper_32_bits(route);
 659         sw->config.route_lo = lower_32_bits(route);
 660         sw->connection_id = connection_id;
 661         sw->connection_key = connection_key;
 662         sw->link = link;
 663         sw->depth = depth;
 664         sw->boot = boot;
 665
 666         /* This switch still exists */
 667         sw->is_unplugged = false;
 668
 669         /* Runtime resume is now complete */
 670         complete(&sw->rpm_complete);
 671 }
 672
 673 static void remove_switch(struct tb_switch *sw)
 674 {
 675         tb_switch_downstream_port(sw)->remote = NULL;
 676         tb_switch_remove(sw);
 677 }
 678
 679 static void add_xdomain(struct tb_switch *sw, u64 route,
 680                         const uuid_t *local_uuid, const uuid_t *remote_uuid,
 681                         u8 link, u8 depth)
 682 {
 683         struct tb_xdomain *xd;
 684
 685         pm_runtime_get_sync(&sw->dev);
 686
 687         xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
 688         if (!xd)
 689                 goto out;
 690
 691         xd->link = link;
 692         xd->depth = depth;
 693
 694         tb_port_at(route, sw)->xdomain = xd;
 695
 696         tb_xdomain_add(xd);
 697
 698 out:
 699         pm_runtime_mark_last_busy(&sw->dev);
 700         pm_runtime_put_autosuspend(&sw->dev);
 701 }
 702
 703 static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
 704 {
 705         xd->link = link;
 706         xd->route = route;
 707         xd->is_unplugged = false;
 708 }
 709
 710 static void remove_xdomain(struct tb_xdomain *xd)
 711 {
 712         struct tb_switch *sw;
 713
 714         sw = tb_to_switch(xd->dev.parent);
 715         tb_port_at(xd->route, sw)->xdomain = NULL;
 716         tb_xdomain_remove(xd);
 717 }
 718
 719 static void
 720 icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 721 {
 722         const struct icm_fr_event_device_connected *pkg =
 723                 (const struct icm_fr_event_device_connected *)hdr;
 724         enum tb_security_level security_level;
 725         struct tb_switch *sw, *parent_sw;
 726         bool boot, dual_lane, speed_gen3;
 727         struct icm *icm = tb_priv(tb);
 728         bool authorized = false;
 729         struct tb_xdomain *xd;
 730         u8 link, depth;
 731         u64 route;
 732         int ret;
 733
 734         icm_postpone_rescan(tb);
 735
 736         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 737         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 738                 ICM_LINK_INFO_DEPTH_SHIFT;
 739         authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
 740         security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
 741                          ICM_FLAGS_SLEVEL_SHIFT;
 742         boot = pkg->link_info & ICM_LINK_INFO_BOOT;
 743         dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
 744         speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
 745
 746         if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
 747                 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
 748                         link, depth);
 749                 return;
 750         }
 751
 752         sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
 753         if (sw) {
 754                 u8 phy_port, sw_phy_port;
 755
 756                 sw_phy_port = tb_phy_port_from_link(sw->link);
 757                 phy_port = tb_phy_port_from_link(link);
 758
 759                 /*
 760                  * On resume ICM will send us connected events for the
 761                  * devices that still are present. However, that
 762                  * information might have changed for example by the
 763                  * fact that a switch on a dual-link connection might
 764                  * have been enumerated using the other link now. Make
 765                  * sure our book keeping matches that.
 766                  */
 767                 if (sw->depth == depth && sw_phy_port == phy_port &&
 768                     !!sw->authorized == authorized) {
 769                         /*
 770                          * It was enumerated through another link so update
 771                          * route string accordingly.
 772                          */
 773                         if (sw->link != link) {
 774                                 ret = icm->get_route(tb, link, depth, &route);
 775                                 if (ret) {
 776                                         tb_err(tb, "failed to update route string for switch at %u.%u\n",
 777                                                link, depth);
 778                                         tb_switch_put(sw);
 779                                         return;
 780                                 }
 781                         } else {
 782                                 route = tb_route(sw);
 783                         }
 784
 785                         update_switch(sw, route, pkg->connection_id,
 786                                       pkg->connection_key, link, depth, boot);
 787                         tb_switch_put(sw);
 788                         return;
 789                 }
 790
 791                 /*
 792                  * User connected the same switch to another physical
 793                  * port or to another part of the topology. Remove the
 794                  * existing switch now before adding the new one.
 795                  */
 796                 remove_switch(sw);
 797                 tb_switch_put(sw);
 798         }
 799
 800         /*
 801          * If the switch was not found by UUID, look for a switch on
 802          * same physical port (taking possible link aggregation into
 803          * account) and depth. If we found one it is definitely a stale
 804          * one so remove it first.
 805          */
 806         sw = tb_switch_find_by_link_depth(tb, link, depth);
 807         if (!sw) {
 808                 u8 dual_link;
 809
 810                 dual_link = dual_link_from_link(link);
 811                 if (dual_link)
 812                         sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
 813         }
 814         if (sw) {
 815                 remove_switch(sw);
 816                 tb_switch_put(sw);
 817         }
 818
 819         /* Remove existing XDomain connection if found */
 820         xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 821         if (xd) {
 822                 remove_xdomain(xd);
 823                 tb_xdomain_put(xd);
 824         }
 825
 826         parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
 827         if (!parent_sw) {
 828                 tb_err(tb, "failed to find parent switch for %u.%u\n",
 829                        link, depth);
 830                 return;
 831         }
 832
 833         ret = icm->get_route(tb, link, depth, &route);
 834         if (ret) {
 835                 tb_err(tb, "failed to find route string for switch at %u.%u\n",
 836                        link, depth);
 837                 tb_switch_put(parent_sw);
 838                 return;
 839         }
 840
 841         pm_runtime_get_sync(&parent_sw->dev);
 842
 843         sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
 844         if (!IS_ERR(sw)) {
 845                 sw->connection_id = pkg->connection_id;
 846                 sw->connection_key = pkg->connection_key;
 847                 sw->link = link;
 848                 sw->depth = depth;
 849                 sw->authorized = authorized;
 850                 sw->security_level = security_level;
 851                 sw->boot = boot;
 852                 sw->link_speed = speed_gen3 ? 20 : 10;
 853                 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
 854                                              TB_LINK_WIDTH_SINGLE;
 855                 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
 856
 857                 if (add_switch(parent_sw, sw))
 858                         tb_switch_put(sw);
 859         }
 860
 861         pm_runtime_mark_last_busy(&parent_sw->dev);
 862         pm_runtime_put_autosuspend(&parent_sw->dev);
 863
 864         tb_switch_put(parent_sw);
 865 }
 866
 867 static void
 868 icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
 869 {
 870         const struct icm_fr_event_device_disconnected *pkg =
 871                 (const struct icm_fr_event_device_disconnected *)hdr;
 872         struct tb_switch *sw;
 873         u8 link, depth;
 874
 875         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 876         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 877                 ICM_LINK_INFO_DEPTH_SHIFT;
 878
 879         if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 880                 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 881                 return;
 882         }
 883
 884         sw = tb_switch_find_by_link_depth(tb, link, depth);
 885         if (!sw) {
 886                 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
 887                         depth);
 888                 return;
 889         }
 890
 891         pm_runtime_get_sync(sw->dev.parent);
 892
 893         remove_switch(sw);
 894
 895         pm_runtime_mark_last_busy(sw->dev.parent);
 896         pm_runtime_put_autosuspend(sw->dev.parent);
 897
 898         tb_switch_put(sw);
 899 }
 900
 901 static void
 902 icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
 903 {
 904         const struct icm_fr_event_xdomain_connected *pkg =
 905                 (const struct icm_fr_event_xdomain_connected *)hdr;
 906         struct tb_xdomain *xd;
 907         struct tb_switch *sw;
 908         u8 link, depth;
 909         u64 route;
 910
 911         link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
 912         depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
 913                 ICM_LINK_INFO_DEPTH_SHIFT;
 914
 915         if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
 916                 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
 917                 return;
 918         }
 919
 920         route = get_route(pkg->local_route_hi, pkg->local_route_lo);
 921
 922         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
 923         if (xd) {
 924                 u8 xd_phy_port, phy_port;
 925
 926                 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
 927                 phy_port = phy_port_from_route(route, depth);
 928
 929                 if (xd->depth == depth && xd_phy_port == phy_port) {
 930                         update_xdomain(xd, route, link);
 931                         tb_xdomain_put(xd);
 932                         return;
 933                 }
 934
 935                 /*
 936                  * If we find an existing XDomain connection remove it
 937                  * now. We need to go through login handshake and
 938                  * everything anyway to be able to re-establish the
 939                  * connection.
 940                  */
 941                 remove_xdomain(xd);
 942                 tb_xdomain_put(xd);
 943         }
 944
 945         /*
 946          * Look if there already exists an XDomain in the same place
 947          * than the new one and in that case remove it because it is
 948          * most likely another host that got disconnected.
 949          */
 950         xd = tb_xdomain_find_by_link_depth(tb, link, depth);
 951         if (!xd) {
 952                 u8 dual_link;
 953
 954                 dual_link = dual_link_from_link(link);
 955                 if (dual_link)
 956                         xd = tb_xdomain_find_by_link_depth(tb, dual_link,
 957                                                            depth);
 958         }
 959         if (xd) {
 960                 remove_xdomain(xd);
 961                 tb_xdomain_put(xd);
 962         }
 963
 964         /*
 965          * If the user disconnected a switch during suspend and
 966          * connected another host to the same port, remove the switch
 967          * first.
 968          */
 969         sw = tb_switch_find_by_route(tb, route);
 970         if (sw) {
 971                 remove_switch(sw);
 972                 tb_switch_put(sw);
 973         }
 974
 975         sw = tb_switch_find_by_link_depth(tb, link, depth);
 976         if (!sw) {
 977                 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
 978                         depth);
 979                 return;
 980         }
 981
 982         add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
 983                     depth);
 984         tb_switch_put(sw);
 985 }
 986
 987 static void
 988 icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
 989 {
 990         const struct icm_fr_event_xdomain_disconnected *pkg =
 991                 (const struct icm_fr_event_xdomain_disconnected *)hdr;
 992         struct tb_xdomain *xd;
 993
 994         /*
 995          * If the connection is through one or multiple devices, the
 996          * XDomain device is removed along with them so it is fine if we
 997          * cannot find it here.
 998          */
 999         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1000         if (xd) {
1001                 remove_xdomain(xd);
1002                 tb_xdomain_put(xd);
1003         }
1004 }
1005
1006 static int icm_tr_cio_reset(struct tb *tb)
1007 {
1008         return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1009 }
1010
1011 static int
1012 icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1013                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1014 {
1015         struct icm_tr_pkg_driver_ready_response reply;
1016         struct icm_pkg_driver_ready request = {
1017                 .hdr.code = ICM_DRIVER_READY,
1018         };
1019         int ret;
1020
1021         memset(&reply, 0, sizeof(reply));
1022         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1023                           1, 10, 250);
1024         if (ret)
1025                 return ret;
1026
1027         if (security_level)
1028                 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1029         if (proto_version)
1030                 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1031                                 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1032         if (nboot_acl)
1033                 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1034                                 ICM_TR_INFO_BOOT_ACL_SHIFT;
1035         if (rpm)
1036                 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1037
1038         return 0;
1039 }
1040
1041 static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1042 {
1043         struct icm_tr_pkg_approve_device request;
1044         struct icm_tr_pkg_approve_device reply;
1045         int ret;
1046
1047         memset(&request, 0, sizeof(request));
1048         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1049         request.hdr.code = ICM_APPROVE_DEVICE;
1050         request.route_lo = sw->config.route_lo;
1051         request.route_hi = sw->config.route_hi;
1052         request.connection_id = sw->connection_id;
1053
1054         memset(&reply, 0, sizeof(reply));
1055         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1056                           1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);
1057         if (ret)
1058                 return ret;
1059
1060         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1061                 tb_warn(tb, "PCIe tunnel creation failed\n");
1062                 return -EIO;
1063         }
1064
1065         return 0;
1066 }
1067
1068 static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1069 {
1070         struct icm_tr_pkg_add_device_key_response reply;
1071         struct icm_tr_pkg_add_device_key request;
1072         int ret;
1073
1074         memset(&request, 0, sizeof(request));
1075         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1076         request.hdr.code = ICM_ADD_DEVICE_KEY;
1077         request.route_lo = sw->config.route_lo;
1078         request.route_hi = sw->config.route_hi;
1079         request.connection_id = sw->connection_id;
1080         memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1081
1082         memset(&reply, 0, sizeof(reply));
1083         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1084                           1, ICM_RETRIES, ICM_TIMEOUT);
1085         if (ret)
1086                 return ret;
1087
1088         if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1089                 tb_warn(tb, "Adding key to switch failed\n");
1090                 return -EIO;
1091         }
1092
1093         return 0;
1094 }
1095
1096 static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1097                                        const u8 *challenge, u8 *response)
1098 {
1099         struct icm_tr_pkg_challenge_device_response reply;
1100         struct icm_tr_pkg_challenge_device request;
1101         int ret;
1102
1103         memset(&request, 0, sizeof(request));
1104         memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1105         request.hdr.code = ICM_CHALLENGE_DEVICE;
1106         request.route_lo = sw->config.route_lo;
1107         request.route_hi = sw->config.route_hi;
1108         request.connection_id = sw->connection_id;
1109         memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1110
1111         memset(&reply, 0, sizeof(reply));
1112         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1113                           1, ICM_RETRIES, ICM_TIMEOUT);
1114         if (ret)
1115                 return ret;
1116
1117         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1118                 return -EKEYREJECTED;
1119         if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1120                 return -ENOKEY;
1121
1122         memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1123
1124         return 0;
1125 }
1126
1127 static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1128                                         int transmit_path, int transmit_ring,
1129                                         int receive_path, int receive_ring)
1130 {
1131         struct icm_tr_pkg_approve_xdomain_response reply;
1132         struct icm_tr_pkg_approve_xdomain request;
1133         int ret;
1134
1135         memset(&request, 0, sizeof(request));
1136         request.hdr.code = ICM_APPROVE_XDOMAIN;
1137         request.route_hi = upper_32_bits(xd->route);
1138         request.route_lo = lower_32_bits(xd->route);
1139         request.transmit_path = transmit_path;
1140         request.transmit_ring = transmit_ring;
1141         request.receive_path = receive_path;
1142         request.receive_ring = receive_ring;
1143         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1144
1145         memset(&reply, 0, sizeof(reply));
1146         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1147                           1, ICM_RETRIES, ICM_TIMEOUT);
1148         if (ret)
1149                 return ret;
1150
1151         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1152                 return -EIO;
1153
1154         return 0;
1155 }
1156
1157 static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1158                                     int stage)
1159 {
1160         struct icm_tr_pkg_disconnect_xdomain_response reply;
1161         struct icm_tr_pkg_disconnect_xdomain request;
1162         int ret;
1163
1164         memset(&request, 0, sizeof(request));
1165         request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1166         request.stage = stage;
1167         request.route_hi = upper_32_bits(xd->route);
1168         request.route_lo = lower_32_bits(xd->route);
1169         memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1170
1171         memset(&reply, 0, sizeof(reply));
1172         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1173                           1, ICM_RETRIES, ICM_TIMEOUT);
1174         if (ret)
1175                 return ret;
1176
1177         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1178                 return -EIO;
1179
1180         return 0;
1181 }
1182
1183 static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
1184                                            int transmit_path, int transmit_ring,
1185                                            int receive_path, int receive_ring)
1186 {
1187         int ret;
1188
1189         ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1190         if (ret)
1191                 return ret;
1192
1193         usleep_range(10, 50);
1194         return icm_tr_xdomain_tear_down(tb, xd, 2);
1195 }
1196
1197 static void
1198 __icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1199                           bool force_rtd3)
1200 {
1201         const struct icm_tr_event_device_connected *pkg =
1202                 (const struct icm_tr_event_device_connected *)hdr;
1203         bool authorized, boot, dual_lane, speed_gen3;
1204         enum tb_security_level security_level;
1205         struct tb_switch *sw, *parent_sw;
1206         struct tb_xdomain *xd;
1207         u64 route;
1208
1209         icm_postpone_rescan(tb);
1210
1211         /*
1212          * Currently we don't use the QoS information coming with the
1213          * device connected message so simply just ignore that extra
1214          * packet for now.
1215          */
1216         if (pkg->hdr.packet_id)
1217                 return;
1218
1219         route = get_route(pkg->route_hi, pkg->route_lo);
1220         authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1221         security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1222                          ICM_FLAGS_SLEVEL_SHIFT;
1223         boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1224         dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1225         speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1226
1227         if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1228                 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1229                         route);
1230                 return;
1231         }
1232
1233         sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1234         if (sw) {
1235                 /* Update the switch if it is still in the same place */
1236                 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1237                         update_switch(sw, route, pkg->connection_id, 0, 0, 0,
1238                                       boot);
1239                         tb_switch_put(sw);
1240                         return;
1241                 }
1242
1243                 remove_switch(sw);
1244                 tb_switch_put(sw);
1245         }
1246
1247         /* Another switch with the same address */
1248         sw = tb_switch_find_by_route(tb, route);
1249         if (sw) {
1250                 remove_switch(sw);
1251                 tb_switch_put(sw);
1252         }
1253
1254         /* XDomain connection with the same address */
1255         xd = tb_xdomain_find_by_route(tb, route);
1256         if (xd) {
1257                 remove_xdomain(xd);
1258                 tb_xdomain_put(xd);
1259         }
1260
1261         parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1262         if (!parent_sw) {
1263                 tb_err(tb, "failed to find parent switch for %llx\n", route);
1264                 return;
1265         }
1266
1267         pm_runtime_get_sync(&parent_sw->dev);
1268
1269         sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1270         if (!IS_ERR(sw)) {
1271                 sw->connection_id = pkg->connection_id;
1272                 sw->authorized = authorized;
1273                 sw->security_level = security_level;
1274                 sw->boot = boot;
1275                 sw->link_speed = speed_gen3 ? 20 : 10;
1276                 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :
1277                                              TB_LINK_WIDTH_SINGLE;
1278                 sw->rpm = force_rtd3;
1279                 if (!sw->rpm)
1280                         sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1281                                                     sizeof(pkg->ep_name));
1282
1283                 if (add_switch(parent_sw, sw))
1284                         tb_switch_put(sw);
1285         }
1286
1287         pm_runtime_mark_last_busy(&parent_sw->dev);
1288         pm_runtime_put_autosuspend(&parent_sw->dev);
1289
1290         tb_switch_put(parent_sw);
1291 }
1292
1293 static void
1294 icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1295 {
1296         __icm_tr_device_connected(tb, hdr, false);
1297 }
1298
1299 static void
1300 icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1301 {
1302         const struct icm_tr_event_device_disconnected *pkg =
1303                 (const struct icm_tr_event_device_disconnected *)hdr;
1304         struct tb_switch *sw;
1305         u64 route;
1306
1307         route = get_route(pkg->route_hi, pkg->route_lo);
1308
1309         sw = tb_switch_find_by_route(tb, route);
1310         if (!sw) {
1311                 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1312                 return;
1313         }
1314         pm_runtime_get_sync(sw->dev.parent);
1315
1316         remove_switch(sw);
1317
1318         pm_runtime_mark_last_busy(sw->dev.parent);
1319         pm_runtime_put_autosuspend(sw->dev.parent);
1320
1321         tb_switch_put(sw);
1322 }
1323
1324 static void
1325 icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1326 {
1327         const struct icm_tr_event_xdomain_connected *pkg =
1328                 (const struct icm_tr_event_xdomain_connected *)hdr;
1329         struct tb_xdomain *xd;
1330         struct tb_switch *sw;
1331         u64 route;
1332
1333         if (!tb->root_switch)
1334                 return;
1335
1336         route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1337
1338         xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1339         if (xd) {
1340                 if (xd->route == route) {
1341                         update_xdomain(xd, route, 0);
1342                         tb_xdomain_put(xd);
1343                         return;
1344                 }
1345
1346                 remove_xdomain(xd);
1347                 tb_xdomain_put(xd);
1348         }
1349
1350         /* An existing xdomain with the same address */
1351         xd = tb_xdomain_find_by_route(tb, route);
1352         if (xd) {
1353                 remove_xdomain(xd);
1354                 tb_xdomain_put(xd);
1355         }
1356
1357         /*
1358          * If the user disconnected a switch during suspend and
1359          * connected another host to the same port, remove the switch
1360          * first.
1361          */
1362         sw = tb_switch_find_by_route(tb, route);
1363         if (sw) {
1364                 remove_switch(sw);
1365                 tb_switch_put(sw);
1366         }
1367
1368         sw = tb_switch_find_by_route(tb, get_parent_route(route));
1369         if (!sw) {
1370                 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1371                 return;
1372         }
1373
1374         add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1375         tb_switch_put(sw);
1376 }
1377
1378 static void
1379 icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1380 {
1381         const struct icm_tr_event_xdomain_disconnected *pkg =
1382                 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1383         struct tb_xdomain *xd;
1384         u64 route;
1385
1386         route = get_route(pkg->route_hi, pkg->route_lo);
1387
1388         xd = tb_xdomain_find_by_route(tb, route);
1389         if (xd) {
1390                 remove_xdomain(xd);
1391                 tb_xdomain_put(xd);
1392         }
1393 }
1394
1395 static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1396 {
1397         struct pci_dev *parent;
1398
1399         parent = pci_upstream_bridge(pdev);
1400         while (parent) {
1401                 if (!pci_is_pcie(parent))
1402                         return NULL;
1403                 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1404                         break;
1405                 parent = pci_upstream_bridge(parent);
1406         }
1407
1408         if (!parent)
1409                 return NULL;
1410
1411         switch (parent->device) {
1412         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1413         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1414         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1415         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1416         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1417         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1418         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1419                 return parent;
1420         }
1421
1422         return NULL;
1423 }
1424
1425 static bool icm_ar_is_supported(struct tb *tb)
1426 {
1427         struct pci_dev *upstream_port;
1428         struct icm *icm = tb_priv(tb);
1429
1430         /*
1431          * Starting from Alpine Ridge we can use ICM on Apple machines
1432          * as well. We just need to reset and re-enable it first.
1433          * However, only start it if explicitly asked by the user.
1434          */
1435         if (icm_firmware_running(tb->nhi))
1436                 return true;
1437         if (!start_icm)
1438                 return false;
1439
1440         /*
1441          * Find the upstream PCIe port in case we need to do reset
1442          * through its vendor specific registers.
1443          */
1444         upstream_port = get_upstream_port(tb->nhi->pdev);
1445         if (upstream_port) {
1446                 int cap;
1447
1448                 cap = pci_find_ext_capability(upstream_port,
1449                                               PCI_EXT_CAP_ID_VNDR);
1450                 if (cap > 0) {
1451                         icm->upstream_port = upstream_port;
1452                         icm->vnd_cap = cap;
1453
1454                         return true;
1455                 }
1456         }
1457
1458         return false;
1459 }
1460
1461 static int icm_ar_cio_reset(struct tb *tb)
1462 {
1463         return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1464 }
1465
1466 static int icm_ar_get_mode(struct tb *tb)
1467 {
1468         struct tb_nhi *nhi = tb->nhi;
1469         int retries = 60;
1470         u32 val;
1471
1472         do {
1473                 val = ioread32(nhi->iobase + REG_FW_STS);
1474                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1475                         break;
1476                 msleep(50);
1477         } while (--retries);
1478
1479         if (!retries) {
1480                 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1481                 return -ENODEV;
1482         }
1483
1484         return nhi_mailbox_mode(nhi);
1485 }
1486
1487 static int
1488 icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1489                     u8 *proto_version, size_t *nboot_acl, bool *rpm)
1490 {
1491         struct icm_ar_pkg_driver_ready_response reply;
1492         struct icm_pkg_driver_ready request = {
1493                 .hdr.code = ICM_DRIVER_READY,
1494         };
1495         int ret;
1496
1497         memset(&reply, 0, sizeof(reply));
1498         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1499                           1, ICM_RETRIES, ICM_TIMEOUT);
1500         if (ret)
1501                 return ret;
1502
1503         if (security_level)
1504                 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1505         if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1506                 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1507                                 ICM_AR_INFO_BOOT_ACL_SHIFT;
1508         if (rpm)
1509                 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1510
1511         return 0;
1512 }
1513
1514 static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1515 {
1516         struct icm_ar_pkg_get_route_response reply;
1517         struct icm_ar_pkg_get_route request = {
1518                 .hdr = { .code = ICM_GET_ROUTE },
1519                 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1520         };
1521         int ret;
1522
1523         memset(&reply, 0, sizeof(reply));
1524         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1525                           1, ICM_RETRIES, ICM_TIMEOUT);
1526         if (ret)
1527                 return ret;
1528
1529         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1530                 return -EIO;
1531
1532         *route = get_route(reply.route_hi, reply.route_lo);
1533         return 0;
1534 }
1535
1536 static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1537 {
1538         struct icm_ar_pkg_preboot_acl_response reply;
1539         struct icm_ar_pkg_preboot_acl request = {
1540                 .hdr = { .code = ICM_PREBOOT_ACL },
1541         };
1542         int ret, i;
1543
1544         memset(&reply, 0, sizeof(reply));
1545         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1546                           1, ICM_RETRIES, ICM_TIMEOUT);
1547         if (ret)
1548                 return ret;
1549
1550         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1551                 return -EIO;
1552
1553         for (i = 0; i < nuuids; i++) {
1554                 u32 *uuid = (u32 *)&uuids[i];
1555
1556                 uuid[0] = reply.acl[i].uuid_lo;
1557                 uuid[1] = reply.acl[i].uuid_hi;
1558
1559                 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1560                         /* Map empty entries to null UUID */
1561                         uuid[0] = 0;
1562                         uuid[1] = 0;
1563                 } else if (uuid[0] != 0 || uuid[1] != 0) {
1564                         /* Upper two DWs are always one's */
1565                         uuid[2] = 0xffffffff;
1566                         uuid[3] = 0xffffffff;
1567                 }
1568         }
1569
1570         return ret;
1571 }
1572
1573 static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1574                                size_t nuuids)
1575 {
1576         struct icm_ar_pkg_preboot_acl_response reply;
1577         struct icm_ar_pkg_preboot_acl request = {
1578                 .hdr = {
1579                         .code = ICM_PREBOOT_ACL,
1580                         .flags = ICM_FLAGS_WRITE,
1581                 },
1582         };
1583         int ret, i;
1584
1585         for (i = 0; i < nuuids; i++) {
1586                 const u32 *uuid = (const u32 *)&uuids[i];
1587
1588                 if (uuid_is_null(&uuids[i])) {
1589                         /*
1590                          * Map null UUID to the empty (all one) entries
1591                          * for ICM.
1592                          */
1593                         request.acl[i].uuid_lo = 0xffffffff;
1594                         request.acl[i].uuid_hi = 0xffffffff;
1595                 } else {
1596                         /* Two high DWs need to be set to all one */
1597                         if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1598                                 return -EINVAL;
1599
1600                         request.acl[i].uuid_lo = uuid[0];
1601                         request.acl[i].uuid_hi = uuid[1];
1602                 }
1603         }
1604
1605         memset(&reply, 0, sizeof(reply));
1606         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1607                           1, ICM_RETRIES, ICM_TIMEOUT);
1608         if (ret)
1609                 return ret;
1610
1611         if (reply.hdr.flags & ICM_FLAGS_ERROR)
1612                 return -EIO;
1613
1614         return 0;
1615 }
1616
1617 static int
1618 icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1619                      u8 *proto_version, size_t *nboot_acl, bool *rpm)
1620 {
1621         struct icm_tr_pkg_driver_ready_response reply;
1622         struct icm_pkg_driver_ready request = {
1623                 .hdr.code = ICM_DRIVER_READY,
1624         };
1625         int ret;
1626
1627         memset(&reply, 0, sizeof(reply));
1628         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1629                           1, ICM_RETRIES, 20000);
1630         if (ret)
1631                 return ret;
1632
1633         if (proto_version)
1634                 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1635                                 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1636
1637         /* Ice Lake always supports RTD3 */
1638         if (rpm)
1639                 *rpm = true;
1640
1641         return 0;
1642 }
1643
1644 static void icm_icl_set_uuid(struct tb *tb)
1645 {
1646         struct tb_nhi *nhi = tb->nhi;
1647         u32 uuid[4];
1648
1649         pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1650         pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1651         uuid[2] = 0xffffffff;
1652         uuid[3] = 0xffffffff;
1653
1654         tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1655 }
1656
1657 static void
1658 icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1659 {
1660         __icm_tr_device_connected(tb, hdr, true);
1661 }
1662
1663 static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1664 {
1665         const struct icm_icl_event_rtd3_veto *pkg =
1666                 (const struct icm_icl_event_rtd3_veto *)hdr;
1667
1668         tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1669
1670         if (pkg->veto_reason)
1671                 icm_veto_begin(tb);
1672         else
1673                 icm_veto_end(tb);
1674 }
1675
1676 static bool icm_tgl_is_supported(struct tb *tb)
1677 {
1678         unsigned long end = jiffies + msecs_to_jiffies(10);
1679
1680         do {
1681                 u32 val;
1682
1683                 val = ioread32(tb->nhi->iobase + REG_FW_STS);
1684                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1685                         return true;
1686                 usleep_range(100, 500);
1687         } while (time_before(jiffies, end));
1688
1689         return false;
1690 }
1691
1692 static void icm_handle_notification(struct work_struct *work)
1693 {
1694         struct icm_notification *n = container_of(work, typeof(*n), work);
1695         struct tb *tb = n->tb;
1696         struct icm *icm = tb_priv(tb);
1697
1698         mutex_lock(&tb->lock);
1699
1700         /*
1701          * When the domain is stopped we flush its workqueue but before
1702          * that the root switch is removed. In that case we should treat
1703          * the queued events as being canceled.
1704          */
1705         if (tb->root_switch) {
1706                 switch (n->pkg->code) {
1707                 case ICM_EVENT_DEVICE_CONNECTED:
1708                         icm->device_connected(tb, n->pkg);
1709                         break;
1710                 case ICM_EVENT_DEVICE_DISCONNECTED:
1711                         icm->device_disconnected(tb, n->pkg);
1712                         break;
1713                 case ICM_EVENT_XDOMAIN_CONNECTED:
1714                         if (tb_is_xdomain_enabled())
1715                                 icm->xdomain_connected(tb, n->pkg);
1716                         break;
1717                 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1718                         if (tb_is_xdomain_enabled())
1719                                 icm->xdomain_disconnected(tb, n->pkg);
1720                         break;
1721                 case ICM_EVENT_RTD3_VETO:
1722                         icm->rtd3_veto(tb, n->pkg);
1723                         break;
1724                 }
1725         }
1726
1727         mutex_unlock(&tb->lock);
1728
1729         kfree(n->pkg);
1730         kfree(n);
1731 }
1732
1733 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1734                              const void *buf, size_t size)
1735 {
1736         struct icm_notification *n;
1737
1738         n = kmalloc(sizeof(*n), GFP_KERNEL);
1739         if (!n)
1740                 return;
1741
1742         n->pkg = kmemdup(buf, size, GFP_KERNEL);
1743         if (!n->pkg) {
1744                 kfree(n);
1745                 return;
1746         }
1747
1748         INIT_WORK(&n->work, icm_handle_notification);
1749         n->tb = tb;
1750
1751         queue_work(tb->wq, &n->work);
1752 }
1753
1754 static int
1755 __icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1756                    u8 *proto_version, size_t *nboot_acl, bool *rpm)
1757 {
1758         struct icm *icm = tb_priv(tb);
1759         unsigned int retries = 50;
1760         int ret;
1761
1762         ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1763                                 rpm);
1764         if (ret) {
1765                 tb_err(tb, "failed to send driver ready to ICM\n");
1766                 return ret;
1767         }
1768
1769         /*
1770          * Hold on here until the switch config space is accessible so
1771          * that we can read root switch config successfully.
1772          */
1773         do {
1774                 struct tb_cfg_result res;
1775                 u32 tmp;
1776
1777                 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1778                                       0, 1, 100);
1779                 if (!res.err)
1780                         return 0;
1781
1782                 msleep(50);
1783         } while (--retries);
1784
1785         tb_err(tb, "failed to read root switch config space, giving up\n");
1786         return -ETIMEDOUT;
1787 }
1788
1789 static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1790 {
1791         struct icm *icm = tb_priv(tb);
1792         u32 val;
1793
1794         if (!icm->upstream_port)
1795                 return -ENODEV;
1796
1797         /* Put ARC to wait for CIO reset event to happen */
1798         val = ioread32(nhi->iobase + REG_FW_STS);
1799         val |= REG_FW_STS_CIO_RESET_REQ;
1800         iowrite32(val, nhi->iobase + REG_FW_STS);
1801
1802         /* Re-start ARC */
1803         val = ioread32(nhi->iobase + REG_FW_STS);
1804         val |= REG_FW_STS_ICM_EN_INVERT;
1805         val |= REG_FW_STS_ICM_EN_CPU;
1806         iowrite32(val, nhi->iobase + REG_FW_STS);
1807
1808         /* Trigger CIO reset now */
1809         return icm->cio_reset(tb);
1810 }
1811
1812 static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1813 {
1814         unsigned int retries = 10;
1815         int ret;
1816         u32 val;
1817
1818         /* Check if the ICM firmware is already running */
1819         if (icm_firmware_running(nhi))
1820                 return 0;
1821
1822         dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1823
1824         ret = icm_firmware_reset(tb, nhi);
1825         if (ret)
1826                 return ret;
1827
1828         /* Wait until the ICM firmware tells us it is up and running */
1829         do {
1830                 /* Check that the ICM firmware is running */
1831                 val = ioread32(nhi->iobase + REG_FW_STS);
1832                 if (val & REG_FW_STS_NVM_AUTH_DONE)
1833                         return 0;
1834
1835                 msleep(300);
1836         } while (--retries);
1837
1838         return -ETIMEDOUT;
1839 }
1840
1841 static int icm_reset_phy_port(struct tb *tb, int phy_port)
1842 {
1843         struct icm *icm = tb_priv(tb);
1844         u32 state0, state1;
1845         int port0, port1;
1846         u32 val0, val1;
1847         int ret;
1848
1849         if (!icm->upstream_port)
1850                 return 0;
1851
1852         if (phy_port) {
1853                 port0 = 3;
1854                 port1 = 4;
1855         } else {
1856                 port0 = 1;
1857                 port1 = 2;
1858         }
1859
1860         /*
1861          * Read link status of both null ports belonging to a single
1862          * physical port.
1863          */
1864         ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1865         if (ret)
1866                 return ret;
1867         ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1868         if (ret)
1869                 return ret;
1870
1871         state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1872         state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1873         state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1874         state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1875
1876         /* If they are both up we need to reset them now */
1877         if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1878                 return 0;
1879
1880         val0 |= PHY_PORT_CS1_LINK_DISABLE;
1881         ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1882         if (ret)
1883                 return ret;
1884
1885         val1 |= PHY_PORT_CS1_LINK_DISABLE;
1886         ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1887         if (ret)
1888                 return ret;
1889
1890         /* Wait a bit and then re-enable both ports */
1891         usleep_range(10, 100);
1892
1893         ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1894         if (ret)
1895                 return ret;
1896         ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1897         if (ret)
1898                 return ret;
1899
1900         val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1901         ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1902         if (ret)
1903                 return ret;
1904
1905         val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1906         return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1907 }
1908
1909 static int icm_firmware_init(struct tb *tb)
1910 {
1911         struct icm *icm = tb_priv(tb);
1912         struct tb_nhi *nhi = tb->nhi;
1913         int ret;
1914
1915         ret = icm_firmware_start(tb, nhi);
1916         if (ret) {
1917                 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1918                 return ret;
1919         }
1920
1921         if (icm->get_mode) {
1922                 ret = icm->get_mode(tb);
1923
1924                 switch (ret) {
1925                 case NHI_FW_SAFE_MODE:
1926                         icm->safe_mode = true;
1927                         break;
1928
1929                 case NHI_FW_CM_MODE:
1930                         /* Ask ICM to accept all Thunderbolt devices */
1931                         nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1932                         break;
1933
1934                 default:
1935                         if (ret < 0)
1936                                 return ret;
1937
1938                         tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1939                         return -ENODEV;
1940                 }
1941         }
1942
1943         /*
1944          * Reset both physical ports if there is anything connected to
1945          * them already.
1946          */
1947         ret = icm_reset_phy_port(tb, 0);
1948         if (ret)
1949                 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1950         ret = icm_reset_phy_port(tb, 1);
1951         if (ret)
1952                 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1953
1954         return 0;
1955 }
1956
1957 static int icm_driver_ready(struct tb *tb)
1958 {
1959         struct icm *icm = tb_priv(tb);
1960         int ret;
1961
1962         ret = icm_firmware_init(tb);
1963         if (ret)
1964                 return ret;
1965
1966         if (icm->safe_mode) {
1967                 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1968                 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1969                 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1970                 return 0;
1971         }
1972
1973         ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1974                                  &tb->nboot_acl, &icm->rpm);
1975         if (ret)
1976                 return ret;
1977
1978         /*
1979          * Make sure the number of supported preboot ACL matches what we
1980          * expect or disable the whole feature.
1981          */
1982         if (tb->nboot_acl > icm->max_boot_acl)
1983                 tb->nboot_acl = 0;
1984
1985         if (icm->proto_version >= 3)
1986                 tb_dbg(tb, "USB4 proxy operations supported\n");
1987
1988         return 0;
1989 }
1990
1991 static int icm_suspend(struct tb *tb)
1992 {
1993         struct icm *icm = tb_priv(tb);
1994
1995         if (icm->save_devices)
1996                 icm->save_devices(tb);
1997
1998         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
1999         return 0;
2000 }
2001
2002 /*
2003  * Mark all switches (except root switch) below this one unplugged. ICM
2004  * firmware will send us an updated list of switches after we have send
2005  * it driver ready command. If a switch is not in that list it will be
2006  * removed when we perform rescan.
2007  */
2008 static void icm_unplug_children(struct tb_switch *sw)
2009 {
2010         struct tb_port *port;
2011
2012         if (tb_route(sw))
2013                 sw->is_unplugged = true;
2014
2015         tb_switch_for_each_port(sw, port) {
2016                 if (port->xdomain)
2017                         port->xdomain->is_unplugged = true;
2018                 else if (tb_port_has_remote(port))
2019                         icm_unplug_children(port->remote->sw);
2020         }
2021 }
2022
2023 static int complete_rpm(struct device *dev, void *data)
2024 {
2025         struct tb_switch *sw = tb_to_switch(dev);
2026
2027         if (sw)
2028                 complete(&sw->rpm_complete);
2029         return 0;
2030 }
2031
2032 static void remove_unplugged_switch(struct tb_switch *sw)
2033 {
2034         struct device *parent = get_device(sw->dev.parent);
2035
2036         pm_runtime_get_sync(parent);
2037
2038         /*
2039          * Signal this and switches below for rpm_complete because
2040          * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2041          * for it.
2042          */
2043         complete_rpm(&sw->dev, NULL);
2044         bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2045         tb_switch_remove(sw);
2046
2047         pm_runtime_mark_last_busy(parent);
2048         pm_runtime_put_autosuspend(parent);
2049
2050         put_device(parent);
2051 }
2052
2053 static void icm_free_unplugged_children(struct tb_switch *sw)
2054 {
2055         struct tb_port *port;
2056
2057         tb_switch_for_each_port(sw, port) {
2058                 if (port->xdomain && port->xdomain->is_unplugged) {
2059                         tb_xdomain_remove(port->xdomain);
2060                         port->xdomain = NULL;
2061                 } else if (tb_port_has_remote(port)) {
2062                         if (port->remote->sw->is_unplugged) {
2063                                 remove_unplugged_switch(port->remote->sw);
2064                                 port->remote = NULL;
2065                         } else {
2066                                 icm_free_unplugged_children(port->remote->sw);
2067                         }
2068                 }
2069         }
2070 }
2071
2072 static void icm_rescan_work(struct work_struct *work)
2073 {
2074         struct icm *icm = container_of(work, struct icm, rescan_work.work);
2075         struct tb *tb = icm_to_tb(icm);
2076
2077         mutex_lock(&tb->lock);
2078         if (tb->root_switch)
2079                 icm_free_unplugged_children(tb->root_switch);
2080         mutex_unlock(&tb->lock);
2081 }
2082
2083 static void icm_complete(struct tb *tb)
2084 {
2085         struct icm *icm = tb_priv(tb);
2086
2087         if (tb->nhi->going_away)
2088                 return;
2089
2090         /*
2091          * If RTD3 was vetoed before we entered system suspend allow it
2092          * again now before driver ready is sent. Firmware sends a new RTD3
2093          * veto if it is still the case after we have sent it driver ready
2094          * command.
2095          */
2096         icm_veto_end(tb);
2097         icm_unplug_children(tb->root_switch);
2098
2099         /*
2100          * Now all existing children should be resumed, start events
2101          * from ICM to get updated status.
2102          */
2103         __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2104
2105         /*
2106          * We do not get notifications of devices that have been
2107          * unplugged during suspend so schedule rescan to clean them up
2108          * if any.
2109          */
2110         queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2111 }
2112
2113 static int icm_runtime_suspend(struct tb *tb)
2114 {
2115         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2116         return 0;
2117 }
2118
2119 static int icm_runtime_suspend_switch(struct tb_switch *sw)
2120 {
2121         if (tb_route(sw))
2122                 reinit_completion(&sw->rpm_complete);
2123         return 0;
2124 }
2125
2126 static int icm_runtime_resume_switch(struct tb_switch *sw)
2127 {
2128         if (tb_route(sw)) {
2129                 if (!wait_for_completion_timeout(&sw->rpm_complete,
2130                                                  msecs_to_jiffies(500))) {
2131                         dev_dbg(&sw->dev, "runtime resuming timed out\n");
2132                 }
2133         }
2134         return 0;
2135 }
2136
2137 static int icm_runtime_resume(struct tb *tb)
2138 {
2139         /*
2140          * We can reuse the same resume functionality than with system
2141          * suspend.
2142          */
2143         icm_complete(tb);
2144         return 0;
2145 }
2146
2147 static int icm_start(struct tb *tb, bool not_used)
2148 {
2149         struct icm *icm = tb_priv(tb);
2150         int ret;
2151
2152         if (icm->safe_mode)
2153                 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2154         else
2155                 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2156         if (IS_ERR(tb->root_switch))
2157                 return PTR_ERR(tb->root_switch);
2158
2159         tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2160         tb->root_switch->rpm = icm->rpm;
2161
2162         if (icm->set_uuid)
2163                 icm->set_uuid(tb);
2164
2165         ret = tb_switch_add(tb->root_switch);
2166         if (ret) {
2167                 tb_switch_put(tb->root_switch);
2168                 tb->root_switch = NULL;
2169         }
2170
2171         return ret;
2172 }
2173
2174 static void icm_stop(struct tb *tb)
2175 {
2176         struct icm *icm = tb_priv(tb);
2177
2178         cancel_delayed_work(&icm->rescan_work);
2179         tb_switch_remove(tb->root_switch);
2180         tb->root_switch = NULL;
2181         nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2182         kfree(icm->last_nvm_auth);
2183         icm->last_nvm_auth = NULL;
2184 }
2185
2186 static int icm_disconnect_pcie_paths(struct tb *tb)
2187 {
2188         return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2189 }
2190
2191 static void icm_usb4_switch_nvm_auth_complete(void *data)
2192 {
2193         struct usb4_switch_nvm_auth *auth = data;
2194         struct icm *icm = auth->icm;
2195         struct tb *tb = icm_to_tb(icm);
2196
2197         tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2198                get_route(auth->reply.route_hi, auth->reply.route_lo),
2199                auth->reply.hdr.flags, auth->reply.status);
2200
2201         mutex_lock(&tb->lock);
2202         if (WARN_ON(icm->last_nvm_auth))
2203                 kfree(icm->last_nvm_auth);
2204         icm->last_nvm_auth = auth;
2205         mutex_unlock(&tb->lock);
2206 }
2207
2208 static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2209 {
2210         struct usb4_switch_nvm_auth *auth;
2211         struct icm *icm = tb_priv(tb);
2212         struct tb_cfg_request *req;
2213         int ret;
2214
2215         auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2216         if (!auth)
2217                 return -ENOMEM;
2218
2219         auth->icm = icm;
2220         auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2221         auth->request.route_hi = upper_32_bits(route);
2222         auth->request.route_lo = lower_32_bits(route);
2223         auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2224
2225         req = tb_cfg_request_alloc();
2226         if (!req) {
2227                 ret = -ENOMEM;
2228                 goto err_free_auth;
2229         }
2230
2231         req->match = icm_match;
2232         req->copy = icm_copy;
2233         req->request = &auth->request;
2234         req->request_size = sizeof(auth->request);
2235         req->request_type = TB_CFG_PKG_ICM_CMD;
2236         req->response = &auth->reply;
2237         req->npackets = 1;
2238         req->response_size = sizeof(auth->reply);
2239         req->response_type = TB_CFG_PKG_ICM_RESP;
2240
2241         tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2242
2243         mutex_lock(&icm->request_lock);
2244         ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2245                              auth);
2246         mutex_unlock(&icm->request_lock);
2247
2248         tb_cfg_request_put(req);
2249         if (ret)
2250                 goto err_free_auth;
2251         return 0;
2252
2253 err_free_auth:
2254         kfree(auth);
2255         return ret;
2256 }
2257
2258 static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2259                               u8 *status, const void *tx_data, size_t tx_data_len,
2260                               void *rx_data, size_t rx_data_len)
2261 {
2262         struct icm_usb4_switch_op_response reply;
2263         struct icm_usb4_switch_op request;
2264         struct tb *tb = sw->tb;
2265         struct icm *icm = tb_priv(tb);
2266         u64 route = tb_route(sw);
2267         int ret;
2268
2269         /*
2270          * USB4 router operation proxy is supported in firmware if the
2271          * protocol version is 3 or higher.
2272          */
2273         if (icm->proto_version < 3)
2274                 return -EOPNOTSUPP;
2275
2276         /*
2277          * NVM_AUTH is a special USB4 proxy operation that does not
2278          * return immediately so handle it separately.
2279          */
2280         if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2281                 return icm_usb4_switch_nvm_authenticate(tb, route);
2282
2283         memset(&request, 0, sizeof(request));
2284         request.hdr.code = ICM_USB4_SWITCH_OP;
2285         request.route_hi = upper_32_bits(route);
2286         request.route_lo = lower_32_bits(route);
2287         request.opcode = opcode;
2288         if (metadata)
2289                 request.metadata = *metadata;
2290
2291         if (tx_data_len) {
2292                 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2293                 if (tx_data_len < ARRAY_SIZE(request.data))
2294                         request.data_len_valid =
2295                                 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2296                 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2297         }
2298
2299         memset(&reply, 0, sizeof(reply));
2300         ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2301                           1, ICM_RETRIES, ICM_TIMEOUT);
2302         if (ret)
2303                 return ret;
2304
2305         if (reply.hdr.flags & ICM_FLAGS_ERROR)
2306                 return -EIO;
2307
2308         if (status)
2309                 *status = reply.status;
2310
2311         if (metadata)
2312                 *metadata = reply.metadata;
2313
2314         if (rx_data_len)
2315                 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2316
2317         return 0;
2318 }
2319
2320 static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2321                                                    u32 *status)
2322 {
2323         struct usb4_switch_nvm_auth *auth;
2324         struct tb *tb = sw->tb;
2325         struct icm *icm = tb_priv(tb);
2326         int ret = 0;
2327
2328         if (icm->proto_version < 3)
2329                 return -EOPNOTSUPP;
2330
2331         auth = icm->last_nvm_auth;
2332         icm->last_nvm_auth = NULL;
2333
2334         if (auth && auth->reply.route_hi == sw->config.route_hi &&
2335             auth->reply.route_lo == sw->config.route_lo) {
2336                 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",
2337                        tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2338                 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2339                         ret = -EIO;
2340                 else
2341                         *status = auth->reply.status;
2342         } else {
2343                 *status = 0;
2344         }
2345
2346         kfree(auth);
2347         return ret;
2348 }
2349
2350 /* Falcon Ridge */
2351 static const struct tb_cm_ops icm_fr_ops = {
2352         .driver_ready = icm_driver_ready,
2353         .start = icm_start,
2354         .stop = icm_stop,
2355         .suspend = icm_suspend,
2356         .complete = icm_complete,
2357         .handle_event = icm_handle_event,
2358         .approve_switch = icm_fr_approve_switch,
2359         .add_switch_key = icm_fr_add_switch_key,
2360         .challenge_switch_key = icm_fr_challenge_switch_key,
2361         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2362         .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2363         .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2364 };
2365
2366 /* Alpine Ridge */
2367 static const struct tb_cm_ops icm_ar_ops = {
2368         .driver_ready = icm_driver_ready,
2369         .start = icm_start,
2370         .stop = icm_stop,
2371         .suspend = icm_suspend,
2372         .complete = icm_complete,
2373         .runtime_suspend = icm_runtime_suspend,
2374         .runtime_resume = icm_runtime_resume,
2375         .runtime_suspend_switch = icm_runtime_suspend_switch,
2376         .runtime_resume_switch = icm_runtime_resume_switch,
2377         .handle_event = icm_handle_event,
2378         .get_boot_acl = icm_ar_get_boot_acl,
2379         .set_boot_acl = icm_ar_set_boot_acl,
2380         .approve_switch = icm_fr_approve_switch,
2381         .add_switch_key = icm_fr_add_switch_key,
2382         .challenge_switch_key = icm_fr_challenge_switch_key,
2383         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2384         .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2385         .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2386 };
2387
2388 /* Titan Ridge */
2389 static const struct tb_cm_ops icm_tr_ops = {
2390         .driver_ready = icm_driver_ready,
2391         .start = icm_start,
2392         .stop = icm_stop,
2393         .suspend = icm_suspend,
2394         .complete = icm_complete,
2395         .runtime_suspend = icm_runtime_suspend,
2396         .runtime_resume = icm_runtime_resume,
2397         .runtime_suspend_switch = icm_runtime_suspend_switch,
2398         .runtime_resume_switch = icm_runtime_resume_switch,
2399         .handle_event = icm_handle_event,
2400         .get_boot_acl = icm_ar_get_boot_acl,
2401         .set_boot_acl = icm_ar_set_boot_acl,
2402         .approve_switch = icm_tr_approve_switch,
2403         .add_switch_key = icm_tr_add_switch_key,
2404         .challenge_switch_key = icm_tr_challenge_switch_key,
2405         .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2406         .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2407         .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2408         .usb4_switch_op = icm_usb4_switch_op,
2409         .usb4_switch_nvm_authenticate_status =
2410                 icm_usb4_switch_nvm_authenticate_status,
2411 };
2412
2413 /* Ice Lake */
2414 static const struct tb_cm_ops icm_icl_ops = {
2415         .driver_ready = icm_driver_ready,
2416         .start = icm_start,
2417         .stop = icm_stop,
2418         .complete = icm_complete,
2419         .runtime_suspend = icm_runtime_suspend,
2420         .runtime_resume = icm_runtime_resume,
2421         .handle_event = icm_handle_event,
2422         .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2423         .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2424         .usb4_switch_op = icm_usb4_switch_op,
2425         .usb4_switch_nvm_authenticate_status =
2426                 icm_usb4_switch_nvm_authenticate_status,
2427 };
2428
2429 struct tb *icm_probe(struct tb_nhi *nhi)
2430 {
2431         struct icm *icm;
2432         struct tb *tb;
2433
2434         tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));
2435         if (!tb)
2436                 return NULL;
2437
2438         icm = tb_priv(tb);
2439         INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2440         mutex_init(&icm->request_lock);
2441
2442         switch (nhi->pdev->device) {
2443         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2444         case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2445                 icm->can_upgrade_nvm = true;
2446                 icm->is_supported = icm_fr_is_supported;
2447                 icm->get_route = icm_fr_get_route;
2448                 icm->save_devices = icm_fr_save_devices;
2449                 icm->driver_ready = icm_fr_driver_ready;
2450                 icm->device_connected = icm_fr_device_connected;
2451                 icm->device_disconnected = icm_fr_device_disconnected;
2452                 icm->xdomain_connected = icm_fr_xdomain_connected;
2453                 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2454                 tb->cm_ops = &icm_fr_ops;
2455                 break;
2456
2457         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2458         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2459         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2460         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2461         case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2462                 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2463                 /*
2464                  * NVM upgrade has not been tested on Apple systems and
2465                  * they don't provide images publicly either. To be on
2466                  * the safe side prevent root switch NVM upgrade on Macs
2467                  * for now.
2468                  */
2469                 icm->can_upgrade_nvm = !x86_apple_machine;
2470                 icm->is_supported = icm_ar_is_supported;
2471                 icm->cio_reset = icm_ar_cio_reset;
2472                 icm->get_mode = icm_ar_get_mode;
2473                 icm->get_route = icm_ar_get_route;
2474                 icm->save_devices = icm_fr_save_devices;
2475                 icm->driver_ready = icm_ar_driver_ready;
2476                 icm->device_connected = icm_fr_device_connected;
2477                 icm->device_disconnected = icm_fr_device_disconnected;
2478                 icm->xdomain_connected = icm_fr_xdomain_connected;
2479                 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2480                 tb->cm_ops = &icm_ar_ops;
2481                 break;
2482
2483         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2484         case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2485                 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2486                 icm->can_upgrade_nvm = !x86_apple_machine;
2487                 icm->is_supported = icm_ar_is_supported;
2488                 icm->cio_reset = icm_tr_cio_reset;
2489                 icm->get_mode = icm_ar_get_mode;
2490                 icm->driver_ready = icm_tr_driver_ready;
2491                 icm->device_connected = icm_tr_device_connected;
2492                 icm->device_disconnected = icm_tr_device_disconnected;
2493                 icm->xdomain_connected = icm_tr_xdomain_connected;
2494                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2495                 tb->cm_ops = &icm_tr_ops;
2496                 break;
2497
2498         case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2499         case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2500                 icm->is_supported = icm_fr_is_supported;
2501                 icm->driver_ready = icm_icl_driver_ready;
2502                 icm->set_uuid = icm_icl_set_uuid;
2503                 icm->device_connected = icm_icl_device_connected;
2504                 icm->device_disconnected = icm_tr_device_disconnected;
2505                 icm->xdomain_connected = icm_tr_xdomain_connected;
2506                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2507                 icm->rtd3_veto = icm_icl_rtd3_veto;
2508                 tb->cm_ops = &icm_icl_ops;
2509                 break;
2510
2511         case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2512         case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2513         case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2514         case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2515         case PCI_DEVICE_ID_INTEL_ADL_NHI0:
2516         case PCI_DEVICE_ID_INTEL_ADL_NHI1:
2517         case PCI_DEVICE_ID_INTEL_RPL_NHI0:
2518         case PCI_DEVICE_ID_INTEL_RPL_NHI1:
2519         case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:
2520         case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:
2521         case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:
2522                 icm->is_supported = icm_tgl_is_supported;
2523                 icm->driver_ready = icm_icl_driver_ready;
2524                 icm->set_uuid = icm_icl_set_uuid;
2525                 icm->device_connected = icm_icl_device_connected;
2526                 icm->device_disconnected = icm_tr_device_disconnected;
2527                 icm->xdomain_connected = icm_tr_xdomain_connected;
2528                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2529                 icm->rtd3_veto = icm_icl_rtd3_veto;
2530                 tb->cm_ops = &icm_icl_ops;
2531                 break;
2532
2533         case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:
2534         case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2535                 icm->can_upgrade_nvm = true;
2536                 icm->is_supported = icm_tgl_is_supported;
2537                 icm->get_mode = icm_ar_get_mode;
2538                 icm->driver_ready = icm_tr_driver_ready;
2539                 icm->device_connected = icm_tr_device_connected;
2540                 icm->device_disconnected = icm_tr_device_disconnected;
2541                 icm->xdomain_connected = icm_tr_xdomain_connected;
2542                 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2543                 tb->cm_ops = &icm_tr_ops;
2544                 break;
2545         }
2546
2547         if (!icm->is_supported || !icm->is_supported(tb)) {
2548                 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2549                 tb_domain_put(tb);
2550                 return NULL;
2551         }
2552
2553         tb_dbg(tb, "using firmware connection manager\n");
2554
2555         return tb;
2556 }