tools/testing/cxl/test/cxl.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 // Copyright(c) 2021 Intel Corporation. All rights reserved.
   3
   4 #include <linux/platform_device.h>
   5 #include <linux/genalloc.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/acpi.h>
   9 #include <linux/pci.h>
  10 #include <linux/mm.h>
  11 #include <cxlmem.h>
  12
  13 #include "../watermark.h"
  14 #include "mock.h"
  15
  16 static int interleave_arithmetic;
  17
  18 #define FAKE_QTG_ID     42
  19
  20 #define NR_CXL_HOST_BRIDGES 2
  21 #define NR_CXL_SINGLE_HOST 1
  22 #define NR_CXL_RCH 1
  23 #define NR_CXL_ROOT_PORTS 2
  24 #define NR_CXL_SWITCH_PORTS 2
  25 #define NR_CXL_PORT_DECODERS 8
  26 #define NR_BRIDGES (NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST + NR_CXL_RCH)
  27
  28 static struct platform_device *cxl_acpi;
  29 static struct platform_device *cxl_host_bridge[NR_CXL_HOST_BRIDGES];
  30 #define NR_MULTI_ROOT (NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS)
  31 static struct platform_device *cxl_root_port[NR_MULTI_ROOT];
  32 static struct platform_device *cxl_switch_uport[NR_MULTI_ROOT];
  33 #define NR_MEM_MULTI \
  34         (NR_CXL_HOST_BRIDGES * NR_CXL_ROOT_PORTS * NR_CXL_SWITCH_PORTS)
  35 static struct platform_device *cxl_switch_dport[NR_MEM_MULTI];
  36
  37 static struct platform_device *cxl_hb_single[NR_CXL_SINGLE_HOST];
  38 static struct platform_device *cxl_root_single[NR_CXL_SINGLE_HOST];
  39 static struct platform_device *cxl_swu_single[NR_CXL_SINGLE_HOST];
  40 #define NR_MEM_SINGLE (NR_CXL_SINGLE_HOST * NR_CXL_SWITCH_PORTS)
  41 static struct platform_device *cxl_swd_single[NR_MEM_SINGLE];
  42
  43 struct platform_device *cxl_mem[NR_MEM_MULTI];
  44 struct platform_device *cxl_mem_single[NR_MEM_SINGLE];
  45
  46 static struct platform_device *cxl_rch[NR_CXL_RCH];
  47 static struct platform_device *cxl_rcd[NR_CXL_RCH];
  48
  49 static inline bool is_multi_bridge(struct device *dev)
  50 {
  51         int i;
  52
  53         for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++)
  54                 if (&cxl_host_bridge[i]->dev == dev)
  55                         return true;
  56         return false;
  57 }
  58
  59 static inline bool is_single_bridge(struct device *dev)
  60 {
  61         int i;
  62
  63         for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++)
  64                 if (&cxl_hb_single[i]->dev == dev)
  65                         return true;
  66         return false;
  67 }
  68
  69 static struct acpi_device acpi0017_mock;
  70 static struct acpi_device host_bridge[NR_BRIDGES] = {
  71         [0] = {
  72                 .handle = &host_bridge[0],
  73                 .pnp.unique_id = "0",
  74         },
  75         [1] = {
  76                 .handle = &host_bridge[1],
  77                 .pnp.unique_id = "1",
  78         },
  79         [2] = {
  80                 .handle = &host_bridge[2],
  81                 .pnp.unique_id = "2",
  82         },
  83         [3] = {
  84                 .handle = &host_bridge[3],
  85                 .pnp.unique_id = "3",
  86         },
  87 };
  88
  89 static bool is_mock_dev(struct device *dev)
  90 {
  91         int i;
  92
  93         for (i = 0; i < ARRAY_SIZE(cxl_mem); i++)
  94                 if (dev == &cxl_mem[i]->dev)
  95                         return true;
  96         for (i = 0; i < ARRAY_SIZE(cxl_mem_single); i++)
  97                 if (dev == &cxl_mem_single[i]->dev)
  98                         return true;
  99         for (i = 0; i < ARRAY_SIZE(cxl_rcd); i++)
 100                 if (dev == &cxl_rcd[i]->dev)
 101                         return true;
 102         if (dev == &cxl_acpi->dev)
 103                 return true;
 104         return false;
 105 }
 106
 107 static bool is_mock_adev(struct acpi_device *adev)
 108 {
 109         int i;
 110
 111         if (adev == &acpi0017_mock)
 112                 return true;
 113
 114         for (i = 0; i < ARRAY_SIZE(host_bridge); i++)
 115                 if (adev == &host_bridge[i])
 116                         return true;
 117
 118         return false;
 119 }
 120
 121 static struct {
 122         struct acpi_table_cedt cedt;
 123         struct acpi_cedt_chbs chbs[NR_BRIDGES];
 124         struct {
 125                 struct acpi_cedt_cfmws cfmws;
 126                 u32 target[1];
 127         } cfmws0;
 128         struct {
 129                 struct acpi_cedt_cfmws cfmws;
 130                 u32 target[2];
 131         } cfmws1;
 132         struct {
 133                 struct acpi_cedt_cfmws cfmws;
 134                 u32 target[1];
 135         } cfmws2;
 136         struct {
 137                 struct acpi_cedt_cfmws cfmws;
 138                 u32 target[2];
 139         } cfmws3;
 140         struct {
 141                 struct acpi_cedt_cfmws cfmws;
 142                 u32 target[1];
 143         } cfmws4;
 144         struct {
 145                 struct acpi_cedt_cfmws cfmws;
 146                 u32 target[1];
 147         } cfmws5;
 148         struct {
 149                 struct acpi_cedt_cfmws cfmws;
 150                 u32 target[1];
 151         } cfmws6;
 152         struct {
 153                 struct acpi_cedt_cfmws cfmws;
 154                 u32 target[2];
 155         } cfmws7;
 156         struct {
 157                 struct acpi_cedt_cfmws cfmws;
 158                 u32 target[4];
 159         } cfmws8;
 160         struct {
 161                 struct acpi_cedt_cxims cxims;
 162                 u64 xormap_list[2];
 163         } cxims0;
 164 } __packed mock_cedt = {
 165         .cedt = {
 166                 .header = {
 167                         .signature = "CEDT",
 168                         .length = sizeof(mock_cedt),
 169                         .revision = 1,
 170                 },
 171         },
 172         .chbs[0] = {
 173                 .header = {
 174                         .type = ACPI_CEDT_TYPE_CHBS,
 175                         .length = sizeof(mock_cedt.chbs[0]),
 176                 },
 177                 .uid = 0,
 178                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
 179         },
 180         .chbs[1] = {
 181                 .header = {
 182                         .type = ACPI_CEDT_TYPE_CHBS,
 183                         .length = sizeof(mock_cedt.chbs[0]),
 184                 },
 185                 .uid = 1,
 186                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
 187         },
 188         .chbs[2] = {
 189                 .header = {
 190                         .type = ACPI_CEDT_TYPE_CHBS,
 191                         .length = sizeof(mock_cedt.chbs[0]),
 192                 },
 193                 .uid = 2,
 194                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL20,
 195         },
 196         .chbs[3] = {
 197                 .header = {
 198                         .type = ACPI_CEDT_TYPE_CHBS,
 199                         .length = sizeof(mock_cedt.chbs[0]),
 200                 },
 201                 .uid = 3,
 202                 .cxl_version = ACPI_CEDT_CHBS_VERSION_CXL11,
 203         },
 204         .cfmws0 = {
 205                 .cfmws = {
 206                         .header = {
 207                                 .type = ACPI_CEDT_TYPE_CFMWS,
 208                                 .length = sizeof(mock_cedt.cfmws0),
 209                         },
 210                         .interleave_ways = 0,
 211                         .granularity = 4,
 212                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 213                                         ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
 214                         .qtg_id = FAKE_QTG_ID,
 215                         .window_size = SZ_256M * 4UL,
 216                 },
 217                 .target = { 0 },
 218         },
 219         .cfmws1 = {
 220                 .cfmws = {
 221                         .header = {
 222                                 .type = ACPI_CEDT_TYPE_CFMWS,
 223                                 .length = sizeof(mock_cedt.cfmws1),
 224                         },
 225                         .interleave_ways = 1,
 226                         .granularity = 4,
 227                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 228                                         ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
 229                         .qtg_id = FAKE_QTG_ID,
 230                         .window_size = SZ_256M * 8UL,
 231                 },
 232                 .target = { 0, 1, },
 233         },
 234         .cfmws2 = {
 235                 .cfmws = {
 236                         .header = {
 237                                 .type = ACPI_CEDT_TYPE_CFMWS,
 238                                 .length = sizeof(mock_cedt.cfmws2),
 239                         },
 240                         .interleave_ways = 0,
 241                         .granularity = 4,
 242                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 243                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 244                         .qtg_id = FAKE_QTG_ID,
 245                         .window_size = SZ_256M * 4UL,
 246                 },
 247                 .target = { 0 },
 248         },
 249         .cfmws3 = {
 250                 .cfmws = {
 251                         .header = {
 252                                 .type = ACPI_CEDT_TYPE_CFMWS,
 253                                 .length = sizeof(mock_cedt.cfmws3),
 254                         },
 255                         .interleave_ways = 1,
 256                         .granularity = 4,
 257                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 258                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 259                         .qtg_id = FAKE_QTG_ID,
 260                         .window_size = SZ_256M * 8UL,
 261                 },
 262                 .target = { 0, 1, },
 263         },
 264         .cfmws4 = {
 265                 .cfmws = {
 266                         .header = {
 267                                 .type = ACPI_CEDT_TYPE_CFMWS,
 268                                 .length = sizeof(mock_cedt.cfmws4),
 269                         },
 270                         .interleave_ways = 0,
 271                         .granularity = 4,
 272                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 273                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 274                         .qtg_id = FAKE_QTG_ID,
 275                         .window_size = SZ_256M * 4UL,
 276                 },
 277                 .target = { 2 },
 278         },
 279         .cfmws5 = {
 280                 .cfmws = {
 281                         .header = {
 282                                 .type = ACPI_CEDT_TYPE_CFMWS,
 283                                 .length = sizeof(mock_cedt.cfmws5),
 284                         },
 285                         .interleave_ways = 0,
 286                         .granularity = 4,
 287                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 288                                         ACPI_CEDT_CFMWS_RESTRICT_VOLATILE,
 289                         .qtg_id = FAKE_QTG_ID,
 290                         .window_size = SZ_256M,
 291                 },
 292                 .target = { 3 },
 293         },
 294         /* .cfmws6,7,8 use ACPI_CEDT_CFMWS_ARITHMETIC_XOR */
 295         .cfmws6 = {
 296                 .cfmws = {
 297                         .header = {
 298                                 .type = ACPI_CEDT_TYPE_CFMWS,
 299                                 .length = sizeof(mock_cedt.cfmws6),
 300                         },
 301                         .interleave_arithmetic = ACPI_CEDT_CFMWS_ARITHMETIC_XOR,
 302                         .interleave_ways = 0,
 303                         .granularity = 4,
 304                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 305                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 306                         .qtg_id = FAKE_QTG_ID,
 307                         .window_size = SZ_256M * 8UL,
 308                 },
 309                 .target = { 0, },
 310         },
 311         .cfmws7 = {
 312                 .cfmws = {
 313                         .header = {
 314                                 .type = ACPI_CEDT_TYPE_CFMWS,
 315                                 .length = sizeof(mock_cedt.cfmws7),
 316                         },
 317                         .interleave_arithmetic = ACPI_CEDT_CFMWS_ARITHMETIC_XOR,
 318                         .interleave_ways = 1,
 319                         .granularity = 0,
 320                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 321                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 322                         .qtg_id = FAKE_QTG_ID,
 323                         .window_size = SZ_256M * 8UL,
 324                 },
 325                 .target = { 0, 1, },
 326         },
 327         .cfmws8 = {
 328                 .cfmws = {
 329                         .header = {
 330                                 .type = ACPI_CEDT_TYPE_CFMWS,
 331                                 .length = sizeof(mock_cedt.cfmws8),
 332                         },
 333                         .interleave_arithmetic = ACPI_CEDT_CFMWS_ARITHMETIC_XOR,
 334                         .interleave_ways = 2,
 335                         .granularity = 0,
 336                         .restrictions = ACPI_CEDT_CFMWS_RESTRICT_TYPE3 |
 337                                         ACPI_CEDT_CFMWS_RESTRICT_PMEM,
 338                         .qtg_id = FAKE_QTG_ID,
 339                         .window_size = SZ_256M * 16UL,
 340                 },
 341                 .target = { 0, 1, 0, 1, },
 342         },
 343         .cxims0 = {
 344                 .cxims = {
 345                         .header = {
 346                                 .type = ACPI_CEDT_TYPE_CXIMS,
 347                                 .length = sizeof(mock_cedt.cxims0),
 348                         },
 349                         .hbig = 0,
 350                         .nr_xormaps = 2,
 351                 },
 352                 .xormap_list = { 0x404100, 0x808200, },
 353         },
 354 };
 355
 356 struct acpi_cedt_cfmws *mock_cfmws[] = {
 357         [0] = &mock_cedt.cfmws0.cfmws,
 358         [1] = &mock_cedt.cfmws1.cfmws,
 359         [2] = &mock_cedt.cfmws2.cfmws,
 360         [3] = &mock_cedt.cfmws3.cfmws,
 361         [4] = &mock_cedt.cfmws4.cfmws,
 362         [5] = &mock_cedt.cfmws5.cfmws,
 363         /* Modulo Math above, XOR Math below */
 364         [6] = &mock_cedt.cfmws6.cfmws,
 365         [7] = &mock_cedt.cfmws7.cfmws,
 366         [8] = &mock_cedt.cfmws8.cfmws,
 367 };
 368
 369 static int cfmws_start;
 370 static int cfmws_end;
 371 #define CFMWS_MOD_ARRAY_START 0
 372 #define CFMWS_MOD_ARRAY_END   5
 373 #define CFMWS_XOR_ARRAY_START 6
 374 #define CFMWS_XOR_ARRAY_END   8
 375
 376 struct acpi_cedt_cxims *mock_cxims[1] = {
 377         [0] = &mock_cedt.cxims0.cxims,
 378 };
 379
 380 struct cxl_mock_res {
 381         struct list_head list;
 382         struct range range;
 383 };
 384
 385 static LIST_HEAD(mock_res);
 386 static DEFINE_MUTEX(mock_res_lock);
 387 static struct gen_pool *cxl_mock_pool;
 388
 389 static void depopulate_all_mock_resources(void)
 390 {
 391         struct cxl_mock_res *res, *_res;
 392
 393         mutex_lock(&mock_res_lock);
 394         list_for_each_entry_safe(res, _res, &mock_res, list) {
 395                 gen_pool_free(cxl_mock_pool, res->range.start,
 396                               range_len(&res->range));
 397                 list_del(&res->list);
 398                 kfree(res);
 399         }
 400         mutex_unlock(&mock_res_lock);
 401 }
 402
 403 static struct cxl_mock_res *alloc_mock_res(resource_size_t size, int align)
 404 {
 405         struct cxl_mock_res *res = kzalloc(sizeof(*res), GFP_KERNEL);
 406         struct genpool_data_align data = {
 407                 .align = align,
 408         };
 409         unsigned long phys;
 410
 411         INIT_LIST_HEAD(&res->list);
 412         phys = gen_pool_alloc_algo(cxl_mock_pool, size,
 413                                    gen_pool_first_fit_align, &data);
 414         if (!phys)
 415                 return NULL;
 416
 417         res->range = (struct range) {
 418                 .start = phys,
 419                 .end = phys + size - 1,
 420         };
 421         mutex_lock(&mock_res_lock);
 422         list_add(&res->list, &mock_res);
 423         mutex_unlock(&mock_res_lock);
 424
 425         return res;
 426 }
 427
 428 static int populate_cedt(void)
 429 {
 430         struct cxl_mock_res *res;
 431         int i;
 432
 433         for (i = 0; i < ARRAY_SIZE(mock_cedt.chbs); i++) {
 434                 struct acpi_cedt_chbs *chbs = &mock_cedt.chbs[i];
 435                 resource_size_t size;
 436
 437                 if (chbs->cxl_version == ACPI_CEDT_CHBS_VERSION_CXL20)
 438                         size = ACPI_CEDT_CHBS_LENGTH_CXL20;
 439                 else
 440                         size = ACPI_CEDT_CHBS_LENGTH_CXL11;
 441
 442                 res = alloc_mock_res(size, size);
 443                 if (!res)
 444                         return -ENOMEM;
 445                 chbs->base = res->range.start;
 446                 chbs->length = size;
 447         }
 448
 449         for (i = cfmws_start; i <= cfmws_end; i++) {
 450                 struct acpi_cedt_cfmws *window = mock_cfmws[i];
 451
 452                 res = alloc_mock_res(window->window_size, SZ_256M);
 453                 if (!res)
 454                         return -ENOMEM;
 455                 window->base_hpa = res->range.start;
 456         }
 457
 458         return 0;
 459 }
 460
 461 static bool is_mock_port(struct device *dev);
 462
 463 /*
 464  * WARNING, this hack assumes the format of 'struct cxl_cfmws_context'
 465  * and 'struct cxl_chbs_context' share the property that the first
 466  * struct member is a cxl_test device being probed by the cxl_acpi
 467  * driver.
 468  */
 469 struct cxl_cedt_context {
 470         struct device *dev;
 471 };
 472
 473 static int mock_acpi_table_parse_cedt(enum acpi_cedt_type id,
 474                                       acpi_tbl_entry_handler_arg handler_arg,
 475                                       void *arg)
 476 {
 477         struct cxl_cedt_context *ctx = arg;
 478         struct device *dev = ctx->dev;
 479         union acpi_subtable_headers *h;
 480         unsigned long end;
 481         int i;
 482
 483         if (!is_mock_port(dev) && !is_mock_dev(dev))
 484                 return acpi_table_parse_cedt(id, handler_arg, arg);
 485
 486         if (id == ACPI_CEDT_TYPE_CHBS)
 487                 for (i = 0; i < ARRAY_SIZE(mock_cedt.chbs); i++) {
 488                         h = (union acpi_subtable_headers *)&mock_cedt.chbs[i];
 489                         end = (unsigned long)&mock_cedt.chbs[i + 1];
 490                         handler_arg(h, arg, end);
 491                 }
 492
 493         if (id == ACPI_CEDT_TYPE_CFMWS)
 494                 for (i = cfmws_start; i <= cfmws_end; i++) {
 495                         h = (union acpi_subtable_headers *) mock_cfmws[i];
 496                         end = (unsigned long) h + mock_cfmws[i]->header.length;
 497                         handler_arg(h, arg, end);
 498                 }
 499
 500         if (id == ACPI_CEDT_TYPE_CXIMS)
 501                 for (i = 0; i < ARRAY_SIZE(mock_cxims); i++) {
 502                         h = (union acpi_subtable_headers *)mock_cxims[i];
 503                         end = (unsigned long)h + mock_cxims[i]->header.length;
 504                         handler_arg(h, arg, end);
 505                 }
 506
 507         return 0;
 508 }
 509
 510 static bool is_mock_bridge(struct device *dev)
 511 {
 512         int i;
 513
 514         for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++)
 515                 if (dev == &cxl_host_bridge[i]->dev)
 516                         return true;
 517         for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++)
 518                 if (dev == &cxl_hb_single[i]->dev)
 519                         return true;
 520         for (i = 0; i < ARRAY_SIZE(cxl_rch); i++)
 521                 if (dev == &cxl_rch[i]->dev)
 522                         return true;
 523
 524         return false;
 525 }
 526
 527 static bool is_mock_port(struct device *dev)
 528 {
 529         int i;
 530
 531         if (is_mock_bridge(dev))
 532                 return true;
 533
 534         for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++)
 535                 if (dev == &cxl_root_port[i]->dev)
 536                         return true;
 537
 538         for (i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++)
 539                 if (dev == &cxl_switch_uport[i]->dev)
 540                         return true;
 541
 542         for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++)
 543                 if (dev == &cxl_switch_dport[i]->dev)
 544                         return true;
 545
 546         for (i = 0; i < ARRAY_SIZE(cxl_root_single); i++)
 547                 if (dev == &cxl_root_single[i]->dev)
 548                         return true;
 549
 550         for (i = 0; i < ARRAY_SIZE(cxl_swu_single); i++)
 551                 if (dev == &cxl_swu_single[i]->dev)
 552                         return true;
 553
 554         for (i = 0; i < ARRAY_SIZE(cxl_swd_single); i++)
 555                 if (dev == &cxl_swd_single[i]->dev)
 556                         return true;
 557
 558         if (is_cxl_memdev(dev))
 559                 return is_mock_dev(dev->parent);
 560
 561         return false;
 562 }
 563
 564 static int host_bridge_index(struct acpi_device *adev)
 565 {
 566         return adev - host_bridge;
 567 }
 568
 569 static struct acpi_device *find_host_bridge(acpi_handle handle)
 570 {
 571         int i;
 572
 573         for (i = 0; i < ARRAY_SIZE(host_bridge); i++)
 574                 if (handle == host_bridge[i].handle)
 575                         return &host_bridge[i];
 576         return NULL;
 577 }
 578
 579 static acpi_status
 580 mock_acpi_evaluate_integer(acpi_handle handle, acpi_string pathname,
 581                            struct acpi_object_list *arguments,
 582                            unsigned long long *data)
 583 {
 584         struct acpi_device *adev = find_host_bridge(handle);
 585
 586         if (!adev || strcmp(pathname, METHOD_NAME__UID) != 0)
 587                 return acpi_evaluate_integer(handle, pathname, arguments, data);
 588
 589         *data = host_bridge_index(adev);
 590         return AE_OK;
 591 }
 592
 593 static struct pci_bus mock_pci_bus[NR_BRIDGES];
 594 static struct acpi_pci_root mock_pci_root[ARRAY_SIZE(mock_pci_bus)] = {
 595         [0] = {
 596                 .bus = &mock_pci_bus[0],
 597         },
 598         [1] = {
 599                 .bus = &mock_pci_bus[1],
 600         },
 601         [2] = {
 602                 .bus = &mock_pci_bus[2],
 603         },
 604         [3] = {
 605                 .bus = &mock_pci_bus[3],
 606         },
 607
 608 };
 609
 610 static bool is_mock_bus(struct pci_bus *bus)
 611 {
 612         int i;
 613
 614         for (i = 0; i < ARRAY_SIZE(mock_pci_bus); i++)
 615                 if (bus == &mock_pci_bus[i])
 616                         return true;
 617         return false;
 618 }
 619
 620 static struct acpi_pci_root *mock_acpi_pci_find_root(acpi_handle handle)
 621 {
 622         struct acpi_device *adev = find_host_bridge(handle);
 623
 624         if (!adev)
 625                 return acpi_pci_find_root(handle);
 626         return &mock_pci_root[host_bridge_index(adev)];
 627 }
 628
 629 static struct cxl_hdm *mock_cxl_setup_hdm(struct cxl_port *port,
 630                                           struct cxl_endpoint_dvsec_info *info)
 631 {
 632         struct cxl_hdm *cxlhdm = devm_kzalloc(&port->dev, sizeof(*cxlhdm), GFP_KERNEL);
 633
 634         if (!cxlhdm)
 635                 return ERR_PTR(-ENOMEM);
 636
 637         cxlhdm->port = port;
 638         return cxlhdm;
 639 }
 640
 641 static int mock_cxl_add_passthrough_decoder(struct cxl_port *port)
 642 {
 643         dev_err(&port->dev, "unexpected passthrough decoder for cxl_test\n");
 644         return -EOPNOTSUPP;
 645 }
 646
 647
 648 struct target_map_ctx {
 649         int *target_map;
 650         int index;
 651         int target_count;
 652 };
 653
 654 static int map_targets(struct device *dev, void *data)
 655 {
 656         struct platform_device *pdev = to_platform_device(dev);
 657         struct target_map_ctx *ctx = data;
 658
 659         ctx->target_map[ctx->index++] = pdev->id;
 660
 661         if (ctx->index > ctx->target_count) {
 662                 dev_WARN_ONCE(dev, 1, "too many targets found?\n");
 663                 return -ENXIO;
 664         }
 665
 666         return 0;
 667 }
 668
 669 static int mock_decoder_commit(struct cxl_decoder *cxld)
 670 {
 671         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 672         int id = cxld->id;
 673
 674         if (cxld->flags & CXL_DECODER_F_ENABLE)
 675                 return 0;
 676
 677         dev_dbg(&port->dev, "%s commit\n", dev_name(&cxld->dev));
 678         if (cxl_num_decoders_committed(port) != id) {
 679                 dev_dbg(&port->dev,
 680                         "%s: out of order commit, expected decoder%d.%d\n",
 681                         dev_name(&cxld->dev), port->id,
 682                         cxl_num_decoders_committed(port));
 683                 return -EBUSY;
 684         }
 685
 686         port->commit_end++;
 687         cxld->flags |= CXL_DECODER_F_ENABLE;
 688
 689         return 0;
 690 }
 691
 692 static int mock_decoder_reset(struct cxl_decoder *cxld)
 693 {
 694         struct cxl_port *port = to_cxl_port(cxld->dev.parent);
 695         int id = cxld->id;
 696
 697         if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
 698                 return 0;
 699
 700         dev_dbg(&port->dev, "%s reset\n", dev_name(&cxld->dev));
 701         if (port->commit_end != id) {
 702                 dev_dbg(&port->dev,
 703                         "%s: out of order reset, expected decoder%d.%d\n",
 704                         dev_name(&cxld->dev), port->id, port->commit_end);
 705                 return -EBUSY;
 706         }
 707
 708         port->commit_end--;
 709         cxld->flags &= ~CXL_DECODER_F_ENABLE;
 710
 711         return 0;
 712 }
 713
 714 static void default_mock_decoder(struct cxl_decoder *cxld)
 715 {
 716         cxld->hpa_range = (struct range){
 717                 .start = 0,
 718                 .end = -1,
 719         };
 720
 721         cxld->interleave_ways = 1;
 722         cxld->interleave_granularity = 256;
 723         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 724         cxld->commit = mock_decoder_commit;
 725         cxld->reset = mock_decoder_reset;
 726 }
 727
 728 static int first_decoder(struct device *dev, void *data)
 729 {
 730         struct cxl_decoder *cxld;
 731
 732         if (!is_switch_decoder(dev))
 733                 return 0;
 734         cxld = to_cxl_decoder(dev);
 735         if (cxld->id == 0)
 736                 return 1;
 737         return 0;
 738 }
 739
 740 static void mock_init_hdm_decoder(struct cxl_decoder *cxld)
 741 {
 742         struct acpi_cedt_cfmws *window = mock_cfmws[0];
 743         struct platform_device *pdev = NULL;
 744         struct cxl_endpoint_decoder *cxled;
 745         struct cxl_switch_decoder *cxlsd;
 746         struct cxl_port *port, *iter;
 747         const int size = SZ_512M;
 748         struct cxl_memdev *cxlmd;
 749         struct cxl_dport *dport;
 750         struct device *dev;
 751         bool hb0 = false;
 752         u64 base;
 753         int i;
 754
 755         if (is_endpoint_decoder(&cxld->dev)) {
 756                 cxled = to_cxl_endpoint_decoder(&cxld->dev);
 757                 cxlmd = cxled_to_memdev(cxled);
 758                 WARN_ON(!dev_is_platform(cxlmd->dev.parent));
 759                 pdev = to_platform_device(cxlmd->dev.parent);
 760
 761                 /* check is endpoint is attach to host-bridge0 */
 762                 port = cxled_to_port(cxled);
 763                 do {
 764                         if (port->uport_dev == &cxl_host_bridge[0]->dev) {
 765                                 hb0 = true;
 766                                 break;
 767                         }
 768                         if (is_cxl_port(port->dev.parent))
 769                                 port = to_cxl_port(port->dev.parent);
 770                         else
 771                                 port = NULL;
 772                 } while (port);
 773                 port = cxled_to_port(cxled);
 774         }
 775
 776         /*
 777          * The first decoder on the first 2 devices on the first switch
 778          * attached to host-bridge0 mock a fake / static RAM region. All
 779          * other decoders are default disabled. Given the round robin
 780          * assignment those devices are named cxl_mem.0, and cxl_mem.4.
 781          *
 782          * See 'cxl list -BMPu -m cxl_mem.0,cxl_mem.4'
 783          */
 784         if (!hb0 || pdev->id % 4 || pdev->id > 4 || cxld->id > 0) {
 785                 default_mock_decoder(cxld);
 786                 return;
 787         }
 788
 789         base = window->base_hpa;
 790         cxld->hpa_range = (struct range) {
 791                 .start = base,
 792                 .end = base + size - 1,
 793         };
 794
 795         cxld->interleave_ways = 2;
 796         eig_to_granularity(window->granularity, &cxld->interleave_granularity);
 797         cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 798         cxld->flags = CXL_DECODER_F_ENABLE;
 799         cxled->state = CXL_DECODER_STATE_AUTO;
 800         port->commit_end = cxld->id;
 801         devm_cxl_dpa_reserve(cxled, 0, size / cxld->interleave_ways, 0);
 802         cxld->commit = mock_decoder_commit;
 803         cxld->reset = mock_decoder_reset;
 804
 805         /*
 806          * Now that endpoint decoder is set up, walk up the hierarchy
 807          * and setup the switch and root port decoders targeting @cxlmd.
 808          */
 809         iter = port;
 810         for (i = 0; i < 2; i++) {
 811                 dport = iter->parent_dport;
 812                 iter = dport->port;
 813                 dev = device_find_child(&iter->dev, NULL, first_decoder);
 814                 /*
 815                  * Ancestor ports are guaranteed to be enumerated before
 816                  * @port, and all ports have at least one decoder.
 817                  */
 818                 if (WARN_ON(!dev))
 819                         continue;
 820                 cxlsd = to_cxl_switch_decoder(dev);
 821                 if (i == 0) {
 822                         /* put cxl_mem.4 second in the decode order */
 823                         if (pdev->id == 4)
 824                                 cxlsd->target[1] = dport;
 825                         else
 826                                 cxlsd->target[0] = dport;
 827                 } else
 828                         cxlsd->target[0] = dport;
 829                 cxld = &cxlsd->cxld;
 830                 cxld->target_type = CXL_DECODER_HOSTONLYMEM;
 831                 cxld->flags = CXL_DECODER_F_ENABLE;
 832                 iter->commit_end = 0;
 833                 /*
 834                  * Switch targets 2 endpoints, while host bridge targets
 835                  * one root port
 836                  */
 837                 if (i == 0)
 838                         cxld->interleave_ways = 2;
 839                 else
 840                         cxld->interleave_ways = 1;
 841                 cxld->interleave_granularity = 4096;
 842                 cxld->hpa_range = (struct range) {
 843                         .start = base,
 844                         .end = base + size - 1,
 845                 };
 846                 put_device(dev);
 847         }
 848 }
 849
 850 static int mock_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
 851                                        struct cxl_endpoint_dvsec_info *info)
 852 {
 853         struct cxl_port *port = cxlhdm->port;
 854         struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
 855         int target_count, i;
 856
 857         if (is_cxl_endpoint(port))
 858                 target_count = 0;
 859         else if (is_cxl_root(parent_port))
 860                 target_count = NR_CXL_ROOT_PORTS;
 861         else
 862                 target_count = NR_CXL_SWITCH_PORTS;
 863
 864         for (i = 0; i < NR_CXL_PORT_DECODERS; i++) {
 865                 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 };
 866                 struct target_map_ctx ctx = {
 867                         .target_map = target_map,
 868                         .target_count = target_count,
 869                 };
 870                 struct cxl_decoder *cxld;
 871                 int rc;
 872
 873                 if (target_count) {
 874                         struct cxl_switch_decoder *cxlsd;
 875
 876                         cxlsd = cxl_switch_decoder_alloc(port, target_count);
 877                         if (IS_ERR(cxlsd)) {
 878                                 dev_warn(&port->dev,
 879                                          "Failed to allocate the decoder\n");
 880                                 return PTR_ERR(cxlsd);
 881                         }
 882                         cxld = &cxlsd->cxld;
 883                 } else {
 884                         struct cxl_endpoint_decoder *cxled;
 885
 886                         cxled = cxl_endpoint_decoder_alloc(port);
 887
 888                         if (IS_ERR(cxled)) {
 889                                 dev_warn(&port->dev,
 890                                          "Failed to allocate the decoder\n");
 891                                 return PTR_ERR(cxled);
 892                         }
 893                         cxld = &cxled->cxld;
 894                 }
 895
 896                 mock_init_hdm_decoder(cxld);
 897
 898                 if (target_count) {
 899                         rc = device_for_each_child(port->uport_dev, &ctx,
 900                                                    map_targets);
 901                         if (rc) {
 902                                 put_device(&cxld->dev);
 903                                 return rc;
 904                         }
 905                 }
 906
 907                 rc = cxl_decoder_add_locked(cxld, target_map);
 908                 if (rc) {
 909                         put_device(&cxld->dev);
 910                         dev_err(&port->dev, "Failed to add decoder\n");
 911                         return rc;
 912                 }
 913
 914                 rc = cxl_decoder_autoremove(&port->dev, cxld);
 915                 if (rc)
 916                         return rc;
 917                 dev_dbg(&cxld->dev, "Added to port %s\n", dev_name(&port->dev));
 918         }
 919
 920         return 0;
 921 }
 922
 923 static int mock_cxl_port_enumerate_dports(struct cxl_port *port)
 924 {
 925         struct platform_device **array;
 926         int i, array_size;
 927
 928         if (port->depth == 1) {
 929                 if (is_multi_bridge(port->uport_dev)) {
 930                         array_size = ARRAY_SIZE(cxl_root_port);
 931                         array = cxl_root_port;
 932                 } else if (is_single_bridge(port->uport_dev)) {
 933                         array_size = ARRAY_SIZE(cxl_root_single);
 934                         array = cxl_root_single;
 935                 } else {
 936                         dev_dbg(&port->dev, "%s: unknown bridge type\n",
 937                                 dev_name(port->uport_dev));
 938                         return -ENXIO;
 939                 }
 940         } else if (port->depth == 2) {
 941                 struct cxl_port *parent = to_cxl_port(port->dev.parent);
 942
 943                 if (is_multi_bridge(parent->uport_dev)) {
 944                         array_size = ARRAY_SIZE(cxl_switch_dport);
 945                         array = cxl_switch_dport;
 946                 } else if (is_single_bridge(parent->uport_dev)) {
 947                         array_size = ARRAY_SIZE(cxl_swd_single);
 948                         array = cxl_swd_single;
 949                 } else {
 950                         dev_dbg(&port->dev, "%s: unknown bridge type\n",
 951                                 dev_name(port->uport_dev));
 952                         return -ENXIO;
 953                 }
 954         } else {
 955                 dev_WARN_ONCE(&port->dev, 1, "unexpected depth %d\n",
 956                               port->depth);
 957                 return -ENXIO;
 958         }
 959
 960         for (i = 0; i < array_size; i++) {
 961                 struct platform_device *pdev = array[i];
 962                 struct cxl_dport *dport;
 963
 964                 if (pdev->dev.parent != port->uport_dev) {
 965                         dev_dbg(&port->dev, "%s: mismatch parent %s\n",
 966                                 dev_name(port->uport_dev),
 967                                 dev_name(pdev->dev.parent));
 968                         continue;
 969                 }
 970
 971                 dport = devm_cxl_add_dport(port, &pdev->dev, pdev->id,
 972                                            CXL_RESOURCE_NONE);
 973
 974                 if (IS_ERR(dport))
 975                         return PTR_ERR(dport);
 976         }
 977
 978         return 0;
 979 }
 980
 981 /*
 982  * Faking the cxl_dpa_perf for the memdev when appropriate.
 983  */
 984 static void dpa_perf_setup(struct cxl_port *endpoint, struct range *range,
 985                            struct cxl_dpa_perf *dpa_perf)
 986 {
 987         dpa_perf->qos_class = FAKE_QTG_ID;
 988         dpa_perf->dpa_range = *range;
 989         for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
 990                 dpa_perf->coord[i].read_latency = 500;
 991                 dpa_perf->coord[i].write_latency = 500;
 992                 dpa_perf->coord[i].read_bandwidth = 1000;
 993                 dpa_perf->coord[i].write_bandwidth = 1000;
 994         }
 995 }
 996
 997 static void mock_cxl_endpoint_parse_cdat(struct cxl_port *port)
 998 {
 999         struct cxl_root *cxl_root __free(put_cxl_root) =
1000                 find_cxl_root(port);
1001         struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev);
1002         struct cxl_dev_state *cxlds = cxlmd->cxlds;
1003         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
1004         struct range pmem_range = {
1005                 .start = cxlds->pmem_res.start,
1006                 .end = cxlds->pmem_res.end,
1007         };
1008         struct range ram_range = {
1009                 .start = cxlds->ram_res.start,
1010                 .end = cxlds->ram_res.end,
1011         };
1012
1013         if (!cxl_root)
1014                 return;
1015
1016         if (range_len(&ram_range))
1017                 dpa_perf_setup(port, &ram_range, &mds->ram_perf);
1018
1019         if (range_len(&pmem_range))
1020                 dpa_perf_setup(port, &pmem_range, &mds->pmem_perf);
1021
1022         cxl_memdev_update_perf(cxlmd);
1023 }
1024
1025 static struct cxl_mock_ops cxl_mock_ops = {
1026         .is_mock_adev = is_mock_adev,
1027         .is_mock_bridge = is_mock_bridge,
1028         .is_mock_bus = is_mock_bus,
1029         .is_mock_port = is_mock_port,
1030         .is_mock_dev = is_mock_dev,
1031         .acpi_table_parse_cedt = mock_acpi_table_parse_cedt,
1032         .acpi_evaluate_integer = mock_acpi_evaluate_integer,
1033         .acpi_pci_find_root = mock_acpi_pci_find_root,
1034         .devm_cxl_port_enumerate_dports = mock_cxl_port_enumerate_dports,
1035         .devm_cxl_setup_hdm = mock_cxl_setup_hdm,
1036         .devm_cxl_add_passthrough_decoder = mock_cxl_add_passthrough_decoder,
1037         .devm_cxl_enumerate_decoders = mock_cxl_enumerate_decoders,
1038         .cxl_endpoint_parse_cdat = mock_cxl_endpoint_parse_cdat,
1039         .list = LIST_HEAD_INIT(cxl_mock_ops.list),
1040 };
1041
1042 static void mock_companion(struct acpi_device *adev, struct device *dev)
1043 {
1044         device_initialize(&adev->dev);
1045         fwnode_init(&adev->fwnode, NULL);
1046         dev->fwnode = &adev->fwnode;
1047         adev->fwnode.dev = dev;
1048 }
1049
1050 #ifndef SZ_64G
1051 #define SZ_64G (SZ_32G * 2)
1052 #endif
1053
1054 static __init int cxl_rch_init(void)
1055 {
1056         int rc, i;
1057
1058         for (i = 0; i < ARRAY_SIZE(cxl_rch); i++) {
1059                 int idx = NR_CXL_HOST_BRIDGES + NR_CXL_SINGLE_HOST + i;
1060                 struct acpi_device *adev = &host_bridge[idx];
1061                 struct platform_device *pdev;
1062
1063                 pdev = platform_device_alloc("cxl_host_bridge", idx);
1064                 if (!pdev)
1065                         goto err_bridge;
1066
1067                 mock_companion(adev, &pdev->dev);
1068                 rc = platform_device_add(pdev);
1069                 if (rc) {
1070                         platform_device_put(pdev);
1071                         goto err_bridge;
1072                 }
1073
1074                 cxl_rch[i] = pdev;
1075                 mock_pci_bus[idx].bridge = &pdev->dev;
1076                 rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
1077                                        "firmware_node");
1078                 if (rc)
1079                         goto err_bridge;
1080         }
1081
1082         for (i = 0; i < ARRAY_SIZE(cxl_rcd); i++) {
1083                 int idx = NR_MEM_MULTI + NR_MEM_SINGLE + i;
1084                 struct platform_device *rch = cxl_rch[i];
1085                 struct platform_device *pdev;
1086
1087                 pdev = platform_device_alloc("cxl_rcd", idx);
1088                 if (!pdev)
1089                         goto err_mem;
1090                 pdev->dev.parent = &rch->dev;
1091                 set_dev_node(&pdev->dev, i % 2);
1092
1093                 rc = platform_device_add(pdev);
1094                 if (rc) {
1095                         platform_device_put(pdev);
1096                         goto err_mem;
1097                 }
1098                 cxl_rcd[i] = pdev;
1099         }
1100
1101         return 0;
1102
1103 err_mem:
1104         for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--)
1105                 platform_device_unregister(cxl_rcd[i]);
1106 err_bridge:
1107         for (i = ARRAY_SIZE(cxl_rch) - 1; i >= 0; i--) {
1108                 struct platform_device *pdev = cxl_rch[i];
1109
1110                 if (!pdev)
1111                         continue;
1112                 sysfs_remove_link(&pdev->dev.kobj, "firmware_node");
1113                 platform_device_unregister(cxl_rch[i]);
1114         }
1115
1116         return rc;
1117 }
1118
1119 static void cxl_rch_exit(void)
1120 {
1121         int i;
1122
1123         for (i = ARRAY_SIZE(cxl_rcd) - 1; i >= 0; i--)
1124                 platform_device_unregister(cxl_rcd[i]);
1125         for (i = ARRAY_SIZE(cxl_rch) - 1; i >= 0; i--) {
1126                 struct platform_device *pdev = cxl_rch[i];
1127
1128                 if (!pdev)
1129                         continue;
1130                 sysfs_remove_link(&pdev->dev.kobj, "firmware_node");
1131                 platform_device_unregister(cxl_rch[i]);
1132         }
1133 }
1134
1135 static __init int cxl_single_init(void)
1136 {
1137         int i, rc;
1138
1139         for (i = 0; i < ARRAY_SIZE(cxl_hb_single); i++) {
1140                 struct acpi_device *adev =
1141                         &host_bridge[NR_CXL_HOST_BRIDGES + i];
1142                 struct platform_device *pdev;
1143
1144                 pdev = platform_device_alloc("cxl_host_bridge",
1145                                              NR_CXL_HOST_BRIDGES + i);
1146                 if (!pdev)
1147                         goto err_bridge;
1148
1149                 mock_companion(adev, &pdev->dev);
1150                 rc = platform_device_add(pdev);
1151                 if (rc) {
1152                         platform_device_put(pdev);
1153                         goto err_bridge;
1154                 }
1155
1156                 cxl_hb_single[i] = pdev;
1157                 mock_pci_bus[i + NR_CXL_HOST_BRIDGES].bridge = &pdev->dev;
1158                 rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
1159                                        "physical_node");
1160                 if (rc)
1161                         goto err_bridge;
1162         }
1163
1164         for (i = 0; i < ARRAY_SIZE(cxl_root_single); i++) {
1165                 struct platform_device *bridge =
1166                         cxl_hb_single[i % ARRAY_SIZE(cxl_hb_single)];
1167                 struct platform_device *pdev;
1168
1169                 pdev = platform_device_alloc("cxl_root_port",
1170                                              NR_MULTI_ROOT + i);
1171                 if (!pdev)
1172                         goto err_port;
1173                 pdev->dev.parent = &bridge->dev;
1174
1175                 rc = platform_device_add(pdev);
1176                 if (rc) {
1177                         platform_device_put(pdev);
1178                         goto err_port;
1179                 }
1180                 cxl_root_single[i] = pdev;
1181         }
1182
1183         for (i = 0; i < ARRAY_SIZE(cxl_swu_single); i++) {
1184                 struct platform_device *root_port = cxl_root_single[i];
1185                 struct platform_device *pdev;
1186
1187                 pdev = platform_device_alloc("cxl_switch_uport",
1188                                              NR_MULTI_ROOT + i);
1189                 if (!pdev)
1190                         goto err_uport;
1191                 pdev->dev.parent = &root_port->dev;
1192
1193                 rc = platform_device_add(pdev);
1194                 if (rc) {
1195                         platform_device_put(pdev);
1196                         goto err_uport;
1197                 }
1198                 cxl_swu_single[i] = pdev;
1199         }
1200
1201         for (i = 0; i < ARRAY_SIZE(cxl_swd_single); i++) {
1202                 struct platform_device *uport =
1203                         cxl_swu_single[i % ARRAY_SIZE(cxl_swu_single)];
1204                 struct platform_device *pdev;
1205
1206                 pdev = platform_device_alloc("cxl_switch_dport",
1207                                              i + NR_MEM_MULTI);
1208                 if (!pdev)
1209                         goto err_dport;
1210                 pdev->dev.parent = &uport->dev;
1211
1212                 rc = platform_device_add(pdev);
1213                 if (rc) {
1214                         platform_device_put(pdev);
1215                         goto err_dport;
1216                 }
1217                 cxl_swd_single[i] = pdev;
1218         }
1219
1220         for (i = 0; i < ARRAY_SIZE(cxl_mem_single); i++) {
1221                 struct platform_device *dport = cxl_swd_single[i];
1222                 struct platform_device *pdev;
1223
1224                 pdev = platform_device_alloc("cxl_mem", NR_MEM_MULTI + i);
1225                 if (!pdev)
1226                         goto err_mem;
1227                 pdev->dev.parent = &dport->dev;
1228                 set_dev_node(&pdev->dev, i % 2);
1229
1230                 rc = platform_device_add(pdev);
1231                 if (rc) {
1232                         platform_device_put(pdev);
1233                         goto err_mem;
1234                 }
1235                 cxl_mem_single[i] = pdev;
1236         }
1237
1238         return 0;
1239
1240 err_mem:
1241         for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--)
1242                 platform_device_unregister(cxl_mem_single[i]);
1243 err_dport:
1244         for (i = ARRAY_SIZE(cxl_swd_single) - 1; i >= 0; i--)
1245                 platform_device_unregister(cxl_swd_single[i]);
1246 err_uport:
1247         for (i = ARRAY_SIZE(cxl_swu_single) - 1; i >= 0; i--)
1248                 platform_device_unregister(cxl_swu_single[i]);
1249 err_port:
1250         for (i = ARRAY_SIZE(cxl_root_single) - 1; i >= 0; i--)
1251                 platform_device_unregister(cxl_root_single[i]);
1252 err_bridge:
1253         for (i = ARRAY_SIZE(cxl_hb_single) - 1; i >= 0; i--) {
1254                 struct platform_device *pdev = cxl_hb_single[i];
1255
1256                 if (!pdev)
1257                         continue;
1258                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1259                 platform_device_unregister(cxl_hb_single[i]);
1260         }
1261
1262         return rc;
1263 }
1264
1265 static void cxl_single_exit(void)
1266 {
1267         int i;
1268
1269         for (i = ARRAY_SIZE(cxl_mem_single) - 1; i >= 0; i--)
1270                 platform_device_unregister(cxl_mem_single[i]);
1271         for (i = ARRAY_SIZE(cxl_swd_single) - 1; i >= 0; i--)
1272                 platform_device_unregister(cxl_swd_single[i]);
1273         for (i = ARRAY_SIZE(cxl_swu_single) - 1; i >= 0; i--)
1274                 platform_device_unregister(cxl_swu_single[i]);
1275         for (i = ARRAY_SIZE(cxl_root_single) - 1; i >= 0; i--)
1276                 platform_device_unregister(cxl_root_single[i]);
1277         for (i = ARRAY_SIZE(cxl_hb_single) - 1; i >= 0; i--) {
1278                 struct platform_device *pdev = cxl_hb_single[i];
1279
1280                 if (!pdev)
1281                         continue;
1282                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1283                 platform_device_unregister(cxl_hb_single[i]);
1284         }
1285 }
1286
1287 static __init int cxl_test_init(void)
1288 {
1289         int rc, i;
1290
1291         cxl_acpi_test();
1292         cxl_core_test();
1293         cxl_mem_test();
1294         cxl_pmem_test();
1295         cxl_port_test();
1296
1297         register_cxl_mock_ops(&cxl_mock_ops);
1298
1299         cxl_mock_pool = gen_pool_create(ilog2(SZ_2M), NUMA_NO_NODE);
1300         if (!cxl_mock_pool) {
1301                 rc = -ENOMEM;
1302                 goto err_gen_pool_create;
1303         }
1304
1305         rc = gen_pool_add(cxl_mock_pool, iomem_resource.end + 1 - SZ_64G,
1306                           SZ_64G, NUMA_NO_NODE);
1307         if (rc)
1308                 goto err_gen_pool_add;
1309
1310         if (interleave_arithmetic == 1) {
1311                 cfmws_start = CFMWS_XOR_ARRAY_START;
1312                 cfmws_end = CFMWS_XOR_ARRAY_END;
1313         } else {
1314                 cfmws_start = CFMWS_MOD_ARRAY_START;
1315                 cfmws_end = CFMWS_MOD_ARRAY_END;
1316         }
1317
1318         rc = populate_cedt();
1319         if (rc)
1320                 goto err_populate;
1321
1322         for (i = 0; i < ARRAY_SIZE(cxl_host_bridge); i++) {
1323                 struct acpi_device *adev = &host_bridge[i];
1324                 struct platform_device *pdev;
1325
1326                 pdev = platform_device_alloc("cxl_host_bridge", i);
1327                 if (!pdev)
1328                         goto err_bridge;
1329
1330                 mock_companion(adev, &pdev->dev);
1331                 rc = platform_device_add(pdev);
1332                 if (rc) {
1333                         platform_device_put(pdev);
1334                         goto err_bridge;
1335                 }
1336
1337                 cxl_host_bridge[i] = pdev;
1338                 mock_pci_bus[i].bridge = &pdev->dev;
1339                 rc = sysfs_create_link(&pdev->dev.kobj, &pdev->dev.kobj,
1340                                        "physical_node");
1341                 if (rc)
1342                         goto err_bridge;
1343         }
1344
1345         for (i = 0; i < ARRAY_SIZE(cxl_root_port); i++) {
1346                 struct platform_device *bridge =
1347                         cxl_host_bridge[i % ARRAY_SIZE(cxl_host_bridge)];
1348                 struct platform_device *pdev;
1349
1350                 pdev = platform_device_alloc("cxl_root_port", i);
1351                 if (!pdev)
1352                         goto err_port;
1353                 pdev->dev.parent = &bridge->dev;
1354
1355                 rc = platform_device_add(pdev);
1356                 if (rc) {
1357                         platform_device_put(pdev);
1358                         goto err_port;
1359                 }
1360                 cxl_root_port[i] = pdev;
1361         }
1362
1363         BUILD_BUG_ON(ARRAY_SIZE(cxl_switch_uport) != ARRAY_SIZE(cxl_root_port));
1364         for (i = 0; i < ARRAY_SIZE(cxl_switch_uport); i++) {
1365                 struct platform_device *root_port = cxl_root_port[i];
1366                 struct platform_device *pdev;
1367
1368                 pdev = platform_device_alloc("cxl_switch_uport", i);
1369                 if (!pdev)
1370                         goto err_uport;
1371                 pdev->dev.parent = &root_port->dev;
1372
1373                 rc = platform_device_add(pdev);
1374                 if (rc) {
1375                         platform_device_put(pdev);
1376                         goto err_uport;
1377                 }
1378                 cxl_switch_uport[i] = pdev;
1379         }
1380
1381         for (i = 0; i < ARRAY_SIZE(cxl_switch_dport); i++) {
1382                 struct platform_device *uport =
1383                         cxl_switch_uport[i % ARRAY_SIZE(cxl_switch_uport)];
1384                 struct platform_device *pdev;
1385
1386                 pdev = platform_device_alloc("cxl_switch_dport", i);
1387                 if (!pdev)
1388                         goto err_dport;
1389                 pdev->dev.parent = &uport->dev;
1390
1391                 rc = platform_device_add(pdev);
1392                 if (rc) {
1393                         platform_device_put(pdev);
1394                         goto err_dport;
1395                 }
1396                 cxl_switch_dport[i] = pdev;
1397         }
1398
1399         for (i = 0; i < ARRAY_SIZE(cxl_mem); i++) {
1400                 struct platform_device *dport = cxl_switch_dport[i];
1401                 struct platform_device *pdev;
1402
1403                 pdev = platform_device_alloc("cxl_mem", i);
1404                 if (!pdev)
1405                         goto err_mem;
1406                 pdev->dev.parent = &dport->dev;
1407                 set_dev_node(&pdev->dev, i % 2);
1408
1409                 rc = platform_device_add(pdev);
1410                 if (rc) {
1411                         platform_device_put(pdev);
1412                         goto err_mem;
1413                 }
1414                 cxl_mem[i] = pdev;
1415         }
1416
1417         rc = cxl_single_init();
1418         if (rc)
1419                 goto err_mem;
1420
1421         rc = cxl_rch_init();
1422         if (rc)
1423                 goto err_single;
1424
1425         cxl_acpi = platform_device_alloc("cxl_acpi", 0);
1426         if (!cxl_acpi)
1427                 goto err_rch;
1428
1429         mock_companion(&acpi0017_mock, &cxl_acpi->dev);
1430         acpi0017_mock.dev.bus = &platform_bus_type;
1431
1432         rc = platform_device_add(cxl_acpi);
1433         if (rc)
1434                 goto err_add;
1435
1436         return 0;
1437
1438 err_add:
1439         platform_device_put(cxl_acpi);
1440 err_rch:
1441         cxl_rch_exit();
1442 err_single:
1443         cxl_single_exit();
1444 err_mem:
1445         for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--)
1446                 platform_device_unregister(cxl_mem[i]);
1447 err_dport:
1448         for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--)
1449                 platform_device_unregister(cxl_switch_dport[i]);
1450 err_uport:
1451         for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--)
1452                 platform_device_unregister(cxl_switch_uport[i]);
1453 err_port:
1454         for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
1455                 platform_device_unregister(cxl_root_port[i]);
1456 err_bridge:
1457         for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) {
1458                 struct platform_device *pdev = cxl_host_bridge[i];
1459
1460                 if (!pdev)
1461                         continue;
1462                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1463                 platform_device_unregister(cxl_host_bridge[i]);
1464         }
1465 err_populate:
1466         depopulate_all_mock_resources();
1467 err_gen_pool_add:
1468         gen_pool_destroy(cxl_mock_pool);
1469 err_gen_pool_create:
1470         unregister_cxl_mock_ops(&cxl_mock_ops);
1471         return rc;
1472 }
1473
1474 static __exit void cxl_test_exit(void)
1475 {
1476         int i;
1477
1478         platform_device_unregister(cxl_acpi);
1479         cxl_rch_exit();
1480         cxl_single_exit();
1481         for (i = ARRAY_SIZE(cxl_mem) - 1; i >= 0; i--)
1482                 platform_device_unregister(cxl_mem[i]);
1483         for (i = ARRAY_SIZE(cxl_switch_dport) - 1; i >= 0; i--)
1484                 platform_device_unregister(cxl_switch_dport[i]);
1485         for (i = ARRAY_SIZE(cxl_switch_uport) - 1; i >= 0; i--)
1486                 platform_device_unregister(cxl_switch_uport[i]);
1487         for (i = ARRAY_SIZE(cxl_root_port) - 1; i >= 0; i--)
1488                 platform_device_unregister(cxl_root_port[i]);
1489         for (i = ARRAY_SIZE(cxl_host_bridge) - 1; i >= 0; i--) {
1490                 struct platform_device *pdev = cxl_host_bridge[i];
1491
1492                 if (!pdev)
1493                         continue;
1494                 sysfs_remove_link(&pdev->dev.kobj, "physical_node");
1495                 platform_device_unregister(cxl_host_bridge[i]);
1496         }
1497         depopulate_all_mock_resources();
1498         gen_pool_destroy(cxl_mock_pool);
1499         unregister_cxl_mock_ops(&cxl_mock_ops);
1500 }
1501
1502 module_param(interleave_arithmetic, int, 0444);
1503 MODULE_PARM_DESC(interleave_arithmetic, "Modulo:0, XOR:1");
1504 module_init(cxl_test_init);
1505 module_exit(cxl_test_exit);
1506 MODULE_LICENSE("GPL v2");
1507 MODULE_IMPORT_NS(ACPI);
1508 MODULE_IMPORT_NS(CXL);