drivers/cxl/core/region.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */
   3 #include <linux/memregion.h>
   4 #include <linux/genalloc.h>
   5 #include <linux/device.h>
   6 #include <linux/module.h>
   7 #include <linux/slab.h>
   8 #include <linux/uuid.h>
   9 #include <linux/sort.h>
  10 #include <linux/idr.h>
  11 #include <cxlmem.h>
  12 #include <cxl.h>
  13 #include "core.h"
  14
  15 /**
  16  * DOC: cxl core region
  17  *
  18  * CXL Regions represent mapped memory capacity in system physical address
  19  * space. Whereas the CXL Root Decoders identify the bounds of potential CXL
  20  * Memory ranges, Regions represent the active mapped capacity by the HDM
  21  * Decoder Capability structures throughout the Host Bridges, Switches, and
  22  * Endpoints in the topology.
  23  *
  24  * Region configuration has ordering constraints. UUID may be set at any time
  25  * but is only visible for persistent regions.
  26  * 1. Interleave granularity
  27  * 2. Interleave size
  28  * 3. Decoder targets
  29  */
  30
  31 static struct cxl_region *to_cxl_region(struct device *dev);
  32
  33 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
  34                          char *buf)
  35 {
  36         struct cxl_region *cxlr = to_cxl_region(dev);
  37         struct cxl_region_params *p = &cxlr->params;
  38         ssize_t rc;
  39
  40         rc = down_read_interruptible(&cxl_region_rwsem);
  41         if (rc)
  42                 return rc;
  43         if (cxlr->mode != CXL_DECODER_PMEM)
  44                 rc = sysfs_emit(buf, "\n");
  45         else
  46                 rc = sysfs_emit(buf, "%pUb\n", &p->uuid);
  47         up_read(&cxl_region_rwsem);
  48
  49         return rc;
  50 }
  51
  52 static int is_dup(struct device *match, void *data)
  53 {
  54         struct cxl_region_params *p;
  55         struct cxl_region *cxlr;
  56         uuid_t *uuid = data;
  57
  58         if (!is_cxl_region(match))
  59                 return 0;
  60
  61         lockdep_assert_held(&cxl_region_rwsem);
  62         cxlr = to_cxl_region(match);
  63         p = &cxlr->params;
  64
  65         if (uuid_equal(&p->uuid, uuid)) {
  66                 dev_dbg(match, "already has uuid: %pUb\n", uuid);
  67                 return -EBUSY;
  68         }
  69
  70         return 0;
  71 }
  72
  73 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr,
  74                           const char *buf, size_t len)
  75 {
  76         struct cxl_region *cxlr = to_cxl_region(dev);
  77         struct cxl_region_params *p = &cxlr->params;
  78         uuid_t temp;
  79         ssize_t rc;
  80
  81         if (len != UUID_STRING_LEN + 1)
  82                 return -EINVAL;
  83
  84         rc = uuid_parse(buf, &temp);
  85         if (rc)
  86                 return rc;
  87
  88         if (uuid_is_null(&temp))
  89                 return -EINVAL;
  90
  91         rc = down_write_killable(&cxl_region_rwsem);
  92         if (rc)
  93                 return rc;
  94
  95         if (uuid_equal(&p->uuid, &temp))
  96                 goto out;
  97
  98         rc = -EBUSY;
  99         if (p->state >= CXL_CONFIG_ACTIVE)
 100                 goto out;
 101
 102         rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup);
 103         if (rc < 0)
 104                 goto out;
 105
 106         uuid_copy(&p->uuid, &temp);
 107 out:
 108         up_write(&cxl_region_rwsem);
 109
 110         if (rc)
 111                 return rc;
 112         return len;
 113 }
 114 static DEVICE_ATTR_RW(uuid);
 115
 116 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port,
 117                                           struct cxl_region *cxlr)
 118 {
 119         return xa_load(&port->regions, (unsigned long)cxlr);
 120 }
 121
 122 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr)
 123 {
 124         if (!cpu_cache_has_invalidate_memregion()) {
 125                 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) {
 126                         dev_info_once(
 127                                 &cxlr->dev,
 128                                 "Bypassing cpu_cache_invalidate_memregion() for testing!\n");
 129                         return 0;
 130                 } else {
 131                         dev_err(&cxlr->dev,
 132                                 "Failed to synchronize CPU cache state\n");
 133                         return -ENXIO;
 134                 }
 135         }
 136
 137         cpu_cache_invalidate_memregion(IORES_DESC_CXL);
 138         return 0;
 139 }
 140
 141 static int cxl_region_decode_reset(struct cxl_region *cxlr, int count)
 142 {
 143         struct cxl_region_params *p = &cxlr->params;
 144         int i, rc = 0;
 145
 146         /*
 147          * Before region teardown attempt to flush, and if the flush
 148          * fails cancel the region teardown for data consistency
 149          * concerns
 150          */
 151         rc = cxl_region_invalidate_memregion(cxlr);
 152         if (rc)
 153                 return rc;
 154
 155         for (i = count - 1; i >= 0; i--) {
 156                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 157                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 158                 struct cxl_port *iter = cxled_to_port(cxled);
 159                 struct cxl_dev_state *cxlds = cxlmd->cxlds;
 160                 struct cxl_ep *ep;
 161
 162                 if (cxlds->rcd)
 163                         goto endpoint_reset;
 164
 165                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
 166                         iter = to_cxl_port(iter->dev.parent);
 167
 168                 for (ep = cxl_ep_load(iter, cxlmd); iter;
 169                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 170                         struct cxl_region_ref *cxl_rr;
 171                         struct cxl_decoder *cxld;
 172
 173                         cxl_rr = cxl_rr_load(iter, cxlr);
 174                         cxld = cxl_rr->decoder;
 175                         if (cxld->reset)
 176                                 rc = cxld->reset(cxld);
 177                         if (rc)
 178                                 return rc;
 179                         set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 180                 }
 181
 182 endpoint_reset:
 183                 rc = cxled->cxld.reset(&cxled->cxld);
 184                 if (rc)
 185                         return rc;
 186                 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 187         }
 188
 189         /* all decoders associated with this region have been torn down */
 190         clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags);
 191
 192         return 0;
 193 }
 194
 195 static int commit_decoder(struct cxl_decoder *cxld)
 196 {
 197         struct cxl_switch_decoder *cxlsd = NULL;
 198
 199         if (cxld->commit)
 200                 return cxld->commit(cxld);
 201
 202         if (is_switch_decoder(&cxld->dev))
 203                 cxlsd = to_cxl_switch_decoder(&cxld->dev);
 204
 205         if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1,
 206                           "->commit() is required\n"))
 207                 return -ENXIO;
 208         return 0;
 209 }
 210
 211 static int cxl_region_decode_commit(struct cxl_region *cxlr)
 212 {
 213         struct cxl_region_params *p = &cxlr->params;
 214         int i, rc = 0;
 215
 216         for (i = 0; i < p->nr_targets; i++) {
 217                 struct cxl_endpoint_decoder *cxled = p->targets[i];
 218                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 219                 struct cxl_region_ref *cxl_rr;
 220                 struct cxl_decoder *cxld;
 221                 struct cxl_port *iter;
 222                 struct cxl_ep *ep;
 223
 224                 /* commit bottom up */
 225                 for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
 226                      iter = to_cxl_port(iter->dev.parent)) {
 227                         cxl_rr = cxl_rr_load(iter, cxlr);
 228                         cxld = cxl_rr->decoder;
 229                         rc = commit_decoder(cxld);
 230                         if (rc)
 231                                 break;
 232                 }
 233
 234                 if (rc) {
 235                         /* programming @iter failed, teardown */
 236                         for (ep = cxl_ep_load(iter, cxlmd); ep && iter;
 237                              iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
 238                                 cxl_rr = cxl_rr_load(iter, cxlr);
 239                                 cxld = cxl_rr->decoder;
 240                                 if (cxld->reset)
 241                                         cxld->reset(cxld);
 242                         }
 243
 244                         cxled->cxld.reset(&cxled->cxld);
 245                         goto err;
 246                 }
 247         }
 248
 249         return 0;
 250
 251 err:
 252         /* undo the targets that were successfully committed */
 253         cxl_region_decode_reset(cxlr, i);
 254         return rc;
 255 }
 256
 257 static ssize_t commit_store(struct device *dev, struct device_attribute *attr,
 258                             const char *buf, size_t len)
 259 {
 260         struct cxl_region *cxlr = to_cxl_region(dev);
 261         struct cxl_region_params *p = &cxlr->params;
 262         bool commit;
 263         ssize_t rc;
 264
 265         rc = kstrtobool(buf, &commit);
 266         if (rc)
 267                 return rc;
 268
 269         rc = down_write_killable(&cxl_region_rwsem);
 270         if (rc)
 271                 return rc;
 272
 273         /* Already in the requested state? */
 274         if (commit && p->state >= CXL_CONFIG_COMMIT)
 275                 goto out;
 276         if (!commit && p->state < CXL_CONFIG_COMMIT)
 277                 goto out;
 278
 279         /* Not ready to commit? */
 280         if (commit && p->state < CXL_CONFIG_ACTIVE) {
 281                 rc = -ENXIO;
 282                 goto out;
 283         }
 284
 285         /*
 286          * Invalidate caches before region setup to drop any speculative
 287          * consumption of this address space
 288          */
 289         rc = cxl_region_invalidate_memregion(cxlr);
 290         if (rc)
 291                 goto out;
 292
 293         if (commit) {
 294                 rc = cxl_region_decode_commit(cxlr);
 295                 if (rc == 0)
 296                         p->state = CXL_CONFIG_COMMIT;
 297         } else {
 298                 p->state = CXL_CONFIG_RESET_PENDING;
 299                 up_write(&cxl_region_rwsem);
 300                 device_release_driver(&cxlr->dev);
 301                 down_write(&cxl_region_rwsem);
 302
 303                 /*
 304                  * The lock was dropped, so need to revalidate that the reset is
 305                  * still pending.
 306                  */
 307                 if (p->state == CXL_CONFIG_RESET_PENDING) {
 308                         rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
 309                         /*
 310                          * Revert to committed since there may still be active
 311                          * decoders associated with this region, or move forward
 312                          * to active to mark the reset successful
 313                          */
 314                         if (rc)
 315                                 p->state = CXL_CONFIG_COMMIT;
 316                         else
 317                                 p->state = CXL_CONFIG_ACTIVE;
 318                 }
 319         }
 320
 321 out:
 322         up_write(&cxl_region_rwsem);
 323
 324         if (rc)
 325                 return rc;
 326         return len;
 327 }
 328
 329 static ssize_t commit_show(struct device *dev, struct device_attribute *attr,
 330                            char *buf)
 331 {
 332         struct cxl_region *cxlr = to_cxl_region(dev);
 333         struct cxl_region_params *p = &cxlr->params;
 334         ssize_t rc;
 335
 336         rc = down_read_interruptible(&cxl_region_rwsem);
 337         if (rc)
 338                 return rc;
 339         rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT);
 340         up_read(&cxl_region_rwsem);
 341
 342         return rc;
 343 }
 344 static DEVICE_ATTR_RW(commit);
 345
 346 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a,
 347                                   int n)
 348 {
 349         struct device *dev = kobj_to_dev(kobj);
 350         struct cxl_region *cxlr = to_cxl_region(dev);
 351
 352         /*
 353          * Support tooling that expects to find a 'uuid' attribute for all
 354          * regions regardless of mode.
 355          */
 356         if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM)
 357                 return 0444;
 358         return a->mode;
 359 }
 360
 361 static ssize_t interleave_ways_show(struct device *dev,
 362                                     struct device_attribute *attr, char *buf)
 363 {
 364         struct cxl_region *cxlr = to_cxl_region(dev);
 365         struct cxl_region_params *p = &cxlr->params;
 366         ssize_t rc;
 367
 368         rc = down_read_interruptible(&cxl_region_rwsem);
 369         if (rc)
 370                 return rc;
 371         rc = sysfs_emit(buf, "%d\n", p->interleave_ways);
 372         up_read(&cxl_region_rwsem);
 373
 374         return rc;
 375 }
 376
 377 static const struct attribute_group *get_cxl_region_target_group(void);
 378
 379 static ssize_t interleave_ways_store(struct device *dev,
 380                                      struct device_attribute *attr,
 381                                      const char *buf, size_t len)
 382 {
 383         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 384         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 385         struct cxl_region *cxlr = to_cxl_region(dev);
 386         struct cxl_region_params *p = &cxlr->params;
 387         unsigned int val, save;
 388         int rc;
 389         u8 iw;
 390
 391         rc = kstrtouint(buf, 0, &val);
 392         if (rc)
 393                 return rc;
 394
 395         rc = ways_to_eiw(val, &iw);
 396         if (rc)
 397                 return rc;
 398
 399         /*
 400          * Even for x3, x6, and x12 interleaves the region interleave must be a
 401          * power of 2 multiple of the host bridge interleave.
 402          */
 403         if (!is_power_of_2(val / cxld->interleave_ways) ||
 404             (val % cxld->interleave_ways)) {
 405                 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val);
 406                 return -EINVAL;
 407         }
 408
 409         rc = down_write_killable(&cxl_region_rwsem);
 410         if (rc)
 411                 return rc;
 412         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 413                 rc = -EBUSY;
 414                 goto out;
 415         }
 416
 417         save = p->interleave_ways;
 418         p->interleave_ways = val;
 419         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
 420         if (rc)
 421                 p->interleave_ways = save;
 422 out:
 423         up_write(&cxl_region_rwsem);
 424         if (rc)
 425                 return rc;
 426         return len;
 427 }
 428 static DEVICE_ATTR_RW(interleave_ways);
 429
 430 static ssize_t interleave_granularity_show(struct device *dev,
 431                                            struct device_attribute *attr,
 432                                            char *buf)
 433 {
 434         struct cxl_region *cxlr = to_cxl_region(dev);
 435         struct cxl_region_params *p = &cxlr->params;
 436         ssize_t rc;
 437
 438         rc = down_read_interruptible(&cxl_region_rwsem);
 439         if (rc)
 440                 return rc;
 441         rc = sysfs_emit(buf, "%d\n", p->interleave_granularity);
 442         up_read(&cxl_region_rwsem);
 443
 444         return rc;
 445 }
 446
 447 static ssize_t interleave_granularity_store(struct device *dev,
 448                                             struct device_attribute *attr,
 449                                             const char *buf, size_t len)
 450 {
 451         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
 452         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
 453         struct cxl_region *cxlr = to_cxl_region(dev);
 454         struct cxl_region_params *p = &cxlr->params;
 455         int rc, val;
 456         u16 ig;
 457
 458         rc = kstrtoint(buf, 0, &val);
 459         if (rc)
 460                 return rc;
 461
 462         rc = granularity_to_eig(val, &ig);
 463         if (rc)
 464                 return rc;
 465
 466         /*
 467          * When the host-bridge is interleaved, disallow region granularity !=
 468          * root granularity. Regions with a granularity less than the root
 469          * interleave result in needing multiple endpoints to support a single
 470          * slot in the interleave (possible to support in the future). Regions
 471          * with a granularity greater than the root interleave result in invalid
 472          * DPA translations (invalid to support).
 473          */
 474         if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity)
 475                 return -EINVAL;
 476
 477         rc = down_write_killable(&cxl_region_rwsem);
 478         if (rc)
 479                 return rc;
 480         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
 481                 rc = -EBUSY;
 482                 goto out;
 483         }
 484
 485         p->interleave_granularity = val;
 486 out:
 487         up_write(&cxl_region_rwsem);
 488         if (rc)
 489                 return rc;
 490         return len;
 491 }
 492 static DEVICE_ATTR_RW(interleave_granularity);
 493
 494 static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
 495                              char *buf)
 496 {
 497         struct cxl_region *cxlr = to_cxl_region(dev);
 498         struct cxl_region_params *p = &cxlr->params;
 499         u64 resource = -1ULL;
 500         ssize_t rc;
 501
 502         rc = down_read_interruptible(&cxl_region_rwsem);
 503         if (rc)
 504                 return rc;
 505         if (p->res)
 506                 resource = p->res->start;
 507         rc = sysfs_emit(buf, "%#llx\n", resource);
 508         up_read(&cxl_region_rwsem);
 509
 510         return rc;
 511 }
 512 static DEVICE_ATTR_RO(resource);
 513
 514 static ssize_t mode_show(struct device *dev, struct device_attribute *attr,
 515                          char *buf)
 516 {
 517         struct cxl_region *cxlr = to_cxl_region(dev);
 518
 519         return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode));
 520 }
 521 static DEVICE_ATTR_RO(mode);
 522
 523 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size)
 524 {
 525         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
 526         struct cxl_region_params *p = &cxlr->params;
 527         struct resource *res;
 528         u64 remainder = 0;
 529
 530         lockdep_assert_held_write(&cxl_region_rwsem);
 531
 532         /* Nothing to do... */
 533         if (p->res && resource_size(p->res) == size)
 534                 return 0;
 535
 536         /* To change size the old size must be freed first */
 537         if (p->res)
 538                 return -EBUSY;
 539
 540         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE)
 541                 return -EBUSY;
 542
 543         /* ways, granularity and uuid (if PMEM) need to be set before HPA */
 544         if (!p->interleave_ways || !p->interleave_granularity ||
 545             (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid)))
 546                 return -ENXIO;
 547
 548         div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder);
 549         if (remainder)
 550                 return -EINVAL;
 551
 552         res = alloc_free_mem_region(cxlrd->res, size, SZ_256M,
 553                                     dev_name(&cxlr->dev));
 554         if (IS_ERR(res)) {
 555                 dev_dbg(&cxlr->dev,
 556                         "HPA allocation error (%ld) for size:%pap in %s %pr\n",
 557                         PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res);
 558                 return PTR_ERR(res);
 559         }
 560
 561         p->res = res;
 562         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
 563
 564         return 0;
 565 }
 566
 567 static void cxl_region_iomem_release(struct cxl_region *cxlr)
 568 {
 569         struct cxl_region_params *p = &cxlr->params;
 570
 571         if (device_is_registered(&cxlr->dev))
 572                 lockdep_assert_held_write(&cxl_region_rwsem);
 573         if (p->res) {
 574                 /*
 575                  * Autodiscovered regions may not have been able to insert their
 576                  * resource.
 577                  */
 578                 if (p->res->parent)
 579                         remove_resource(p->res);
 580                 kfree(p->res);
 581                 p->res = NULL;
 582         }
 583 }
 584
 585 static int free_hpa(struct cxl_region *cxlr)
 586 {
 587         struct cxl_region_params *p = &cxlr->params;
 588
 589         lockdep_assert_held_write(&cxl_region_rwsem);
 590
 591         if (!p->res)
 592                 return 0;
 593
 594         if (p->state >= CXL_CONFIG_ACTIVE)
 595                 return -EBUSY;
 596
 597         cxl_region_iomem_release(cxlr);
 598         p->state = CXL_CONFIG_IDLE;
 599         return 0;
 600 }
 601
 602 static ssize_t size_store(struct device *dev, struct device_attribute *attr,
 603                           const char *buf, size_t len)
 604 {
 605         struct cxl_region *cxlr = to_cxl_region(dev);
 606         u64 val;
 607         int rc;
 608
 609         rc = kstrtou64(buf, 0, &val);
 610         if (rc)
 611                 return rc;
 612
 613         rc = down_write_killable(&cxl_region_rwsem);
 614         if (rc)
 615                 return rc;
 616
 617         if (val)
 618                 rc = alloc_hpa(cxlr, val);
 619         else
 620                 rc = free_hpa(cxlr);
 621         up_write(&cxl_region_rwsem);
 622
 623         if (rc)
 624                 return rc;
 625
 626         return len;
 627 }
 628
 629 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
 630                          char *buf)
 631 {
 632         struct cxl_region *cxlr = to_cxl_region(dev);
 633         struct cxl_region_params *p = &cxlr->params;
 634         u64 size = 0;
 635         ssize_t rc;
 636
 637         rc = down_read_interruptible(&cxl_region_rwsem);
 638         if (rc)
 639                 return rc;
 640         if (p->res)
 641                 size = resource_size(p->res);
 642         rc = sysfs_emit(buf, "%#llx\n", size);
 643         up_read(&cxl_region_rwsem);
 644
 645         return rc;
 646 }
 647 static DEVICE_ATTR_RW(size);
 648
 649 static struct attribute *cxl_region_attrs[] = {
 650         &dev_attr_uuid.attr,
 651         &dev_attr_commit.attr,
 652         &dev_attr_interleave_ways.attr,
 653         &dev_attr_interleave_granularity.attr,
 654         &dev_attr_resource.attr,
 655         &dev_attr_size.attr,
 656         &dev_attr_mode.attr,
 657         NULL,
 658 };
 659
 660 static const struct attribute_group cxl_region_group = {
 661         .attrs = cxl_region_attrs,
 662         .is_visible = cxl_region_visible,
 663 };
 664
 665 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos)
 666 {
 667         struct cxl_region_params *p = &cxlr->params;
 668         struct cxl_endpoint_decoder *cxled;
 669         int rc;
 670
 671         rc = down_read_interruptible(&cxl_region_rwsem);
 672         if (rc)
 673                 return rc;
 674
 675         if (pos >= p->interleave_ways) {
 676                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
 677                         p->interleave_ways);
 678                 rc = -ENXIO;
 679                 goto out;
 680         }
 681
 682         cxled = p->targets[pos];
 683         if (!cxled)
 684                 rc = sysfs_emit(buf, "\n");
 685         else
 686                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev));
 687 out:
 688         up_read(&cxl_region_rwsem);
 689
 690         return rc;
 691 }
 692
 693 static int match_free_decoder(struct device *dev, void *data)
 694 {
 695         struct cxl_decoder *cxld;
 696         int *id = data;
 697
 698         if (!is_switch_decoder(dev))
 699                 return 0;
 700
 701         cxld = to_cxl_decoder(dev);
 702
 703         /* enforce ordered allocation */
 704         if (cxld->id != *id)
 705                 return 0;
 706
 707         if (!cxld->region)
 708                 return 1;
 709
 710         (*id)++;
 711
 712         return 0;
 713 }
 714
 715 static int match_auto_decoder(struct device *dev, void *data)
 716 {
 717         struct cxl_region_params *p = data;
 718         struct cxl_decoder *cxld;
 719         struct range *r;
 720
 721         if (!is_switch_decoder(dev))
 722                 return 0;
 723
 724         cxld = to_cxl_decoder(dev);
 725         r = &cxld->hpa_range;
 726
 727         if (p->res && p->res->start == r->start && p->res->end == r->end)
 728                 return 1;
 729
 730         return 0;
 731 }
 732
 733 static struct cxl_decoder *
 734 cxl_region_find_decoder(struct cxl_port *port,
 735                         struct cxl_endpoint_decoder *cxled,
 736                         struct cxl_region *cxlr)
 737 {
 738         struct device *dev;
 739         int id = 0;
 740
 741         if (port == cxled_to_port(cxled))
 742                 return &cxled->cxld;
 743
 744         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
 745                 dev = device_find_child(&port->dev, &cxlr->params,
 746                                         match_auto_decoder);
 747         else
 748                 dev = device_find_child(&port->dev, &id, match_free_decoder);
 749         if (!dev)
 750                 return NULL;
 751         /*
 752          * This decoder is pinned registered as long as the endpoint decoder is
 753          * registered, and endpoint decoder unregistration holds the
 754          * cxl_region_rwsem over unregister events, so no need to hold on to
 755          * this extra reference.
 756          */
 757         put_device(dev);
 758         return to_cxl_decoder(dev);
 759 }
 760
 761 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter,
 762                           struct cxl_decoder *cxld)
 763 {
 764         struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter);
 765         struct cxl_decoder *cxld_iter = rr->decoder;
 766
 767         /*
 768          * Allow the out of order assembly of auto-discovered regions.
 769          * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders
 770          * in HPA order. Confirm that the decoder with the lesser HPA
 771          * starting address has the lesser id.
 772          */
 773         dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n",
 774                 dev_name(&cxld->dev), cxld->id,
 775                 dev_name(&cxld_iter->dev), cxld_iter->id);
 776
 777         if (cxld_iter->id > cxld->id)
 778                 return true;
 779
 780         return false;
 781 }
 782
 783 static struct cxl_region_ref *
 784 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr,
 785                  struct cxl_endpoint_decoder *cxled)
 786 {
 787         struct cxl_region_params *p = &cxlr->params;
 788         struct cxl_region_ref *cxl_rr, *iter;
 789         unsigned long index;
 790         int rc;
 791
 792         xa_for_each(&port->regions, index, iter) {
 793                 struct cxl_region_params *ip = &iter->region->params;
 794
 795                 if (!ip->res || ip->res->start < p->res->start)
 796                         continue;
 797
 798                 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
 799                         struct cxl_decoder *cxld;
 800
 801                         cxld = cxl_region_find_decoder(port, cxled, cxlr);
 802                         if (auto_order_ok(port, iter->region, cxld))
 803                                 continue;
 804                 }
 805                 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n",
 806                         dev_name(&port->dev),
 807                         dev_name(&iter->region->dev), ip->res, p->res);
 808
 809                 return ERR_PTR(-EBUSY);
 810         }
 811
 812         cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL);
 813         if (!cxl_rr)
 814                 return ERR_PTR(-ENOMEM);
 815         cxl_rr->port = port;
 816         cxl_rr->region = cxlr;
 817         cxl_rr->nr_targets = 1;
 818         xa_init(&cxl_rr->endpoints);
 819
 820         rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL);
 821         if (rc) {
 822                 dev_dbg(&cxlr->dev,
 823                         "%s: failed to track region reference: %d\n",
 824                         dev_name(&port->dev), rc);
 825                 kfree(cxl_rr);
 826                 return ERR_PTR(rc);
 827         }
 828
 829         return cxl_rr;
 830 }
 831
 832 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr)
 833 {
 834         struct cxl_region *cxlr = cxl_rr->region;
 835         struct cxl_decoder *cxld = cxl_rr->decoder;
 836
 837         if (!cxld)
 838                 return;
 839
 840         dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n");
 841         if (cxld->region == cxlr) {
 842                 cxld->region = NULL;
 843                 put_device(&cxlr->dev);
 844         }
 845 }
 846
 847 static void free_region_ref(struct cxl_region_ref *cxl_rr)
 848 {
 849         struct cxl_port *port = cxl_rr->port;
 850         struct cxl_region *cxlr = cxl_rr->region;
 851
 852         cxl_rr_free_decoder(cxl_rr);
 853         xa_erase(&port->regions, (unsigned long)cxlr);
 854         xa_destroy(&cxl_rr->endpoints);
 855         kfree(cxl_rr);
 856 }
 857
 858 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr,
 859                          struct cxl_endpoint_decoder *cxled)
 860 {
 861         int rc;
 862         struct cxl_port *port = cxl_rr->port;
 863         struct cxl_region *cxlr = cxl_rr->region;
 864         struct cxl_decoder *cxld = cxl_rr->decoder;
 865         struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled));
 866
 867         if (ep) {
 868                 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep,
 869                                GFP_KERNEL);
 870                 if (rc)
 871                         return rc;
 872         }
 873         cxl_rr->nr_eps++;
 874
 875         if (!cxld->region) {
 876                 cxld->region = cxlr;
 877                 get_device(&cxlr->dev);
 878         }
 879
 880         return 0;
 881 }
 882
 883 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr,
 884                                 struct cxl_endpoint_decoder *cxled,
 885                                 struct cxl_region_ref *cxl_rr)
 886 {
 887         struct cxl_decoder *cxld;
 888
 889         cxld = cxl_region_find_decoder(port, cxled, cxlr);
 890         if (!cxld) {
 891                 dev_dbg(&cxlr->dev, "%s: no decoder available\n",
 892                         dev_name(&port->dev));
 893                 return -EBUSY;
 894         }
 895
 896         if (cxld->region) {
 897                 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n",
 898                         dev_name(&port->dev), dev_name(&cxld->dev),
 899                         dev_name(&cxld->region->dev));
 900                 return -EBUSY;
 901         }
 902
 903         /*
 904          * Endpoints should already match the region type, but backstop that
 905          * assumption with an assertion. Switch-decoders change mapping-type
 906          * based on what is mapped when they are assigned to a region.
 907          */
 908         dev_WARN_ONCE(&cxlr->dev,
 909                       port == cxled_to_port(cxled) &&
 910                               cxld->target_type != cxlr->type,
 911                       "%s:%s mismatch decoder type %d -> %d\n",
 912                       dev_name(&cxled_to_memdev(cxled)->dev),
 913                       dev_name(&cxld->dev), cxld->target_type, cxlr->type);
 914         cxld->target_type = cxlr->type;
 915         cxl_rr->decoder = cxld;
 916         return 0;
 917 }
 918
 919 /**
 920  * cxl_port_attach_region() - track a region's interest in a port by endpoint
 921  * @port: port to add a new region reference 'struct cxl_region_ref'
 922  * @cxlr: region to attach to @port
 923  * @cxled: endpoint decoder used to create or further pin a region reference
 924  * @pos: interleave position of @cxled in @cxlr
 925  *
 926  * The attach event is an opportunity to validate CXL decode setup
 927  * constraints and record metadata needed for programming HDM decoders,
 928  * in particular decoder target lists.
 929  *
 930  * The steps are:
 931  *
 932  * - validate that there are no other regions with a higher HPA already
 933  *   associated with @port
 934  * - establish a region reference if one is not already present
 935  *
 936  *   - additionally allocate a decoder instance that will host @cxlr on
 937  *     @port
 938  *
 939  * - pin the region reference by the endpoint
 940  * - account for how many entries in @port's target list are needed to
 941  *   cover all of the added endpoints.
 942  */
 943 static int cxl_port_attach_region(struct cxl_port *port,
 944                                   struct cxl_region *cxlr,
 945                                   struct cxl_endpoint_decoder *cxled, int pos)
 946 {
 947         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
 948         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
 949         struct cxl_region_ref *cxl_rr;
 950         bool nr_targets_inc = false;
 951         struct cxl_decoder *cxld;
 952         unsigned long index;
 953         int rc = -EBUSY;
 954
 955         lockdep_assert_held_write(&cxl_region_rwsem);
 956
 957         cxl_rr = cxl_rr_load(port, cxlr);
 958         if (cxl_rr) {
 959                 struct cxl_ep *ep_iter;
 960                 int found = 0;
 961
 962                 /*
 963                  * Walk the existing endpoints that have been attached to
 964                  * @cxlr at @port and see if they share the same 'next' port
 965                  * in the downstream direction. I.e. endpoints that share common
 966                  * upstream switch.
 967                  */
 968                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
 969                         if (ep_iter == ep)
 970                                 continue;
 971                         if (ep_iter->next == ep->next) {
 972                                 found++;
 973                                 break;
 974                         }
 975                 }
 976
 977                 /*
 978                  * New target port, or @port is an endpoint port that always
 979                  * accounts its own local decode as a target.
 980                  */
 981                 if (!found || !ep->next) {
 982                         cxl_rr->nr_targets++;
 983                         nr_targets_inc = true;
 984                 }
 985         } else {
 986                 cxl_rr = alloc_region_ref(port, cxlr, cxled);
 987                 if (IS_ERR(cxl_rr)) {
 988                         dev_dbg(&cxlr->dev,
 989                                 "%s: failed to allocate region reference\n",
 990                                 dev_name(&port->dev));
 991                         return PTR_ERR(cxl_rr);
 992                 }
 993                 nr_targets_inc = true;
 994
 995                 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr);
 996                 if (rc)
 997                         goto out_erase;
 998         }
 999         cxld = cxl_rr->decoder;
1000
1001         rc = cxl_rr_ep_add(cxl_rr, cxled);
1002         if (rc) {
1003                 dev_dbg(&cxlr->dev,
1004                         "%s: failed to track endpoint %s:%s reference\n",
1005                         dev_name(&port->dev), dev_name(&cxlmd->dev),
1006                         dev_name(&cxld->dev));
1007                 goto out_erase;
1008         }
1009
1010         dev_dbg(&cxlr->dev,
1011                 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n",
1012                 dev_name(port->uport_dev), dev_name(&port->dev),
1013                 dev_name(&cxld->dev), dev_name(&cxlmd->dev),
1014                 dev_name(&cxled->cxld.dev), pos,
1015                 ep ? ep->next ? dev_name(ep->next->uport_dev) :
1016                                       dev_name(&cxlmd->dev) :
1017                            "none",
1018                 cxl_rr->nr_eps, cxl_rr->nr_targets);
1019
1020         return 0;
1021 out_erase:
1022         if (nr_targets_inc)
1023                 cxl_rr->nr_targets--;
1024         if (cxl_rr->nr_eps == 0)
1025                 free_region_ref(cxl_rr);
1026         return rc;
1027 }
1028
1029 static void cxl_port_detach_region(struct cxl_port *port,
1030                                    struct cxl_region *cxlr,
1031                                    struct cxl_endpoint_decoder *cxled)
1032 {
1033         struct cxl_region_ref *cxl_rr;
1034         struct cxl_ep *ep = NULL;
1035
1036         lockdep_assert_held_write(&cxl_region_rwsem);
1037
1038         cxl_rr = cxl_rr_load(port, cxlr);
1039         if (!cxl_rr)
1040                 return;
1041
1042         /*
1043          * Endpoint ports do not carry cxl_ep references, and they
1044          * never target more than one endpoint by definition
1045          */
1046         if (cxl_rr->decoder == &cxled->cxld)
1047                 cxl_rr->nr_eps--;
1048         else
1049                 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled);
1050         if (ep) {
1051                 struct cxl_ep *ep_iter;
1052                 unsigned long index;
1053                 int found = 0;
1054
1055                 cxl_rr->nr_eps--;
1056                 xa_for_each(&cxl_rr->endpoints, index, ep_iter) {
1057                         if (ep_iter->next == ep->next) {
1058                                 found++;
1059                                 break;
1060                         }
1061                 }
1062                 if (!found)
1063                         cxl_rr->nr_targets--;
1064         }
1065
1066         if (cxl_rr->nr_eps == 0)
1067                 free_region_ref(cxl_rr);
1068 }
1069
1070 static int check_last_peer(struct cxl_endpoint_decoder *cxled,
1071                            struct cxl_ep *ep, struct cxl_region_ref *cxl_rr,
1072                            int distance)
1073 {
1074         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1075         struct cxl_region *cxlr = cxl_rr->region;
1076         struct cxl_region_params *p = &cxlr->params;
1077         struct cxl_endpoint_decoder *cxled_peer;
1078         struct cxl_port *port = cxl_rr->port;
1079         struct cxl_memdev *cxlmd_peer;
1080         struct cxl_ep *ep_peer;
1081         int pos = cxled->pos;
1082
1083         /*
1084          * If this position wants to share a dport with the last endpoint mapped
1085          * then that endpoint, at index 'position - distance', must also be
1086          * mapped by this dport.
1087          */
1088         if (pos < distance) {
1089                 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
1090                         dev_name(port->uport_dev), dev_name(&port->dev),
1091                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1092                 return -ENXIO;
1093         }
1094         cxled_peer = p->targets[pos - distance];
1095         cxlmd_peer = cxled_to_memdev(cxled_peer);
1096         ep_peer = cxl_ep_load(port, cxlmd_peer);
1097         if (ep->dport != ep_peer->dport) {
1098                 dev_dbg(&cxlr->dev,
1099                         "%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
1100                         dev_name(port->uport_dev), dev_name(&port->dev),
1101                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos,
1102                         dev_name(&cxlmd_peer->dev),
1103                         dev_name(&cxled_peer->cxld.dev));
1104                 return -ENXIO;
1105         }
1106
1107         return 0;
1108 }
1109
1110 static int cxl_port_setup_targets(struct cxl_port *port,
1111                                   struct cxl_region *cxlr,
1112                                   struct cxl_endpoint_decoder *cxled)
1113 {
1114         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1115         int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos;
1116         struct cxl_port *parent_port = to_cxl_port(port->dev.parent);
1117         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1118         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1119         struct cxl_ep *ep = cxl_ep_load(port, cxlmd);
1120         struct cxl_region_params *p = &cxlr->params;
1121         struct cxl_decoder *cxld = cxl_rr->decoder;
1122         struct cxl_switch_decoder *cxlsd;
1123         u16 eig, peig;
1124         u8 eiw, peiw;
1125
1126         /*
1127          * While root level decoders support x3, x6, x12, switch level
1128          * decoders only support powers of 2 up to x16.
1129          */
1130         if (!is_power_of_2(cxl_rr->nr_targets)) {
1131                 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n",
1132                         dev_name(port->uport_dev), dev_name(&port->dev),
1133                         cxl_rr->nr_targets);
1134                 return -EINVAL;
1135         }
1136
1137         cxlsd = to_cxl_switch_decoder(&cxld->dev);
1138         if (cxl_rr->nr_targets_set) {
1139                 int i, distance;
1140
1141                 /*
1142                  * Passthrough decoders impose no distance requirements between
1143                  * peers
1144                  */
1145                 if (cxl_rr->nr_targets == 1)
1146                         distance = 0;
1147                 else
1148                         distance = p->nr_targets / cxl_rr->nr_targets;
1149                 for (i = 0; i < cxl_rr->nr_targets_set; i++)
1150                         if (ep->dport == cxlsd->target[i]) {
1151                                 rc = check_last_peer(cxled, ep, cxl_rr,
1152                                                      distance);
1153                                 if (rc)
1154                                         return rc;
1155                                 goto out_target_set;
1156                         }
1157                 goto add_target;
1158         }
1159
1160         if (is_cxl_root(parent_port)) {
1161                 /*
1162                  * Root decoder IG is always set to value in CFMWS which
1163                  * may be different than this region's IG.  We can use the
1164                  * region's IG here since interleave_granularity_store()
1165                  * does not allow interleaved host-bridges with
1166                  * root IG != region IG.
1167                  */
1168                 parent_ig = p->interleave_granularity;
1169                 parent_iw = cxlrd->cxlsd.cxld.interleave_ways;
1170                 /*
1171                  * For purposes of address bit routing, use power-of-2 math for
1172                  * switch ports.
1173                  */
1174                 if (!is_power_of_2(parent_iw))
1175                         parent_iw /= 3;
1176         } else {
1177                 struct cxl_region_ref *parent_rr;
1178                 struct cxl_decoder *parent_cxld;
1179
1180                 parent_rr = cxl_rr_load(parent_port, cxlr);
1181                 parent_cxld = parent_rr->decoder;
1182                 parent_ig = parent_cxld->interleave_granularity;
1183                 parent_iw = parent_cxld->interleave_ways;
1184         }
1185
1186         rc = granularity_to_eig(parent_ig, &peig);
1187         if (rc) {
1188                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n",
1189                         dev_name(parent_port->uport_dev),
1190                         dev_name(&parent_port->dev), parent_ig);
1191                 return rc;
1192         }
1193
1194         rc = ways_to_eiw(parent_iw, &peiw);
1195         if (rc) {
1196                 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n",
1197                         dev_name(parent_port->uport_dev),
1198                         dev_name(&parent_port->dev), parent_iw);
1199                 return rc;
1200         }
1201
1202         iw = cxl_rr->nr_targets;
1203         rc = ways_to_eiw(iw, &eiw);
1204         if (rc) {
1205                 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n",
1206                         dev_name(port->uport_dev), dev_name(&port->dev), iw);
1207                 return rc;
1208         }
1209
1210         /*
1211          * Interleave granularity is a multiple of @parent_port granularity.
1212          * Multiplier is the parent port interleave ways.
1213          */
1214         rc = granularity_to_eig(parent_ig * parent_iw, &eig);
1215         if (rc) {
1216                 dev_dbg(&cxlr->dev,
1217                         "%s: invalid granularity calculation (%d * %d)\n",
1218                         dev_name(&parent_port->dev), parent_ig, parent_iw);
1219                 return rc;
1220         }
1221
1222         rc = eig_to_granularity(eig, &ig);
1223         if (rc) {
1224                 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n",
1225                         dev_name(port->uport_dev), dev_name(&port->dev),
1226                         256 << eig);
1227                 return rc;
1228         }
1229
1230         if (iw > 8 || iw > cxlsd->nr_targets) {
1231                 dev_dbg(&cxlr->dev,
1232                         "%s:%s:%s: ways: %d overflows targets: %d\n",
1233                         dev_name(port->uport_dev), dev_name(&port->dev),
1234                         dev_name(&cxld->dev), iw, cxlsd->nr_targets);
1235                 return -ENXIO;
1236         }
1237
1238         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1239                 if (cxld->interleave_ways != iw ||
1240                     cxld->interleave_granularity != ig ||
1241                     cxld->hpa_range.start != p->res->start ||
1242                     cxld->hpa_range.end != p->res->end ||
1243                     ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) {
1244                         dev_err(&cxlr->dev,
1245                                 "%s:%s %s expected iw: %d ig: %d %pr\n",
1246                                 dev_name(port->uport_dev), dev_name(&port->dev),
1247                                 __func__, iw, ig, p->res);
1248                         dev_err(&cxlr->dev,
1249                                 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n",
1250                                 dev_name(port->uport_dev), dev_name(&port->dev),
1251                                 __func__, cxld->interleave_ways,
1252                                 cxld->interleave_granularity,
1253                                 (cxld->flags & CXL_DECODER_F_ENABLE) ?
1254                                         "enabled" :
1255                                         "disabled",
1256                                 cxld->hpa_range.start, cxld->hpa_range.end);
1257                         return -ENXIO;
1258                 }
1259         } else {
1260                 cxld->interleave_ways = iw;
1261                 cxld->interleave_granularity = ig;
1262                 cxld->hpa_range = (struct range) {
1263                         .start = p->res->start,
1264                         .end = p->res->end,
1265                 };
1266         }
1267         dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev),
1268                 dev_name(&port->dev), iw, ig);
1269 add_target:
1270         if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) {
1271                 dev_dbg(&cxlr->dev,
1272                         "%s:%s: targets full trying to add %s:%s at %d\n",
1273                         dev_name(port->uport_dev), dev_name(&port->dev),
1274                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1275                 return -ENXIO;
1276         }
1277         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1278                 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) {
1279                         dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n",
1280                                 dev_name(port->uport_dev), dev_name(&port->dev),
1281                                 dev_name(&cxlsd->cxld.dev),
1282                                 dev_name(ep->dport->dport_dev),
1283                                 cxl_rr->nr_targets_set);
1284                         return -ENXIO;
1285                 }
1286         } else
1287                 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport;
1288         inc = 1;
1289 out_target_set:
1290         cxl_rr->nr_targets_set += inc;
1291         dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n",
1292                 dev_name(port->uport_dev), dev_name(&port->dev),
1293                 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev),
1294                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos);
1295
1296         return 0;
1297 }
1298
1299 static void cxl_port_reset_targets(struct cxl_port *port,
1300                                    struct cxl_region *cxlr)
1301 {
1302         struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr);
1303         struct cxl_decoder *cxld;
1304
1305         /*
1306          * After the last endpoint has been detached the entire cxl_rr may now
1307          * be gone.
1308          */
1309         if (!cxl_rr)
1310                 return;
1311         cxl_rr->nr_targets_set = 0;
1312
1313         cxld = cxl_rr->decoder;
1314         cxld->hpa_range = (struct range) {
1315                 .start = 0,
1316                 .end = -1,
1317         };
1318 }
1319
1320 static void cxl_region_teardown_targets(struct cxl_region *cxlr)
1321 {
1322         struct cxl_region_params *p = &cxlr->params;
1323         struct cxl_endpoint_decoder *cxled;
1324         struct cxl_dev_state *cxlds;
1325         struct cxl_memdev *cxlmd;
1326         struct cxl_port *iter;
1327         struct cxl_ep *ep;
1328         int i;
1329
1330         /*
1331          * In the auto-discovery case skip automatic teardown since the
1332          * address space is already active
1333          */
1334         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags))
1335                 return;
1336
1337         for (i = 0; i < p->nr_targets; i++) {
1338                 cxled = p->targets[i];
1339                 cxlmd = cxled_to_memdev(cxled);
1340                 cxlds = cxlmd->cxlds;
1341
1342                 if (cxlds->rcd)
1343                         continue;
1344
1345                 iter = cxled_to_port(cxled);
1346                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1347                         iter = to_cxl_port(iter->dev.parent);
1348
1349                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1350                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd))
1351                         cxl_port_reset_targets(iter, cxlr);
1352         }
1353 }
1354
1355 static int cxl_region_setup_targets(struct cxl_region *cxlr)
1356 {
1357         struct cxl_region_params *p = &cxlr->params;
1358         struct cxl_endpoint_decoder *cxled;
1359         struct cxl_dev_state *cxlds;
1360         int i, rc, rch = 0, vh = 0;
1361         struct cxl_memdev *cxlmd;
1362         struct cxl_port *iter;
1363         struct cxl_ep *ep;
1364
1365         for (i = 0; i < p->nr_targets; i++) {
1366                 cxled = p->targets[i];
1367                 cxlmd = cxled_to_memdev(cxled);
1368                 cxlds = cxlmd->cxlds;
1369
1370                 /* validate that all targets agree on topology */
1371                 if (!cxlds->rcd) {
1372                         vh++;
1373                 } else {
1374                         rch++;
1375                         continue;
1376                 }
1377
1378                 iter = cxled_to_port(cxled);
1379                 while (!is_cxl_root(to_cxl_port(iter->dev.parent)))
1380                         iter = to_cxl_port(iter->dev.parent);
1381
1382                 /*
1383                  * Descend the topology tree programming / validating
1384                  * targets while looking for conflicts.
1385                  */
1386                 for (ep = cxl_ep_load(iter, cxlmd); iter;
1387                      iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) {
1388                         rc = cxl_port_setup_targets(iter, cxlr, cxled);
1389                         if (rc) {
1390                                 cxl_region_teardown_targets(cxlr);
1391                                 return rc;
1392                         }
1393                 }
1394         }
1395
1396         if (rch && vh) {
1397                 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n");
1398                 cxl_region_teardown_targets(cxlr);
1399                 return -ENXIO;
1400         }
1401
1402         return 0;
1403 }
1404
1405 static int cxl_region_validate_position(struct cxl_region *cxlr,
1406                                         struct cxl_endpoint_decoder *cxled,
1407                                         int pos)
1408 {
1409         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1410         struct cxl_region_params *p = &cxlr->params;
1411         int i;
1412
1413         if (pos < 0 || pos >= p->interleave_ways) {
1414                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1415                         p->interleave_ways);
1416                 return -ENXIO;
1417         }
1418
1419         if (p->targets[pos] == cxled)
1420                 return 0;
1421
1422         if (p->targets[pos]) {
1423                 struct cxl_endpoint_decoder *cxled_target = p->targets[pos];
1424                 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target);
1425
1426                 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n",
1427                         pos, dev_name(&cxlmd_target->dev),
1428                         dev_name(&cxled_target->cxld.dev));
1429                 return -EBUSY;
1430         }
1431
1432         for (i = 0; i < p->interleave_ways; i++) {
1433                 struct cxl_endpoint_decoder *cxled_target;
1434                 struct cxl_memdev *cxlmd_target;
1435
1436                 cxled_target = p->targets[i];
1437                 if (!cxled_target)
1438                         continue;
1439
1440                 cxlmd_target = cxled_to_memdev(cxled_target);
1441                 if (cxlmd_target == cxlmd) {
1442                         dev_dbg(&cxlr->dev,
1443                                 "%s already specified at position %d via: %s\n",
1444                                 dev_name(&cxlmd->dev), pos,
1445                                 dev_name(&cxled_target->cxld.dev));
1446                         return -EBUSY;
1447                 }
1448         }
1449
1450         return 0;
1451 }
1452
1453 static int cxl_region_attach_position(struct cxl_region *cxlr,
1454                                       struct cxl_root_decoder *cxlrd,
1455                                       struct cxl_endpoint_decoder *cxled,
1456                                       const struct cxl_dport *dport, int pos)
1457 {
1458         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1459         struct cxl_port *iter;
1460         int rc;
1461
1462         if (cxlrd->calc_hb(cxlrd, pos) != dport) {
1463                 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n",
1464                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1465                         dev_name(&cxlrd->cxlsd.cxld.dev));
1466                 return -ENXIO;
1467         }
1468
1469         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1470              iter = to_cxl_port(iter->dev.parent)) {
1471                 rc = cxl_port_attach_region(iter, cxlr, cxled, pos);
1472                 if (rc)
1473                         goto err;
1474         }
1475
1476         return 0;
1477
1478 err:
1479         for (iter = cxled_to_port(cxled); !is_cxl_root(iter);
1480              iter = to_cxl_port(iter->dev.parent))
1481                 cxl_port_detach_region(iter, cxlr, cxled);
1482         return rc;
1483 }
1484
1485 static int cxl_region_attach_auto(struct cxl_region *cxlr,
1486                                   struct cxl_endpoint_decoder *cxled, int pos)
1487 {
1488         struct cxl_region_params *p = &cxlr->params;
1489
1490         if (cxled->state != CXL_DECODER_STATE_AUTO) {
1491                 dev_err(&cxlr->dev,
1492                         "%s: unable to add decoder to autodetected region\n",
1493                         dev_name(&cxled->cxld.dev));
1494                 return -EINVAL;
1495         }
1496
1497         if (pos >= 0) {
1498                 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n",
1499                         dev_name(&cxled->cxld.dev), pos);
1500                 return -EINVAL;
1501         }
1502
1503         if (p->nr_targets >= p->interleave_ways) {
1504                 dev_err(&cxlr->dev, "%s: no more target slots available\n",
1505                         dev_name(&cxled->cxld.dev));
1506                 return -ENXIO;
1507         }
1508
1509         /*
1510          * Temporarily record the endpoint decoder into the target array. Yes,
1511          * this means that userspace can view devices in the wrong position
1512          * before the region activates, and must be careful to understand when
1513          * it might be racing region autodiscovery.
1514          */
1515         pos = p->nr_targets;
1516         p->targets[pos] = cxled;
1517         cxled->pos = pos;
1518         p->nr_targets++;
1519
1520         return 0;
1521 }
1522
1523 static int cmp_interleave_pos(const void *a, const void *b)
1524 {
1525         struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a;
1526         struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b;
1527
1528         return cxled_a->pos - cxled_b->pos;
1529 }
1530
1531 static struct cxl_port *next_port(struct cxl_port *port)
1532 {
1533         if (!port->parent_dport)
1534                 return NULL;
1535         return port->parent_dport->port;
1536 }
1537
1538 static int match_switch_decoder_by_range(struct device *dev, void *data)
1539 {
1540         struct cxl_switch_decoder *cxlsd;
1541         struct range *r1, *r2 = data;
1542
1543         if (!is_switch_decoder(dev))
1544                 return 0;
1545
1546         cxlsd = to_cxl_switch_decoder(dev);
1547         r1 = &cxlsd->cxld.hpa_range;
1548
1549         if (is_root_decoder(dev))
1550                 return range_contains(r1, r2);
1551         return (r1->start == r2->start && r1->end == r2->end);
1552 }
1553
1554 static int find_pos_and_ways(struct cxl_port *port, struct range *range,
1555                              int *pos, int *ways)
1556 {
1557         struct cxl_switch_decoder *cxlsd;
1558         struct cxl_port *parent;
1559         struct device *dev;
1560         int rc = -ENXIO;
1561
1562         parent = next_port(port);
1563         if (!parent)
1564                 return rc;
1565
1566         dev = device_find_child(&parent->dev, range,
1567                                 match_switch_decoder_by_range);
1568         if (!dev) {
1569                 dev_err(port->uport_dev,
1570                         "failed to find decoder mapping %#llx-%#llx\n",
1571                         range->start, range->end);
1572                 return rc;
1573         }
1574         cxlsd = to_cxl_switch_decoder(dev);
1575         *ways = cxlsd->cxld.interleave_ways;
1576
1577         for (int i = 0; i < *ways; i++) {
1578                 if (cxlsd->target[i] == port->parent_dport) {
1579                         *pos = i;
1580                         rc = 0;
1581                         break;
1582                 }
1583         }
1584         put_device(dev);
1585
1586         return rc;
1587 }
1588
1589 /**
1590  * cxl_calc_interleave_pos() - calculate an endpoint position in a region
1591  * @cxled: endpoint decoder member of given region
1592  *
1593  * The endpoint position is calculated by traversing the topology from
1594  * the endpoint to the root decoder and iteratively applying this
1595  * calculation:
1596  *
1597  *    position = position * parent_ways + parent_pos;
1598  *
1599  * ...where @position is inferred from switch and root decoder target lists.
1600  *
1601  * Return: position >= 0 on success
1602  *         -ENXIO on failure
1603  */
1604 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled)
1605 {
1606         struct cxl_port *iter, *port = cxled_to_port(cxled);
1607         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1608         struct range *range = &cxled->cxld.hpa_range;
1609         int parent_ways = 0, parent_pos = 0, pos = 0;
1610         int rc;
1611
1612         /*
1613          * Example: the expected interleave order of the 4-way region shown
1614          * below is: mem0, mem2, mem1, mem3
1615          *
1616          *                root_port
1617          *                 /      \
1618          *      host_bridge_0    host_bridge_1
1619          *        |    |           |    |
1620          *       mem0 mem1        mem2 mem3
1621          *
1622          * In the example the calculator will iterate twice. The first iteration
1623          * uses the mem position in the host-bridge and the ways of the host-
1624          * bridge to generate the first, or local, position. The second
1625          * iteration uses the host-bridge position in the root_port and the ways
1626          * of the root_port to refine the position.
1627          *
1628          * A trace of the calculation per endpoint looks like this:
1629          * mem0: pos = 0 * 2 + 0    mem2: pos = 0 * 2 + 0
1630          *       pos = 0 * 2 + 0          pos = 0 * 2 + 1
1631          *       pos: 0                   pos: 1
1632          *
1633          * mem1: pos = 0 * 2 + 1    mem3: pos = 0 * 2 + 1
1634          *       pos = 1 * 2 + 0          pos = 1 * 2 + 1
1635          *       pos: 2                   pos = 3
1636          *
1637          * Note that while this example is simple, the method applies to more
1638          * complex topologies, including those with switches.
1639          */
1640
1641         /* Iterate from endpoint to root_port refining the position */
1642         for (iter = port; iter; iter = next_port(iter)) {
1643                 if (is_cxl_root(iter))
1644                         break;
1645
1646                 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways);
1647                 if (rc)
1648                         return rc;
1649
1650                 pos = pos * parent_ways + parent_pos;
1651         }
1652
1653         dev_dbg(&cxlmd->dev,
1654                 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n",
1655                 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent),
1656                 dev_name(&port->dev), range->start, range->end, pos);
1657
1658         return pos;
1659 }
1660
1661 static int cxl_region_sort_targets(struct cxl_region *cxlr)
1662 {
1663         struct cxl_region_params *p = &cxlr->params;
1664         int i, rc = 0;
1665
1666         for (i = 0; i < p->nr_targets; i++) {
1667                 struct cxl_endpoint_decoder *cxled = p->targets[i];
1668
1669                 cxled->pos = cxl_calc_interleave_pos(cxled);
1670                 /*
1671                  * Record that sorting failed, but still continue to calc
1672                  * cxled->pos so that follow-on code paths can reliably
1673                  * do p->targets[cxled->pos] to self-reference their entry.
1674                  */
1675                 if (cxled->pos < 0)
1676                         rc = -ENXIO;
1677         }
1678         /* Keep the cxlr target list in interleave position order */
1679         sort(p->targets, p->nr_targets, sizeof(p->targets[0]),
1680              cmp_interleave_pos, NULL);
1681
1682         dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful");
1683         return rc;
1684 }
1685
1686 static int cxl_region_attach(struct cxl_region *cxlr,
1687                              struct cxl_endpoint_decoder *cxled, int pos)
1688 {
1689         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent);
1690         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1691         struct cxl_region_params *p = &cxlr->params;
1692         struct cxl_port *ep_port, *root_port;
1693         struct cxl_dport *dport;
1694         int rc = -ENXIO;
1695
1696         if (cxled->mode != cxlr->mode) {
1697                 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n",
1698                         dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode);
1699                 return -EINVAL;
1700         }
1701
1702         if (cxled->mode == CXL_DECODER_DEAD) {
1703                 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev));
1704                 return -ENODEV;
1705         }
1706
1707         /* all full of members, or interleave config not established? */
1708         if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) {
1709                 dev_dbg(&cxlr->dev, "region already active\n");
1710                 return -EBUSY;
1711         } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) {
1712                 dev_dbg(&cxlr->dev, "interleave config missing\n");
1713                 return -ENXIO;
1714         }
1715
1716         if (p->nr_targets >= p->interleave_ways) {
1717                 dev_dbg(&cxlr->dev, "region already has %d endpoints\n",
1718                         p->nr_targets);
1719                 return -EINVAL;
1720         }
1721
1722         ep_port = cxled_to_port(cxled);
1723         root_port = cxlrd_to_port(cxlrd);
1724         dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge);
1725         if (!dport) {
1726                 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n",
1727                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1728                         dev_name(cxlr->dev.parent));
1729                 return -ENXIO;
1730         }
1731
1732         if (cxled->cxld.target_type != cxlr->type) {
1733                 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n",
1734                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1735                         cxled->cxld.target_type, cxlr->type);
1736                 return -ENXIO;
1737         }
1738
1739         if (!cxled->dpa_res) {
1740                 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
1741                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
1742                 return -ENXIO;
1743         }
1744
1745         if (resource_size(cxled->dpa_res) * p->interleave_ways !=
1746             resource_size(p->res)) {
1747                 dev_dbg(&cxlr->dev,
1748                         "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n",
1749                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1750                         (u64)resource_size(cxled->dpa_res), p->interleave_ways,
1751                         (u64)resource_size(p->res));
1752                 return -EINVAL;
1753         }
1754
1755         if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) {
1756                 int i;
1757
1758                 rc = cxl_region_attach_auto(cxlr, cxled, pos);
1759                 if (rc)
1760                         return rc;
1761
1762                 /* await more targets to arrive... */
1763                 if (p->nr_targets < p->interleave_ways)
1764                         return 0;
1765
1766                 /*
1767                  * All targets are here, which implies all PCI enumeration that
1768                  * affects this region has been completed. Walk the topology to
1769                  * sort the devices into their relative region decode position.
1770                  */
1771                 rc = cxl_region_sort_targets(cxlr);
1772                 if (rc)
1773                         return rc;
1774
1775                 for (i = 0; i < p->nr_targets; i++) {
1776                         cxled = p->targets[i];
1777                         ep_port = cxled_to_port(cxled);
1778                         dport = cxl_find_dport_by_dev(root_port,
1779                                                       ep_port->host_bridge);
1780                         rc = cxl_region_attach_position(cxlr, cxlrd, cxled,
1781                                                         dport, i);
1782                         if (rc)
1783                                 return rc;
1784                 }
1785
1786                 rc = cxl_region_setup_targets(cxlr);
1787                 if (rc)
1788                         return rc;
1789
1790                 /*
1791                  * If target setup succeeds in the autodiscovery case
1792                  * then the region is already committed.
1793                  */
1794                 p->state = CXL_CONFIG_COMMIT;
1795
1796                 return 0;
1797         }
1798
1799         rc = cxl_region_validate_position(cxlr, cxled, pos);
1800         if (rc)
1801                 return rc;
1802
1803         rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos);
1804         if (rc)
1805                 return rc;
1806
1807         p->targets[pos] = cxled;
1808         cxled->pos = pos;
1809         p->nr_targets++;
1810
1811         if (p->nr_targets == p->interleave_ways) {
1812                 rc = cxl_region_setup_targets(cxlr);
1813                 if (rc)
1814                         return rc;
1815                 p->state = CXL_CONFIG_ACTIVE;
1816         }
1817
1818         cxled->cxld.interleave_ways = p->interleave_ways;
1819         cxled->cxld.interleave_granularity = p->interleave_granularity;
1820         cxled->cxld.hpa_range = (struct range) {
1821                 .start = p->res->start,
1822                 .end = p->res->end,
1823         };
1824
1825         if (p->nr_targets != p->interleave_ways)
1826                 return 0;
1827
1828         /*
1829          * Test the auto-discovery position calculator function
1830          * against this successfully created user-defined region.
1831          * A fail message here means that this interleave config
1832          * will fail when presented as CXL_REGION_F_AUTO.
1833          */
1834         for (int i = 0; i < p->nr_targets; i++) {
1835                 struct cxl_endpoint_decoder *cxled = p->targets[i];
1836                 int test_pos;
1837
1838                 test_pos = cxl_calc_interleave_pos(cxled);
1839                 dev_dbg(&cxled->cxld.dev,
1840                         "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n",
1841                         (test_pos == cxled->pos) ? "success" : "fail",
1842                         test_pos, cxled->pos);
1843         }
1844
1845         return 0;
1846 }
1847
1848 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled)
1849 {
1850         struct cxl_port *iter, *ep_port = cxled_to_port(cxled);
1851         struct cxl_region *cxlr = cxled->cxld.region;
1852         struct cxl_region_params *p;
1853         int rc = 0;
1854
1855         lockdep_assert_held_write(&cxl_region_rwsem);
1856
1857         if (!cxlr)
1858                 return 0;
1859
1860         p = &cxlr->params;
1861         get_device(&cxlr->dev);
1862
1863         if (p->state > CXL_CONFIG_ACTIVE) {
1864                 /*
1865                  * TODO: tear down all impacted regions if a device is
1866                  * removed out of order
1867                  */
1868                 rc = cxl_region_decode_reset(cxlr, p->interleave_ways);
1869                 if (rc)
1870                         goto out;
1871                 p->state = CXL_CONFIG_ACTIVE;
1872         }
1873
1874         for (iter = ep_port; !is_cxl_root(iter);
1875              iter = to_cxl_port(iter->dev.parent))
1876                 cxl_port_detach_region(iter, cxlr, cxled);
1877
1878         if (cxled->pos < 0 || cxled->pos >= p->interleave_ways ||
1879             p->targets[cxled->pos] != cxled) {
1880                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
1881
1882                 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n",
1883                               dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
1884                               cxled->pos);
1885                 goto out;
1886         }
1887
1888         if (p->state == CXL_CONFIG_ACTIVE) {
1889                 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
1890                 cxl_region_teardown_targets(cxlr);
1891         }
1892         p->targets[cxled->pos] = NULL;
1893         p->nr_targets--;
1894         cxled->cxld.hpa_range = (struct range) {
1895                 .start = 0,
1896                 .end = -1,
1897         };
1898
1899         /* notify the region driver that one of its targets has departed */
1900         up_write(&cxl_region_rwsem);
1901         device_release_driver(&cxlr->dev);
1902         down_write(&cxl_region_rwsem);
1903 out:
1904         put_device(&cxlr->dev);
1905         return rc;
1906 }
1907
1908 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled)
1909 {
1910         down_write(&cxl_region_rwsem);
1911         cxled->mode = CXL_DECODER_DEAD;
1912         cxl_region_detach(cxled);
1913         up_write(&cxl_region_rwsem);
1914 }
1915
1916 static int attach_target(struct cxl_region *cxlr,
1917                          struct cxl_endpoint_decoder *cxled, int pos,
1918                          unsigned int state)
1919 {
1920         int rc = 0;
1921
1922         if (state == TASK_INTERRUPTIBLE)
1923                 rc = down_write_killable(&cxl_region_rwsem);
1924         else
1925                 down_write(&cxl_region_rwsem);
1926         if (rc)
1927                 return rc;
1928
1929         down_read(&cxl_dpa_rwsem);
1930         rc = cxl_region_attach(cxlr, cxled, pos);
1931         up_read(&cxl_dpa_rwsem);
1932         up_write(&cxl_region_rwsem);
1933         return rc;
1934 }
1935
1936 static int detach_target(struct cxl_region *cxlr, int pos)
1937 {
1938         struct cxl_region_params *p = &cxlr->params;
1939         int rc;
1940
1941         rc = down_write_killable(&cxl_region_rwsem);
1942         if (rc)
1943                 return rc;
1944
1945         if (pos >= p->interleave_ways) {
1946                 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos,
1947                         p->interleave_ways);
1948                 rc = -ENXIO;
1949                 goto out;
1950         }
1951
1952         if (!p->targets[pos]) {
1953                 rc = 0;
1954                 goto out;
1955         }
1956
1957         rc = cxl_region_detach(p->targets[pos]);
1958 out:
1959         up_write(&cxl_region_rwsem);
1960         return rc;
1961 }
1962
1963 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos,
1964                             size_t len)
1965 {
1966         int rc;
1967
1968         if (sysfs_streq(buf, "\n"))
1969                 rc = detach_target(cxlr, pos);
1970         else {
1971                 struct device *dev;
1972
1973                 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf);
1974                 if (!dev)
1975                         return -ENODEV;
1976
1977                 if (!is_endpoint_decoder(dev)) {
1978                         rc = -EINVAL;
1979                         goto out;
1980                 }
1981
1982                 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos,
1983                                    TASK_INTERRUPTIBLE);
1984 out:
1985                 put_device(dev);
1986         }
1987
1988         if (rc < 0)
1989                 return rc;
1990         return len;
1991 }
1992
1993 #define TARGET_ATTR_RW(n)                                              \
1994 static ssize_t target##n##_show(                                       \
1995         struct device *dev, struct device_attribute *attr, char *buf)  \
1996 {                                                                      \
1997         return show_targetN(to_cxl_region(dev), buf, (n));             \
1998 }                                                                      \
1999 static ssize_t target##n##_store(struct device *dev,                   \
2000                                  struct device_attribute *attr,        \
2001                                  const char *buf, size_t len)          \
2002 {                                                                      \
2003         return store_targetN(to_cxl_region(dev), buf, (n), len);       \
2004 }                                                                      \
2005 static DEVICE_ATTR_RW(target##n)
2006
2007 TARGET_ATTR_RW(0);
2008 TARGET_ATTR_RW(1);
2009 TARGET_ATTR_RW(2);
2010 TARGET_ATTR_RW(3);
2011 TARGET_ATTR_RW(4);
2012 TARGET_ATTR_RW(5);
2013 TARGET_ATTR_RW(6);
2014 TARGET_ATTR_RW(7);
2015 TARGET_ATTR_RW(8);
2016 TARGET_ATTR_RW(9);
2017 TARGET_ATTR_RW(10);
2018 TARGET_ATTR_RW(11);
2019 TARGET_ATTR_RW(12);
2020 TARGET_ATTR_RW(13);
2021 TARGET_ATTR_RW(14);
2022 TARGET_ATTR_RW(15);
2023
2024 static struct attribute *target_attrs[] = {
2025         &dev_attr_target0.attr,
2026         &dev_attr_target1.attr,
2027         &dev_attr_target2.attr,
2028         &dev_attr_target3.attr,
2029         &dev_attr_target4.attr,
2030         &dev_attr_target5.attr,
2031         &dev_attr_target6.attr,
2032         &dev_attr_target7.attr,
2033         &dev_attr_target8.attr,
2034         &dev_attr_target9.attr,
2035         &dev_attr_target10.attr,
2036         &dev_attr_target11.attr,
2037         &dev_attr_target12.attr,
2038         &dev_attr_target13.attr,
2039         &dev_attr_target14.attr,
2040         &dev_attr_target15.attr,
2041         NULL,
2042 };
2043
2044 static umode_t cxl_region_target_visible(struct kobject *kobj,
2045                                          struct attribute *a, int n)
2046 {
2047         struct device *dev = kobj_to_dev(kobj);
2048         struct cxl_region *cxlr = to_cxl_region(dev);
2049         struct cxl_region_params *p = &cxlr->params;
2050
2051         if (n < p->interleave_ways)
2052                 return a->mode;
2053         return 0;
2054 }
2055
2056 static const struct attribute_group cxl_region_target_group = {
2057         .attrs = target_attrs,
2058         .is_visible = cxl_region_target_visible,
2059 };
2060
2061 static const struct attribute_group *get_cxl_region_target_group(void)
2062 {
2063         return &cxl_region_target_group;
2064 }
2065
2066 static const struct attribute_group *region_groups[] = {
2067         &cxl_base_attribute_group,
2068         &cxl_region_group,
2069         &cxl_region_target_group,
2070         NULL,
2071 };
2072
2073 static void cxl_region_release(struct device *dev)
2074 {
2075         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent);
2076         struct cxl_region *cxlr = to_cxl_region(dev);
2077         int id = atomic_read(&cxlrd->region_id);
2078
2079         /*
2080          * Try to reuse the recently idled id rather than the cached
2081          * next id to prevent the region id space from increasing
2082          * unnecessarily.
2083          */
2084         if (cxlr->id < id)
2085                 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) {
2086                         memregion_free(id);
2087                         goto out;
2088                 }
2089
2090         memregion_free(cxlr->id);
2091 out:
2092         put_device(dev->parent);
2093         kfree(cxlr);
2094 }
2095
2096 const struct device_type cxl_region_type = {
2097         .name = "cxl_region",
2098         .release = cxl_region_release,
2099         .groups = region_groups
2100 };
2101
2102 bool is_cxl_region(struct device *dev)
2103 {
2104         return dev->type == &cxl_region_type;
2105 }
2106 EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL);
2107
2108 static struct cxl_region *to_cxl_region(struct device *dev)
2109 {
2110         if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type,
2111                           "not a cxl_region device\n"))
2112                 return NULL;
2113
2114         return container_of(dev, struct cxl_region, dev);
2115 }
2116
2117 static void unregister_region(void *_cxlr)
2118 {
2119         struct cxl_region *cxlr = _cxlr;
2120         struct cxl_region_params *p = &cxlr->params;
2121         int i;
2122
2123         device_del(&cxlr->dev);
2124
2125         /*
2126          * Now that region sysfs is shutdown, the parameter block is now
2127          * read-only, so no need to hold the region rwsem to access the
2128          * region parameters.
2129          */
2130         for (i = 0; i < p->interleave_ways; i++)
2131                 detach_target(cxlr, i);
2132
2133         cxl_region_iomem_release(cxlr);
2134         put_device(&cxlr->dev);
2135 }
2136
2137 static struct lock_class_key cxl_region_key;
2138
2139 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id)
2140 {
2141         struct cxl_region *cxlr;
2142         struct device *dev;
2143
2144         cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL);
2145         if (!cxlr) {
2146                 memregion_free(id);
2147                 return ERR_PTR(-ENOMEM);
2148         }
2149
2150         dev = &cxlr->dev;
2151         device_initialize(dev);
2152         lockdep_set_class(&dev->mutex, &cxl_region_key);
2153         dev->parent = &cxlrd->cxlsd.cxld.dev;
2154         /*
2155          * Keep root decoder pinned through cxl_region_release to fixup
2156          * region id allocations
2157          */
2158         get_device(dev->parent);
2159         device_set_pm_not_required(dev);
2160         dev->bus = &cxl_bus_type;
2161         dev->type = &cxl_region_type;
2162         cxlr->id = id;
2163
2164         return cxlr;
2165 }
2166
2167 /**
2168  * devm_cxl_add_region - Adds a region to a decoder
2169  * @cxlrd: root decoder
2170  * @id: memregion id to create, or memregion_free() on failure
2171  * @mode: mode for the endpoint decoders of this region
2172  * @type: select whether this is an expander or accelerator (type-2 or type-3)
2173  *
2174  * This is the second step of region initialization. Regions exist within an
2175  * address space which is mapped by a @cxlrd.
2176  *
2177  * Return: 0 if the region was added to the @cxlrd, else returns negative error
2178  * code. The region will be named "regionZ" where Z is the unique region number.
2179  */
2180 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
2181                                               int id,
2182                                               enum cxl_decoder_mode mode,
2183                                               enum cxl_decoder_type type)
2184 {
2185         struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent);
2186         struct cxl_region *cxlr;
2187         struct device *dev;
2188         int rc;
2189
2190         switch (mode) {
2191         case CXL_DECODER_RAM:
2192         case CXL_DECODER_PMEM:
2193                 break;
2194         default:
2195                 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode);
2196                 return ERR_PTR(-EINVAL);
2197         }
2198
2199         cxlr = cxl_region_alloc(cxlrd, id);
2200         if (IS_ERR(cxlr))
2201                 return cxlr;
2202         cxlr->mode = mode;
2203         cxlr->type = type;
2204
2205         dev = &cxlr->dev;
2206         rc = dev_set_name(dev, "region%d", id);
2207         if (rc)
2208                 goto err;
2209
2210         rc = device_add(dev);
2211         if (rc)
2212                 goto err;
2213
2214         rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr);
2215         if (rc)
2216                 return ERR_PTR(rc);
2217
2218         dev_dbg(port->uport_dev, "%s: created %s\n",
2219                 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev));
2220         return cxlr;
2221
2222 err:
2223         put_device(dev);
2224         return ERR_PTR(rc);
2225 }
2226
2227 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf)
2228 {
2229         return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id));
2230 }
2231
2232 static ssize_t create_pmem_region_show(struct device *dev,
2233                                        struct device_attribute *attr, char *buf)
2234 {
2235         return __create_region_show(to_cxl_root_decoder(dev), buf);
2236 }
2237
2238 static ssize_t create_ram_region_show(struct device *dev,
2239                                       struct device_attribute *attr, char *buf)
2240 {
2241         return __create_region_show(to_cxl_root_decoder(dev), buf);
2242 }
2243
2244 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd,
2245                                           enum cxl_decoder_mode mode, int id)
2246 {
2247         int rc;
2248
2249         rc = memregion_alloc(GFP_KERNEL);
2250         if (rc < 0)
2251                 return ERR_PTR(rc);
2252
2253         if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) {
2254                 memregion_free(rc);
2255                 return ERR_PTR(-EBUSY);
2256         }
2257
2258         return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM);
2259 }
2260
2261 static ssize_t create_pmem_region_store(struct device *dev,
2262                                         struct device_attribute *attr,
2263                                         const char *buf, size_t len)
2264 {
2265         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2266         struct cxl_region *cxlr;
2267         int rc, id;
2268
2269         rc = sscanf(buf, "region%d\n", &id);
2270         if (rc != 1)
2271                 return -EINVAL;
2272
2273         cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id);
2274         if (IS_ERR(cxlr))
2275                 return PTR_ERR(cxlr);
2276
2277         return len;
2278 }
2279 DEVICE_ATTR_RW(create_pmem_region);
2280
2281 static ssize_t create_ram_region_store(struct device *dev,
2282                                        struct device_attribute *attr,
2283                                        const char *buf, size_t len)
2284 {
2285         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2286         struct cxl_region *cxlr;
2287         int rc, id;
2288
2289         rc = sscanf(buf, "region%d\n", &id);
2290         if (rc != 1)
2291                 return -EINVAL;
2292
2293         cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id);
2294         if (IS_ERR(cxlr))
2295                 return PTR_ERR(cxlr);
2296
2297         return len;
2298 }
2299 DEVICE_ATTR_RW(create_ram_region);
2300
2301 static ssize_t region_show(struct device *dev, struct device_attribute *attr,
2302                            char *buf)
2303 {
2304         struct cxl_decoder *cxld = to_cxl_decoder(dev);
2305         ssize_t rc;
2306
2307         rc = down_read_interruptible(&cxl_region_rwsem);
2308         if (rc)
2309                 return rc;
2310
2311         if (cxld->region)
2312                 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev));
2313         else
2314                 rc = sysfs_emit(buf, "\n");
2315         up_read(&cxl_region_rwsem);
2316
2317         return rc;
2318 }
2319 DEVICE_ATTR_RO(region);
2320
2321 static struct cxl_region *
2322 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name)
2323 {
2324         struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld;
2325         struct device *region_dev;
2326
2327         region_dev = device_find_child_by_name(&cxld->dev, name);
2328         if (!region_dev)
2329                 return ERR_PTR(-ENODEV);
2330
2331         return to_cxl_region(region_dev);
2332 }
2333
2334 static ssize_t delete_region_store(struct device *dev,
2335                                    struct device_attribute *attr,
2336                                    const char *buf, size_t len)
2337 {
2338         struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev);
2339         struct cxl_port *port = to_cxl_port(dev->parent);
2340         struct cxl_region *cxlr;
2341
2342         cxlr = cxl_find_region_by_name(cxlrd, buf);
2343         if (IS_ERR(cxlr))
2344                 return PTR_ERR(cxlr);
2345
2346         devm_release_action(port->uport_dev, unregister_region, cxlr);
2347         put_device(&cxlr->dev);
2348
2349         return len;
2350 }
2351 DEVICE_ATTR_WO(delete_region);
2352
2353 static void cxl_pmem_region_release(struct device *dev)
2354 {
2355         struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev);
2356         int i;
2357
2358         for (i = 0; i < cxlr_pmem->nr_mappings; i++) {
2359                 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd;
2360
2361                 put_device(&cxlmd->dev);
2362         }
2363
2364         kfree(cxlr_pmem);
2365 }
2366
2367 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = {
2368         &cxl_base_attribute_group,
2369         NULL,
2370 };
2371
2372 const struct device_type cxl_pmem_region_type = {
2373         .name = "cxl_pmem_region",
2374         .release = cxl_pmem_region_release,
2375         .groups = cxl_pmem_region_attribute_groups,
2376 };
2377
2378 bool is_cxl_pmem_region(struct device *dev)
2379 {
2380         return dev->type == &cxl_pmem_region_type;
2381 }
2382 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL);
2383
2384 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev)
2385 {
2386         if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev),
2387                           "not a cxl_pmem_region device\n"))
2388                 return NULL;
2389         return container_of(dev, struct cxl_pmem_region, dev);
2390 }
2391 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL);
2392
2393 struct cxl_poison_context {
2394         struct cxl_port *port;
2395         enum cxl_decoder_mode mode;
2396         u64 offset;
2397 };
2398
2399 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
2400                                    struct cxl_poison_context *ctx)
2401 {
2402         struct cxl_dev_state *cxlds = cxlmd->cxlds;
2403         u64 offset, length;
2404         int rc = 0;
2405
2406         /*
2407          * Collect poison for the remaining unmapped resources
2408          * after poison is collected by committed endpoints.
2409          *
2410          * Knowing that PMEM must always follow RAM, get poison
2411          * for unmapped resources based on the last decoder's mode:
2412          *      ram: scan remains of ram range, then any pmem range
2413          *      pmem: scan remains of pmem range
2414          */
2415
2416         if (ctx->mode == CXL_DECODER_RAM) {
2417                 offset = ctx->offset;
2418                 length = resource_size(&cxlds->ram_res) - offset;
2419                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2420                 if (rc == -EFAULT)
2421                         rc = 0;
2422                 if (rc)
2423                         return rc;
2424         }
2425         if (ctx->mode == CXL_DECODER_PMEM) {
2426                 offset = ctx->offset;
2427                 length = resource_size(&cxlds->dpa_res) - offset;
2428                 if (!length)
2429                         return 0;
2430         } else if (resource_size(&cxlds->pmem_res)) {
2431                 offset = cxlds->pmem_res.start;
2432                 length = resource_size(&cxlds->pmem_res);
2433         } else {
2434                 return 0;
2435         }
2436
2437         return cxl_mem_get_poison(cxlmd, offset, length, NULL);
2438 }
2439
2440 static int poison_by_decoder(struct device *dev, void *arg)
2441 {
2442         struct cxl_poison_context *ctx = arg;
2443         struct cxl_endpoint_decoder *cxled;
2444         struct cxl_memdev *cxlmd;
2445         u64 offset, length;
2446         int rc = 0;
2447
2448         if (!is_endpoint_decoder(dev))
2449                 return rc;
2450
2451         cxled = to_cxl_endpoint_decoder(dev);
2452         if (!cxled->dpa_res || !resource_size(cxled->dpa_res))
2453                 return rc;
2454
2455         /*
2456          * Regions are only created with single mode decoders: pmem or ram.
2457          * Linux does not support mixed mode decoders. This means that
2458          * reading poison per endpoint decoder adheres to the requirement
2459          * that poison reads of pmem and ram must be separated.
2460          * CXL 3.0 Spec 8.2.9.8.4.1
2461          */
2462         if (cxled->mode == CXL_DECODER_MIXED) {
2463                 dev_dbg(dev, "poison list read unsupported in mixed mode\n");
2464                 return rc;
2465         }
2466
2467         cxlmd = cxled_to_memdev(cxled);
2468         if (cxled->skip) {
2469                 offset = cxled->dpa_res->start - cxled->skip;
2470                 length = cxled->skip;
2471                 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
2472                 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2473                         rc = 0;
2474                 if (rc)
2475                         return rc;
2476         }
2477
2478         offset = cxled->dpa_res->start;
2479         length = cxled->dpa_res->end - offset + 1;
2480         rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
2481         if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM)
2482                 rc = 0;
2483         if (rc)
2484                 return rc;
2485
2486         /* Iterate until commit_end is reached */
2487         if (cxled->cxld.id == ctx->port->commit_end) {
2488                 ctx->offset = cxled->dpa_res->end + 1;
2489                 ctx->mode = cxled->mode;
2490                 return 1;
2491         }
2492
2493         return 0;
2494 }
2495
2496 int cxl_get_poison_by_endpoint(struct cxl_port *port)
2497 {
2498         struct cxl_poison_context ctx;
2499         int rc = 0;
2500
2501         ctx = (struct cxl_poison_context) {
2502                 .port = port
2503         };
2504
2505         rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
2506         if (rc == 1)
2507                 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
2508                                              &ctx);
2509
2510         return rc;
2511 }
2512
2513 static struct lock_class_key cxl_pmem_region_key;
2514
2515 static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr)
2516 {
2517         struct cxl_region_params *p = &cxlr->params;
2518         struct cxl_nvdimm_bridge *cxl_nvb;
2519         struct cxl_pmem_region *cxlr_pmem;
2520         struct device *dev;
2521         int i;
2522
2523         down_read(&cxl_region_rwsem);
2524         if (p->state != CXL_CONFIG_COMMIT) {
2525                 cxlr_pmem = ERR_PTR(-ENXIO);
2526                 goto out;
2527         }
2528
2529         cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets),
2530                             GFP_KERNEL);
2531         if (!cxlr_pmem) {
2532                 cxlr_pmem = ERR_PTR(-ENOMEM);
2533                 goto out;
2534         }
2535
2536         cxlr_pmem->hpa_range.start = p->res->start;
2537         cxlr_pmem->hpa_range.end = p->res->end;
2538
2539         /* Snapshot the region configuration underneath the cxl_region_rwsem */
2540         cxlr_pmem->nr_mappings = p->nr_targets;
2541         for (i = 0; i < p->nr_targets; i++) {
2542                 struct cxl_endpoint_decoder *cxled = p->targets[i];
2543                 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2544                 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i];
2545
2546                 /*
2547                  * Regions never span CXL root devices, so by definition the
2548                  * bridge for one device is the same for all.
2549                  */
2550                 if (i == 0) {
2551                         cxl_nvb = cxl_find_nvdimm_bridge(cxlmd);
2552                         if (!cxl_nvb) {
2553                                 cxlr_pmem = ERR_PTR(-ENODEV);
2554                                 goto out;
2555                         }
2556                         cxlr->cxl_nvb = cxl_nvb;
2557                 }
2558                 m->cxlmd = cxlmd;
2559                 get_device(&cxlmd->dev);
2560                 m->start = cxled->dpa_res->start;
2561                 m->size = resource_size(cxled->dpa_res);
2562                 m->position = i;
2563         }
2564
2565         dev = &cxlr_pmem->dev;
2566         cxlr_pmem->cxlr = cxlr;
2567         cxlr->cxlr_pmem = cxlr_pmem;
2568         device_initialize(dev);
2569         lockdep_set_class(&dev->mutex, &cxl_pmem_region_key);
2570         device_set_pm_not_required(dev);
2571         dev->parent = &cxlr->dev;
2572         dev->bus = &cxl_bus_type;
2573         dev->type = &cxl_pmem_region_type;
2574 out:
2575         up_read(&cxl_region_rwsem);
2576
2577         return cxlr_pmem;
2578 }
2579
2580 static void cxl_dax_region_release(struct device *dev)
2581 {
2582         struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev);
2583
2584         kfree(cxlr_dax);
2585 }
2586
2587 static const struct attribute_group *cxl_dax_region_attribute_groups[] = {
2588         &cxl_base_attribute_group,
2589         NULL,
2590 };
2591
2592 const struct device_type cxl_dax_region_type = {
2593         .name = "cxl_dax_region",
2594         .release = cxl_dax_region_release,
2595         .groups = cxl_dax_region_attribute_groups,
2596 };
2597
2598 static bool is_cxl_dax_region(struct device *dev)
2599 {
2600         return dev->type == &cxl_dax_region_type;
2601 }
2602
2603 struct cxl_dax_region *to_cxl_dax_region(struct device *dev)
2604 {
2605         if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev),
2606                           "not a cxl_dax_region device\n"))
2607                 return NULL;
2608         return container_of(dev, struct cxl_dax_region, dev);
2609 }
2610 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL);
2611
2612 static struct lock_class_key cxl_dax_region_key;
2613
2614 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr)
2615 {
2616         struct cxl_region_params *p = &cxlr->params;
2617         struct cxl_dax_region *cxlr_dax;
2618         struct device *dev;
2619
2620         down_read(&cxl_region_rwsem);
2621         if (p->state != CXL_CONFIG_COMMIT) {
2622                 cxlr_dax = ERR_PTR(-ENXIO);
2623                 goto out;
2624         }
2625
2626         cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL);
2627         if (!cxlr_dax) {
2628                 cxlr_dax = ERR_PTR(-ENOMEM);
2629                 goto out;
2630         }
2631
2632         cxlr_dax->hpa_range.start = p->res->start;
2633         cxlr_dax->hpa_range.end = p->res->end;
2634
2635         dev = &cxlr_dax->dev;
2636         cxlr_dax->cxlr = cxlr;
2637         device_initialize(dev);
2638         lockdep_set_class(&dev->mutex, &cxl_dax_region_key);
2639         device_set_pm_not_required(dev);
2640         dev->parent = &cxlr->dev;
2641         dev->bus = &cxl_bus_type;
2642         dev->type = &cxl_dax_region_type;
2643 out:
2644         up_read(&cxl_region_rwsem);
2645
2646         return cxlr_dax;
2647 }
2648
2649 static void cxlr_pmem_unregister(void *_cxlr_pmem)
2650 {
2651         struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem;
2652         struct cxl_region *cxlr = cxlr_pmem->cxlr;
2653         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2654
2655         /*
2656          * Either the bridge is in ->remove() context under the device_lock(),
2657          * or cxlr_release_nvdimm() is cancelling the bridge's release action
2658          * for @cxlr_pmem and doing it itself (while manually holding the bridge
2659          * lock).
2660          */
2661         device_lock_assert(&cxl_nvb->dev);
2662         cxlr->cxlr_pmem = NULL;
2663         cxlr_pmem->cxlr = NULL;
2664         device_unregister(&cxlr_pmem->dev);
2665 }
2666
2667 static void cxlr_release_nvdimm(void *_cxlr)
2668 {
2669         struct cxl_region *cxlr = _cxlr;
2670         struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb;
2671
2672         device_lock(&cxl_nvb->dev);
2673         if (cxlr->cxlr_pmem)
2674                 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister,
2675                                     cxlr->cxlr_pmem);
2676         device_unlock(&cxl_nvb->dev);
2677         cxlr->cxl_nvb = NULL;
2678         put_device(&cxl_nvb->dev);
2679 }
2680
2681 /**
2682  * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge
2683  * @cxlr: parent CXL region for this pmem region bridge device
2684  *
2685  * Return: 0 on success negative error code on failure.
2686  */
2687 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
2688 {
2689         struct cxl_pmem_region *cxlr_pmem;
2690         struct cxl_nvdimm_bridge *cxl_nvb;
2691         struct device *dev;
2692         int rc;
2693
2694         cxlr_pmem = cxl_pmem_region_alloc(cxlr);
2695         if (IS_ERR(cxlr_pmem))
2696                 return PTR_ERR(cxlr_pmem);
2697         cxl_nvb = cxlr->cxl_nvb;
2698
2699         dev = &cxlr_pmem->dev;
2700         rc = dev_set_name(dev, "pmem_region%d", cxlr->id);
2701         if (rc)
2702                 goto err;
2703
2704         rc = device_add(dev);
2705         if (rc)
2706                 goto err;
2707
2708         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2709                 dev_name(dev));
2710
2711         device_lock(&cxl_nvb->dev);
2712         if (cxl_nvb->dev.driver)
2713                 rc = devm_add_action_or_reset(&cxl_nvb->dev,
2714                                               cxlr_pmem_unregister, cxlr_pmem);
2715         else
2716                 rc = -ENXIO;
2717         device_unlock(&cxl_nvb->dev);
2718
2719         if (rc)
2720                 goto err_bridge;
2721
2722         /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */
2723         return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr);
2724
2725 err:
2726         put_device(dev);
2727 err_bridge:
2728         put_device(&cxl_nvb->dev);
2729         cxlr->cxl_nvb = NULL;
2730         return rc;
2731 }
2732
2733 static void cxlr_dax_unregister(void *_cxlr_dax)
2734 {
2735         struct cxl_dax_region *cxlr_dax = _cxlr_dax;
2736
2737         device_unregister(&cxlr_dax->dev);
2738 }
2739
2740 static int devm_cxl_add_dax_region(struct cxl_region *cxlr)
2741 {
2742         struct cxl_dax_region *cxlr_dax;
2743         struct device *dev;
2744         int rc;
2745
2746         cxlr_dax = cxl_dax_region_alloc(cxlr);
2747         if (IS_ERR(cxlr_dax))
2748                 return PTR_ERR(cxlr_dax);
2749
2750         dev = &cxlr_dax->dev;
2751         rc = dev_set_name(dev, "dax_region%d", cxlr->id);
2752         if (rc)
2753                 goto err;
2754
2755         rc = device_add(dev);
2756         if (rc)
2757                 goto err;
2758
2759         dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent),
2760                 dev_name(dev));
2761
2762         return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister,
2763                                         cxlr_dax);
2764 err:
2765         put_device(dev);
2766         return rc;
2767 }
2768
2769 static int match_root_decoder_by_range(struct device *dev, void *data)
2770 {
2771         struct range *r1, *r2 = data;
2772         struct cxl_root_decoder *cxlrd;
2773
2774         if (!is_root_decoder(dev))
2775                 return 0;
2776
2777         cxlrd = to_cxl_root_decoder(dev);
2778         r1 = &cxlrd->cxlsd.cxld.hpa_range;
2779         return range_contains(r1, r2);
2780 }
2781
2782 static int match_region_by_range(struct device *dev, void *data)
2783 {
2784         struct cxl_region_params *p;
2785         struct cxl_region *cxlr;
2786         struct range *r = data;
2787         int rc = 0;
2788
2789         if (!is_cxl_region(dev))
2790                 return 0;
2791
2792         cxlr = to_cxl_region(dev);
2793         p = &cxlr->params;
2794
2795         down_read(&cxl_region_rwsem);
2796         if (p->res && p->res->start == r->start && p->res->end == r->end)
2797                 rc = 1;
2798         up_read(&cxl_region_rwsem);
2799
2800         return rc;
2801 }
2802
2803 /* Establish an empty region covering the given HPA range */
2804 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd,
2805                                            struct cxl_endpoint_decoder *cxled)
2806 {
2807         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2808         struct cxl_port *port = cxlrd_to_port(cxlrd);
2809         struct range *hpa = &cxled->cxld.hpa_range;
2810         struct cxl_region_params *p;
2811         struct cxl_region *cxlr;
2812         struct resource *res;
2813         int rc;
2814
2815         do {
2816                 cxlr = __create_region(cxlrd, cxled->mode,
2817                                        atomic_read(&cxlrd->region_id));
2818         } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY);
2819
2820         if (IS_ERR(cxlr)) {
2821                 dev_err(cxlmd->dev.parent,
2822                         "%s:%s: %s failed assign region: %ld\n",
2823                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2824                         __func__, PTR_ERR(cxlr));
2825                 return cxlr;
2826         }
2827
2828         down_write(&cxl_region_rwsem);
2829         p = &cxlr->params;
2830         if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) {
2831                 dev_err(cxlmd->dev.parent,
2832                         "%s:%s: %s autodiscovery interrupted\n",
2833                         dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2834                         __func__);
2835                 rc = -EBUSY;
2836                 goto err;
2837         }
2838
2839         set_bit(CXL_REGION_F_AUTO, &cxlr->flags);
2840
2841         res = kmalloc(sizeof(*res), GFP_KERNEL);
2842         if (!res) {
2843                 rc = -ENOMEM;
2844                 goto err;
2845         }
2846
2847         *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa),
2848                                     dev_name(&cxlr->dev));
2849         rc = insert_resource(cxlrd->res, res);
2850         if (rc) {
2851                 /*
2852                  * Platform-firmware may not have split resources like "System
2853                  * RAM" on CXL window boundaries see cxl_region_iomem_release()
2854                  */
2855                 dev_warn(cxlmd->dev.parent,
2856                          "%s:%s: %s %s cannot insert resource\n",
2857                          dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev),
2858                          __func__, dev_name(&cxlr->dev));
2859         }
2860
2861         p->res = res;
2862         p->interleave_ways = cxled->cxld.interleave_ways;
2863         p->interleave_granularity = cxled->cxld.interleave_granularity;
2864         p->state = CXL_CONFIG_INTERLEAVE_ACTIVE;
2865
2866         rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group());
2867         if (rc)
2868                 goto err;
2869
2870         dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n",
2871                 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__,
2872                 dev_name(&cxlr->dev), p->res, p->interleave_ways,
2873                 p->interleave_granularity);
2874
2875         /* ...to match put_device() in cxl_add_to_region() */
2876         get_device(&cxlr->dev);
2877         up_write(&cxl_region_rwsem);
2878
2879         return cxlr;
2880
2881 err:
2882         up_write(&cxl_region_rwsem);
2883         devm_release_action(port->uport_dev, unregister_region, cxlr);
2884         return ERR_PTR(rc);
2885 }
2886
2887 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled)
2888 {
2889         struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
2890         struct range *hpa = &cxled->cxld.hpa_range;
2891         struct cxl_decoder *cxld = &cxled->cxld;
2892         struct device *cxlrd_dev, *region_dev;
2893         struct cxl_root_decoder *cxlrd;
2894         struct cxl_region_params *p;
2895         struct cxl_region *cxlr;
2896         bool attach = false;
2897         int rc;
2898
2899         cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range,
2900                                       match_root_decoder_by_range);
2901         if (!cxlrd_dev) {
2902                 dev_err(cxlmd->dev.parent,
2903                         "%s:%s no CXL window for range %#llx:%#llx\n",
2904                         dev_name(&cxlmd->dev), dev_name(&cxld->dev),
2905                         cxld->hpa_range.start, cxld->hpa_range.end);
2906                 return -ENXIO;
2907         }
2908
2909         cxlrd = to_cxl_root_decoder(cxlrd_dev);
2910
2911         /*
2912          * Ensure that if multiple threads race to construct_region() for @hpa
2913          * one does the construction and the others add to that.
2914          */
2915         mutex_lock(&cxlrd->range_lock);
2916         region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa,
2917                                        match_region_by_range);
2918         if (!region_dev) {
2919                 cxlr = construct_region(cxlrd, cxled);
2920                 region_dev = &cxlr->dev;
2921         } else
2922                 cxlr = to_cxl_region(region_dev);
2923         mutex_unlock(&cxlrd->range_lock);
2924
2925         rc = PTR_ERR_OR_ZERO(cxlr);
2926         if (rc)
2927                 goto out;
2928
2929         attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE);
2930
2931         down_read(&cxl_region_rwsem);
2932         p = &cxlr->params;
2933         attach = p->state == CXL_CONFIG_COMMIT;
2934         up_read(&cxl_region_rwsem);
2935
2936         if (attach) {
2937                 /*
2938                  * If device_attach() fails the range may still be active via
2939                  * the platform-firmware memory map, otherwise the driver for
2940                  * regions is local to this file, so driver matching can't fail.
2941                  */
2942                 if (device_attach(&cxlr->dev) < 0)
2943                         dev_err(&cxlr->dev, "failed to enable, range: %pr\n",
2944                                 p->res);
2945         }
2946
2947         put_device(region_dev);
2948 out:
2949         put_device(cxlrd_dev);
2950         return rc;
2951 }
2952 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL);
2953
2954 static int is_system_ram(struct resource *res, void *arg)
2955 {
2956         struct cxl_region *cxlr = arg;
2957         struct cxl_region_params *p = &cxlr->params;
2958
2959         dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res);
2960         return 1;
2961 }
2962
2963 static int cxl_region_probe(struct device *dev)
2964 {
2965         struct cxl_region *cxlr = to_cxl_region(dev);
2966         struct cxl_region_params *p = &cxlr->params;
2967         int rc;
2968
2969         rc = down_read_interruptible(&cxl_region_rwsem);
2970         if (rc) {
2971                 dev_dbg(&cxlr->dev, "probe interrupted\n");
2972                 return rc;
2973         }
2974
2975         if (p->state < CXL_CONFIG_COMMIT) {
2976                 dev_dbg(&cxlr->dev, "config state: %d\n", p->state);
2977                 rc = -ENXIO;
2978                 goto out;
2979         }
2980
2981         if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) {
2982                 dev_err(&cxlr->dev,
2983                         "failed to activate, re-commit region and retry\n");
2984                 rc = -ENXIO;
2985                 goto out;
2986         }
2987
2988         /*
2989          * From this point on any path that changes the region's state away from
2990          * CXL_CONFIG_COMMIT is also responsible for releasing the driver.
2991          */
2992 out:
2993         up_read(&cxl_region_rwsem);
2994
2995         if (rc)
2996                 return rc;
2997
2998         switch (cxlr->mode) {
2999         case CXL_DECODER_PMEM:
3000                 return devm_cxl_add_pmem_region(cxlr);
3001         case CXL_DECODER_RAM:
3002                 /*
3003                  * The region can not be manged by CXL if any portion of
3004                  * it is already online as 'System RAM'
3005                  */
3006                 if (walk_iomem_res_desc(IORES_DESC_NONE,
3007                                         IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY,
3008                                         p->res->start, p->res->end, cxlr,
3009                                         is_system_ram) > 0)
3010                         return 0;
3011                 return devm_cxl_add_dax_region(cxlr);
3012         default:
3013                 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n",
3014                         cxlr->mode);
3015                 return -ENXIO;
3016         }
3017 }
3018
3019 static struct cxl_driver cxl_region_driver = {
3020         .name = "cxl_region",
3021         .probe = cxl_region_probe,
3022         .id = CXL_DEVICE_REGION,
3023 };
3024
3025 int cxl_region_init(void)
3026 {
3027         return cxl_driver_register(&cxl_region_driver);
3028 }
3029
3030 void cxl_region_exit(void)
3031 {
3032         cxl_driver_unregister(&cxl_region_driver);
3033 }
3034
3035 MODULE_IMPORT_NS(CXL);
3036 MODULE_IMPORT_NS(DEVMEM);
3037 MODULE_ALIAS_CXL(CXL_DEVICE_REGION);