drivers/cxl/core/mbox.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright(c) 2020 Intel Corporation. All rights reserved. */
   3 #include <linux/security.h>
   4 #include <linux/debugfs.h>
   5 #include <linux/ktime.h>
   6 #include <linux/mutex.h>
   7 #include <asm/unaligned.h>
   8 #include <cxlpci.h>
   9 #include <cxlmem.h>
  10 #include <cxl.h>
  11
  12 #include "core.h"
  13 #include "trace.h"
  14
  15 static bool cxl_raw_allow_all;
  16
  17 /**
  18  * DOC: cxl mbox
  19  *
  20  * Core implementation of the CXL 2.0 Type-3 Memory Device Mailbox. The
  21  * implementation is used by the cxl_pci driver to initialize the device
  22  * and implement the cxl_mem.h IOCTL UAPI. It also implements the
  23  * backend of the cxl_pmem_ctl() transport for LIBNVDIMM.
  24  */
  25
  26 #define cxl_for_each_cmd(cmd)                                                  \
  27         for ((cmd) = &cxl_mem_commands[0];                                     \
  28              ((cmd) - cxl_mem_commands) < ARRAY_SIZE(cxl_mem_commands); (cmd)++)
  29
  30 #define CXL_CMD(_id, sin, sout, _flags)                                        \
  31         [CXL_MEM_COMMAND_ID_##_id] = {                                         \
  32         .info = {                                                              \
  33                         .id = CXL_MEM_COMMAND_ID_##_id,                        \
  34                         .size_in = sin,                                        \
  35                         .size_out = sout,                                      \
  36                 },                                                             \
  37         .opcode = CXL_MBOX_OP_##_id,                                           \
  38         .flags = _flags,                                                       \
  39         }
  40
  41 #define CXL_VARIABLE_PAYLOAD    ~0U
  42 /*
  43  * This table defines the supported mailbox commands for the driver. This table
  44  * is made up of a UAPI structure. Non-negative values as parameters in the
  45  * table will be validated against the user's input. For example, if size_in is
  46  * 0, and the user passed in 1, it is an error.
  47  */
  48 static struct cxl_mem_command cxl_mem_commands[CXL_MEM_COMMAND_ID_MAX] = {
  49         CXL_CMD(IDENTIFY, 0, 0x43, CXL_CMD_FLAG_FORCE_ENABLE),
  50 #ifdef CONFIG_CXL_MEM_RAW_COMMANDS
  51         CXL_CMD(RAW, CXL_VARIABLE_PAYLOAD, CXL_VARIABLE_PAYLOAD, 0),
  52 #endif
  53         CXL_CMD(GET_SUPPORTED_LOGS, 0, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
  54         CXL_CMD(GET_FW_INFO, 0, 0x50, 0),
  55         CXL_CMD(GET_PARTITION_INFO, 0, 0x20, 0),
  56         CXL_CMD(GET_LSA, 0x8, CXL_VARIABLE_PAYLOAD, 0),
  57         CXL_CMD(GET_HEALTH_INFO, 0, 0x12, 0),
  58         CXL_CMD(GET_LOG, 0x18, CXL_VARIABLE_PAYLOAD, CXL_CMD_FLAG_FORCE_ENABLE),
  59         CXL_CMD(SET_PARTITION_INFO, 0x0a, 0, 0),
  60         CXL_CMD(SET_LSA, CXL_VARIABLE_PAYLOAD, 0, 0),
  61         CXL_CMD(GET_ALERT_CONFIG, 0, 0x10, 0),
  62         CXL_CMD(SET_ALERT_CONFIG, 0xc, 0, 0),
  63         CXL_CMD(GET_SHUTDOWN_STATE, 0, 0x1, 0),
  64         CXL_CMD(SET_SHUTDOWN_STATE, 0x1, 0, 0),
  65         CXL_CMD(GET_SCAN_MEDIA_CAPS, 0x10, 0x4, 0),
  66 };
  67
  68 /*
  69  * Commands that RAW doesn't permit. The rationale for each:
  70  *
  71  * CXL_MBOX_OP_ACTIVATE_FW: Firmware activation requires adjustment /
  72  * coordination of transaction timeout values at the root bridge level.
  73  *
  74  * CXL_MBOX_OP_SET_PARTITION_INFO: The device memory map may change live
  75  * and needs to be coordinated with HDM updates.
  76  *
  77  * CXL_MBOX_OP_SET_LSA: The label storage area may be cached by the
  78  * driver and any writes from userspace invalidates those contents.
  79  *
  80  * CXL_MBOX_OP_SET_SHUTDOWN_STATE: Set shutdown state assumes no writes
  81  * to the device after it is marked clean, userspace can not make that
  82  * assertion.
  83  *
  84  * CXL_MBOX_OP_[GET_]SCAN_MEDIA: The kernel provides a native error list that
  85  * is kept up to date with patrol notifications and error management.
  86  *
  87  * CXL_MBOX_OP_[GET_,INJECT_,CLEAR_]POISON: These commands require kernel
  88  * driver orchestration for safety.
  89  */
  90 static u16 cxl_disabled_raw_commands[] = {
  91         CXL_MBOX_OP_ACTIVATE_FW,
  92         CXL_MBOX_OP_SET_PARTITION_INFO,
  93         CXL_MBOX_OP_SET_LSA,
  94         CXL_MBOX_OP_SET_SHUTDOWN_STATE,
  95         CXL_MBOX_OP_SCAN_MEDIA,
  96         CXL_MBOX_OP_GET_SCAN_MEDIA,
  97         CXL_MBOX_OP_GET_POISON,
  98         CXL_MBOX_OP_INJECT_POISON,
  99         CXL_MBOX_OP_CLEAR_POISON,
 100 };
 101
 102 /*
 103  * Command sets that RAW doesn't permit. All opcodes in this set are
 104  * disabled because they pass plain text security payloads over the
 105  * user/kernel boundary. This functionality is intended to be wrapped
 106  * behind the keys ABI which allows for encrypted payloads in the UAPI
 107  */
 108 static u8 security_command_sets[] = {
 109         0x44, /* Sanitize */
 110         0x45, /* Persistent Memory Data-at-rest Security */
 111         0x46, /* Security Passthrough */
 112 };
 113
 114 static bool cxl_is_security_command(u16 opcode)
 115 {
 116         int i;
 117
 118         for (i = 0; i < ARRAY_SIZE(security_command_sets); i++)
 119                 if (security_command_sets[i] == (opcode >> 8))
 120                         return true;
 121         return false;
 122 }
 123
 124 static void cxl_set_security_cmd_enabled(struct cxl_security_state *security,
 125                                          u16 opcode)
 126 {
 127         switch (opcode) {
 128         case CXL_MBOX_OP_SANITIZE:
 129                 set_bit(CXL_SEC_ENABLED_SANITIZE, security->enabled_cmds);
 130                 break;
 131         case CXL_MBOX_OP_SECURE_ERASE:
 132                 set_bit(CXL_SEC_ENABLED_SECURE_ERASE,
 133                         security->enabled_cmds);
 134                 break;
 135         case CXL_MBOX_OP_GET_SECURITY_STATE:
 136                 set_bit(CXL_SEC_ENABLED_GET_SECURITY_STATE,
 137                         security->enabled_cmds);
 138                 break;
 139         case CXL_MBOX_OP_SET_PASSPHRASE:
 140                 set_bit(CXL_SEC_ENABLED_SET_PASSPHRASE,
 141                         security->enabled_cmds);
 142                 break;
 143         case CXL_MBOX_OP_DISABLE_PASSPHRASE:
 144                 set_bit(CXL_SEC_ENABLED_DISABLE_PASSPHRASE,
 145                         security->enabled_cmds);
 146                 break;
 147         case CXL_MBOX_OP_UNLOCK:
 148                 set_bit(CXL_SEC_ENABLED_UNLOCK, security->enabled_cmds);
 149                 break;
 150         case CXL_MBOX_OP_FREEZE_SECURITY:
 151                 set_bit(CXL_SEC_ENABLED_FREEZE_SECURITY,
 152                         security->enabled_cmds);
 153                 break;
 154         case CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE:
 155                 set_bit(CXL_SEC_ENABLED_PASSPHRASE_SECURE_ERASE,
 156                         security->enabled_cmds);
 157                 break;
 158         default:
 159                 break;
 160         }
 161 }
 162
 163 static bool cxl_is_poison_command(u16 opcode)
 164 {
 165 #define CXL_MBOX_OP_POISON_CMDS 0x43
 166
 167         if ((opcode >> 8) == CXL_MBOX_OP_POISON_CMDS)
 168                 return true;
 169
 170         return false;
 171 }
 172
 173 static void cxl_set_poison_cmd_enabled(struct cxl_poison_state *poison,
 174                                        u16 opcode)
 175 {
 176         switch (opcode) {
 177         case CXL_MBOX_OP_GET_POISON:
 178                 set_bit(CXL_POISON_ENABLED_LIST, poison->enabled_cmds);
 179                 break;
 180         case CXL_MBOX_OP_INJECT_POISON:
 181                 set_bit(CXL_POISON_ENABLED_INJECT, poison->enabled_cmds);
 182                 break;
 183         case CXL_MBOX_OP_CLEAR_POISON:
 184                 set_bit(CXL_POISON_ENABLED_CLEAR, poison->enabled_cmds);
 185                 break;
 186         case CXL_MBOX_OP_GET_SCAN_MEDIA_CAPS:
 187                 set_bit(CXL_POISON_ENABLED_SCAN_CAPS, poison->enabled_cmds);
 188                 break;
 189         case CXL_MBOX_OP_SCAN_MEDIA:
 190                 set_bit(CXL_POISON_ENABLED_SCAN_MEDIA, poison->enabled_cmds);
 191                 break;
 192         case CXL_MBOX_OP_GET_SCAN_MEDIA:
 193                 set_bit(CXL_POISON_ENABLED_SCAN_RESULTS, poison->enabled_cmds);
 194                 break;
 195         default:
 196                 break;
 197         }
 198 }
 199
 200 static struct cxl_mem_command *cxl_mem_find_command(u16 opcode)
 201 {
 202         struct cxl_mem_command *c;
 203
 204         cxl_for_each_cmd(c)
 205                 if (c->opcode == opcode)
 206                         return c;
 207
 208         return NULL;
 209 }
 210
 211 static const char *cxl_mem_opcode_to_name(u16 opcode)
 212 {
 213         struct cxl_mem_command *c;
 214
 215         c = cxl_mem_find_command(opcode);
 216         if (!c)
 217                 return NULL;
 218
 219         return cxl_command_names[c->info.id].name;
 220 }
 221
 222 /**
 223  * cxl_internal_send_cmd() - Kernel internal interface to send a mailbox command
 224  * @mds: The driver data for the operation
 225  * @mbox_cmd: initialized command to execute
 226  *
 227  * Context: Any context.
 228  * Return:
 229  *  * %>=0      - Number of bytes returned in @out.
 230  *  * %-E2BIG   - Payload is too large for hardware.
 231  *  * %-EBUSY   - Couldn't acquire exclusive mailbox access.
 232  *  * %-EFAULT  - Hardware error occurred.
 233  *  * %-ENXIO   - Command completed, but device reported an error.
 234  *  * %-EIO     - Unexpected output size.
 235  *
 236  * Mailbox commands may execute successfully yet the device itself reported an
 237  * error. While this distinction can be useful for commands from userspace, the
 238  * kernel will only be able to use results when both are successful.
 239  */
 240 int cxl_internal_send_cmd(struct cxl_memdev_state *mds,
 241                           struct cxl_mbox_cmd *mbox_cmd)
 242 {
 243         size_t out_size, min_out;
 244         int rc;
 245
 246         if (mbox_cmd->size_in > mds->payload_size ||
 247             mbox_cmd->size_out > mds->payload_size)
 248                 return -E2BIG;
 249
 250         out_size = mbox_cmd->size_out;
 251         min_out = mbox_cmd->min_out;
 252         rc = mds->mbox_send(mds, mbox_cmd);
 253         /*
 254          * EIO is reserved for a payload size mismatch and mbox_send()
 255          * may not return this error.
 256          */
 257         if (WARN_ONCE(rc == -EIO, "Bad return code: -EIO"))
 258                 return -ENXIO;
 259         if (rc)
 260                 return rc;
 261
 262         if (mbox_cmd->return_code != CXL_MBOX_CMD_RC_SUCCESS &&
 263             mbox_cmd->return_code != CXL_MBOX_CMD_RC_BACKGROUND)
 264                 return cxl_mbox_cmd_rc2errno(mbox_cmd);
 265
 266         if (!out_size)
 267                 return 0;
 268
 269         /*
 270          * Variable sized output needs to at least satisfy the caller's
 271          * minimum if not the fully requested size.
 272          */
 273         if (min_out == 0)
 274                 min_out = out_size;
 275
 276         if (mbox_cmd->size_out < min_out)
 277                 return -EIO;
 278         return 0;
 279 }
 280 EXPORT_SYMBOL_NS_GPL(cxl_internal_send_cmd, CXL);
 281
 282 static bool cxl_mem_raw_command_allowed(u16 opcode)
 283 {
 284         int i;
 285
 286         if (!IS_ENABLED(CONFIG_CXL_MEM_RAW_COMMANDS))
 287                 return false;
 288
 289         if (security_locked_down(LOCKDOWN_PCI_ACCESS))
 290                 return false;
 291
 292         if (cxl_raw_allow_all)
 293                 return true;
 294
 295         if (cxl_is_security_command(opcode))
 296                 return false;
 297
 298         for (i = 0; i < ARRAY_SIZE(cxl_disabled_raw_commands); i++)
 299                 if (cxl_disabled_raw_commands[i] == opcode)
 300                         return false;
 301
 302         return true;
 303 }
 304
 305 /**
 306  * cxl_payload_from_user_allowed() - Check contents of in_payload.
 307  * @opcode: The mailbox command opcode.
 308  * @payload_in: Pointer to the input payload passed in from user space.
 309  *
 310  * Return:
 311  *  * true      - payload_in passes check for @opcode.
 312  *  * false     - payload_in contains invalid or unsupported values.
 313  *
 314  * The driver may inspect payload contents before sending a mailbox
 315  * command from user space to the device. The intent is to reject
 316  * commands with input payloads that are known to be unsafe. This
 317  * check is not intended to replace the users careful selection of
 318  * mailbox command parameters and makes no guarantee that the user
 319  * command will succeed, nor that it is appropriate.
 320  *
 321  * The specific checks are determined by the opcode.
 322  */
 323 static bool cxl_payload_from_user_allowed(u16 opcode, void *payload_in)
 324 {
 325         switch (opcode) {
 326         case CXL_MBOX_OP_SET_PARTITION_INFO: {
 327                 struct cxl_mbox_set_partition_info *pi = payload_in;
 328
 329                 if (pi->flags & CXL_SET_PARTITION_IMMEDIATE_FLAG)
 330                         return false;
 331                 break;
 332         }
 333         default:
 334                 break;
 335         }
 336         return true;
 337 }
 338
 339 static int cxl_mbox_cmd_ctor(struct cxl_mbox_cmd *mbox,
 340                              struct cxl_memdev_state *mds, u16 opcode,
 341                              size_t in_size, size_t out_size, u64 in_payload)
 342 {
 343         *mbox = (struct cxl_mbox_cmd) {
 344                 .opcode = opcode,
 345                 .size_in = in_size,
 346         };
 347
 348         if (in_size) {
 349                 mbox->payload_in = vmemdup_user(u64_to_user_ptr(in_payload),
 350                                                 in_size);
 351                 if (IS_ERR(mbox->payload_in))
 352                         return PTR_ERR(mbox->payload_in);
 353
 354                 if (!cxl_payload_from_user_allowed(opcode, mbox->payload_in)) {
 355                         dev_dbg(mds->cxlds.dev, "%s: input payload not allowed\n",
 356                                 cxl_mem_opcode_to_name(opcode));
 357                         kvfree(mbox->payload_in);
 358                         return -EBUSY;
 359                 }
 360         }
 361
 362         /* Prepare to handle a full payload for variable sized output */
 363         if (out_size == CXL_VARIABLE_PAYLOAD)
 364                 mbox->size_out = mds->payload_size;
 365         else
 366                 mbox->size_out = out_size;
 367
 368         if (mbox->size_out) {
 369                 mbox->payload_out = kvzalloc(mbox->size_out, GFP_KERNEL);
 370                 if (!mbox->payload_out) {
 371                         kvfree(mbox->payload_in);
 372                         return -ENOMEM;
 373                 }
 374         }
 375         return 0;
 376 }
 377
 378 static void cxl_mbox_cmd_dtor(struct cxl_mbox_cmd *mbox)
 379 {
 380         kvfree(mbox->payload_in);
 381         kvfree(mbox->payload_out);
 382 }
 383
 384 static int cxl_to_mem_cmd_raw(struct cxl_mem_command *mem_cmd,
 385                               const struct cxl_send_command *send_cmd,
 386                               struct cxl_memdev_state *mds)
 387 {
 388         if (send_cmd->raw.rsvd)
 389                 return -EINVAL;
 390
 391         /*
 392          * Unlike supported commands, the output size of RAW commands
 393          * gets passed along without further checking, so it must be
 394          * validated here.
 395          */
 396         if (send_cmd->out.size > mds->payload_size)
 397                 return -EINVAL;
 398
 399         if (!cxl_mem_raw_command_allowed(send_cmd->raw.opcode))
 400                 return -EPERM;
 401
 402         dev_WARN_ONCE(mds->cxlds.dev, true, "raw command path used\n");
 403
 404         *mem_cmd = (struct cxl_mem_command) {
 405                 .info = {
 406                         .id = CXL_MEM_COMMAND_ID_RAW,
 407                         .size_in = send_cmd->in.size,
 408                         .size_out = send_cmd->out.size,
 409                 },
 410                 .opcode = send_cmd->raw.opcode
 411         };
 412
 413         return 0;
 414 }
 415
 416 static int cxl_to_mem_cmd(struct cxl_mem_command *mem_cmd,
 417                           const struct cxl_send_command *send_cmd,
 418                           struct cxl_memdev_state *mds)
 419 {
 420         struct cxl_mem_command *c = &cxl_mem_commands[send_cmd->id];
 421         const struct cxl_command_info *info = &c->info;
 422
 423         if (send_cmd->flags & ~CXL_MEM_COMMAND_FLAG_MASK)
 424                 return -EINVAL;
 425
 426         if (send_cmd->rsvd)
 427                 return -EINVAL;
 428
 429         if (send_cmd->in.rsvd || send_cmd->out.rsvd)
 430                 return -EINVAL;
 431
 432         /* Check that the command is enabled for hardware */
 433         if (!test_bit(info->id, mds->enabled_cmds))
 434                 return -ENOTTY;
 435
 436         /* Check that the command is not claimed for exclusive kernel use */
 437         if (test_bit(info->id, mds->exclusive_cmds))
 438                 return -EBUSY;
 439
 440         /* Check the input buffer is the expected size */
 441         if ((info->size_in != CXL_VARIABLE_PAYLOAD) &&
 442             (info->size_in != send_cmd->in.size))
 443                 return -ENOMEM;
 444
 445         /* Check the output buffer is at least large enough */
 446         if ((info->size_out != CXL_VARIABLE_PAYLOAD) &&
 447             (send_cmd->out.size < info->size_out))
 448                 return -ENOMEM;
 449
 450         *mem_cmd = (struct cxl_mem_command) {
 451                 .info = {
 452                         .id = info->id,
 453                         .flags = info->flags,
 454                         .size_in = send_cmd->in.size,
 455                         .size_out = send_cmd->out.size,
 456                 },
 457                 .opcode = c->opcode
 458         };
 459
 460         return 0;
 461 }
 462
 463 /**
 464  * cxl_validate_cmd_from_user() - Check fields for CXL_MEM_SEND_COMMAND.
 465  * @mbox_cmd: Sanitized and populated &struct cxl_mbox_cmd.
 466  * @mds: The driver data for the operation
 467  * @send_cmd: &struct cxl_send_command copied in from userspace.
 468  *
 469  * Return:
 470  *  * %0        - @out_cmd is ready to send.
 471  *  * %-ENOTTY  - Invalid command specified.
 472  *  * %-EINVAL  - Reserved fields or invalid values were used.
 473  *  * %-ENOMEM  - Input or output buffer wasn't sized properly.
 474  *  * %-EPERM   - Attempted to use a protected command.
 475  *  * %-EBUSY   - Kernel has claimed exclusive access to this opcode
 476  *
 477  * The result of this command is a fully validated command in @mbox_cmd that is
 478  * safe to send to the hardware.
 479  */
 480 static int cxl_validate_cmd_from_user(struct cxl_mbox_cmd *mbox_cmd,
 481                                       struct cxl_memdev_state *mds,
 482                                       const struct cxl_send_command *send_cmd)
 483 {
 484         struct cxl_mem_command mem_cmd;
 485         int rc;
 486
 487         if (send_cmd->id == 0 || send_cmd->id >= CXL_MEM_COMMAND_ID_MAX)
 488                 return -ENOTTY;
 489
 490         /*
 491          * The user can never specify an input payload larger than what hardware
 492          * supports, but output can be arbitrarily large (simply write out as
 493          * much data as the hardware provides).
 494          */
 495         if (send_cmd->in.size > mds->payload_size)
 496                 return -EINVAL;
 497
 498         /* Sanitize and construct a cxl_mem_command */
 499         if (send_cmd->id == CXL_MEM_COMMAND_ID_RAW)
 500                 rc = cxl_to_mem_cmd_raw(&mem_cmd, send_cmd, mds);
 501         else
 502                 rc = cxl_to_mem_cmd(&mem_cmd, send_cmd, mds);
 503
 504         if (rc)
 505                 return rc;
 506
 507         /* Sanitize and construct a cxl_mbox_cmd */
 508         return cxl_mbox_cmd_ctor(mbox_cmd, mds, mem_cmd.opcode,
 509                                  mem_cmd.info.size_in, mem_cmd.info.size_out,
 510                                  send_cmd->in.payload);
 511 }
 512
 513 int cxl_query_cmd(struct cxl_memdev *cxlmd,
 514                   struct cxl_mem_query_commands __user *q)
 515 {
 516         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 517         struct device *dev = &cxlmd->dev;
 518         struct cxl_mem_command *cmd;
 519         u32 n_commands;
 520         int j = 0;
 521
 522         dev_dbg(dev, "Query IOCTL\n");
 523
 524         if (get_user(n_commands, &q->n_commands))
 525                 return -EFAULT;
 526
 527         /* returns the total number if 0 elements are requested. */
 528         if (n_commands == 0)
 529                 return put_user(ARRAY_SIZE(cxl_mem_commands), &q->n_commands);
 530
 531         /*
 532          * otherwise, return max(n_commands, total commands) cxl_command_info
 533          * structures.
 534          */
 535         cxl_for_each_cmd(cmd) {
 536                 struct cxl_command_info info = cmd->info;
 537
 538                 if (test_bit(info.id, mds->enabled_cmds))
 539                         info.flags |= CXL_MEM_COMMAND_FLAG_ENABLED;
 540                 if (test_bit(info.id, mds->exclusive_cmds))
 541                         info.flags |= CXL_MEM_COMMAND_FLAG_EXCLUSIVE;
 542
 543                 if (copy_to_user(&q->commands[j++], &info, sizeof(info)))
 544                         return -EFAULT;
 545
 546                 if (j == n_commands)
 547                         break;
 548         }
 549
 550         return 0;
 551 }
 552
 553 /**
 554  * handle_mailbox_cmd_from_user() - Dispatch a mailbox command for userspace.
 555  * @mds: The driver data for the operation
 556  * @mbox_cmd: The validated mailbox command.
 557  * @out_payload: Pointer to userspace's output payload.
 558  * @size_out: (Input) Max payload size to copy out.
 559  *            (Output) Payload size hardware generated.
 560  * @retval: Hardware generated return code from the operation.
 561  *
 562  * Return:
 563  *  * %0        - Mailbox transaction succeeded. This implies the mailbox
 564  *                protocol completed successfully not that the operation itself
 565  *                was successful.
 566  *  * %-ENOMEM  - Couldn't allocate a bounce buffer.
 567  *  * %-EFAULT  - Something happened with copy_to/from_user.
 568  *  * %-EINTR   - Mailbox acquisition interrupted.
 569  *  * %-EXXX    - Transaction level failures.
 570  *
 571  * Dispatches a mailbox command on behalf of a userspace request.
 572  * The output payload is copied to userspace.
 573  *
 574  * See cxl_send_cmd().
 575  */
 576 static int handle_mailbox_cmd_from_user(struct cxl_memdev_state *mds,
 577                                         struct cxl_mbox_cmd *mbox_cmd,
 578                                         u64 out_payload, s32 *size_out,
 579                                         u32 *retval)
 580 {
 581         struct device *dev = mds->cxlds.dev;
 582         int rc;
 583
 584         dev_dbg(dev,
 585                 "Submitting %s command for user\n"
 586                 "\topcode: %x\n"
 587                 "\tsize: %zx\n",
 588                 cxl_mem_opcode_to_name(mbox_cmd->opcode),
 589                 mbox_cmd->opcode, mbox_cmd->size_in);
 590
 591         rc = mds->mbox_send(mds, mbox_cmd);
 592         if (rc)
 593                 goto out;
 594
 595         /*
 596          * @size_out contains the max size that's allowed to be written back out
 597          * to userspace. While the payload may have written more output than
 598          * this it will have to be ignored.
 599          */
 600         if (mbox_cmd->size_out) {
 601                 dev_WARN_ONCE(dev, mbox_cmd->size_out > *size_out,
 602                               "Invalid return size\n");
 603                 if (copy_to_user(u64_to_user_ptr(out_payload),
 604                                  mbox_cmd->payload_out, mbox_cmd->size_out)) {
 605                         rc = -EFAULT;
 606                         goto out;
 607                 }
 608         }
 609
 610         *size_out = mbox_cmd->size_out;
 611         *retval = mbox_cmd->return_code;
 612
 613 out:
 614         cxl_mbox_cmd_dtor(mbox_cmd);
 615         return rc;
 616 }
 617
 618 int cxl_send_cmd(struct cxl_memdev *cxlmd, struct cxl_send_command __user *s)
 619 {
 620         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
 621         struct device *dev = &cxlmd->dev;
 622         struct cxl_send_command send;
 623         struct cxl_mbox_cmd mbox_cmd;
 624         int rc;
 625
 626         dev_dbg(dev, "Send IOCTL\n");
 627
 628         if (copy_from_user(&send, s, sizeof(send)))
 629                 return -EFAULT;
 630
 631         rc = cxl_validate_cmd_from_user(&mbox_cmd, mds, &send);
 632         if (rc)
 633                 return rc;
 634
 635         rc = handle_mailbox_cmd_from_user(mds, &mbox_cmd, send.out.payload,
 636                                           &send.out.size, &send.retval);
 637         if (rc)
 638                 return rc;
 639
 640         if (copy_to_user(s, &send, sizeof(send)))
 641                 return -EFAULT;
 642
 643         return 0;
 644 }
 645
 646 static int cxl_xfer_log(struct cxl_memdev_state *mds, uuid_t *uuid,
 647                         u32 *size, u8 *out)
 648 {
 649         u32 remaining = *size;
 650         u32 offset = 0;
 651
 652         while (remaining) {
 653                 u32 xfer_size = min_t(u32, remaining, mds->payload_size);
 654                 struct cxl_mbox_cmd mbox_cmd;
 655                 struct cxl_mbox_get_log log;
 656                 int rc;
 657
 658                 log = (struct cxl_mbox_get_log) {
 659                         .uuid = *uuid,
 660                         .offset = cpu_to_le32(offset),
 661                         .length = cpu_to_le32(xfer_size),
 662                 };
 663
 664                 mbox_cmd = (struct cxl_mbox_cmd) {
 665                         .opcode = CXL_MBOX_OP_GET_LOG,
 666                         .size_in = sizeof(log),
 667                         .payload_in = &log,
 668                         .size_out = xfer_size,
 669                         .payload_out = out,
 670                 };
 671
 672                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 673
 674                 /*
 675                  * The output payload length that indicates the number
 676                  * of valid bytes can be smaller than the Log buffer
 677                  * size.
 678                  */
 679                 if (rc == -EIO && mbox_cmd.size_out < xfer_size) {
 680                         offset += mbox_cmd.size_out;
 681                         break;
 682                 }
 683
 684                 if (rc < 0)
 685                         return rc;
 686
 687                 out += xfer_size;
 688                 remaining -= xfer_size;
 689                 offset += xfer_size;
 690         }
 691
 692         *size = offset;
 693
 694         return 0;
 695 }
 696
 697 /**
 698  * cxl_walk_cel() - Walk through the Command Effects Log.
 699  * @mds: The driver data for the operation
 700  * @size: Length of the Command Effects Log.
 701  * @cel: CEL
 702  *
 703  * Iterate over each entry in the CEL and determine if the driver supports the
 704  * command. If so, the command is enabled for the device and can be used later.
 705  */
 706 static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
 707 {
 708         struct cxl_cel_entry *cel_entry;
 709         const int cel_entries = size / sizeof(*cel_entry);
 710         struct device *dev = mds->cxlds.dev;
 711         int i;
 712
 713         cel_entry = (struct cxl_cel_entry *) cel;
 714
 715         for (i = 0; i < cel_entries; i++) {
 716                 u16 opcode = le16_to_cpu(cel_entry[i].opcode);
 717                 struct cxl_mem_command *cmd = cxl_mem_find_command(opcode);
 718                 int enabled = 0;
 719
 720                 if (cmd) {
 721                         set_bit(cmd->info.id, mds->enabled_cmds);
 722                         enabled++;
 723                 }
 724
 725                 if (cxl_is_poison_command(opcode)) {
 726                         cxl_set_poison_cmd_enabled(&mds->poison, opcode);
 727                         enabled++;
 728                 }
 729
 730                 if (cxl_is_security_command(opcode)) {
 731                         cxl_set_security_cmd_enabled(&mds->security, opcode);
 732                         enabled++;
 733                 }
 734
 735                 dev_dbg(dev, "Opcode 0x%04x %s\n", opcode,
 736                         enabled ? "enabled" : "unsupported by driver");
 737         }
 738 }
 739
 740 static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
 741 {
 742         struct cxl_mbox_get_supported_logs *ret;
 743         struct cxl_mbox_cmd mbox_cmd;
 744         int rc;
 745
 746         ret = kvmalloc(mds->payload_size, GFP_KERNEL);
 747         if (!ret)
 748                 return ERR_PTR(-ENOMEM);
 749
 750         mbox_cmd = (struct cxl_mbox_cmd) {
 751                 .opcode = CXL_MBOX_OP_GET_SUPPORTED_LOGS,
 752                 .size_out = mds->payload_size,
 753                 .payload_out = ret,
 754                 /* At least the record number field must be valid */
 755                 .min_out = 2,
 756         };
 757         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 758         if (rc < 0) {
 759                 kvfree(ret);
 760                 return ERR_PTR(rc);
 761         }
 762
 763
 764         return ret;
 765 }
 766
 767 enum {
 768         CEL_UUID,
 769         VENDOR_DEBUG_UUID,
 770 };
 771
 772 /* See CXL 2.0 Table 170. Get Log Input Payload */
 773 static const uuid_t log_uuid[] = {
 774         [CEL_UUID] = DEFINE_CXL_CEL_UUID,
 775         [VENDOR_DEBUG_UUID] = DEFINE_CXL_VENDOR_DEBUG_UUID,
 776 };
 777
 778 /**
 779  * cxl_enumerate_cmds() - Enumerate commands for a device.
 780  * @mds: The driver data for the operation
 781  *
 782  * Returns 0 if enumerate completed successfully.
 783  *
 784  * CXL devices have optional support for certain commands. This function will
 785  * determine the set of supported commands for the hardware and update the
 786  * enabled_cmds bitmap in the @mds.
 787  */
 788 int cxl_enumerate_cmds(struct cxl_memdev_state *mds)
 789 {
 790         struct cxl_mbox_get_supported_logs *gsl;
 791         struct device *dev = mds->cxlds.dev;
 792         struct cxl_mem_command *cmd;
 793         int i, rc;
 794
 795         gsl = cxl_get_gsl(mds);
 796         if (IS_ERR(gsl))
 797                 return PTR_ERR(gsl);
 798
 799         rc = -ENOENT;
 800         for (i = 0; i < le16_to_cpu(gsl->entries); i++) {
 801                 u32 size = le32_to_cpu(gsl->entry[i].size);
 802                 uuid_t uuid = gsl->entry[i].uuid;
 803                 u8 *log;
 804
 805                 dev_dbg(dev, "Found LOG type %pU of size %d", &uuid, size);
 806
 807                 if (!uuid_equal(&uuid, &log_uuid[CEL_UUID]))
 808                         continue;
 809
 810                 log = kvmalloc(size, GFP_KERNEL);
 811                 if (!log) {
 812                         rc = -ENOMEM;
 813                         goto out;
 814                 }
 815
 816                 rc = cxl_xfer_log(mds, &uuid, &size, log);
 817                 if (rc) {
 818                         kvfree(log);
 819                         goto out;
 820                 }
 821
 822                 cxl_walk_cel(mds, size, log);
 823                 kvfree(log);
 824
 825                 /* In case CEL was bogus, enable some default commands. */
 826                 cxl_for_each_cmd(cmd)
 827                         if (cmd->flags & CXL_CMD_FLAG_FORCE_ENABLE)
 828                                 set_bit(cmd->info.id, mds->enabled_cmds);
 829
 830                 /* Found the required CEL */
 831                 rc = 0;
 832         }
 833 out:
 834         kvfree(gsl);
 835         return rc;
 836 }
 837 EXPORT_SYMBOL_NS_GPL(cxl_enumerate_cmds, CXL);
 838
 839 /*
 840  * General Media Event Record
 841  * CXL rev 3.0 Section 8.2.9.2.1.1; Table 8-43
 842  */
 843 static const uuid_t gen_media_event_uuid =
 844         UUID_INIT(0xfbcd0a77, 0xc260, 0x417f,
 845                   0x85, 0xa9, 0x08, 0x8b, 0x16, 0x21, 0xeb, 0xa6);
 846
 847 /*
 848  * DRAM Event Record
 849  * CXL rev 3.0 section 8.2.9.2.1.2; Table 8-44
 850  */
 851 static const uuid_t dram_event_uuid =
 852         UUID_INIT(0x601dcbb3, 0x9c06, 0x4eab,
 853                   0xb8, 0xaf, 0x4e, 0x9b, 0xfb, 0x5c, 0x96, 0x24);
 854
 855 /*
 856  * Memory Module Event Record
 857  * CXL rev 3.0 section 8.2.9.2.1.3; Table 8-45
 858  */
 859 static const uuid_t mem_mod_event_uuid =
 860         UUID_INIT(0xfe927475, 0xdd59, 0x4339,
 861                   0xa5, 0x86, 0x79, 0xba, 0xb1, 0x13, 0xb7, 0x74);
 862
 863 static void cxl_event_trace_record(const struct cxl_memdev *cxlmd,
 864                                    enum cxl_event_log_type type,
 865                                    struct cxl_event_record_raw *record)
 866 {
 867         uuid_t *id = &record->hdr.id;
 868
 869         if (uuid_equal(id, &gen_media_event_uuid)) {
 870                 struct cxl_event_gen_media *rec =
 871                                 (struct cxl_event_gen_media *)record;
 872
 873                 trace_cxl_general_media(cxlmd, type, rec);
 874         } else if (uuid_equal(id, &dram_event_uuid)) {
 875                 struct cxl_event_dram *rec = (struct cxl_event_dram *)record;
 876
 877                 trace_cxl_dram(cxlmd, type, rec);
 878         } else if (uuid_equal(id, &mem_mod_event_uuid)) {
 879                 struct cxl_event_mem_module *rec =
 880                                 (struct cxl_event_mem_module *)record;
 881
 882                 trace_cxl_memory_module(cxlmd, type, rec);
 883         } else {
 884                 /* For unknown record types print just the header */
 885                 trace_cxl_generic_event(cxlmd, type, record);
 886         }
 887 }
 888
 889 static int cxl_clear_event_record(struct cxl_memdev_state *mds,
 890                                   enum cxl_event_log_type log,
 891                                   struct cxl_get_event_payload *get_pl)
 892 {
 893         struct cxl_mbox_clear_event_payload *payload;
 894         u16 total = le16_to_cpu(get_pl->record_count);
 895         u8 max_handles = CXL_CLEAR_EVENT_MAX_HANDLES;
 896         size_t pl_size = struct_size(payload, handles, max_handles);
 897         struct cxl_mbox_cmd mbox_cmd;
 898         u16 cnt;
 899         int rc = 0;
 900         int i;
 901
 902         /* Payload size may limit the max handles */
 903         if (pl_size > mds->payload_size) {
 904                 max_handles = (mds->payload_size - sizeof(*payload)) /
 905                               sizeof(__le16);
 906                 pl_size = struct_size(payload, handles, max_handles);
 907         }
 908
 909         payload = kvzalloc(pl_size, GFP_KERNEL);
 910         if (!payload)
 911                 return -ENOMEM;
 912
 913         *payload = (struct cxl_mbox_clear_event_payload) {
 914                 .event_log = log,
 915         };
 916
 917         mbox_cmd = (struct cxl_mbox_cmd) {
 918                 .opcode = CXL_MBOX_OP_CLEAR_EVENT_RECORD,
 919                 .payload_in = payload,
 920                 .size_in = pl_size,
 921         };
 922
 923         /*
 924          * Clear Event Records uses u8 for the handle cnt while Get Event
 925          * Record can return up to 0xffff records.
 926          */
 927         i = 0;
 928         for (cnt = 0; cnt < total; cnt++) {
 929                 payload->handles[i++] = get_pl->records[cnt].hdr.handle;
 930                 dev_dbg(mds->cxlds.dev, "Event log '%d': Clearing %u\n", log,
 931                         le16_to_cpu(payload->handles[i]));
 932
 933                 if (i == max_handles) {
 934                         payload->nr_recs = i;
 935                         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 936                         if (rc)
 937                                 goto free_pl;
 938                         i = 0;
 939                 }
 940         }
 941
 942         /* Clear what is left if any */
 943         if (i) {
 944                 payload->nr_recs = i;
 945                 mbox_cmd.size_in = struct_size(payload, handles, i);
 946                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 947                 if (rc)
 948                         goto free_pl;
 949         }
 950
 951 free_pl:
 952         kvfree(payload);
 953         return rc;
 954 }
 955
 956 static void cxl_mem_get_records_log(struct cxl_memdev_state *mds,
 957                                     enum cxl_event_log_type type)
 958 {
 959         struct cxl_memdev *cxlmd = mds->cxlds.cxlmd;
 960         struct device *dev = mds->cxlds.dev;
 961         struct cxl_get_event_payload *payload;
 962         struct cxl_mbox_cmd mbox_cmd;
 963         u8 log_type = type;
 964         u16 nr_rec;
 965
 966         mutex_lock(&mds->event.log_lock);
 967         payload = mds->event.buf;
 968
 969         mbox_cmd = (struct cxl_mbox_cmd) {
 970                 .opcode = CXL_MBOX_OP_GET_EVENT_RECORD,
 971                 .payload_in = &log_type,
 972                 .size_in = sizeof(log_type),
 973                 .payload_out = payload,
 974                 .size_out = mds->payload_size,
 975                 .min_out = struct_size(payload, records, 0),
 976         };
 977
 978         do {
 979                 int rc, i;
 980
 981                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
 982                 if (rc) {
 983                         dev_err_ratelimited(dev,
 984                                 "Event log '%d': Failed to query event records : %d",
 985                                 type, rc);
 986                         break;
 987                 }
 988
 989                 nr_rec = le16_to_cpu(payload->record_count);
 990                 if (!nr_rec)
 991                         break;
 992
 993                 for (i = 0; i < nr_rec; i++)
 994                         cxl_event_trace_record(cxlmd, type,
 995                                                &payload->records[i]);
 996
 997                 if (payload->flags & CXL_GET_EVENT_FLAG_OVERFLOW)
 998                         trace_cxl_overflow(cxlmd, type, payload);
 999
1000                 rc = cxl_clear_event_record(mds, type, payload);
1001                 if (rc) {
1002                         dev_err_ratelimited(dev,
1003                                 "Event log '%d': Failed to clear events : %d",
1004                                 type, rc);
1005                         break;
1006                 }
1007         } while (nr_rec);
1008
1009         mutex_unlock(&mds->event.log_lock);
1010 }
1011
1012 /**
1013  * cxl_mem_get_event_records - Get Event Records from the device
1014  * @mds: The driver data for the operation
1015  * @status: Event Status register value identifying which events are available.
1016  *
1017  * Retrieve all event records available on the device, report them as trace
1018  * events, and clear them.
1019  *
1020  * See CXL rev 3.0 @8.2.9.2.2 Get Event Records
1021  * See CXL rev 3.0 @8.2.9.2.3 Clear Event Records
1022  */
1023 void cxl_mem_get_event_records(struct cxl_memdev_state *mds, u32 status)
1024 {
1025         dev_dbg(mds->cxlds.dev, "Reading event logs: %x\n", status);
1026
1027         if (status & CXLDEV_EVENT_STATUS_FATAL)
1028                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FATAL);
1029         if (status & CXLDEV_EVENT_STATUS_FAIL)
1030                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_FAIL);
1031         if (status & CXLDEV_EVENT_STATUS_WARN)
1032                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_WARN);
1033         if (status & CXLDEV_EVENT_STATUS_INFO)
1034                 cxl_mem_get_records_log(mds, CXL_EVENT_TYPE_INFO);
1035 }
1036 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_event_records, CXL);
1037
1038 /**
1039  * cxl_mem_get_partition_info - Get partition info
1040  * @mds: The driver data for the operation
1041  *
1042  * Retrieve the current partition info for the device specified.  The active
1043  * values are the current capacity in bytes.  If not 0, the 'next' values are
1044  * the pending values, in bytes, which take affect on next cold reset.
1045  *
1046  * Return: 0 if no error: or the result of the mailbox command.
1047  *
1048  * See CXL @8.2.9.5.2.1 Get Partition Info
1049  */
1050 static int cxl_mem_get_partition_info(struct cxl_memdev_state *mds)
1051 {
1052         struct cxl_mbox_get_partition_info pi;
1053         struct cxl_mbox_cmd mbox_cmd;
1054         int rc;
1055
1056         mbox_cmd = (struct cxl_mbox_cmd) {
1057                 .opcode = CXL_MBOX_OP_GET_PARTITION_INFO,
1058                 .size_out = sizeof(pi),
1059                 .payload_out = &pi,
1060         };
1061         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1062         if (rc)
1063                 return rc;
1064
1065         mds->active_volatile_bytes =
1066                 le64_to_cpu(pi.active_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1067         mds->active_persistent_bytes =
1068                 le64_to_cpu(pi.active_persistent_cap) * CXL_CAPACITY_MULTIPLIER;
1069         mds->next_volatile_bytes =
1070                 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1071         mds->next_persistent_bytes =
1072                 le64_to_cpu(pi.next_volatile_cap) * CXL_CAPACITY_MULTIPLIER;
1073
1074         return 0;
1075 }
1076
1077 /**
1078  * cxl_dev_state_identify() - Send the IDENTIFY command to the device.
1079  * @mds: The driver data for the operation
1080  *
1081  * Return: 0 if identify was executed successfully or media not ready.
1082  *
1083  * This will dispatch the identify command to the device and on success populate
1084  * structures to be exported to sysfs.
1085  */
1086 int cxl_dev_state_identify(struct cxl_memdev_state *mds)
1087 {
1088         /* See CXL 2.0 Table 175 Identify Memory Device Output Payload */
1089         struct cxl_mbox_identify id;
1090         struct cxl_mbox_cmd mbox_cmd;
1091         u32 val;
1092         int rc;
1093
1094         if (!mds->cxlds.media_ready)
1095                 return 0;
1096
1097         mbox_cmd = (struct cxl_mbox_cmd) {
1098                 .opcode = CXL_MBOX_OP_IDENTIFY,
1099                 .size_out = sizeof(id),
1100                 .payload_out = &id,
1101         };
1102         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1103         if (rc < 0)
1104                 return rc;
1105
1106         mds->total_bytes =
1107                 le64_to_cpu(id.total_capacity) * CXL_CAPACITY_MULTIPLIER;
1108         mds->volatile_only_bytes =
1109                 le64_to_cpu(id.volatile_capacity) * CXL_CAPACITY_MULTIPLIER;
1110         mds->persistent_only_bytes =
1111                 le64_to_cpu(id.persistent_capacity) * CXL_CAPACITY_MULTIPLIER;
1112         mds->partition_align_bytes =
1113                 le64_to_cpu(id.partition_align) * CXL_CAPACITY_MULTIPLIER;
1114
1115         mds->lsa_size = le32_to_cpu(id.lsa_size);
1116         memcpy(mds->firmware_version, id.fw_revision,
1117                sizeof(id.fw_revision));
1118
1119         if (test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds)) {
1120                 val = get_unaligned_le24(id.poison_list_max_mer);
1121                 mds->poison.max_errors = min_t(u32, val, CXL_POISON_LIST_MAX);
1122         }
1123
1124         return 0;
1125 }
1126 EXPORT_SYMBOL_NS_GPL(cxl_dev_state_identify, CXL);
1127
1128 static int __cxl_mem_sanitize(struct cxl_memdev_state *mds, u16 cmd)
1129 {
1130         int rc;
1131         u32 sec_out = 0;
1132         struct cxl_get_security_output {
1133                 __le32 flags;
1134         } out;
1135         struct cxl_mbox_cmd sec_cmd = {
1136                 .opcode = CXL_MBOX_OP_GET_SECURITY_STATE,
1137                 .payload_out = &out,
1138                 .size_out = sizeof(out),
1139         };
1140         struct cxl_mbox_cmd mbox_cmd = { .opcode = cmd };
1141         struct cxl_dev_state *cxlds = &mds->cxlds;
1142
1143         if (cmd != CXL_MBOX_OP_SANITIZE && cmd != CXL_MBOX_OP_SECURE_ERASE)
1144                 return -EINVAL;
1145
1146         rc = cxl_internal_send_cmd(mds, &sec_cmd);
1147         if (rc < 0) {
1148                 dev_err(cxlds->dev, "Failed to get security state : %d", rc);
1149                 return rc;
1150         }
1151
1152         /*
1153          * Prior to using these commands, any security applied to
1154          * the user data areas of the device shall be DISABLED (or
1155          * UNLOCKED for secure erase case).
1156          */
1157         sec_out = le32_to_cpu(out.flags);
1158         if (sec_out & CXL_PMEM_SEC_STATE_USER_PASS_SET)
1159                 return -EINVAL;
1160
1161         if (cmd == CXL_MBOX_OP_SECURE_ERASE &&
1162             sec_out & CXL_PMEM_SEC_STATE_LOCKED)
1163                 return -EINVAL;
1164
1165         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1166         if (rc < 0) {
1167                 dev_err(cxlds->dev, "Failed to sanitize device : %d", rc);
1168                 return rc;
1169         }
1170
1171         return 0;
1172 }
1173
1174
1175 /**
1176  * cxl_mem_sanitize() - Send a sanitization command to the device.
1177  * @cxlmd: The device for the operation
1178  * @cmd: The specific sanitization command opcode
1179  *
1180  * Return: 0 if the command was executed successfully, regardless of
1181  * whether or not the actual security operation is done in the background,
1182  * such as for the Sanitize case.
1183  * Error return values can be the result of the mailbox command, -EINVAL
1184  * when security requirements are not met or invalid contexts, or -EBUSY
1185  * if the sanitize operation is already in flight.
1186  *
1187  * See CXL 3.0 @8.2.9.8.5.1 Sanitize and @8.2.9.8.5.2 Secure Erase.
1188  */
1189 int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
1190 {
1191         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1192         struct cxl_port  *endpoint;
1193         int rc;
1194
1195         /* synchronize with cxl_mem_probe() and decoder write operations */
1196         device_lock(&cxlmd->dev);
1197         endpoint = cxlmd->endpoint;
1198         down_read(&cxl_region_rwsem);
1199         /*
1200          * Require an endpoint to be safe otherwise the driver can not
1201          * be sure that the device is unmapped.
1202          */
1203         if (endpoint && endpoint->commit_end == -1)
1204                 rc = __cxl_mem_sanitize(mds, cmd);
1205         else
1206                 rc = -EBUSY;
1207         up_read(&cxl_region_rwsem);
1208         device_unlock(&cxlmd->dev);
1209
1210         return rc;
1211 }
1212
1213 static int add_dpa_res(struct device *dev, struct resource *parent,
1214                        struct resource *res, resource_size_t start,
1215                        resource_size_t size, const char *type)
1216 {
1217         int rc;
1218
1219         res->name = type;
1220         res->start = start;
1221         res->end = start + size - 1;
1222         res->flags = IORESOURCE_MEM;
1223         if (resource_size(res) == 0) {
1224                 dev_dbg(dev, "DPA(%s): no capacity\n", res->name);
1225                 return 0;
1226         }
1227         rc = request_resource(parent, res);
1228         if (rc) {
1229                 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name,
1230                         res, rc);
1231                 return rc;
1232         }
1233
1234         dev_dbg(dev, "DPA(%s): %pr\n", res->name, res);
1235
1236         return 0;
1237 }
1238
1239 int cxl_mem_create_range_info(struct cxl_memdev_state *mds)
1240 {
1241         struct cxl_dev_state *cxlds = &mds->cxlds;
1242         struct device *dev = cxlds->dev;
1243         int rc;
1244
1245         if (!cxlds->media_ready) {
1246                 cxlds->dpa_res = DEFINE_RES_MEM(0, 0);
1247                 cxlds->ram_res = DEFINE_RES_MEM(0, 0);
1248                 cxlds->pmem_res = DEFINE_RES_MEM(0, 0);
1249                 return 0;
1250         }
1251
1252         cxlds->dpa_res = DEFINE_RES_MEM(0, mds->total_bytes);
1253
1254         if (mds->partition_align_bytes == 0) {
1255                 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
1256                                  mds->volatile_only_bytes, "ram");
1257                 if (rc)
1258                         return rc;
1259                 return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
1260                                    mds->volatile_only_bytes,
1261                                    mds->persistent_only_bytes, "pmem");
1262         }
1263
1264         rc = cxl_mem_get_partition_info(mds);
1265         if (rc) {
1266                 dev_err(dev, "Failed to query partition information\n");
1267                 return rc;
1268         }
1269
1270         rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->ram_res, 0,
1271                          mds->active_volatile_bytes, "ram");
1272         if (rc)
1273                 return rc;
1274         return add_dpa_res(dev, &cxlds->dpa_res, &cxlds->pmem_res,
1275                            mds->active_volatile_bytes,
1276                            mds->active_persistent_bytes, "pmem");
1277 }
1278 EXPORT_SYMBOL_NS_GPL(cxl_mem_create_range_info, CXL);
1279
1280 int cxl_set_timestamp(struct cxl_memdev_state *mds)
1281 {
1282         struct cxl_mbox_cmd mbox_cmd;
1283         struct cxl_mbox_set_timestamp_in pi;
1284         int rc;
1285
1286         pi.timestamp = cpu_to_le64(ktime_get_real_ns());
1287         mbox_cmd = (struct cxl_mbox_cmd) {
1288                 .opcode = CXL_MBOX_OP_SET_TIMESTAMP,
1289                 .size_in = sizeof(pi),
1290                 .payload_in = &pi,
1291         };
1292
1293         rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1294         /*
1295          * Command is optional. Devices may have another way of providing
1296          * a timestamp, or may return all 0s in timestamp fields.
1297          * Don't report an error if this command isn't supported
1298          */
1299         if (rc && (mbox_cmd.return_code != CXL_MBOX_CMD_RC_UNSUPPORTED))
1300                 return rc;
1301
1302         return 0;
1303 }
1304 EXPORT_SYMBOL_NS_GPL(cxl_set_timestamp, CXL);
1305
1306 int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
1307                        struct cxl_region *cxlr)
1308 {
1309         struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
1310         struct cxl_mbox_poison_out *po;
1311         struct cxl_mbox_poison_in pi;
1312         struct cxl_mbox_cmd mbox_cmd;
1313         int nr_records = 0;
1314         int rc;
1315
1316         rc = mutex_lock_interruptible(&mds->poison.lock);
1317         if (rc)
1318                 return rc;
1319
1320         po = mds->poison.list_out;
1321         pi.offset = cpu_to_le64(offset);
1322         pi.length = cpu_to_le64(len / CXL_POISON_LEN_MULT);
1323
1324         mbox_cmd = (struct cxl_mbox_cmd) {
1325                 .opcode = CXL_MBOX_OP_GET_POISON,
1326                 .size_in = sizeof(pi),
1327                 .payload_in = &pi,
1328                 .size_out = mds->payload_size,
1329                 .payload_out = po,
1330                 .min_out = struct_size(po, record, 0),
1331         };
1332
1333         do {
1334                 rc = cxl_internal_send_cmd(mds, &mbox_cmd);
1335                 if (rc)
1336                         break;
1337
1338                 for (int i = 0; i < le16_to_cpu(po->count); i++)
1339                         trace_cxl_poison(cxlmd, cxlr, &po->record[i],
1340                                          po->flags, po->overflow_ts,
1341                                          CXL_POISON_TRACE_LIST);
1342
1343                 /* Protect against an uncleared _FLAG_MORE */
1344                 nr_records = nr_records + le16_to_cpu(po->count);
1345                 if (nr_records >= mds->poison.max_errors) {
1346                         dev_dbg(&cxlmd->dev, "Max Error Records reached: %d\n",
1347                                 nr_records);
1348                         break;
1349                 }
1350         } while (po->flags & CXL_POISON_FLAG_MORE);
1351
1352         mutex_unlock(&mds->poison.lock);
1353         return rc;
1354 }
1355 EXPORT_SYMBOL_NS_GPL(cxl_mem_get_poison, CXL);
1356
1357 static void free_poison_buf(void *buf)
1358 {
1359         kvfree(buf);
1360 }
1361
1362 /* Get Poison List output buffer is protected by mds->poison.lock */
1363 static int cxl_poison_alloc_buf(struct cxl_memdev_state *mds)
1364 {
1365         mds->poison.list_out = kvmalloc(mds->payload_size, GFP_KERNEL);
1366         if (!mds->poison.list_out)
1367                 return -ENOMEM;
1368
1369         return devm_add_action_or_reset(mds->cxlds.dev, free_poison_buf,
1370                                         mds->poison.list_out);
1371 }
1372
1373 int cxl_poison_state_init(struct cxl_memdev_state *mds)
1374 {
1375         int rc;
1376
1377         if (!test_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds))
1378                 return 0;
1379
1380         rc = cxl_poison_alloc_buf(mds);
1381         if (rc) {
1382                 clear_bit(CXL_POISON_ENABLED_LIST, mds->poison.enabled_cmds);
1383                 return rc;
1384         }
1385
1386         mutex_init(&mds->poison.lock);
1387         return 0;
1388 }
1389 EXPORT_SYMBOL_NS_GPL(cxl_poison_state_init, CXL);
1390
1391 struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev)
1392 {
1393         struct cxl_memdev_state *mds;
1394
1395         mds = devm_kzalloc(dev, sizeof(*mds), GFP_KERNEL);
1396         if (!mds) {
1397                 dev_err(dev, "No memory available\n");
1398                 return ERR_PTR(-ENOMEM);
1399         }
1400
1401         mutex_init(&mds->mbox_mutex);
1402         mutex_init(&mds->event.log_lock);
1403         mds->cxlds.dev = dev;
1404         mds->cxlds.reg_map.host = dev;
1405         mds->cxlds.reg_map.resource = CXL_RESOURCE_NONE;
1406         mds->cxlds.type = CXL_DEVTYPE_CLASSMEM;
1407
1408         return mds;
1409 }
1410 EXPORT_SYMBOL_NS_GPL(cxl_memdev_state_create, CXL);
1411
1412 void __init cxl_mbox_init(void)
1413 {
1414         struct dentry *mbox_debugfs;
1415
1416         mbox_debugfs = cxl_debugfs_create_dir("mbox");
1417         debugfs_create_bool("raw_allow_all", 0600, mbox_debugfs,
1418                             &cxl_raw_allow_all);
1419 }