sound/soc/sof/ipc4-pcm.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 //
   3 // This file is provided under a dual BSD/GPLv2 license.  When using or
   4 // redistributing this file, you may do so under either license.
   5 //
   6 // Copyright(c) 2022 Intel Corporation. All rights reserved.
   7 //
   8
   9 #include <sound/pcm_params.h>
  10 #include <sound/sof/ipc4/header.h>
  11 #include "sof-audio.h"
  12 #include "sof-priv.h"
  13 #include "ops.h"
  14 #include "ipc4-priv.h"
  15 #include "ipc4-topology.h"
  16 #include "ipc4-fw-reg.h"
  17
  18 static int sof_ipc4_set_multi_pipeline_state(struct snd_sof_dev *sdev, u32 state,
  19                                              struct ipc4_pipeline_set_state_data *trigger_list)
  20 {
  21         struct sof_ipc4_msg msg = {{ 0 }};
  22         u32 primary, ipc_size;
  23
  24         /* trigger a single pipeline */
  25         if (trigger_list->count == 1)
  26                 return sof_ipc4_set_pipeline_state(sdev, trigger_list->pipeline_instance_ids[0],
  27                                                    state);
  28
  29         primary = state;
  30         primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_GLB_SET_PIPELINE_STATE);
  31         primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
  32         primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_FW_GEN_MSG);
  33         msg.primary = primary;
  34
  35         /* trigger multiple pipelines with a single IPC */
  36         msg.extension = SOF_IPC4_GLB_PIPE_STATE_EXT_MULTI;
  37
  38         /* ipc_size includes the count and the pipeline IDs for the number of pipelines */
  39         ipc_size = sizeof(u32) * (trigger_list->count + 1);
  40         msg.data_size = ipc_size;
  41         msg.data_ptr = trigger_list;
  42
  43         return sof_ipc_tx_message_no_reply(sdev->ipc, &msg, ipc_size);
  44 }
  45
  46 int sof_ipc4_set_pipeline_state(struct snd_sof_dev *sdev, u32 instance_id, u32 state)
  47 {
  48         struct sof_ipc4_msg msg = {{ 0 }};
  49         u32 primary;
  50
  51         dev_dbg(sdev->dev, "ipc4 set pipeline instance %d state %d", instance_id, state);
  52
  53         primary = state;
  54         primary |= SOF_IPC4_GLB_PIPE_STATE_ID(instance_id);
  55         primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_GLB_SET_PIPELINE_STATE);
  56         primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
  57         primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_FW_GEN_MSG);
  58
  59         msg.primary = primary;
  60
  61         return sof_ipc_tx_message_no_reply(sdev->ipc, &msg, 0);
  62 }
  63 EXPORT_SYMBOL(sof_ipc4_set_pipeline_state);
  64
  65 static void sof_ipc4_add_pipeline_by_priority(struct ipc4_pipeline_set_state_data *trigger_list,
  66                                               struct snd_sof_widget *pipe_widget,
  67                                               s8 *pipe_priority, bool ascend)
  68 {
  69         struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
  70         int i, j;
  71
  72         for (i = 0; i < trigger_list->count; i++) {
  73                 /* add pipeline from low priority to high */
  74                 if (ascend && pipeline->priority < pipe_priority[i])
  75                         break;
  76                 /* add pipeline from high priority to low */
  77                 else if (!ascend && pipeline->priority > pipe_priority[i])
  78                         break;
  79         }
  80
  81         for (j = trigger_list->count - 1; j >= i; j--) {
  82                 trigger_list->pipeline_instance_ids[j + 1] = trigger_list->pipeline_instance_ids[j];
  83                 pipe_priority[j + 1] = pipe_priority[j];
  84         }
  85
  86         trigger_list->pipeline_instance_ids[i] = pipe_widget->instance_id;
  87         trigger_list->count++;
  88         pipe_priority[i] = pipeline->priority;
  89 }
  90
  91 static void
  92 sof_ipc4_add_pipeline_to_trigger_list(struct snd_sof_dev *sdev, int state,
  93                                       struct snd_sof_pipeline *spipe,
  94                                       struct ipc4_pipeline_set_state_data *trigger_list,
  95                                       s8 *pipe_priority)
  96 {
  97         struct snd_sof_widget *pipe_widget = spipe->pipe_widget;
  98         struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
  99
 100         if (pipeline->skip_during_fe_trigger && state != SOF_IPC4_PIPE_RESET)
 101                 return;
 102
 103         switch (state) {
 104         case SOF_IPC4_PIPE_RUNNING:
 105                 /*
 106                  * Trigger pipeline if all PCMs containing it are paused or if it is RUNNING
 107                  * for the first time
 108                  */
 109                 if (spipe->started_count == spipe->paused_count)
 110                         sof_ipc4_add_pipeline_by_priority(trigger_list, pipe_widget, pipe_priority,
 111                                                           false);
 112                 break;
 113         case SOF_IPC4_PIPE_RESET:
 114                 /* RESET if the pipeline is neither running nor paused */
 115                 if (!spipe->started_count && !spipe->paused_count)
 116                         sof_ipc4_add_pipeline_by_priority(trigger_list, pipe_widget, pipe_priority,
 117                                                           true);
 118                 break;
 119         case SOF_IPC4_PIPE_PAUSED:
 120                 /* Pause the pipeline only when its started_count is 1 more than paused_count */
 121                 if (spipe->paused_count == (spipe->started_count - 1))
 122                         sof_ipc4_add_pipeline_by_priority(trigger_list, pipe_widget, pipe_priority,
 123                                                           true);
 124                 break;
 125         default:
 126                 break;
 127         }
 128 }
 129
 130 static void
 131 sof_ipc4_update_pipeline_state(struct snd_sof_dev *sdev, int state, int cmd,
 132                                struct snd_sof_pipeline *spipe,
 133                                struct ipc4_pipeline_set_state_data *trigger_list)
 134 {
 135         struct snd_sof_widget *pipe_widget = spipe->pipe_widget;
 136         struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
 137         int i;
 138
 139         if (pipeline->skip_during_fe_trigger && state != SOF_IPC4_PIPE_RESET)
 140                 return;
 141
 142         /* set state for pipeline if it was just triggered */
 143         for (i = 0; i < trigger_list->count; i++) {
 144                 if (trigger_list->pipeline_instance_ids[i] == pipe_widget->instance_id) {
 145                         pipeline->state = state;
 146                         break;
 147                 }
 148         }
 149
 150         switch (state) {
 151         case SOF_IPC4_PIPE_PAUSED:
 152                 switch (cmd) {
 153                 case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
 154                         /*
 155                          * increment paused_count if the PAUSED is the final state during
 156                          * the PAUSE trigger
 157                          */
 158                         spipe->paused_count++;
 159                         break;
 160                 case SNDRV_PCM_TRIGGER_STOP:
 161                 case SNDRV_PCM_TRIGGER_SUSPEND:
 162                         /*
 163                          * decrement started_count if PAUSED is the final state during the
 164                          * STOP trigger
 165                          */
 166                         spipe->started_count--;
 167                         break;
 168                 default:
 169                         break;
 170                 }
 171                 break;
 172         case SOF_IPC4_PIPE_RUNNING:
 173                 switch (cmd) {
 174                 case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
 175                         /* decrement paused_count for RELEASE */
 176                         spipe->paused_count--;
 177                         break;
 178                 case SNDRV_PCM_TRIGGER_START:
 179                 case SNDRV_PCM_TRIGGER_RESUME:
 180                         /* increment started_count for START/RESUME */
 181                         spipe->started_count++;
 182                         break;
 183                 default:
 184                         break;
 185                 }
 186                 break;
 187         default:
 188                 break;
 189         }
 190 }
 191
 192 /*
 193  * The picture below represents the pipeline state machine wrt PCM actions corresponding to the
 194  * triggers and ioctls
 195  *                              +---------------+
 196  *                              |               |
 197  *                              |    INIT       |
 198  *                              |               |
 199  *                              +-------+-------+
 200  *                                      |
 201  *                                      |
 202  *                                      | START
 203  *                                      |
 204  *                                      |
 205  * +----------------+              +------v-------+               +-------------+
 206  * |                |   START     |              |   HW_FREE      |             |
 207  * |   RUNNING      <-------------+  PAUSED      +--------------> +   RESET     |
 208  * |                |   PAUSE     |              |                |             |
 209  * +------+---------+   RELEASE   +---------+----+                +-------------+
 210  *        |                                  ^
 211  *        |                                  |
 212  *        |                                  |
 213  *        |                                  |
 214  *        |             PAUSE                |
 215  *        +---------------------------------+
 216  *                      STOP/SUSPEND
 217  *
 218  * Note that during system suspend, the suspend trigger is followed by a hw_free in
 219  * sof_pcm_trigger(). So, the final state during suspend would be RESET.
 220  * Also, since the SOF driver doesn't support full resume, streams would be restarted with the
 221  * prepare ioctl before the START trigger.
 222  */
 223
 224 /*
 225  * Chained DMA is a special case where there is no processing on
 226  * DSP. The samples are just moved over by host side DMA to a single
 227  * buffer on DSP and directly from there to link DMA. However, the
 228  * model on SOF driver has two notional pipelines, one at host DAI,
 229  * and another at link DAI. They both shall have the use_chain_dma
 230  * attribute.
 231  */
 232
 233 static int sof_ipc4_chain_dma_trigger(struct snd_sof_dev *sdev,
 234                                       int direction,
 235                                       struct snd_sof_pcm_stream_pipeline_list *pipeline_list,
 236                                       int state, int cmd)
 237 {
 238         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
 239         bool allocate, enable, set_fifo_size;
 240         struct sof_ipc4_msg msg = {{ 0 }};
 241         int i;
 242
 243         switch (state) {
 244         case SOF_IPC4_PIPE_RUNNING: /* Allocate and start chained dma */
 245                 allocate = true;
 246                 enable = true;
 247                 /*
 248                  * SOF assumes creation of a new stream from the presence of fifo_size
 249                  * in the message, so we must leave it out in pause release case.
 250                  */
 251                 if (cmd == SNDRV_PCM_TRIGGER_PAUSE_RELEASE)
 252                         set_fifo_size = false;
 253                 else
 254                         set_fifo_size = true;
 255                 break;
 256         case SOF_IPC4_PIPE_PAUSED: /* Disable chained DMA. */
 257                 allocate = true;
 258                 enable = false;
 259                 set_fifo_size = false;
 260                 break;
 261         case SOF_IPC4_PIPE_RESET: /* Disable and free chained DMA. */
 262                 allocate = false;
 263                 enable = false;
 264                 set_fifo_size = false;
 265                 break;
 266         default:
 267                 dev_err(sdev->dev, "Unexpected state %d", state);
 268                 return -EINVAL;
 269         }
 270
 271         msg.primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_GLB_CHAIN_DMA);
 272         msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
 273         msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_FW_GEN_MSG);
 274
 275         /*
 276          * To set-up the DMA chain, the host DMA ID and SCS setting
 277          * are retrieved from the host pipeline configuration. Likewise
 278          * the link DMA ID and fifo_size are retrieved from the link
 279          * pipeline configuration.
 280          */
 281         for (i = 0; i < pipeline_list->count; i++) {
 282                 struct snd_sof_pipeline *spipe = pipeline_list->pipelines[i];
 283                 struct snd_sof_widget *pipe_widget = spipe->pipe_widget;
 284                 struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
 285
 286                 if (!pipeline->use_chain_dma) {
 287                         dev_err(sdev->dev,
 288                                 "All pipelines in chained DMA stream should have use_chain_dma attribute set.");
 289                         return -EINVAL;
 290                 }
 291
 292                 msg.primary |= pipeline->msg.primary;
 293
 294                 /* Add fifo_size (actually DMA buffer size) field to the message */
 295                 if (set_fifo_size)
 296                         msg.extension |= pipeline->msg.extension;
 297         }
 298
 299         if (direction == SNDRV_PCM_STREAM_CAPTURE) {
 300                 /*
 301                  * For ChainDMA the DMA ids are unique with the following mapping:
 302                  * playback:  0 - (num_playback_streams - 1)
 303                  * capture:   num_playback_streams - (num_playback_streams +
 304                  *                                    num_capture_streams - 1)
 305                  *
 306                  * Add the num_playback_streams offset to the DMA ids stored in
 307                  * msg.primary in case capture
 308                  */
 309                 msg.primary +=  SOF_IPC4_GLB_CHAIN_DMA_HOST_ID(ipc4_data->num_playback_streams);
 310                 msg.primary +=  SOF_IPC4_GLB_CHAIN_DMA_LINK_ID(ipc4_data->num_playback_streams);
 311         }
 312
 313         if (allocate)
 314                 msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_ALLOCATE_MASK;
 315
 316         if (enable)
 317                 msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_ENABLE_MASK;
 318
 319         return sof_ipc_tx_message_no_reply(sdev->ipc, &msg, 0);
 320 }
 321
 322 static int sof_ipc4_trigger_pipelines(struct snd_soc_component *component,
 323                                       struct snd_pcm_substream *substream, int state, int cmd)
 324 {
 325         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
 326         struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
 327         struct snd_sof_pcm_stream_pipeline_list *pipeline_list;
 328         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
 329         struct ipc4_pipeline_set_state_data *trigger_list;
 330         struct snd_sof_widget *pipe_widget;
 331         struct sof_ipc4_pipeline *pipeline;
 332         struct snd_sof_pipeline *spipe;
 333         struct snd_sof_pcm *spcm;
 334         u8 *pipe_priority;
 335         int ret;
 336         int i;
 337
 338         dev_dbg(sdev->dev, "trigger cmd: %d state: %d\n", cmd, state);
 339
 340         spcm = snd_sof_find_spcm_dai(component, rtd);
 341         if (!spcm)
 342                 return -EINVAL;
 343
 344         pipeline_list = &spcm->stream[substream->stream].pipeline_list;
 345
 346         /* nothing to trigger if the list is empty */
 347         if (!pipeline_list->pipelines || !pipeline_list->count)
 348                 return 0;
 349
 350         spipe = pipeline_list->pipelines[0];
 351         pipe_widget = spipe->pipe_widget;
 352         pipeline = pipe_widget->private;
 353
 354         /*
 355          * If use_chain_dma attribute is set we proceed to chained DMA
 356          * trigger function that handles the rest for the substream.
 357          */
 358         if (pipeline->use_chain_dma)
 359                 return sof_ipc4_chain_dma_trigger(sdev, substream->stream,
 360                                                   pipeline_list, state, cmd);
 361
 362         /* allocate memory for the pipeline data */
 363         trigger_list = kzalloc(struct_size(trigger_list, pipeline_instance_ids,
 364                                            pipeline_list->count), GFP_KERNEL);
 365         if (!trigger_list)
 366                 return -ENOMEM;
 367
 368         pipe_priority = kzalloc(pipeline_list->count, GFP_KERNEL);
 369         if (!pipe_priority) {
 370                 kfree(trigger_list);
 371                 return -ENOMEM;
 372         }
 373
 374         mutex_lock(&ipc4_data->pipeline_state_mutex);
 375
 376         /*
 377          * IPC4 requires pipelines to be triggered in order starting at the sink and
 378          * walking all the way to the source. So traverse the pipeline_list in the order
 379          * sink->source when starting PCM's and in the reverse order to pause/stop PCM's.
 380          * Skip the pipelines that have their skip_during_fe_trigger flag set. If there is a fork
 381          * in the pipeline, the order of triggering between the left/right paths will be
 382          * indeterministic. But the sink->source trigger order sink->source would still be
 383          * guaranteed for each fork independently.
 384          */
 385         if (state == SOF_IPC4_PIPE_RUNNING || state == SOF_IPC4_PIPE_RESET)
 386                 for (i = pipeline_list->count - 1; i >= 0; i--) {
 387                         spipe = pipeline_list->pipelines[i];
 388                         sof_ipc4_add_pipeline_to_trigger_list(sdev, state, spipe, trigger_list,
 389                                                               pipe_priority);
 390                 }
 391         else
 392                 for (i = 0; i < pipeline_list->count; i++) {
 393                         spipe = pipeline_list->pipelines[i];
 394                         sof_ipc4_add_pipeline_to_trigger_list(sdev, state, spipe, trigger_list,
 395                                                               pipe_priority);
 396                 }
 397
 398         /* return if all pipelines are in the requested state already */
 399         if (!trigger_list->count) {
 400                 ret = 0;
 401                 goto free;
 402         }
 403
 404         /* no need to pause before reset or before pause release */
 405         if (state == SOF_IPC4_PIPE_RESET || cmd == SNDRV_PCM_TRIGGER_PAUSE_RELEASE)
 406                 goto skip_pause_transition;
 407
 408         /*
 409          * set paused state for pipelines if the final state is PAUSED or when the pipeline
 410          * is set to RUNNING for the first time after the PCM is started.
 411          */
 412         ret = sof_ipc4_set_multi_pipeline_state(sdev, SOF_IPC4_PIPE_PAUSED, trigger_list);
 413         if (ret < 0) {
 414                 dev_err(sdev->dev, "failed to pause all pipelines\n");
 415                 goto free;
 416         }
 417
 418         /* update PAUSED state for all pipelines just triggered */
 419         for (i = 0; i < pipeline_list->count ; i++) {
 420                 spipe = pipeline_list->pipelines[i];
 421                 sof_ipc4_update_pipeline_state(sdev, SOF_IPC4_PIPE_PAUSED, cmd, spipe,
 422                                                trigger_list);
 423         }
 424
 425         /* return if this is the final state */
 426         if (state == SOF_IPC4_PIPE_PAUSED)
 427                 goto free;
 428 skip_pause_transition:
 429         /* else set the RUNNING/RESET state in the DSP */
 430         ret = sof_ipc4_set_multi_pipeline_state(sdev, state, trigger_list);
 431         if (ret < 0) {
 432                 dev_err(sdev->dev, "failed to set final state %d for all pipelines\n", state);
 433                 /*
 434                  * workaround: if the firmware is crashed while setting the
 435                  * pipelines to reset state we must ignore the error code and
 436                  * reset it to 0.
 437                  * Since the firmware is crashed we will not send IPC messages
 438                  * and we are going to see errors printed, but the state of the
 439                  * widgets will be correct for the next boot.
 440                  */
 441                 if (sdev->fw_state != SOF_FW_CRASHED || state != SOF_IPC4_PIPE_RESET)
 442                         goto free;
 443
 444                 ret = 0;
 445         }
 446
 447         /* update RUNNING/RESET state for all pipelines that were just triggered */
 448         for (i = 0; i < pipeline_list->count; i++) {
 449                 spipe = pipeline_list->pipelines[i];
 450                 sof_ipc4_update_pipeline_state(sdev, state, cmd, spipe, trigger_list);
 451         }
 452
 453 free:
 454         mutex_unlock(&ipc4_data->pipeline_state_mutex);
 455         kfree(trigger_list);
 456         kfree(pipe_priority);
 457         return ret;
 458 }
 459
 460 static int sof_ipc4_pcm_trigger(struct snd_soc_component *component,
 461                                 struct snd_pcm_substream *substream, int cmd)
 462 {
 463         int state;
 464
 465         /* determine the pipeline state */
 466         switch (cmd) {
 467         case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
 468                 state = SOF_IPC4_PIPE_PAUSED;
 469                 break;
 470         case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
 471         case SNDRV_PCM_TRIGGER_RESUME:
 472         case SNDRV_PCM_TRIGGER_START:
 473                 state = SOF_IPC4_PIPE_RUNNING;
 474                 break;
 475         case SNDRV_PCM_TRIGGER_SUSPEND:
 476         case SNDRV_PCM_TRIGGER_STOP:
 477                 state = SOF_IPC4_PIPE_PAUSED;
 478                 break;
 479         default:
 480                 dev_err(component->dev, "%s: unhandled trigger cmd %d\n", __func__, cmd);
 481                 return -EINVAL;
 482         }
 483
 484         /* set the pipeline state */
 485         return sof_ipc4_trigger_pipelines(component, substream, state, cmd);
 486 }
 487
 488 static int sof_ipc4_pcm_hw_free(struct snd_soc_component *component,
 489                                 struct snd_pcm_substream *substream)
 490 {
 491         /* command is not relevant with RESET, so just pass 0 */
 492         return sof_ipc4_trigger_pipelines(component, substream, SOF_IPC4_PIPE_RESET, 0);
 493 }
 494
 495 static void ipc4_ssp_dai_config_pcm_params_match(struct snd_sof_dev *sdev, const char *link_name,
 496                                                  struct snd_pcm_hw_params *params)
 497 {
 498         struct snd_sof_dai_link *slink;
 499         struct snd_sof_dai *dai;
 500         bool dai_link_found = false;
 501         int i;
 502
 503         list_for_each_entry(slink, &sdev->dai_link_list, list) {
 504                 if (!strcmp(slink->link->name, link_name)) {
 505                         dai_link_found = true;
 506                         break;
 507                 }
 508         }
 509
 510         if (!dai_link_found)
 511                 return;
 512
 513         for (i = 0; i < slink->num_hw_configs; i++) {
 514                 struct snd_soc_tplg_hw_config *hw_config = &slink->hw_configs[i];
 515
 516                 if (params_rate(params) == le32_to_cpu(hw_config->fsync_rate)) {
 517                         /* set current config for all DAI's with matching name */
 518                         list_for_each_entry(dai, &sdev->dai_list, list)
 519                                 if (!strcmp(slink->link->name, dai->name))
 520                                         dai->current_config = le32_to_cpu(hw_config->id);
 521                         break;
 522                 }
 523         }
 524 }
 525
 526 /*
 527  * Fixup DAI link parameters for sampling rate based on
 528  * DAI copier configuration.
 529  */
 530 static int sof_ipc4_pcm_dai_link_fixup_rate(struct snd_sof_dev *sdev,
 531                                             struct snd_pcm_hw_params *params,
 532                                             struct sof_ipc4_copier *ipc4_copier)
 533 {
 534         struct sof_ipc4_pin_format *pin_fmts = ipc4_copier->available_fmt.input_pin_fmts;
 535         struct snd_interval *rate = hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE);
 536         int num_input_formats = ipc4_copier->available_fmt.num_input_formats;
 537         unsigned int fe_rate = params_rate(params);
 538         bool fe_be_rate_match = false;
 539         bool single_be_rate = true;
 540         unsigned int be_rate;
 541         int i;
 542
 543         /*
 544          * Copier does not change sampling rate, so we
 545          * need to only consider the input pin information.
 546          */
 547         for (i = 0; i < num_input_formats; i++) {
 548                 unsigned int val = pin_fmts[i].audio_fmt.sampling_frequency;
 549
 550                 if (i == 0)
 551                         be_rate = val;
 552                 else if (val != be_rate)
 553                         single_be_rate = false;
 554
 555                 if (val == fe_rate) {
 556                         fe_be_rate_match = true;
 557                         break;
 558                 }
 559         }
 560
 561         /*
 562          * If rate is different than FE rate, topology must
 563          * contain an SRC. But we do require topology to
 564          * define a single rate in the DAI copier config in
 565          * this case (FE rate may be variable).
 566          */
 567         if (!fe_be_rate_match) {
 568                 if (!single_be_rate) {
 569                         dev_err(sdev->dev, "Unable to select sampling rate for DAI link\n");
 570                         return -EINVAL;
 571                 }
 572
 573                 rate->min = be_rate;
 574                 rate->max = rate->min;
 575         }
 576
 577         return 0;
 578 }
 579
 580 static int sof_ipc4_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd,
 581                                        struct snd_pcm_hw_params *params)
 582 {
 583         struct snd_soc_component *component = snd_soc_rtdcom_lookup(rtd, SOF_AUDIO_PCM_DRV_NAME);
 584         struct snd_sof_dai *dai = snd_sof_find_dai(component, rtd->dai_link->name);
 585         struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
 586         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
 587         struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0);
 588         struct sof_ipc4_audio_format *ipc4_fmt;
 589         struct sof_ipc4_copier *ipc4_copier;
 590         bool single_fmt = false;
 591         u32 valid_bits = 0;
 592         int dir, ret;
 593
 594         if (!dai) {
 595                 dev_err(component->dev, "%s: No DAI found with name %s\n", __func__,
 596                         rtd->dai_link->name);
 597                 return -EINVAL;
 598         }
 599
 600         ipc4_copier = dai->private;
 601         if (!ipc4_copier) {
 602                 dev_err(component->dev, "%s: No private data found for DAI %s\n",
 603                         __func__, rtd->dai_link->name);
 604                 return -EINVAL;
 605         }
 606
 607         for_each_pcm_streams(dir) {
 608                 struct snd_soc_dapm_widget *w = snd_soc_dai_get_widget(cpu_dai, dir);
 609
 610                 if (w) {
 611                         struct sof_ipc4_available_audio_format *available_fmt =
 612                                 &ipc4_copier->available_fmt;
 613                         struct snd_sof_widget *swidget = w->dobj.private;
 614                         struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
 615                         struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
 616
 617                         /* Chain DMA does not use copiers, so no fixup needed */
 618                         if (pipeline->use_chain_dma)
 619                                 return 0;
 620
 621                         if (dir == SNDRV_PCM_STREAM_PLAYBACK) {
 622                                 if (sof_ipc4_copier_is_single_format(sdev,
 623                                         available_fmt->output_pin_fmts,
 624                                         available_fmt->num_output_formats)) {
 625                                         ipc4_fmt = &available_fmt->output_pin_fmts->audio_fmt;
 626                                         single_fmt = true;
 627                                 }
 628                         } else {
 629                                 if (sof_ipc4_copier_is_single_format(sdev,
 630                                         available_fmt->input_pin_fmts,
 631                                         available_fmt->num_input_formats)) {
 632                                         ipc4_fmt = &available_fmt->input_pin_fmts->audio_fmt;
 633                                         single_fmt = true;
 634                                 }
 635                         }
 636                 }
 637         }
 638
 639         ret = sof_ipc4_pcm_dai_link_fixup_rate(sdev, params, ipc4_copier);
 640         if (ret)
 641                 return ret;
 642
 643         if (single_fmt) {
 644                 snd_mask_none(fmt);
 645                 valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(ipc4_fmt->fmt_cfg);
 646                 dev_dbg(component->dev, "Set %s to %d bit format\n", dai->name, valid_bits);
 647         }
 648
 649         /* Set format if it is specified */
 650         switch (valid_bits) {
 651         case 16:
 652                 snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S16_LE);
 653                 break;
 654         case 24:
 655                 snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S24_LE);
 656                 break;
 657         case 32:
 658                 snd_mask_set_format(fmt, SNDRV_PCM_FORMAT_S32_LE);
 659                 break;
 660         default:
 661                 break;
 662         }
 663
 664         switch (ipc4_copier->dai_type) {
 665         case SOF_DAI_INTEL_SSP:
 666                 ipc4_ssp_dai_config_pcm_params_match(sdev, (char *)rtd->dai_link->name, params);
 667                 break;
 668         default:
 669                 break;
 670         }
 671
 672         return 0;
 673 }
 674
 675 static void sof_ipc4_pcm_free(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm)
 676 {
 677         struct snd_sof_pcm_stream_pipeline_list *pipeline_list;
 678         int stream;
 679
 680         for_each_pcm_streams(stream) {
 681                 pipeline_list = &spcm->stream[stream].pipeline_list;
 682                 kfree(pipeline_list->pipelines);
 683                 pipeline_list->pipelines = NULL;
 684                 kfree(spcm->stream[stream].private);
 685                 spcm->stream[stream].private = NULL;
 686         }
 687 }
 688
 689 static int sof_ipc4_pcm_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm)
 690 {
 691         struct snd_sof_pcm_stream_pipeline_list *pipeline_list;
 692         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
 693         struct sof_ipc4_timestamp_info *stream_info;
 694         bool support_info = true;
 695         u32 abi_version;
 696         u32 abi_offset;
 697         int stream;
 698
 699         abi_offset = offsetof(struct sof_ipc4_fw_registers, abi_ver);
 700         sof_mailbox_read(sdev, sdev->fw_info_box.offset + abi_offset, &abi_version,
 701                          sizeof(abi_version));
 702
 703         if (abi_version < SOF_IPC4_FW_REGS_ABI_VER)
 704                 support_info = false;
 705
 706         for_each_pcm_streams(stream) {
 707                 pipeline_list = &spcm->stream[stream].pipeline_list;
 708
 709                 /* allocate memory for max number of pipeline IDs */
 710                 pipeline_list->pipelines = kcalloc(ipc4_data->max_num_pipelines,
 711                                                    sizeof(struct snd_sof_widget *), GFP_KERNEL);
 712                 if (!pipeline_list->pipelines) {
 713                         sof_ipc4_pcm_free(sdev, spcm);
 714                         return -ENOMEM;
 715                 }
 716
 717                 if (!support_info)
 718                         continue;
 719
 720                 stream_info = kzalloc(sizeof(*stream_info), GFP_KERNEL);
 721                 if (!stream_info) {
 722                         sof_ipc4_pcm_free(sdev, spcm);
 723                         return -ENOMEM;
 724                 }
 725
 726                 spcm->stream[stream].private = stream_info;
 727         }
 728
 729         return 0;
 730 }
 731
 732 static void sof_ipc4_build_time_info(struct snd_sof_dev *sdev, struct snd_sof_pcm_stream *spcm)
 733 {
 734         struct sof_ipc4_copier *host_copier = NULL;
 735         struct sof_ipc4_copier *dai_copier = NULL;
 736         struct sof_ipc4_llp_reading_slot llp_slot;
 737         struct sof_ipc4_timestamp_info *info;
 738         struct snd_soc_dapm_widget *widget;
 739         struct snd_sof_dai *dai;
 740         int i;
 741
 742         /* find host & dai to locate info in memory window */
 743         for_each_dapm_widgets(spcm->list, i, widget) {
 744                 struct snd_sof_widget *swidget = widget->dobj.private;
 745
 746                 if (!swidget)
 747                         continue;
 748
 749                 if (WIDGET_IS_AIF(swidget->widget->id)) {
 750                         host_copier = swidget->private;
 751                 } else if (WIDGET_IS_DAI(swidget->widget->id)) {
 752                         dai = swidget->private;
 753                         dai_copier = dai->private;
 754                 }
 755         }
 756
 757         /* both host and dai copier must be valid for time_info */
 758         if (!host_copier || !dai_copier) {
 759                 dev_err(sdev->dev, "host or dai copier are not found\n");
 760                 return;
 761         }
 762
 763         info = spcm->private;
 764         info->host_copier = host_copier;
 765         info->dai_copier = dai_copier;
 766         info->llp_offset = offsetof(struct sof_ipc4_fw_registers, llp_gpdma_reading_slots) +
 767                                     sdev->fw_info_box.offset;
 768
 769         /* find llp slot used by current dai */
 770         for (i = 0; i < SOF_IPC4_MAX_LLP_GPDMA_READING_SLOTS; i++) {
 771                 sof_mailbox_read(sdev, info->llp_offset, &llp_slot, sizeof(llp_slot));
 772                 if (llp_slot.node_id == dai_copier->data.gtw_cfg.node_id)
 773                         break;
 774
 775                 info->llp_offset += sizeof(llp_slot);
 776         }
 777
 778         if (i < SOF_IPC4_MAX_LLP_GPDMA_READING_SLOTS)
 779                 return;
 780
 781         /* if no llp gpdma slot is used, check aggregated sdw slot */
 782         info->llp_offset = offsetof(struct sof_ipc4_fw_registers, llp_sndw_reading_slots) +
 783                                         sdev->fw_info_box.offset;
 784         for (i = 0; i < SOF_IPC4_MAX_LLP_SNDW_READING_SLOTS; i++) {
 785                 sof_mailbox_read(sdev, info->llp_offset, &llp_slot, sizeof(llp_slot));
 786                 if (llp_slot.node_id == dai_copier->data.gtw_cfg.node_id)
 787                         break;
 788
 789                 info->llp_offset += sizeof(llp_slot);
 790         }
 791
 792         if (i < SOF_IPC4_MAX_LLP_SNDW_READING_SLOTS)
 793                 return;
 794
 795         /* check EVAD slot */
 796         info->llp_offset = offsetof(struct sof_ipc4_fw_registers, llp_evad_reading_slot) +
 797                                         sdev->fw_info_box.offset;
 798         sof_mailbox_read(sdev, info->llp_offset, &llp_slot, sizeof(llp_slot));
 799         if (llp_slot.node_id != dai_copier->data.gtw_cfg.node_id)
 800                 info->llp_offset = 0;
 801 }
 802
 803 static int sof_ipc4_pcm_hw_params(struct snd_soc_component *component,
 804                                   struct snd_pcm_substream *substream,
 805                                   struct snd_pcm_hw_params *params,
 806                                   struct snd_sof_platform_stream_params *platform_params)
 807 {
 808         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
 809         struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
 810         struct sof_ipc4_timestamp_info *time_info;
 811         struct snd_sof_pcm *spcm;
 812
 813         spcm = snd_sof_find_spcm_dai(component, rtd);
 814         if (!spcm)
 815                 return -EINVAL;
 816
 817         time_info = spcm->stream[substream->stream].private;
 818         /* delay calculation is not supported by current fw_reg ABI */
 819         if (!time_info)
 820                 return 0;
 821
 822         time_info->stream_start_offset = SOF_IPC4_INVALID_STREAM_POSITION;
 823         time_info->llp_offset = 0;
 824
 825         sof_ipc4_build_time_info(sdev, &spcm->stream[substream->stream]);
 826
 827         return 0;
 828 }
 829
 830 static int sof_ipc4_get_stream_start_offset(struct snd_sof_dev *sdev,
 831                                             struct snd_pcm_substream *substream,
 832                                             struct snd_sof_pcm_stream *stream,
 833                                             struct sof_ipc4_timestamp_info *time_info)
 834 {
 835         struct sof_ipc4_copier *host_copier = time_info->host_copier;
 836         struct sof_ipc4_copier *dai_copier = time_info->dai_copier;
 837         struct sof_ipc4_pipeline_registers ppl_reg;
 838         u64 stream_start_position;
 839         u32 dai_sample_size;
 840         u32 ch, node_index;
 841         u32 offset;
 842
 843         if (!host_copier || !dai_copier)
 844                 return -EINVAL;
 845
 846         if (host_copier->data.gtw_cfg.node_id == SOF_IPC4_INVALID_NODE_ID)
 847                 return -EINVAL;
 848
 849         node_index = SOF_IPC4_NODE_INDEX(host_copier->data.gtw_cfg.node_id);
 850         offset = offsetof(struct sof_ipc4_fw_registers, pipeline_regs) + node_index * sizeof(ppl_reg);
 851         sof_mailbox_read(sdev, sdev->fw_info_box.offset + offset, &ppl_reg, sizeof(ppl_reg));
 852         if (ppl_reg.stream_start_offset == SOF_IPC4_INVALID_STREAM_POSITION)
 853                 return -EINVAL;
 854
 855         stream_start_position = ppl_reg.stream_start_offset;
 856         ch = dai_copier->data.out_format.fmt_cfg;
 857         ch = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(ch);
 858         dai_sample_size = (dai_copier->data.out_format.bit_depth >> 3) * ch;
 859         /* convert offset to sample count */
 860         do_div(stream_start_position, dai_sample_size);
 861         time_info->stream_start_offset = stream_start_position;
 862
 863         return 0;
 864 }
 865
 866 static snd_pcm_sframes_t sof_ipc4_pcm_delay(struct snd_soc_component *component,
 867                                             struct snd_pcm_substream *substream)
 868 {
 869         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
 870         struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
 871         struct sof_ipc4_timestamp_info *time_info;
 872         struct sof_ipc4_llp_reading_slot llp;
 873         snd_pcm_uframes_t head_ptr, tail_ptr;
 874         struct snd_sof_pcm_stream *stream;
 875         struct snd_sof_pcm *spcm;
 876         u64 tmp_ptr;
 877         int ret;
 878
 879         spcm = snd_sof_find_spcm_dai(component, rtd);
 880         if (!spcm)
 881                 return 0;
 882
 883         stream = &spcm->stream[substream->stream];
 884         time_info = stream->private;
 885         if (!time_info)
 886                 return 0;
 887
 888         /*
 889          * stream_start_offset is updated to memory window by FW based on
 890          * pipeline statistics and it may be invalid if host query happens before
 891          * the statistics is complete. And it will not change after the first initiailization.
 892          */
 893         if (time_info->stream_start_offset == SOF_IPC4_INVALID_STREAM_POSITION) {
 894                 ret = sof_ipc4_get_stream_start_offset(sdev, substream, stream, time_info);
 895                 if (ret < 0)
 896                         return 0;
 897         }
 898
 899         /*
 900          * HDaudio links don't support the LLP counter reported by firmware
 901          * the link position is read directly from hardware registers.
 902          */
 903         if (!time_info->llp_offset) {
 904                 tmp_ptr = snd_sof_pcm_get_stream_position(sdev, component, substream);
 905                 if (!tmp_ptr)
 906                         return 0;
 907         } else {
 908                 sof_mailbox_read(sdev, time_info->llp_offset, &llp, sizeof(llp));
 909                 tmp_ptr = ((u64)llp.reading.llp_u << 32) | llp.reading.llp_l;
 910         }
 911
 912         /* In two cases dai dma position is not accurate
 913          * (1) dai pipeline is started before host pipeline
 914          * (2) multiple streams mixed into one. Each stream has the same dai dma position
 915          *
 916          * Firmware calculates correct stream_start_offset for all cases including above two.
 917          * Driver subtracts stream_start_offset from dai dma position to get accurate one
 918          */
 919         tmp_ptr -= time_info->stream_start_offset;
 920
 921         /* Calculate the delay taking into account that both pointer can wrap */
 922         div64_u64_rem(tmp_ptr, substream->runtime->boundary, &tmp_ptr);
 923         if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
 924                 head_ptr = substream->runtime->status->hw_ptr;
 925                 tail_ptr = tmp_ptr;
 926         } else {
 927                 head_ptr = tmp_ptr;
 928                 tail_ptr = substream->runtime->status->hw_ptr;
 929         }
 930
 931         if (head_ptr < tail_ptr)
 932                 return substream->runtime->boundary - tail_ptr + head_ptr;
 933
 934         return head_ptr - tail_ptr;
 935 }
 936
 937 const struct sof_ipc_pcm_ops ipc4_pcm_ops = {
 938         .hw_params = sof_ipc4_pcm_hw_params,
 939         .trigger = sof_ipc4_pcm_trigger,
 940         .hw_free = sof_ipc4_pcm_hw_free,
 941         .dai_link_fixup = sof_ipc4_pcm_dai_link_fixup,
 942         .pcm_setup = sof_ipc4_pcm_setup,
 943         .pcm_free = sof_ipc4_pcm_free,
 944         .delay = sof_ipc4_pcm_delay,
 945         .ipc_first_on_start = true,
 946         .platform_stop_during_hw_free = true,
 947 };