sound/soc/sof/ipc4-topology.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 //
   3 // This file is provided under a dual BSD/GPLv2 license.  When using or
   4 // redistributing this file, you may do so under either license.
   5 //
   6 // Copyright(c) 2022 Intel Corporation. All rights reserved.
   7 //
   8 //
   9 #include <linux/bitfield.h>
  10 #include <uapi/sound/sof/tokens.h>
  11 #include <sound/pcm_params.h>
  12 #include <sound/sof/ext_manifest4.h>
  13 #include <sound/intel-nhlt.h>
  14 #include "sof-priv.h"
  15 #include "sof-audio.h"
  16 #include "ipc4-priv.h"
  17 #include "ipc4-topology.h"
  18 #include "ops.h"
  19
  20 /*
  21  * The ignore_cpc flag can be used to ignore the CPC value for all modules by
  22  * using 0 instead.
  23  * The CPC is sent to the firmware along with the SOF_IPC4_MOD_INIT_INSTANCE
  24  * message and it is used for clock scaling.
  25  * 0 as CPC value will instruct the firmware to use maximum frequency, thus
  26  * deactivating the clock scaling.
  27  */
  28 static bool ignore_cpc;
  29 module_param_named(ipc4_ignore_cpc, ignore_cpc, bool, 0444);
  30 MODULE_PARM_DESC(ipc4_ignore_cpc,
  31                  "Ignore CPC values. This option will disable clock scaling in firmware.");
  32
  33 #define SOF_IPC4_GAIN_PARAM_ID  0
  34 #define SOF_IPC4_TPLG_ABI_SIZE 6
  35 #define SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS 2
  36
  37 static DEFINE_IDA(alh_group_ida);
  38 static DEFINE_IDA(pipeline_ida);
  39
  40 static const struct sof_topology_token ipc4_sched_tokens[] = {
  41         {SOF_TKN_SCHED_LP_MODE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  42                 offsetof(struct sof_ipc4_pipeline, lp_mode)},
  43         {SOF_TKN_SCHED_USE_CHAIN_DMA, SND_SOC_TPLG_TUPLE_TYPE_BOOL, get_token_u16,
  44                 offsetof(struct sof_ipc4_pipeline, use_chain_dma)},
  45         {SOF_TKN_SCHED_CORE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  46                 offsetof(struct sof_ipc4_pipeline, core_id)},
  47         {SOF_TKN_SCHED_PRIORITY, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  48                 offsetof(struct sof_ipc4_pipeline, priority)},
  49 };
  50
  51 static const struct sof_topology_token pipeline_tokens[] = {
  52         {SOF_TKN_SCHED_DYNAMIC_PIPELINE, SND_SOC_TPLG_TUPLE_TYPE_BOOL, get_token_u16,
  53                 offsetof(struct snd_sof_widget, dynamic_pipeline_widget)},
  54 };
  55
  56 static const struct sof_topology_token ipc4_comp_tokens[] = {
  57         {SOF_TKN_COMP_IS_PAGES, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  58                 offsetof(struct sof_ipc4_base_module_cfg, is_pages)},
  59 };
  60
  61 static const struct sof_topology_token ipc4_in_audio_format_tokens[] = {
  62         {SOF_TKN_CAVS_AUDIO_FORMAT_IN_RATE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  63                 offsetof(struct sof_ipc4_pin_format, audio_fmt.sampling_frequency)},
  64         {SOF_TKN_CAVS_AUDIO_FORMAT_IN_BIT_DEPTH, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  65                 offsetof(struct sof_ipc4_pin_format, audio_fmt.bit_depth)},
  66         {SOF_TKN_CAVS_AUDIO_FORMAT_IN_CH_MAP, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  67                 offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_map)},
  68         {SOF_TKN_CAVS_AUDIO_FORMAT_IN_CH_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  69                 offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_cfg)},
  70         {SOF_TKN_CAVS_AUDIO_FORMAT_IN_INTERLEAVING_STYLE, SND_SOC_TPLG_TUPLE_TYPE_WORD,
  71                 get_token_u32, offsetof(struct sof_ipc4_pin_format,
  72                 audio_fmt.interleaving_style)},
  73         {SOF_TKN_CAVS_AUDIO_FORMAT_IN_FMT_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  74                 offsetof(struct sof_ipc4_pin_format, audio_fmt.fmt_cfg)},
  75         {SOF_TKN_CAVS_AUDIO_FORMAT_INPUT_PIN_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  76                 offsetof(struct sof_ipc4_pin_format, pin_index)},
  77         {SOF_TKN_CAVS_AUDIO_FORMAT_IBS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  78                 offsetof(struct sof_ipc4_pin_format, buffer_size)},
  79 };
  80
  81 static const struct sof_topology_token ipc4_out_audio_format_tokens[] = {
  82         {SOF_TKN_CAVS_AUDIO_FORMAT_OUT_RATE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  83                 offsetof(struct sof_ipc4_pin_format, audio_fmt.sampling_frequency)},
  84         {SOF_TKN_CAVS_AUDIO_FORMAT_OUT_BIT_DEPTH, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  85                 offsetof(struct sof_ipc4_pin_format, audio_fmt.bit_depth)},
  86         {SOF_TKN_CAVS_AUDIO_FORMAT_OUT_CH_MAP, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  87                 offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_map)},
  88         {SOF_TKN_CAVS_AUDIO_FORMAT_OUT_CH_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  89                 offsetof(struct sof_ipc4_pin_format, audio_fmt.ch_cfg)},
  90         {SOF_TKN_CAVS_AUDIO_FORMAT_OUT_INTERLEAVING_STYLE, SND_SOC_TPLG_TUPLE_TYPE_WORD,
  91                 get_token_u32, offsetof(struct sof_ipc4_pin_format,
  92                 audio_fmt.interleaving_style)},
  93         {SOF_TKN_CAVS_AUDIO_FORMAT_OUT_FMT_CFG, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  94                 offsetof(struct sof_ipc4_pin_format, audio_fmt.fmt_cfg)},
  95         {SOF_TKN_CAVS_AUDIO_FORMAT_OUTPUT_PIN_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  96                 offsetof(struct sof_ipc4_pin_format, pin_index)},
  97         {SOF_TKN_CAVS_AUDIO_FORMAT_OBS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
  98                 offsetof(struct sof_ipc4_pin_format, buffer_size)},
  99 };
 100
 101 static const struct sof_topology_token ipc4_copier_deep_buffer_tokens[] = {
 102         {SOF_TKN_INTEL_COPIER_DEEP_BUFFER_DMA_MS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, 0},
 103 };
 104
 105 static const struct sof_topology_token ipc4_copier_tokens[] = {
 106         {SOF_TKN_INTEL_COPIER_NODE_TYPE, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, 0},
 107 };
 108
 109 static const struct sof_topology_token ipc4_audio_fmt_num_tokens[] = {
 110         {SOF_TKN_COMP_NUM_INPUT_AUDIO_FORMATS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 111                 offsetof(struct sof_ipc4_available_audio_format, num_input_formats)},
 112         {SOF_TKN_COMP_NUM_OUTPUT_AUDIO_FORMATS, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 113                 offsetof(struct sof_ipc4_available_audio_format, num_output_formats)},
 114 };
 115
 116 static const struct sof_topology_token dai_tokens[] = {
 117         {SOF_TKN_DAI_TYPE, SND_SOC_TPLG_TUPLE_TYPE_STRING, get_token_dai_type,
 118                 offsetof(struct sof_ipc4_copier, dai_type)},
 119         {SOF_TKN_DAI_INDEX, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 120                 offsetof(struct sof_ipc4_copier, dai_index)},
 121 };
 122
 123 /* Component extended tokens */
 124 static const struct sof_topology_token comp_ext_tokens[] = {
 125         {SOF_TKN_COMP_UUID, SND_SOC_TPLG_TUPLE_TYPE_UUID, get_token_uuid,
 126                 offsetof(struct snd_sof_widget, uuid)},
 127         {SOF_TKN_COMP_CORE_ID, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 128                 offsetof(struct snd_sof_widget, core)},
 129 };
 130
 131 static const struct sof_topology_token gain_tokens[] = {
 132         {SOF_TKN_GAIN_RAMP_TYPE, SND_SOC_TPLG_TUPLE_TYPE_WORD,
 133                 get_token_u32, offsetof(struct sof_ipc4_gain_params, curve_type)},
 134         {SOF_TKN_GAIN_RAMP_DURATION,
 135                 SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 136                 offsetof(struct sof_ipc4_gain_params, curve_duration_l)},
 137         {SOF_TKN_GAIN_VAL, SND_SOC_TPLG_TUPLE_TYPE_WORD,
 138                 get_token_u32, offsetof(struct sof_ipc4_gain_params, init_val)},
 139 };
 140
 141 /* SRC */
 142 static const struct sof_topology_token src_tokens[] = {
 143         {SOF_TKN_SRC_RATE_OUT, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32,
 144                 offsetof(struct sof_ipc4_src_data, sink_rate)},
 145 };
 146
 147 static const struct sof_token_info ipc4_token_list[SOF_TOKEN_COUNT] = {
 148         [SOF_DAI_TOKENS] = {"DAI tokens", dai_tokens, ARRAY_SIZE(dai_tokens)},
 149         [SOF_PIPELINE_TOKENS] = {"Pipeline tokens", pipeline_tokens, ARRAY_SIZE(pipeline_tokens)},
 150         [SOF_SCHED_TOKENS] = {"Scheduler tokens", ipc4_sched_tokens,
 151                 ARRAY_SIZE(ipc4_sched_tokens)},
 152         [SOF_COMP_EXT_TOKENS] = {"Comp extended tokens", comp_ext_tokens,
 153                 ARRAY_SIZE(comp_ext_tokens)},
 154         [SOF_COMP_TOKENS] = {"IPC4 Component tokens",
 155                 ipc4_comp_tokens, ARRAY_SIZE(ipc4_comp_tokens)},
 156         [SOF_IN_AUDIO_FORMAT_TOKENS] = {"IPC4 Input Audio format tokens",
 157                 ipc4_in_audio_format_tokens, ARRAY_SIZE(ipc4_in_audio_format_tokens)},
 158         [SOF_OUT_AUDIO_FORMAT_TOKENS] = {"IPC4 Output Audio format tokens",
 159                 ipc4_out_audio_format_tokens, ARRAY_SIZE(ipc4_out_audio_format_tokens)},
 160         [SOF_COPIER_DEEP_BUFFER_TOKENS] = {"IPC4 Copier deep buffer tokens",
 161                 ipc4_copier_deep_buffer_tokens, ARRAY_SIZE(ipc4_copier_deep_buffer_tokens)},
 162         [SOF_COPIER_TOKENS] = {"IPC4 Copier tokens", ipc4_copier_tokens,
 163                 ARRAY_SIZE(ipc4_copier_tokens)},
 164         [SOF_AUDIO_FMT_NUM_TOKENS] = {"IPC4 Audio format number tokens",
 165                 ipc4_audio_fmt_num_tokens, ARRAY_SIZE(ipc4_audio_fmt_num_tokens)},
 166         [SOF_GAIN_TOKENS] = {"Gain tokens", gain_tokens, ARRAY_SIZE(gain_tokens)},
 167         [SOF_SRC_TOKENS] = {"SRC tokens", src_tokens, ARRAY_SIZE(src_tokens)},
 168 };
 169
 170 struct snd_sof_widget *sof_ipc4_find_swidget_by_ids(struct snd_sof_dev *sdev,
 171                                                     u32 module_id, int instance_id)
 172 {
 173         struct snd_sof_widget *swidget;
 174
 175         list_for_each_entry(swidget, &sdev->widget_list, list) {
 176                 struct sof_ipc4_fw_module *fw_module = swidget->module_info;
 177
 178                 /* Only active module instances have valid instance_id */
 179                 if (!swidget->use_count)
 180                         continue;
 181
 182                 if (fw_module && fw_module->man4_module_entry.id == module_id &&
 183                     swidget->instance_id == instance_id)
 184                         return swidget;
 185         }
 186
 187         return NULL;
 188 }
 189
 190 static void sof_ipc4_dbg_audio_format(struct device *dev, struct sof_ipc4_pin_format *pin_fmt,
 191                                       int num_formats)
 192 {
 193         int i;
 194
 195         for (i = 0; i < num_formats; i++) {
 196                 struct sof_ipc4_audio_format *fmt = &pin_fmt[i].audio_fmt;
 197                 dev_dbg(dev,
 198                         "Pin index #%d: %uHz, %ubit (ch_map %#x ch_cfg %u interleaving_style %u fmt_cfg %#x) buffer size %d\n",
 199                         pin_fmt[i].pin_index, fmt->sampling_frequency, fmt->bit_depth, fmt->ch_map,
 200                         fmt->ch_cfg, fmt->interleaving_style, fmt->fmt_cfg,
 201                         pin_fmt[i].buffer_size);
 202         }
 203 }
 204
 205 static const struct sof_ipc4_audio_format *
 206 sof_ipc4_get_input_pin_audio_fmt(struct snd_sof_widget *swidget, int pin_index)
 207 {
 208         struct sof_ipc4_base_module_cfg_ext *base_cfg_ext;
 209         struct sof_ipc4_process *process;
 210         int i;
 211
 212         if (swidget->id != snd_soc_dapm_effect) {
 213                 struct sof_ipc4_base_module_cfg *base = swidget->private;
 214
 215                 /* For non-process modules, base module config format is used for all input pins */
 216                 return &base->audio_fmt;
 217         }
 218
 219         process = swidget->private;
 220         base_cfg_ext = process->base_config_ext;
 221
 222         /*
 223          * If there are multiple input formats available for a pin, the first available format
 224          * is chosen.
 225          */
 226         for (i = 0; i < base_cfg_ext->num_input_pin_fmts; i++) {
 227                 struct sof_ipc4_pin_format *pin_format = &base_cfg_ext->pin_formats[i];
 228
 229                 if (pin_format->pin_index == pin_index)
 230                         return &pin_format->audio_fmt;
 231         }
 232
 233         return NULL;
 234 }
 235
 236 /**
 237  * sof_ipc4_get_audio_fmt - get available audio formats from swidget->tuples
 238  * @scomp: pointer to pointer to SOC component
 239  * @swidget: pointer to struct snd_sof_widget containing tuples
 240  * @available_fmt: pointer to struct sof_ipc4_available_audio_format being filling in
 241  * @module_base_cfg: Pointer to the base_config in the module init IPC payload
 242  *
 243  * Return: 0 if successful
 244  */
 245 static int sof_ipc4_get_audio_fmt(struct snd_soc_component *scomp,
 246                                   struct snd_sof_widget *swidget,
 247                                   struct sof_ipc4_available_audio_format *available_fmt,
 248                                   struct sof_ipc4_base_module_cfg *module_base_cfg)
 249 {
 250         struct sof_ipc4_pin_format *in_format = NULL;
 251         struct sof_ipc4_pin_format *out_format;
 252         int ret;
 253
 254         ret = sof_update_ipc_object(scomp, available_fmt,
 255                                     SOF_AUDIO_FMT_NUM_TOKENS, swidget->tuples,
 256                                     swidget->num_tuples, sizeof(*available_fmt), 1);
 257         if (ret) {
 258                 dev_err(scomp->dev, "Failed to parse audio format token count\n");
 259                 return ret;
 260         }
 261
 262         if (!available_fmt->num_input_formats && !available_fmt->num_output_formats) {
 263                 dev_err(scomp->dev, "No input/output pin formats set in topology\n");
 264                 return -EINVAL;
 265         }
 266
 267         dev_dbg(scomp->dev,
 268                 "Number of input audio formats: %d. Number of output audio formats: %d\n",
 269                 available_fmt->num_input_formats, available_fmt->num_output_formats);
 270
 271         /* set is_pages in the module's base_config */
 272         ret = sof_update_ipc_object(scomp, module_base_cfg, SOF_COMP_TOKENS, swidget->tuples,
 273                                     swidget->num_tuples, sizeof(*module_base_cfg), 1);
 274         if (ret) {
 275                 dev_err(scomp->dev, "parse comp tokens for %s failed, error: %d\n",
 276                         swidget->widget->name, ret);
 277                 return ret;
 278         }
 279
 280         dev_dbg(scomp->dev, "widget %s: is_pages: %d\n", swidget->widget->name,
 281                 module_base_cfg->is_pages);
 282
 283         if (available_fmt->num_input_formats) {
 284                 in_format = kcalloc(available_fmt->num_input_formats,
 285                                     sizeof(*in_format), GFP_KERNEL);
 286                 if (!in_format)
 287                         return -ENOMEM;
 288                 available_fmt->input_pin_fmts = in_format;
 289
 290                 ret = sof_update_ipc_object(scomp, in_format,
 291                                             SOF_IN_AUDIO_FORMAT_TOKENS, swidget->tuples,
 292                                             swidget->num_tuples, sizeof(*in_format),
 293                                             available_fmt->num_input_formats);
 294                 if (ret) {
 295                         dev_err(scomp->dev, "parse input audio fmt tokens failed %d\n", ret);
 296                         goto err_in;
 297                 }
 298
 299                 dev_dbg(scomp->dev, "Input audio formats for %s\n", swidget->widget->name);
 300                 sof_ipc4_dbg_audio_format(scomp->dev, in_format,
 301                                           available_fmt->num_input_formats);
 302         }
 303
 304         if (available_fmt->num_output_formats) {
 305                 out_format = kcalloc(available_fmt->num_output_formats, sizeof(*out_format),
 306                                      GFP_KERNEL);
 307                 if (!out_format) {
 308                         ret = -ENOMEM;
 309                         goto err_in;
 310                 }
 311
 312                 ret = sof_update_ipc_object(scomp, out_format,
 313                                             SOF_OUT_AUDIO_FORMAT_TOKENS, swidget->tuples,
 314                                             swidget->num_tuples, sizeof(*out_format),
 315                                             available_fmt->num_output_formats);
 316                 if (ret) {
 317                         dev_err(scomp->dev, "parse output audio fmt tokens failed\n");
 318                         goto err_out;
 319                 }
 320
 321                 available_fmt->output_pin_fmts = out_format;
 322                 dev_dbg(scomp->dev, "Output audio formats for %s\n", swidget->widget->name);
 323                 sof_ipc4_dbg_audio_format(scomp->dev, out_format,
 324                                           available_fmt->num_output_formats);
 325         }
 326
 327         return 0;
 328
 329 err_out:
 330         kfree(out_format);
 331 err_in:
 332         kfree(in_format);
 333         available_fmt->input_pin_fmts = NULL;
 334         return ret;
 335 }
 336
 337 /* release the memory allocated in sof_ipc4_get_audio_fmt */
 338 static void sof_ipc4_free_audio_fmt(struct sof_ipc4_available_audio_format *available_fmt)
 339
 340 {
 341         kfree(available_fmt->output_pin_fmts);
 342         available_fmt->output_pin_fmts = NULL;
 343         kfree(available_fmt->input_pin_fmts);
 344         available_fmt->input_pin_fmts = NULL;
 345 }
 346
 347 static void sof_ipc4_widget_free_comp_pipeline(struct snd_sof_widget *swidget)
 348 {
 349         kfree(swidget->private);
 350 }
 351
 352 static int sof_ipc4_widget_set_module_info(struct snd_sof_widget *swidget)
 353 {
 354         struct snd_soc_component *scomp = swidget->scomp;
 355         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
 356
 357         swidget->module_info = sof_ipc4_find_module_by_uuid(sdev, &swidget->uuid);
 358
 359         if (swidget->module_info)
 360                 return 0;
 361
 362         dev_err(sdev->dev, "failed to find module info for widget %s with UUID %pUL\n",
 363                 swidget->widget->name, &swidget->uuid);
 364         return -EINVAL;
 365 }
 366
 367 static int sof_ipc4_widget_setup_msg(struct snd_sof_widget *swidget, struct sof_ipc4_msg *msg)
 368 {
 369         struct sof_ipc4_fw_module *fw_module;
 370         uint32_t type;
 371         int ret;
 372
 373         ret = sof_ipc4_widget_set_module_info(swidget);
 374         if (ret)
 375                 return ret;
 376
 377         fw_module = swidget->module_info;
 378
 379         msg->primary = fw_module->man4_module_entry.id;
 380         msg->primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_INIT_INSTANCE);
 381         msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
 382         msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
 383
 384         msg->extension = SOF_IPC4_MOD_EXT_CORE_ID(swidget->core);
 385
 386         type = (fw_module->man4_module_entry.type & SOF_IPC4_MODULE_DP) ? 1 : 0;
 387         msg->extension |= SOF_IPC4_MOD_EXT_DOMAIN(type);
 388
 389         return 0;
 390 }
 391
 392 static void sof_ipc4_widget_update_kcontrol_module_id(struct snd_sof_widget *swidget)
 393 {
 394         struct snd_soc_component *scomp = swidget->scomp;
 395         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
 396         struct sof_ipc4_fw_module *fw_module = swidget->module_info;
 397         struct snd_sof_control *scontrol;
 398
 399         /* update module ID for all kcontrols for this widget */
 400         list_for_each_entry(scontrol, &sdev->kcontrol_list, list) {
 401                 if (scontrol->comp_id == swidget->comp_id) {
 402                         struct sof_ipc4_control_data *cdata = scontrol->ipc_control_data;
 403                         struct sof_ipc4_msg *msg = &cdata->msg;
 404
 405                         msg->primary |= fw_module->man4_module_entry.id;
 406                 }
 407         }
 408 }
 409
 410 static int sof_ipc4_widget_setup_pcm(struct snd_sof_widget *swidget)
 411 {
 412         struct sof_ipc4_available_audio_format *available_fmt;
 413         struct snd_soc_component *scomp = swidget->scomp;
 414         struct sof_ipc4_copier *ipc4_copier;
 415         struct snd_sof_pcm *spcm;
 416         int node_type = 0;
 417         int ret, dir;
 418
 419         ipc4_copier = kzalloc(sizeof(*ipc4_copier), GFP_KERNEL);
 420         if (!ipc4_copier)
 421                 return -ENOMEM;
 422
 423         swidget->private = ipc4_copier;
 424         available_fmt = &ipc4_copier->available_fmt;
 425
 426         dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
 427
 428         ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt,
 429                                      &ipc4_copier->data.base_config);
 430         if (ret)
 431                 goto free_copier;
 432
 433         /*
 434          * This callback is used by host copier and module-to-module copier,
 435          * and only host copier needs to set gtw_cfg.
 436          */
 437         if (!WIDGET_IS_AIF(swidget->id))
 438                 goto skip_gtw_cfg;
 439
 440         ret = sof_update_ipc_object(scomp, &node_type,
 441                                     SOF_COPIER_TOKENS, swidget->tuples,
 442                                     swidget->num_tuples, sizeof(node_type), 1);
 443
 444         if (ret) {
 445                 dev_err(scomp->dev, "parse host copier node type token failed %d\n",
 446                         ret);
 447                 goto free_available_fmt;
 448         }
 449         dev_dbg(scomp->dev, "host copier '%s' node_type %u\n", swidget->widget->name, node_type);
 450
 451         spcm = snd_sof_find_spcm_comp(scomp, swidget->comp_id, &dir);
 452         if (!spcm)
 453                 goto skip_gtw_cfg;
 454
 455         if (dir == SNDRV_PCM_STREAM_PLAYBACK) {
 456                 struct snd_sof_pcm_stream *sps = &spcm->stream[dir];
 457
 458                 sof_update_ipc_object(scomp, &sps->dsp_max_burst_size_in_ms,
 459                                       SOF_COPIER_DEEP_BUFFER_TOKENS,
 460                                       swidget->tuples,
 461                                       swidget->num_tuples, sizeof(u32), 1);
 462                 /* Set default DMA buffer size if it is not specified in topology */
 463                 if (!sps->dsp_max_burst_size_in_ms)
 464                         sps->dsp_max_burst_size_in_ms = SOF_IPC4_MIN_DMA_BUFFER_SIZE;
 465         } else {
 466                 /* Capture data is copied from DSP to host in 1ms bursts */
 467                 spcm->stream[dir].dsp_max_burst_size_in_ms = 1;
 468         }
 469
 470 skip_gtw_cfg:
 471         ipc4_copier->gtw_attr = kzalloc(sizeof(*ipc4_copier->gtw_attr), GFP_KERNEL);
 472         if (!ipc4_copier->gtw_attr) {
 473                 ret = -ENOMEM;
 474                 goto free_available_fmt;
 475         }
 476
 477         ipc4_copier->copier_config = (uint32_t *)ipc4_copier->gtw_attr;
 478         ipc4_copier->data.gtw_cfg.config_length =
 479                 sizeof(struct sof_ipc4_gtw_attributes) >> 2;
 480
 481         switch (swidget->id) {
 482         case snd_soc_dapm_aif_in:
 483         case snd_soc_dapm_aif_out:
 484                 ipc4_copier->data.gtw_cfg.node_id = SOF_IPC4_NODE_TYPE(node_type);
 485                 break;
 486         case snd_soc_dapm_buffer:
 487                 ipc4_copier->data.gtw_cfg.node_id = SOF_IPC4_INVALID_NODE_ID;
 488                 ipc4_copier->ipc_config_size = 0;
 489                 break;
 490         default:
 491                 dev_err(scomp->dev, "invalid widget type %d\n", swidget->id);
 492                 ret = -EINVAL;
 493                 goto free_gtw_attr;
 494         }
 495
 496         /* set up module info and message header */
 497         ret = sof_ipc4_widget_setup_msg(swidget, &ipc4_copier->msg);
 498         if (ret)
 499                 goto free_gtw_attr;
 500
 501         return 0;
 502
 503 free_gtw_attr:
 504         kfree(ipc4_copier->gtw_attr);
 505 free_available_fmt:
 506         sof_ipc4_free_audio_fmt(available_fmt);
 507 free_copier:
 508         kfree(ipc4_copier);
 509         swidget->private = NULL;
 510         return ret;
 511 }
 512
 513 static void sof_ipc4_widget_free_comp_pcm(struct snd_sof_widget *swidget)
 514 {
 515         struct sof_ipc4_copier *ipc4_copier = swidget->private;
 516         struct sof_ipc4_available_audio_format *available_fmt;
 517
 518         if (!ipc4_copier)
 519                 return;
 520
 521         available_fmt = &ipc4_copier->available_fmt;
 522         kfree(available_fmt->output_pin_fmts);
 523         kfree(ipc4_copier->gtw_attr);
 524         kfree(ipc4_copier);
 525         swidget->private = NULL;
 526 }
 527
 528 static int sof_ipc4_widget_setup_comp_dai(struct snd_sof_widget *swidget)
 529 {
 530         struct sof_ipc4_available_audio_format *available_fmt;
 531         struct snd_soc_component *scomp = swidget->scomp;
 532         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
 533         struct snd_sof_dai *dai = swidget->private;
 534         struct sof_ipc4_copier *ipc4_copier;
 535         struct snd_sof_widget *pipe_widget;
 536         struct sof_ipc4_pipeline *pipeline;
 537         int node_type = 0;
 538         int ret;
 539
 540         ipc4_copier = kzalloc(sizeof(*ipc4_copier), GFP_KERNEL);
 541         if (!ipc4_copier)
 542                 return -ENOMEM;
 543
 544         available_fmt = &ipc4_copier->available_fmt;
 545
 546         dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
 547
 548         ret = sof_ipc4_get_audio_fmt(scomp, swidget, available_fmt,
 549                                      &ipc4_copier->data.base_config);
 550         if (ret)
 551                 goto free_copier;
 552
 553         ret = sof_update_ipc_object(scomp, &node_type,
 554                                     SOF_COPIER_TOKENS, swidget->tuples,
 555                                     swidget->num_tuples, sizeof(node_type), 1);
 556         if (ret) {
 557                 dev_err(scomp->dev, "parse dai node type failed %d\n", ret);
 558                 goto free_available_fmt;
 559         }
 560
 561         ret = sof_update_ipc_object(scomp, ipc4_copier,
 562                                     SOF_DAI_TOKENS, swidget->tuples,
 563                                     swidget->num_tuples, sizeof(u32), 1);
 564         if (ret) {
 565                 dev_err(scomp->dev, "parse dai copier node token failed %d\n", ret);
 566                 goto free_available_fmt;
 567         }
 568
 569         dev_dbg(scomp->dev, "dai %s node_type %u dai_type %u dai_index %d\n", swidget->widget->name,
 570                 node_type, ipc4_copier->dai_type, ipc4_copier->dai_index);
 571
 572         dai->type = ipc4_copier->dai_type;
 573         ipc4_copier->data.gtw_cfg.node_id = SOF_IPC4_NODE_TYPE(node_type);
 574
 575         pipe_widget = swidget->spipe->pipe_widget;
 576         pipeline = pipe_widget->private;
 577
 578         if (pipeline->use_chain_dma &&
 579             !snd_sof_is_chain_dma_supported(sdev, ipc4_copier->dai_type)) {
 580                 dev_err(scomp->dev, "Bad DAI type '%d', Chain DMA is not supported\n",
 581                         ipc4_copier->dai_type);
 582                 ret = -ENODEV;
 583                 goto free_available_fmt;
 584         }
 585
 586         switch (ipc4_copier->dai_type) {
 587         case SOF_DAI_INTEL_ALH:
 588         {
 589                 struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
 590                 struct sof_ipc4_alh_configuration_blob *blob;
 591                 struct snd_soc_dapm_path *p;
 592                 struct snd_sof_widget *w;
 593                 int src_num = 0;
 594
 595                 snd_soc_dapm_widget_for_each_source_path(swidget->widget, p)
 596                         src_num++;
 597
 598                 if (swidget->id == snd_soc_dapm_dai_in && src_num == 0) {
 599                         /*
 600                          * The blob will not be used if the ALH copier is playback direction
 601                          * and doesn't connect to any source.
 602                          * It is fine to call kfree(ipc4_copier->copier_config) since
 603                          * ipc4_copier->copier_config is null.
 604                          */
 605                         ret = 0;
 606                         break;
 607                 }
 608
 609                 blob = kzalloc(sizeof(*blob), GFP_KERNEL);
 610                 if (!blob) {
 611                         ret = -ENOMEM;
 612                         goto free_available_fmt;
 613                 }
 614
 615                 list_for_each_entry(w, &sdev->widget_list, list) {
 616                         if (w->widget->sname &&
 617                             strcmp(w->widget->sname, swidget->widget->sname))
 618                                 continue;
 619
 620                         blob->alh_cfg.device_count++;
 621                 }
 622
 623                 ipc4_copier->copier_config = (uint32_t *)blob;
 624                 /* set data.gtw_cfg.config_length based on device_count */
 625                 ipc4_copier->data.gtw_cfg.config_length = (sizeof(blob->gw_attr) +
 626                                                            sizeof(blob->alh_cfg.device_count) +
 627                                                            sizeof(*blob->alh_cfg.mapping) *
 628                                                            blob->alh_cfg.device_count) >> 2;
 629                 break;
 630         }
 631         case SOF_DAI_INTEL_SSP:
 632                 /* set SSP DAI index as the node_id */
 633                 ipc4_copier->data.gtw_cfg.node_id |=
 634                         SOF_IPC4_NODE_INDEX_INTEL_SSP(ipc4_copier->dai_index);
 635                 break;
 636         case SOF_DAI_INTEL_DMIC:
 637                 /* set DMIC DAI index as the node_id */
 638                 ipc4_copier->data.gtw_cfg.node_id |=
 639                         SOF_IPC4_NODE_INDEX_INTEL_DMIC(ipc4_copier->dai_index);
 640                 break;
 641         default:
 642                 ipc4_copier->gtw_attr = kzalloc(sizeof(*ipc4_copier->gtw_attr), GFP_KERNEL);
 643                 if (!ipc4_copier->gtw_attr) {
 644                         ret = -ENOMEM;
 645                         goto free_available_fmt;
 646                 }
 647
 648                 ipc4_copier->copier_config = (uint32_t *)ipc4_copier->gtw_attr;
 649                 ipc4_copier->data.gtw_cfg.config_length =
 650                         sizeof(struct sof_ipc4_gtw_attributes) >> 2;
 651                 break;
 652         }
 653
 654         dai->scomp = scomp;
 655         dai->private = ipc4_copier;
 656
 657         /* set up module info and message header */
 658         ret = sof_ipc4_widget_setup_msg(swidget, &ipc4_copier->msg);
 659         if (ret)
 660                 goto free_copier_config;
 661
 662         return 0;
 663
 664 free_copier_config:
 665         kfree(ipc4_copier->copier_config);
 666 free_available_fmt:
 667         sof_ipc4_free_audio_fmt(available_fmt);
 668 free_copier:
 669         kfree(ipc4_copier);
 670         dai->private = NULL;
 671         dai->scomp = NULL;
 672         return ret;
 673 }
 674
 675 static void sof_ipc4_widget_free_comp_dai(struct snd_sof_widget *swidget)
 676 {
 677         struct sof_ipc4_available_audio_format *available_fmt;
 678         struct snd_sof_dai *dai = swidget->private;
 679         struct sof_ipc4_copier *ipc4_copier;
 680
 681         if (!dai)
 682                 return;
 683
 684         if (!dai->private) {
 685                 kfree(dai);
 686                 swidget->private = NULL;
 687                 return;
 688         }
 689
 690         ipc4_copier = dai->private;
 691         available_fmt = &ipc4_copier->available_fmt;
 692
 693         kfree(available_fmt->output_pin_fmts);
 694         if (ipc4_copier->dai_type != SOF_DAI_INTEL_SSP &&
 695             ipc4_copier->dai_type != SOF_DAI_INTEL_DMIC)
 696                 kfree(ipc4_copier->copier_config);
 697         kfree(dai->private);
 698         kfree(dai);
 699         swidget->private = NULL;
 700 }
 701
 702 static int sof_ipc4_widget_setup_comp_pipeline(struct snd_sof_widget *swidget)
 703 {
 704         struct snd_soc_component *scomp = swidget->scomp;
 705         struct sof_ipc4_pipeline *pipeline;
 706         struct snd_sof_pipeline *spipe = swidget->spipe;
 707         int ret;
 708
 709         pipeline = kzalloc(sizeof(*pipeline), GFP_KERNEL);
 710         if (!pipeline)
 711                 return -ENOMEM;
 712
 713         ret = sof_update_ipc_object(scomp, pipeline, SOF_SCHED_TOKENS, swidget->tuples,
 714                                     swidget->num_tuples, sizeof(*pipeline), 1);
 715         if (ret) {
 716                 dev_err(scomp->dev, "parsing scheduler tokens failed\n");
 717                 goto err;
 718         }
 719
 720         swidget->core = pipeline->core_id;
 721         spipe->core_mask |= BIT(pipeline->core_id);
 722
 723         if (pipeline->use_chain_dma) {
 724                 dev_dbg(scomp->dev, "Set up chain DMA for %s\n", swidget->widget->name);
 725                 swidget->private = pipeline;
 726                 return 0;
 727         }
 728
 729         /* parse one set of pipeline tokens */
 730         ret = sof_update_ipc_object(scomp, swidget, SOF_PIPELINE_TOKENS, swidget->tuples,
 731                                     swidget->num_tuples, sizeof(*swidget), 1);
 732         if (ret) {
 733                 dev_err(scomp->dev, "parsing pipeline tokens failed\n");
 734                 goto err;
 735         }
 736
 737         dev_dbg(scomp->dev, "pipeline '%s': id %d, pri %d, core_id %u, lp mode %d\n",
 738                 swidget->widget->name, swidget->pipeline_id,
 739                 pipeline->priority, pipeline->core_id, pipeline->lp_mode);
 740
 741         swidget->private = pipeline;
 742
 743         pipeline->msg.primary = SOF_IPC4_GLB_PIPE_PRIORITY(pipeline->priority);
 744         pipeline->msg.primary |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_GLB_CREATE_PIPELINE);
 745         pipeline->msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
 746         pipeline->msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_FW_GEN_MSG);
 747
 748         pipeline->msg.extension = pipeline->lp_mode;
 749         pipeline->msg.extension |= SOF_IPC4_GLB_PIPE_EXT_CORE_ID(pipeline->core_id);
 750         pipeline->state = SOF_IPC4_PIPE_UNINITIALIZED;
 751
 752         return 0;
 753 err:
 754         kfree(pipeline);
 755         return ret;
 756 }
 757
 758 static int sof_ipc4_widget_setup_comp_pga(struct snd_sof_widget *swidget)
 759 {
 760         struct snd_soc_component *scomp = swidget->scomp;
 761         struct sof_ipc4_gain *gain;
 762         int ret;
 763
 764         gain = kzalloc(sizeof(*gain), GFP_KERNEL);
 765         if (!gain)
 766                 return -ENOMEM;
 767
 768         swidget->private = gain;
 769
 770         gain->data.params.channels = SOF_IPC4_GAIN_ALL_CHANNELS_MASK;
 771         gain->data.params.init_val = SOF_IPC4_VOL_ZERO_DB;
 772
 773         ret = sof_ipc4_get_audio_fmt(scomp, swidget, &gain->available_fmt, &gain->data.base_config);
 774         if (ret)
 775                 goto err;
 776
 777         ret = sof_update_ipc_object(scomp, &gain->data.params, SOF_GAIN_TOKENS,
 778                                     swidget->tuples, swidget->num_tuples, sizeof(gain->data), 1);
 779         if (ret) {
 780                 dev_err(scomp->dev, "Parsing gain tokens failed\n");
 781                 goto err;
 782         }
 783
 784         dev_dbg(scomp->dev,
 785                 "pga widget %s: ramp type: %d, ramp duration %d, initial gain value: %#x\n",
 786                 swidget->widget->name, gain->data.params.curve_type,
 787                 gain->data.params.curve_duration_l, gain->data.params.init_val);
 788
 789         ret = sof_ipc4_widget_setup_msg(swidget, &gain->msg);
 790         if (ret)
 791                 goto err;
 792
 793         sof_ipc4_widget_update_kcontrol_module_id(swidget);
 794
 795         return 0;
 796 err:
 797         sof_ipc4_free_audio_fmt(&gain->available_fmt);
 798         kfree(gain);
 799         swidget->private = NULL;
 800         return ret;
 801 }
 802
 803 static void sof_ipc4_widget_free_comp_pga(struct snd_sof_widget *swidget)
 804 {
 805         struct sof_ipc4_gain *gain = swidget->private;
 806
 807         if (!gain)
 808                 return;
 809
 810         sof_ipc4_free_audio_fmt(&gain->available_fmt);
 811         kfree(swidget->private);
 812         swidget->private = NULL;
 813 }
 814
 815 static int sof_ipc4_widget_setup_comp_mixer(struct snd_sof_widget *swidget)
 816 {
 817         struct snd_soc_component *scomp = swidget->scomp;
 818         struct sof_ipc4_mixer *mixer;
 819         int ret;
 820
 821         dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
 822
 823         mixer = kzalloc(sizeof(*mixer), GFP_KERNEL);
 824         if (!mixer)
 825                 return -ENOMEM;
 826
 827         swidget->private = mixer;
 828
 829         ret = sof_ipc4_get_audio_fmt(scomp, swidget, &mixer->available_fmt,
 830                                      &mixer->base_config);
 831         if (ret)
 832                 goto err;
 833
 834         ret = sof_ipc4_widget_setup_msg(swidget, &mixer->msg);
 835         if (ret)
 836                 goto err;
 837
 838         return 0;
 839 err:
 840         sof_ipc4_free_audio_fmt(&mixer->available_fmt);
 841         kfree(mixer);
 842         swidget->private = NULL;
 843         return ret;
 844 }
 845
 846 static int sof_ipc4_widget_setup_comp_src(struct snd_sof_widget *swidget)
 847 {
 848         struct snd_soc_component *scomp = swidget->scomp;
 849         struct snd_sof_pipeline *spipe = swidget->spipe;
 850         struct sof_ipc4_src *src;
 851         int ret;
 852
 853         dev_dbg(scomp->dev, "Updating IPC structure for %s\n", swidget->widget->name);
 854
 855         src = kzalloc(sizeof(*src), GFP_KERNEL);
 856         if (!src)
 857                 return -ENOMEM;
 858
 859         swidget->private = src;
 860
 861         ret = sof_ipc4_get_audio_fmt(scomp, swidget, &src->available_fmt,
 862                                      &src->data.base_config);
 863         if (ret)
 864                 goto err;
 865
 866         ret = sof_update_ipc_object(scomp, &src->data, SOF_SRC_TOKENS, swidget->tuples,
 867                                     swidget->num_tuples, sizeof(*src), 1);
 868         if (ret) {
 869                 dev_err(scomp->dev, "Parsing SRC tokens failed\n");
 870                 goto err;
 871         }
 872
 873         spipe->core_mask |= BIT(swidget->core);
 874
 875         dev_dbg(scomp->dev, "SRC sink rate %d\n", src->data.sink_rate);
 876
 877         ret = sof_ipc4_widget_setup_msg(swidget, &src->msg);
 878         if (ret)
 879                 goto err;
 880
 881         return 0;
 882 err:
 883         sof_ipc4_free_audio_fmt(&src->available_fmt);
 884         kfree(src);
 885         swidget->private = NULL;
 886         return ret;
 887 }
 888
 889 static void sof_ipc4_widget_free_comp_src(struct snd_sof_widget *swidget)
 890 {
 891         struct sof_ipc4_src *src = swidget->private;
 892
 893         if (!src)
 894                 return;
 895
 896         sof_ipc4_free_audio_fmt(&src->available_fmt);
 897         kfree(swidget->private);
 898         swidget->private = NULL;
 899 }
 900
 901 static void sof_ipc4_widget_free_comp_mixer(struct snd_sof_widget *swidget)
 902 {
 903         struct sof_ipc4_mixer *mixer = swidget->private;
 904
 905         if (!mixer)
 906                 return;
 907
 908         sof_ipc4_free_audio_fmt(&mixer->available_fmt);
 909         kfree(swidget->private);
 910         swidget->private = NULL;
 911 }
 912
 913 /*
 914  * Add the process modules support. The process modules are defined as snd_soc_dapm_effect modules.
 915  */
 916 static int sof_ipc4_widget_setup_comp_process(struct snd_sof_widget *swidget)
 917 {
 918         struct snd_soc_component *scomp = swidget->scomp;
 919         struct sof_ipc4_fw_module *fw_module;
 920         struct snd_sof_pipeline *spipe = swidget->spipe;
 921         struct sof_ipc4_process *process;
 922         void *cfg;
 923         int ret;
 924
 925         process = kzalloc(sizeof(*process), GFP_KERNEL);
 926         if (!process)
 927                 return -ENOMEM;
 928
 929         swidget->private = process;
 930
 931         ret = sof_ipc4_get_audio_fmt(scomp, swidget, &process->available_fmt,
 932                                      &process->base_config);
 933         if (ret)
 934                 goto err;
 935
 936         ret = sof_ipc4_widget_setup_msg(swidget, &process->msg);
 937         if (ret)
 938                 goto err;
 939
 940         /* parse process init module payload config type from module info */
 941         fw_module = swidget->module_info;
 942         process->init_config = FIELD_GET(SOF_IPC4_MODULE_INIT_CONFIG_MASK,
 943                                          fw_module->man4_module_entry.type);
 944
 945         process->ipc_config_size = sizeof(struct sof_ipc4_base_module_cfg);
 946
 947         /* allocate memory for base config extension if needed */
 948         if (process->init_config == SOF_IPC4_MODULE_INIT_CONFIG_TYPE_BASE_CFG_WITH_EXT) {
 949                 struct sof_ipc4_base_module_cfg_ext *base_cfg_ext;
 950                 u32 ext_size = struct_size(base_cfg_ext, pin_formats,
 951                                            size_add(swidget->num_input_pins,
 952                                                     swidget->num_output_pins));
 953
 954                 base_cfg_ext = kzalloc(ext_size, GFP_KERNEL);
 955                 if (!base_cfg_ext) {
 956                         ret = -ENOMEM;
 957                         goto free_available_fmt;
 958                 }
 959
 960                 base_cfg_ext->num_input_pin_fmts = swidget->num_input_pins;
 961                 base_cfg_ext->num_output_pin_fmts = swidget->num_output_pins;
 962                 process->base_config_ext = base_cfg_ext;
 963                 process->base_config_ext_size = ext_size;
 964                 process->ipc_config_size += ext_size;
 965         }
 966
 967         cfg = kzalloc(process->ipc_config_size, GFP_KERNEL);
 968         if (!cfg) {
 969                 ret = -ENOMEM;
 970                 goto free_base_cfg_ext;
 971         }
 972
 973         process->ipc_config_data = cfg;
 974
 975         sof_ipc4_widget_update_kcontrol_module_id(swidget);
 976
 977         /* set pipeline core mask to keep track of the core the module is scheduled to run on */
 978         spipe->core_mask |= BIT(swidget->core);
 979
 980         return 0;
 981 free_base_cfg_ext:
 982         kfree(process->base_config_ext);
 983         process->base_config_ext = NULL;
 984 free_available_fmt:
 985         sof_ipc4_free_audio_fmt(&process->available_fmt);
 986 err:
 987         kfree(process);
 988         swidget->private = NULL;
 989         return ret;
 990 }
 991
 992 static void sof_ipc4_widget_free_comp_process(struct snd_sof_widget *swidget)
 993 {
 994         struct sof_ipc4_process *process = swidget->private;
 995
 996         if (!process)
 997                 return;
 998
 999         kfree(process->ipc_config_data);
1000         kfree(process->base_config_ext);
1001         sof_ipc4_free_audio_fmt(&process->available_fmt);
1002         kfree(swidget->private);
1003         swidget->private = NULL;
1004 }
1005
1006 static void
1007 sof_ipc4_update_resource_usage(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget,
1008                                struct sof_ipc4_base_module_cfg *base_config)
1009 {
1010         struct sof_ipc4_fw_module *fw_module = swidget->module_info;
1011         struct snd_sof_widget *pipe_widget;
1012         struct sof_ipc4_pipeline *pipeline;
1013         int task_mem, queue_mem;
1014         int ibs, bss, total;
1015
1016         ibs = base_config->ibs;
1017         bss = base_config->is_pages;
1018
1019         task_mem = SOF_IPC4_PIPELINE_OBJECT_SIZE;
1020         task_mem += SOF_IPC4_MODULE_INSTANCE_LIST_ITEM_SIZE + bss;
1021
1022         if (fw_module->man4_module_entry.type & SOF_IPC4_MODULE_LL) {
1023                 task_mem += SOF_IPC4_FW_ROUNDUP(SOF_IPC4_LL_TASK_OBJECT_SIZE);
1024                 task_mem += SOF_IPC4_FW_MAX_QUEUE_COUNT * SOF_IPC4_MODULE_INSTANCE_LIST_ITEM_SIZE;
1025                 task_mem += SOF_IPC4_LL_TASK_LIST_ITEM_SIZE;
1026         } else {
1027                 task_mem += SOF_IPC4_FW_ROUNDUP(SOF_IPC4_DP_TASK_OBJECT_SIZE);
1028                 task_mem += SOF_IPC4_DP_TASK_LIST_SIZE;
1029         }
1030
1031         ibs = SOF_IPC4_FW_ROUNDUP(ibs);
1032         queue_mem = SOF_IPC4_FW_MAX_QUEUE_COUNT * (SOF_IPC4_DATA_QUEUE_OBJECT_SIZE +  ibs);
1033
1034         total = SOF_IPC4_FW_PAGE(task_mem + queue_mem);
1035
1036         pipe_widget = swidget->spipe->pipe_widget;
1037         pipeline = pipe_widget->private;
1038         pipeline->mem_usage += total;
1039
1040         /* Update base_config->cpc from the module manifest */
1041         sof_ipc4_update_cpc_from_manifest(sdev, fw_module, base_config);
1042
1043         if (ignore_cpc) {
1044                 dev_dbg(sdev->dev, "%s: ibs / obs: %u / %u, forcing cpc to 0 from %u\n",
1045                         swidget->widget->name, base_config->ibs, base_config->obs,
1046                         base_config->cpc);
1047                 base_config->cpc = 0;
1048         } else {
1049                 dev_dbg(sdev->dev, "%s: ibs / obs / cpc: %u / %u / %u\n",
1050                         swidget->widget->name, base_config->ibs, base_config->obs,
1051                         base_config->cpc);
1052         }
1053 }
1054
1055 static int sof_ipc4_widget_assign_instance_id(struct snd_sof_dev *sdev,
1056                                               struct snd_sof_widget *swidget)
1057 {
1058         struct sof_ipc4_fw_module *fw_module = swidget->module_info;
1059         int max_instances = fw_module->man4_module_entry.instance_max_count;
1060
1061         swidget->instance_id = ida_alloc_max(&fw_module->m_ida, max_instances, GFP_KERNEL);
1062         if (swidget->instance_id < 0) {
1063                 dev_err(sdev->dev, "failed to assign instance id for widget %s",
1064                         swidget->widget->name);
1065                 return swidget->instance_id;
1066         }
1067
1068         return 0;
1069 }
1070
1071 /* update hw_params based on the audio stream format */
1072 static int sof_ipc4_update_hw_params(struct snd_sof_dev *sdev, struct snd_pcm_hw_params *params,
1073                                      struct sof_ipc4_audio_format *fmt)
1074 {
1075         snd_pcm_format_t snd_fmt;
1076         struct snd_interval *i;
1077         struct snd_mask *m;
1078         int valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1079         unsigned int channels, rate;
1080
1081         switch (valid_bits) {
1082         case 16:
1083                 snd_fmt = SNDRV_PCM_FORMAT_S16_LE;
1084                 break;
1085         case 24:
1086                 snd_fmt = SNDRV_PCM_FORMAT_S24_LE;
1087                 break;
1088         case 32:
1089                 snd_fmt = SNDRV_PCM_FORMAT_S32_LE;
1090                 break;
1091         default:
1092                 dev_err(sdev->dev, "invalid PCM valid_bits %d\n", valid_bits);
1093                 return -EINVAL;
1094         }
1095
1096         m = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT);
1097         snd_mask_none(m);
1098         snd_mask_set_format(m, snd_fmt);
1099
1100         rate = fmt->sampling_frequency;
1101         i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE);
1102         i->min = rate;
1103         i->max = rate;
1104
1105         channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1106         i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_CHANNELS);
1107         i->min = channels;
1108         i->max = channels;
1109
1110         return 0;
1111 }
1112
1113 static bool sof_ipc4_is_single_format(struct snd_sof_dev *sdev,
1114                                       struct sof_ipc4_pin_format *pin_fmts, u32 pin_fmts_size)
1115 {
1116         struct sof_ipc4_audio_format *fmt;
1117         u32 rate, channels, valid_bits;
1118         int i;
1119
1120         fmt = &pin_fmts[0].audio_fmt;
1121         rate = fmt->sampling_frequency;
1122         channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1123         valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1124
1125         /* check if all output formats in topology are the same */
1126         for (i = 1; i < pin_fmts_size; i++) {
1127                 u32 _rate, _channels, _valid_bits;
1128
1129                 fmt = &pin_fmts[i].audio_fmt;
1130                 _rate = fmt->sampling_frequency;
1131                 _channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1132                 _valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1133
1134                 if (_rate != rate || _channels != channels || _valid_bits != valid_bits)
1135                         return false;
1136         }
1137
1138         return true;
1139 }
1140
1141 static int sof_ipc4_init_output_audio_fmt(struct snd_sof_dev *sdev,
1142                                           struct sof_ipc4_base_module_cfg *base_config,
1143                                           struct sof_ipc4_available_audio_format *available_fmt,
1144                                           u32 out_ref_rate, u32 out_ref_channels,
1145                                           u32 out_ref_valid_bits)
1146 {
1147         struct sof_ipc4_audio_format *out_fmt;
1148         bool single_format;
1149         int i;
1150
1151         if (!available_fmt->num_output_formats)
1152                 return -EINVAL;
1153
1154         single_format = sof_ipc4_is_single_format(sdev, available_fmt->output_pin_fmts,
1155                                                   available_fmt->num_output_formats);
1156
1157         /* pick the first format if there's only one available or if all formats are the same */
1158         if (single_format) {
1159                 base_config->obs = available_fmt->output_pin_fmts[0].buffer_size;
1160                 return 0;
1161         }
1162
1163         /*
1164          * if there are multiple output formats, then choose the output format that matches
1165          * the reference params
1166          */
1167         for (i = 0; i < available_fmt->num_output_formats; i++) {
1168                 u32 _out_rate, _out_channels, _out_valid_bits;
1169
1170                 out_fmt = &available_fmt->output_pin_fmts[i].audio_fmt;
1171                 _out_rate = out_fmt->sampling_frequency;
1172                 _out_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(out_fmt->fmt_cfg);
1173                 _out_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(out_fmt->fmt_cfg);
1174
1175                 if (_out_rate == out_ref_rate && _out_channels == out_ref_channels &&
1176                     _out_valid_bits == out_ref_valid_bits) {
1177                         base_config->obs = available_fmt->output_pin_fmts[i].buffer_size;
1178                         return i;
1179                 }
1180         }
1181
1182         return -EINVAL;
1183 }
1184
1185 static int sof_ipc4_get_valid_bits(struct snd_sof_dev *sdev, struct snd_pcm_hw_params *params)
1186 {
1187         switch (params_format(params)) {
1188         case SNDRV_PCM_FORMAT_S16_LE:
1189                 return 16;
1190         case SNDRV_PCM_FORMAT_S24_LE:
1191                 return 24;
1192         case SNDRV_PCM_FORMAT_S32_LE:
1193                 return 32;
1194         default:
1195                 dev_err(sdev->dev, "invalid pcm frame format %d\n", params_format(params));
1196                 return -EINVAL;
1197         }
1198 }
1199
1200 static int sof_ipc4_init_input_audio_fmt(struct snd_sof_dev *sdev,
1201                                          struct snd_sof_widget *swidget,
1202                                          struct sof_ipc4_base_module_cfg *base_config,
1203                                          struct snd_pcm_hw_params *params,
1204                                          struct sof_ipc4_available_audio_format *available_fmt)
1205 {
1206         struct sof_ipc4_pin_format *pin_fmts = available_fmt->input_pin_fmts;
1207         u32 pin_fmts_size = available_fmt->num_input_formats;
1208         u32 valid_bits;
1209         u32 channels;
1210         u32 rate;
1211         bool single_format;
1212         int sample_valid_bits;
1213         int i = 0;
1214
1215         if (!available_fmt->num_input_formats) {
1216                 dev_err(sdev->dev, "no input formats for %s\n", swidget->widget->name);
1217                 return -EINVAL;
1218         }
1219
1220         single_format = sof_ipc4_is_single_format(sdev, available_fmt->input_pin_fmts,
1221                                                   available_fmt->num_input_formats);
1222         if (single_format)
1223                 goto in_fmt;
1224
1225         sample_valid_bits = sof_ipc4_get_valid_bits(sdev, params);
1226         if (sample_valid_bits < 0)
1227                 return sample_valid_bits;
1228
1229         /*
1230          * Search supported input audio formats with pin index 0 to match rate, channels and
1231          * sample_valid_bits from reference params
1232          */
1233         for (i = 0; i < pin_fmts_size; i++) {
1234                 struct sof_ipc4_audio_format *fmt = &pin_fmts[i].audio_fmt;
1235
1236                 if (pin_fmts[i].pin_index)
1237                         continue;
1238
1239                 rate = fmt->sampling_frequency;
1240                 channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(fmt->fmt_cfg);
1241                 valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1242                 if (params_rate(params) == rate && params_channels(params) == channels &&
1243                     sample_valid_bits == valid_bits) {
1244                         dev_dbg(sdev->dev, "matched audio format index for %uHz, %ubit, %u channels: %d\n",
1245                                 rate, valid_bits, channels, i);
1246                         break;
1247                 }
1248         }
1249
1250         if (i == pin_fmts_size) {
1251                 dev_err(sdev->dev, "%s: Unsupported audio format: %uHz, %ubit, %u channels\n",
1252                         __func__, params_rate(params), sample_valid_bits, params_channels(params));
1253                 return -EINVAL;
1254         }
1255
1256 in_fmt:
1257         /* copy input format */
1258         if (available_fmt->num_input_formats && i < available_fmt->num_input_formats) {
1259                 memcpy(&base_config->audio_fmt, &available_fmt->input_pin_fmts[i].audio_fmt,
1260                        sizeof(struct sof_ipc4_audio_format));
1261
1262                 /* set base_cfg ibs/obs */
1263                 base_config->ibs = available_fmt->input_pin_fmts[i].buffer_size;
1264
1265                 dev_dbg(sdev->dev, "Init input audio formats for %s\n", swidget->widget->name);
1266                 sof_ipc4_dbg_audio_format(sdev->dev, &available_fmt->input_pin_fmts[i], 1);
1267         }
1268
1269         return i;
1270 }
1271
1272 static void sof_ipc4_unprepare_copier_module(struct snd_sof_widget *swidget)
1273 {
1274         struct sof_ipc4_copier *ipc4_copier = NULL;
1275         struct snd_sof_widget *pipe_widget;
1276         struct sof_ipc4_pipeline *pipeline;
1277
1278         /* reset pipeline memory usage */
1279         pipe_widget = swidget->spipe->pipe_widget;
1280         pipeline = pipe_widget->private;
1281         pipeline->mem_usage = 0;
1282
1283         if (WIDGET_IS_AIF(swidget->id) || swidget->id == snd_soc_dapm_buffer) {
1284                 if (pipeline->use_chain_dma) {
1285                         pipeline->msg.primary = 0;
1286                         pipeline->msg.extension = 0;
1287                 }
1288                 ipc4_copier = swidget->private;
1289         } else if (WIDGET_IS_DAI(swidget->id)) {
1290                 struct snd_sof_dai *dai = swidget->private;
1291
1292                 ipc4_copier = dai->private;
1293
1294                 if (pipeline->use_chain_dma) {
1295                         pipeline->msg.primary = 0;
1296                         pipeline->msg.extension = 0;
1297                 }
1298
1299                 if (ipc4_copier->dai_type == SOF_DAI_INTEL_ALH) {
1300                         struct sof_ipc4_copier_data *copier_data = &ipc4_copier->data;
1301                         struct sof_ipc4_alh_configuration_blob *blob;
1302                         unsigned int group_id;
1303
1304                         blob = (struct sof_ipc4_alh_configuration_blob *)ipc4_copier->copier_config;
1305                         if (blob->alh_cfg.device_count > 1) {
1306                                 group_id = SOF_IPC4_NODE_INDEX(ipc4_copier->data.gtw_cfg.node_id) -
1307                                            ALH_MULTI_GTW_BASE;
1308                                 ida_free(&alh_group_ida, group_id);
1309                         }
1310
1311                         /* clear the node ID */
1312                         copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
1313                 }
1314         }
1315
1316         if (ipc4_copier) {
1317                 kfree(ipc4_copier->ipc_config_data);
1318                 ipc4_copier->ipc_config_data = NULL;
1319                 ipc4_copier->ipc_config_size = 0;
1320         }
1321 }
1322
1323 #if IS_ENABLED(CONFIG_ACPI) && IS_ENABLED(CONFIG_SND_INTEL_NHLT)
1324 static int snd_sof_get_hw_config_params(struct snd_sof_dev *sdev, struct snd_sof_dai *dai,
1325                                         int *sample_rate, int *channel_count, int *bit_depth)
1326 {
1327         struct snd_soc_tplg_hw_config *hw_config;
1328         struct snd_sof_dai_link *slink;
1329         bool dai_link_found = false;
1330         bool hw_cfg_found = false;
1331         int i;
1332
1333         /* get current hw_config from link */
1334         list_for_each_entry(slink, &sdev->dai_link_list, list) {
1335                 if (!strcmp(slink->link->name, dai->name)) {
1336                         dai_link_found = true;
1337                         break;
1338                 }
1339         }
1340
1341         if (!dai_link_found) {
1342                 dev_err(sdev->dev, "%s: no DAI link found for DAI %s\n", __func__, dai->name);
1343                 return -EINVAL;
1344         }
1345
1346         for (i = 0; i < slink->num_hw_configs; i++) {
1347                 hw_config = &slink->hw_configs[i];
1348                 if (dai->current_config == le32_to_cpu(hw_config->id)) {
1349                         hw_cfg_found = true;
1350                         break;
1351                 }
1352         }
1353
1354         if (!hw_cfg_found) {
1355                 dev_err(sdev->dev, "%s: no matching hw_config found for DAI %s\n", __func__,
1356                         dai->name);
1357                 return -EINVAL;
1358         }
1359
1360         *bit_depth = le32_to_cpu(hw_config->tdm_slot_width);
1361         *channel_count = le32_to_cpu(hw_config->tdm_slots);
1362         *sample_rate = le32_to_cpu(hw_config->fsync_rate);
1363
1364         dev_dbg(sdev->dev, "sample rate: %d sample width: %d channels: %d\n",
1365                 *sample_rate, *bit_depth, *channel_count);
1366
1367         return 0;
1368 }
1369
1370 static int snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai,
1371                                           struct snd_pcm_hw_params *params, u32 dai_index,
1372                                           u32 linktype, u8 dir, u32 **dst, u32 *len)
1373 {
1374         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
1375         struct nhlt_specific_cfg *cfg;
1376         int sample_rate, channel_count;
1377         int bit_depth, ret;
1378         u32 nhlt_type;
1379         int dev_type = 0;
1380
1381         /* convert to NHLT type */
1382         switch (linktype) {
1383         case SOF_DAI_INTEL_DMIC:
1384                 nhlt_type = NHLT_LINK_DMIC;
1385                 bit_depth = params_width(params);
1386                 channel_count = params_channels(params);
1387                 sample_rate = params_rate(params);
1388                 break;
1389         case SOF_DAI_INTEL_SSP:
1390                 nhlt_type = NHLT_LINK_SSP;
1391                 ret = snd_sof_get_hw_config_params(sdev, dai, &sample_rate, &channel_count,
1392                                                    &bit_depth);
1393                 if (ret < 0)
1394                         return ret;
1395
1396                 /*
1397                  * We need to know the type of the external device attached to a SSP
1398                  * port to retrieve the blob from NHLT. However, device type is not
1399                  * specified in topology.
1400                  * Query the type for the port and then pass that information back
1401                  * to the blob lookup function.
1402                  */
1403                 dev_type = intel_nhlt_ssp_device_type(sdev->dev, ipc4_data->nhlt,
1404                                                       dai_index);
1405                 if (dev_type < 0)
1406                         return dev_type;
1407                 break;
1408         default:
1409                 return 0;
1410         }
1411
1412         dev_dbg(sdev->dev, "dai index %d nhlt type %d direction %d dev type %d\n",
1413                 dai_index, nhlt_type, dir, dev_type);
1414
1415         /* find NHLT blob with matching params */
1416         cfg = intel_nhlt_get_endpoint_blob(sdev->dev, ipc4_data->nhlt, dai_index, nhlt_type,
1417                                            bit_depth, bit_depth, channel_count, sample_rate,
1418                                            dir, dev_type);
1419
1420         if (!cfg) {
1421                 dev_err(sdev->dev,
1422                         "no matching blob for sample rate: %d sample width: %d channels: %d\n",
1423                         sample_rate, bit_depth, channel_count);
1424                 return -EINVAL;
1425         }
1426
1427         /* config length should be in dwords */
1428         *len = cfg->size >> 2;
1429         *dst = (u32 *)cfg->caps;
1430
1431         return 0;
1432 }
1433 #else
1434 static int snd_sof_get_nhlt_endpoint_data(struct snd_sof_dev *sdev, struct snd_sof_dai *dai,
1435                                           struct snd_pcm_hw_params *params, u32 dai_index,
1436                                           u32 linktype, u8 dir, u32 **dst, u32 *len)
1437 {
1438         return 0;
1439 }
1440 #endif
1441
1442 bool sof_ipc4_copier_is_single_format(struct snd_sof_dev *sdev,
1443                                       struct sof_ipc4_pin_format *pin_fmts,
1444                                       u32 pin_fmts_size)
1445 {
1446         struct sof_ipc4_audio_format *fmt;
1447         u32 valid_bits;
1448         int i;
1449
1450         fmt = &pin_fmts[0].audio_fmt;
1451         valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1452
1453         /* check if all formats in topology are the same */
1454         for (i = 1; i < pin_fmts_size; i++) {
1455                 u32 _valid_bits;
1456
1457                 fmt = &pin_fmts[i].audio_fmt;
1458                 _valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(fmt->fmt_cfg);
1459
1460                 if (_valid_bits != valid_bits)
1461                         return false;
1462         }
1463
1464         return true;
1465 }
1466
1467 static int
1468 sof_ipc4_prepare_copier_module(struct snd_sof_widget *swidget,
1469                                struct snd_pcm_hw_params *fe_params,
1470                                struct snd_sof_platform_stream_params *platform_params,
1471                                struct snd_pcm_hw_params *pipeline_params, int dir)
1472 {
1473         struct sof_ipc4_available_audio_format *available_fmt;
1474         struct snd_soc_component *scomp = swidget->scomp;
1475         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
1476         struct sof_ipc4_copier_data *copier_data;
1477         struct snd_pcm_hw_params *ref_params;
1478         struct sof_ipc4_copier *ipc4_copier;
1479         struct snd_sof_dai *dai;
1480         u32 gtw_cfg_config_length;
1481         u32 dma_config_tlv_size = 0;
1482         void **ipc_config_data;
1483         int *ipc_config_size;
1484         u32 **data;
1485         int ipc_size, ret, out_ref_valid_bits;
1486         u32 out_ref_rate, out_ref_channels;
1487         u32 deep_buffer_dma_ms = 0;
1488         int output_fmt_index;
1489         bool single_output_format;
1490
1491         dev_dbg(sdev->dev, "copier %s, type %d", swidget->widget->name, swidget->id);
1492
1493         switch (swidget->id) {
1494         case snd_soc_dapm_aif_in:
1495         case snd_soc_dapm_aif_out:
1496         {
1497                 struct sof_ipc4_gtw_attributes *gtw_attr;
1498                 struct snd_sof_widget *pipe_widget;
1499                 struct sof_ipc4_pipeline *pipeline;
1500
1501                 /* parse the deep buffer dma size */
1502                 ret = sof_update_ipc_object(scomp, &deep_buffer_dma_ms,
1503                                             SOF_COPIER_DEEP_BUFFER_TOKENS, swidget->tuples,
1504                                             swidget->num_tuples, sizeof(u32), 1);
1505                 if (ret) {
1506                         dev_err(scomp->dev, "Failed to parse deep buffer dma size for %s\n",
1507                                 swidget->widget->name);
1508                         return ret;
1509                 }
1510
1511                 ipc4_copier = (struct sof_ipc4_copier *)swidget->private;
1512                 gtw_attr = ipc4_copier->gtw_attr;
1513                 copier_data = &ipc4_copier->data;
1514                 available_fmt = &ipc4_copier->available_fmt;
1515
1516                 pipe_widget = swidget->spipe->pipe_widget;
1517                 pipeline = pipe_widget->private;
1518
1519                 if (pipeline->use_chain_dma) {
1520                         u32 host_dma_id;
1521                         u32 fifo_size;
1522
1523                         host_dma_id = platform_params->stream_tag - 1;
1524                         pipeline->msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_HOST_ID(host_dma_id);
1525
1526                         /* Set SCS bit for S16_LE format only */
1527                         if (params_format(fe_params) == SNDRV_PCM_FORMAT_S16_LE)
1528                                 pipeline->msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_SCS_MASK;
1529
1530                         /*
1531                          * Despite its name the bitfield 'fifo_size' is used to define DMA buffer
1532                          * size. The expression calculates 2ms buffer size.
1533                          */
1534                         fifo_size = DIV_ROUND_UP((SOF_IPC4_CHAIN_DMA_BUF_SIZE_MS *
1535                                                   params_rate(fe_params) *
1536                                                   params_channels(fe_params) *
1537                                                   params_physical_width(fe_params)), 8000);
1538                         pipeline->msg.extension |= SOF_IPC4_GLB_EXT_CHAIN_DMA_FIFO_SIZE(fifo_size);
1539
1540                         /*
1541                          * Chain DMA does not support stream timestamping, set node_id to invalid
1542                          * to skip the code in sof_ipc4_get_stream_start_offset().
1543                          */
1544                         copier_data->gtw_cfg.node_id = SOF_IPC4_INVALID_NODE_ID;
1545
1546                         return 0;
1547                 }
1548
1549                 /*
1550                  * Use the input_pin_fmts to match pcm params for playback and the output_pin_fmts
1551                  * for capture.
1552                  */
1553                 if (dir == SNDRV_PCM_STREAM_PLAYBACK)
1554                         ref_params = fe_params;
1555                 else
1556                         ref_params = pipeline_params;
1557
1558                 copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
1559                 copier_data->gtw_cfg.node_id |=
1560                         SOF_IPC4_NODE_INDEX(platform_params->stream_tag - 1);
1561
1562                 /* set gateway attributes */
1563                 gtw_attr->lp_buffer_alloc = pipeline->lp_mode;
1564                 break;
1565         }
1566         case snd_soc_dapm_dai_in:
1567         case snd_soc_dapm_dai_out:
1568         {
1569                 struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
1570                 struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
1571
1572                 if (pipeline->use_chain_dma)
1573                         return 0;
1574
1575                 dai = swidget->private;
1576
1577                 ipc4_copier = (struct sof_ipc4_copier *)dai->private;
1578                 copier_data = &ipc4_copier->data;
1579                 available_fmt = &ipc4_copier->available_fmt;
1580
1581                 /*
1582                  * When there is format conversion within a pipeline, the number of supported
1583                  * output formats is typically limited to just 1 for the DAI copiers. But when there
1584                  * is no format conversion, the DAI copiers input format must match that of the
1585                  * FE hw_params for capture and the pipeline params for playback.
1586                  */
1587                 if (dir == SNDRV_PCM_STREAM_PLAYBACK)
1588                         ref_params = pipeline_params;
1589                 else
1590                         ref_params = fe_params;
1591
1592                 ret = snd_sof_get_nhlt_endpoint_data(sdev, dai, fe_params, ipc4_copier->dai_index,
1593                                                      ipc4_copier->dai_type, dir,
1594                                                      &ipc4_copier->copier_config,
1595                                                      &copier_data->gtw_cfg.config_length);
1596                 if (ret < 0)
1597                         return ret;
1598
1599                 break;
1600         }
1601         case snd_soc_dapm_buffer:
1602         {
1603                 ipc4_copier = (struct sof_ipc4_copier *)swidget->private;
1604                 copier_data = &ipc4_copier->data;
1605                 available_fmt = &ipc4_copier->available_fmt;
1606                 ref_params = pipeline_params;
1607
1608                 break;
1609         }
1610         default:
1611                 dev_err(sdev->dev, "unsupported type %d for copier %s",
1612                         swidget->id, swidget->widget->name);
1613                 return -EINVAL;
1614         }
1615
1616         /* set input and output audio formats */
1617         ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, &copier_data->base_config, ref_params,
1618                                             available_fmt);
1619         if (ret < 0)
1620                 return ret;
1621
1622         /* set the reference params for output format selection */
1623         single_output_format = sof_ipc4_copier_is_single_format(sdev,
1624                                                                 available_fmt->output_pin_fmts,
1625                                                                 available_fmt->num_output_formats);
1626         switch (swidget->id) {
1627         case snd_soc_dapm_aif_in:
1628         case snd_soc_dapm_dai_out:
1629         case snd_soc_dapm_buffer:
1630         {
1631                 struct sof_ipc4_audio_format *in_fmt;
1632
1633                 in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
1634                 out_ref_rate = in_fmt->sampling_frequency;
1635                 out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
1636
1637                 if (!single_output_format)
1638                         out_ref_valid_bits =
1639                                 SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
1640                 break;
1641         }
1642         case snd_soc_dapm_aif_out:
1643         case snd_soc_dapm_dai_in:
1644                 out_ref_rate = params_rate(fe_params);
1645                 out_ref_channels = params_channels(fe_params);
1646                 if (!single_output_format) {
1647                         out_ref_valid_bits = sof_ipc4_get_valid_bits(sdev, fe_params);
1648                         if (out_ref_valid_bits < 0)
1649                                 return out_ref_valid_bits;
1650                 }
1651                 break;
1652         default:
1653                 /*
1654                  * Unsupported type should be caught by the former switch default
1655                  * case, this should never happen in reality.
1656                  */
1657                 return -EINVAL;
1658         }
1659
1660         /*
1661          * if the output format is the same across all available output formats, choose
1662          * that as the reference.
1663          */
1664         if (single_output_format) {
1665                 struct sof_ipc4_audio_format *out_fmt;
1666
1667                 out_fmt = &available_fmt->output_pin_fmts[0].audio_fmt;
1668                 out_ref_valid_bits =
1669                         SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(out_fmt->fmt_cfg);
1670         }
1671
1672         dev_dbg(sdev->dev, "copier %s: reference output rate %d, channels %d valid_bits %d\n",
1673                 swidget->widget->name, out_ref_rate, out_ref_channels, out_ref_valid_bits);
1674
1675         output_fmt_index = sof_ipc4_init_output_audio_fmt(sdev, &copier_data->base_config,
1676                                                           available_fmt, out_ref_rate,
1677                                                           out_ref_channels, out_ref_valid_bits);
1678         if (output_fmt_index < 0) {
1679                 dev_err(sdev->dev, "Failed to initialize output format for %s",
1680                         swidget->widget->name);
1681                 return output_fmt_index;
1682         }
1683
1684         /*
1685          * Set the output format. Current topology defines pin 0 input and output formats in pairs.
1686          * This assumes that the pin 0 formats are defined before all other pins.
1687          * So pick the output audio format with the same index as the chosen
1688          * input format. This logic will need to be updated when the format definitions
1689          * in topology change.
1690          */
1691         memcpy(&copier_data->out_format,
1692                &available_fmt->output_pin_fmts[output_fmt_index].audio_fmt,
1693                sizeof(struct sof_ipc4_audio_format));
1694         dev_dbg(sdev->dev, "Output audio format for %s\n", swidget->widget->name);
1695         sof_ipc4_dbg_audio_format(sdev->dev, &available_fmt->output_pin_fmts[output_fmt_index], 1);
1696
1697         switch (swidget->id) {
1698         case snd_soc_dapm_dai_in:
1699         case snd_soc_dapm_dai_out:
1700         {
1701                 /*
1702                  * Only SOF_DAI_INTEL_ALH needs copier_data to set blob.
1703                  * That's why only ALH dai's blob is set after sof_ipc4_init_input_audio_fmt
1704                  */
1705                 if (ipc4_copier->dai_type == SOF_DAI_INTEL_ALH) {
1706                         struct sof_ipc4_alh_configuration_blob *blob;
1707                         struct sof_ipc4_copier_data *alh_data;
1708                         struct sof_ipc4_copier *alh_copier;
1709                         struct snd_sof_widget *w;
1710                         u32 ch_count = 0;
1711                         u32 ch_mask = 0;
1712                         u32 ch_map;
1713                         u32 step;
1714                         u32 mask;
1715                         int i;
1716
1717                         blob = (struct sof_ipc4_alh_configuration_blob *)ipc4_copier->copier_config;
1718
1719                         blob->gw_attr.lp_buffer_alloc = 0;
1720
1721                         /* Get channel_mask from ch_map */
1722                         ch_map = copier_data->base_config.audio_fmt.ch_map;
1723                         for (i = 0; ch_map; i++) {
1724                                 if ((ch_map & 0xf) != 0xf) {
1725                                         ch_mask |= BIT(i);
1726                                         ch_count++;
1727                                 }
1728                                 ch_map >>= 4;
1729                         }
1730
1731                         step = ch_count / blob->alh_cfg.device_count;
1732                         mask =  GENMASK(step - 1, 0);
1733                         /*
1734                          * Set each gtw_cfg.node_id to blob->alh_cfg.mapping[]
1735                          * for all widgets with the same stream name
1736                          */
1737                         i = 0;
1738                         list_for_each_entry(w, &sdev->widget_list, list) {
1739                                 if (w->widget->sname &&
1740                                     strcmp(w->widget->sname, swidget->widget->sname))
1741                                         continue;
1742
1743                                 dai = w->private;
1744                                 alh_copier = (struct sof_ipc4_copier *)dai->private;
1745                                 alh_data = &alh_copier->data;
1746                                 blob->alh_cfg.mapping[i].device = alh_data->gtw_cfg.node_id;
1747                                 /*
1748                                  * Set the same channel mask for playback as the audio data is
1749                                  * duplicated for all speakers. For capture, split the channels
1750                                  * among the aggregated DAIs. For example, with 4 channels on 2
1751                                  * aggregated DAIs, the channel_mask should be 0x3 and 0xc for the
1752                                  * two DAI's.
1753                                  * The channel masks used depend on the cpu_dais used in the
1754                                  * dailink at the machine driver level, which actually comes from
1755                                  * the tables in soc_acpi files depending on the _ADR and devID
1756                                  * registers for each codec.
1757                                  */
1758                                 if (w->id == snd_soc_dapm_dai_in)
1759                                         blob->alh_cfg.mapping[i].channel_mask = ch_mask;
1760                                 else
1761                                         blob->alh_cfg.mapping[i].channel_mask = mask << (step * i);
1762
1763                                 i++;
1764                         }
1765                         if (blob->alh_cfg.device_count > 1) {
1766                                 int group_id;
1767
1768                                 group_id = ida_alloc_max(&alh_group_ida, ALH_MULTI_GTW_COUNT - 1,
1769                                                          GFP_KERNEL);
1770
1771                                 if (group_id < 0)
1772                                         return group_id;
1773
1774                                 /* add multi-gateway base */
1775                                 group_id += ALH_MULTI_GTW_BASE;
1776                                 copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
1777                                 copier_data->gtw_cfg.node_id |= SOF_IPC4_NODE_INDEX(group_id);
1778                         }
1779                 }
1780         }
1781         }
1782
1783         /* modify the input params for the next widget */
1784         ret = sof_ipc4_update_hw_params(sdev, pipeline_params, &copier_data->out_format);
1785         if (ret)
1786                 return ret;
1787
1788         /*
1789          * Set the gateway dma_buffer_size to 2ms buffer size to meet the FW expectation. In the
1790          * deep buffer case, set the dma_buffer_size depending on the deep_buffer_dma_ms set
1791          * in topology.
1792          */
1793         switch (swidget->id) {
1794         case snd_soc_dapm_dai_in:
1795                 copier_data->gtw_cfg.dma_buffer_size =
1796                         SOF_IPC4_MIN_DMA_BUFFER_SIZE * copier_data->base_config.ibs;
1797                 break;
1798         case snd_soc_dapm_aif_in:
1799                 copier_data->gtw_cfg.dma_buffer_size =
1800                         max((u32)SOF_IPC4_MIN_DMA_BUFFER_SIZE, deep_buffer_dma_ms) *
1801                                 copier_data->base_config.ibs;
1802                 dev_dbg(sdev->dev, "copier %s, dma buffer%s: %u ms (%u bytes)",
1803                         swidget->widget->name,
1804                         deep_buffer_dma_ms ? " (using Deep Buffer)" : "",
1805                         max((u32)SOF_IPC4_MIN_DMA_BUFFER_SIZE, deep_buffer_dma_ms),
1806                         copier_data->gtw_cfg.dma_buffer_size);
1807                 break;
1808         case snd_soc_dapm_dai_out:
1809         case snd_soc_dapm_aif_out:
1810                 copier_data->gtw_cfg.dma_buffer_size =
1811                         SOF_IPC4_MIN_DMA_BUFFER_SIZE * copier_data->base_config.obs;
1812                 break;
1813         default:
1814                 break;
1815         }
1816
1817         data = &ipc4_copier->copier_config;
1818         ipc_config_size = &ipc4_copier->ipc_config_size;
1819         ipc_config_data = &ipc4_copier->ipc_config_data;
1820
1821         /* config_length is DWORD based */
1822         gtw_cfg_config_length = copier_data->gtw_cfg.config_length * 4;
1823         ipc_size = sizeof(*copier_data) + gtw_cfg_config_length;
1824
1825         if (ipc4_copier->dma_config_tlv.type == SOF_IPC4_GTW_DMA_CONFIG_ID &&
1826             ipc4_copier->dma_config_tlv.length) {
1827                 dma_config_tlv_size = sizeof(ipc4_copier->dma_config_tlv) +
1828                         ipc4_copier->dma_config_tlv.dma_config.dma_priv_config_size;
1829
1830                 /* paranoia check on TLV size/length */
1831                 if (dma_config_tlv_size != ipc4_copier->dma_config_tlv.length +
1832                     sizeof(uint32_t) * 2) {
1833                         dev_err(sdev->dev, "Invalid configuration, TLV size %d length %d\n",
1834                                 dma_config_tlv_size, ipc4_copier->dma_config_tlv.length);
1835                         return -EINVAL;
1836                 }
1837
1838                 ipc_size += dma_config_tlv_size;
1839
1840                 /* we also need to increase the size at the gtw level */
1841                 copier_data->gtw_cfg.config_length += dma_config_tlv_size / 4;
1842         }
1843
1844         dev_dbg(sdev->dev, "copier %s, IPC size is %d", swidget->widget->name, ipc_size);
1845
1846         *ipc_config_data = kzalloc(ipc_size, GFP_KERNEL);
1847         if (!*ipc_config_data)
1848                 return -ENOMEM;
1849
1850         *ipc_config_size = ipc_size;
1851
1852         /* update pipeline memory usage */
1853         sof_ipc4_update_resource_usage(sdev, swidget, &copier_data->base_config);
1854
1855         /* copy IPC data */
1856         memcpy(*ipc_config_data, (void *)copier_data, sizeof(*copier_data));
1857         if (gtw_cfg_config_length)
1858                 memcpy(*ipc_config_data + sizeof(*copier_data),
1859                        *data, gtw_cfg_config_length);
1860
1861         /* add DMA Config TLV, if configured */
1862         if (dma_config_tlv_size)
1863                 memcpy(*ipc_config_data + sizeof(*copier_data) +
1864                        gtw_cfg_config_length,
1865                        &ipc4_copier->dma_config_tlv, dma_config_tlv_size);
1866
1867         /*
1868          * Restore gateway config length now that IPC payload is prepared. This avoids
1869          * counting the DMA CONFIG TLV multiple times
1870          */
1871         copier_data->gtw_cfg.config_length = gtw_cfg_config_length / 4;
1872
1873         return 0;
1874 }
1875
1876 static int sof_ipc4_prepare_gain_module(struct snd_sof_widget *swidget,
1877                                         struct snd_pcm_hw_params *fe_params,
1878                                         struct snd_sof_platform_stream_params *platform_params,
1879                                         struct snd_pcm_hw_params *pipeline_params, int dir)
1880 {
1881         struct snd_soc_component *scomp = swidget->scomp;
1882         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
1883         struct sof_ipc4_gain *gain = swidget->private;
1884         struct sof_ipc4_available_audio_format *available_fmt = &gain->available_fmt;
1885         struct sof_ipc4_audio_format *in_fmt;
1886         u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
1887         int ret;
1888
1889         ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, &gain->data.base_config,
1890                                             pipeline_params, available_fmt);
1891         if (ret < 0)
1892                 return ret;
1893
1894         in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
1895         out_ref_rate = in_fmt->sampling_frequency;
1896         out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
1897         out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
1898
1899         ret = sof_ipc4_init_output_audio_fmt(sdev, &gain->data.base_config, available_fmt,
1900                                              out_ref_rate, out_ref_channels, out_ref_valid_bits);
1901         if (ret < 0) {
1902                 dev_err(sdev->dev, "Failed to initialize output format for %s",
1903                         swidget->widget->name);
1904                 return ret;
1905         }
1906
1907         /* update pipeline memory usage */
1908         sof_ipc4_update_resource_usage(sdev, swidget, &gain->data.base_config);
1909
1910         return 0;
1911 }
1912
1913 static int sof_ipc4_prepare_mixer_module(struct snd_sof_widget *swidget,
1914                                          struct snd_pcm_hw_params *fe_params,
1915                                          struct snd_sof_platform_stream_params *platform_params,
1916                                          struct snd_pcm_hw_params *pipeline_params, int dir)
1917 {
1918         struct snd_soc_component *scomp = swidget->scomp;
1919         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
1920         struct sof_ipc4_mixer *mixer = swidget->private;
1921         struct sof_ipc4_available_audio_format *available_fmt = &mixer->available_fmt;
1922         struct sof_ipc4_audio_format *in_fmt;
1923         u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
1924         int ret;
1925
1926         ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, &mixer->base_config,
1927                                             pipeline_params, available_fmt);
1928         if (ret < 0)
1929                 return ret;
1930
1931         in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
1932         out_ref_rate = in_fmt->sampling_frequency;
1933         out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
1934         out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
1935
1936         ret = sof_ipc4_init_output_audio_fmt(sdev, &mixer->base_config, available_fmt,
1937                                              out_ref_rate, out_ref_channels, out_ref_valid_bits);
1938         if (ret < 0) {
1939                 dev_err(sdev->dev, "Failed to initialize output format for %s",
1940                         swidget->widget->name);
1941                 return ret;
1942         }
1943
1944         /* update pipeline memory usage */
1945         sof_ipc4_update_resource_usage(sdev, swidget, &mixer->base_config);
1946
1947         return 0;
1948 }
1949
1950 static int sof_ipc4_prepare_src_module(struct snd_sof_widget *swidget,
1951                                        struct snd_pcm_hw_params *fe_params,
1952                                        struct snd_sof_platform_stream_params *platform_params,
1953                                        struct snd_pcm_hw_params *pipeline_params, int dir)
1954 {
1955         struct snd_soc_component *scomp = swidget->scomp;
1956         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
1957         struct sof_ipc4_src *src = swidget->private;
1958         struct sof_ipc4_available_audio_format *available_fmt = &src->available_fmt;
1959         struct sof_ipc4_audio_format *out_audio_fmt;
1960         struct sof_ipc4_audio_format *in_audio_fmt;
1961         u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
1962         int output_format_index, input_format_index;
1963
1964         input_format_index = sof_ipc4_init_input_audio_fmt(sdev, swidget, &src->data.base_config,
1965                                                            pipeline_params, available_fmt);
1966         if (input_format_index < 0)
1967                 return input_format_index;
1968
1969         /*
1970          * For playback, the SRC sink rate will be configured based on the requested output
1971          * format, which is restricted to only deal with DAI's with a single format for now.
1972          */
1973         if (dir == SNDRV_PCM_STREAM_PLAYBACK && available_fmt->num_output_formats > 1) {
1974                 dev_err(sdev->dev, "Invalid number of output formats: %d for SRC %s\n",
1975                         available_fmt->num_output_formats, swidget->widget->name);
1976                 return -EINVAL;
1977         }
1978
1979         /*
1980          * SRC does not perform format conversion, so the output channels and valid bit depth must
1981          * be the same as that of the input.
1982          */
1983         in_audio_fmt = &available_fmt->input_pin_fmts[input_format_index].audio_fmt;
1984         out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_audio_fmt->fmt_cfg);
1985         out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_audio_fmt->fmt_cfg);
1986
1987         /*
1988          * For capture, the SRC module should convert the rate to match the rate requested by the
1989          * PCM hw_params. Set the reference params based on the fe_params unconditionally as it
1990          * will be ignored for playback anyway.
1991          */
1992         out_ref_rate = params_rate(fe_params);
1993
1994         output_format_index = sof_ipc4_init_output_audio_fmt(sdev, &src->data.base_config,
1995                                                              available_fmt, out_ref_rate,
1996                                                              out_ref_channels, out_ref_valid_bits);
1997         if (output_format_index < 0) {
1998                 dev_err(sdev->dev, "Failed to initialize output format for %s",
1999                         swidget->widget->name);
2000                 return output_format_index;
2001         }
2002
2003         /* update pipeline memory usage */
2004         sof_ipc4_update_resource_usage(sdev, swidget, &src->data.base_config);
2005
2006         out_audio_fmt = &available_fmt->output_pin_fmts[output_format_index].audio_fmt;
2007         src->data.sink_rate = out_audio_fmt->sampling_frequency;
2008
2009         /* update pipeline_params for sink widgets */
2010         return sof_ipc4_update_hw_params(sdev, pipeline_params, out_audio_fmt);
2011 }
2012
2013 static int
2014 sof_ipc4_process_set_pin_formats(struct snd_sof_widget *swidget, int pin_type)
2015 {
2016         struct sof_ipc4_process *process = swidget->private;
2017         struct sof_ipc4_base_module_cfg_ext *base_cfg_ext = process->base_config_ext;
2018         struct sof_ipc4_available_audio_format *available_fmt = &process->available_fmt;
2019         struct sof_ipc4_pin_format *pin_format, *format_list_to_search;
2020         struct snd_soc_component *scomp = swidget->scomp;
2021         int num_pins, format_list_count;
2022         int pin_format_offset = 0;
2023         int i, j;
2024
2025         /* set number of pins, offset of pin format and format list to search based on pin type */
2026         if (pin_type == SOF_PIN_TYPE_INPUT) {
2027                 num_pins = swidget->num_input_pins;
2028                 format_list_to_search = available_fmt->input_pin_fmts;
2029                 format_list_count = available_fmt->num_input_formats;
2030         } else {
2031                 num_pins = swidget->num_output_pins;
2032                 pin_format_offset = swidget->num_input_pins;
2033                 format_list_to_search = available_fmt->output_pin_fmts;
2034                 format_list_count = available_fmt->num_output_formats;
2035         }
2036
2037         for (i = pin_format_offset; i < num_pins + pin_format_offset; i++) {
2038                 pin_format = &base_cfg_ext->pin_formats[i];
2039
2040                 /* Pin 0 audio formats are derived from the base config input/output format */
2041                 if (i == pin_format_offset) {
2042                         if (pin_type == SOF_PIN_TYPE_INPUT) {
2043                                 pin_format->buffer_size = process->base_config.ibs;
2044                                 pin_format->audio_fmt = process->base_config.audio_fmt;
2045                         } else {
2046                                 pin_format->buffer_size = process->base_config.obs;
2047                                 pin_format->audio_fmt = process->output_format;
2048                         }
2049                         continue;
2050                 }
2051
2052                 /*
2053                  * For all other pins, find the pin formats from those set in topology. If there
2054                  * is more than one format specified for a pin, this will pick the first available
2055                  * one.
2056                  */
2057                 for (j = 0; j < format_list_count; j++) {
2058                         struct sof_ipc4_pin_format *pin_format_item = &format_list_to_search[j];
2059
2060                         if (pin_format_item->pin_index == i - pin_format_offset) {
2061                                 *pin_format = *pin_format_item;
2062                                 break;
2063                         }
2064                 }
2065
2066                 if (j == format_list_count) {
2067                         dev_err(scomp->dev, "%s pin %d format not found for %s\n",
2068                                 (pin_type == SOF_PIN_TYPE_INPUT) ? "input" : "output",
2069                                 i - pin_format_offset, swidget->widget->name);
2070                         return -EINVAL;
2071                 }
2072         }
2073
2074         return 0;
2075 }
2076
2077 static int sof_ipc4_process_add_base_cfg_extn(struct snd_sof_widget *swidget)
2078 {
2079         int ret, i;
2080
2081         /* copy input and output pin formats */
2082         for (i = 0; i <= SOF_PIN_TYPE_OUTPUT; i++) {
2083                 ret = sof_ipc4_process_set_pin_formats(swidget, i);
2084                 if (ret < 0)
2085                         return ret;
2086         }
2087
2088         return 0;
2089 }
2090
2091 static int sof_ipc4_prepare_process_module(struct snd_sof_widget *swidget,
2092                                            struct snd_pcm_hw_params *fe_params,
2093                                            struct snd_sof_platform_stream_params *platform_params,
2094                                            struct snd_pcm_hw_params *pipeline_params, int dir)
2095 {
2096         struct snd_soc_component *scomp = swidget->scomp;
2097         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
2098         struct sof_ipc4_process *process = swidget->private;
2099         struct sof_ipc4_available_audio_format *available_fmt = &process->available_fmt;
2100         struct sof_ipc4_audio_format *in_fmt;
2101         u32 out_ref_rate, out_ref_channels, out_ref_valid_bits;
2102         void *cfg = process->ipc_config_data;
2103         int output_fmt_index;
2104         int ret;
2105
2106         ret = sof_ipc4_init_input_audio_fmt(sdev, swidget, &process->base_config,
2107                                             pipeline_params, available_fmt);
2108         if (ret < 0)
2109                 return ret;
2110
2111         in_fmt = &available_fmt->input_pin_fmts[ret].audio_fmt;
2112         out_ref_rate = in_fmt->sampling_frequency;
2113         out_ref_channels = SOF_IPC4_AUDIO_FORMAT_CFG_CHANNELS_COUNT(in_fmt->fmt_cfg);
2114         out_ref_valid_bits = SOF_IPC4_AUDIO_FORMAT_CFG_V_BIT_DEPTH(in_fmt->fmt_cfg);
2115
2116         output_fmt_index = sof_ipc4_init_output_audio_fmt(sdev, &process->base_config,
2117                                                           available_fmt, out_ref_rate,
2118                                                           out_ref_channels, out_ref_valid_bits);
2119         if (output_fmt_index < 0 && available_fmt->num_output_formats) {
2120                 dev_err(sdev->dev, "Failed to initialize output format for %s",
2121                         swidget->widget->name);
2122                 return output_fmt_index;
2123         }
2124
2125         /* copy Pin 0 output format */
2126         if (available_fmt->num_output_formats &&
2127             output_fmt_index < available_fmt->num_output_formats &&
2128             !available_fmt->output_pin_fmts[output_fmt_index].pin_index) {
2129                 memcpy(&process->output_format,
2130                        &available_fmt->output_pin_fmts[output_fmt_index].audio_fmt,
2131                        sizeof(struct sof_ipc4_audio_format));
2132
2133                 /* modify the pipeline params with the pin 0 output format */
2134                 ret = sof_ipc4_update_hw_params(sdev, pipeline_params, &process->output_format);
2135                 if (ret)
2136                         return ret;
2137         }
2138
2139         /* update pipeline memory usage */
2140         sof_ipc4_update_resource_usage(sdev, swidget, &process->base_config);
2141
2142         /* ipc_config_data is composed of the base_config followed by an optional extension */
2143         memcpy(cfg, &process->base_config, sizeof(struct sof_ipc4_base_module_cfg));
2144         cfg += sizeof(struct sof_ipc4_base_module_cfg);
2145
2146         if (process->init_config == SOF_IPC4_MODULE_INIT_CONFIG_TYPE_BASE_CFG_WITH_EXT) {
2147                 struct sof_ipc4_base_module_cfg_ext *base_cfg_ext = process->base_config_ext;
2148
2149                 ret = sof_ipc4_process_add_base_cfg_extn(swidget);
2150                 if (ret < 0)
2151                         return ret;
2152
2153                 memcpy(cfg, base_cfg_ext, process->base_config_ext_size);
2154         }
2155
2156         return 0;
2157 }
2158
2159 static int sof_ipc4_control_load_volume(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2160 {
2161         struct sof_ipc4_control_data *control_data;
2162         struct sof_ipc4_msg *msg;
2163         int i;
2164
2165         scontrol->size = struct_size(control_data, chanv, scontrol->num_channels);
2166
2167         /* scontrol->ipc_control_data will be freed in sof_control_unload */
2168         scontrol->ipc_control_data = kzalloc(scontrol->size, GFP_KERNEL);
2169         if (!scontrol->ipc_control_data)
2170                 return -ENOMEM;
2171
2172         control_data = scontrol->ipc_control_data;
2173         control_data->index = scontrol->index;
2174
2175         msg = &control_data->msg;
2176         msg->primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
2177         msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2178         msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2179
2180         /* volume controls with range 0-1 (off/on) are switch controls */
2181         if (scontrol->max == 1)
2182                 msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_SWITCH_CONTROL_PARAM_ID);
2183         else
2184                 msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_GAIN_PARAM_ID);
2185
2186         for (i = 0; i < scontrol->num_channels; i++) {
2187                 control_data->chanv[i].channel = i;
2188                 /*
2189                  * Default, initial values:
2190                  * - 0dB for volume controls
2191                  * - off (0) for switch controls - value already zero after
2192                  *                                 memory allocation
2193                  */
2194                 if (scontrol->max > 1)
2195                         control_data->chanv[i].value = SOF_IPC4_VOL_ZERO_DB;
2196         }
2197
2198         return 0;
2199 }
2200
2201 static int sof_ipc4_control_load_enum(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2202 {
2203         struct sof_ipc4_control_data *control_data;
2204         struct sof_ipc4_msg *msg;
2205         int i;
2206
2207         scontrol->size = struct_size(control_data, chanv, scontrol->num_channels);
2208
2209         /* scontrol->ipc_control_data will be freed in sof_control_unload */
2210         scontrol->ipc_control_data = kzalloc(scontrol->size, GFP_KERNEL);
2211         if (!scontrol->ipc_control_data)
2212                 return -ENOMEM;
2213
2214         control_data = scontrol->ipc_control_data;
2215         control_data->index = scontrol->index;
2216
2217         msg = &control_data->msg;
2218         msg->primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
2219         msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2220         msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2221
2222         msg->extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_ENUM_CONTROL_PARAM_ID);
2223
2224         /* Default, initial value for enums: first enum entry is selected (0) */
2225         for (i = 0; i < scontrol->num_channels; i++)
2226                 control_data->chanv[i].channel = i;
2227
2228         return 0;
2229 }
2230
2231 static int sof_ipc4_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2232 {
2233         struct sof_ipc4_control_data *control_data;
2234         struct sof_ipc4_msg *msg;
2235         int ret;
2236
2237         if (scontrol->max_size < (sizeof(*control_data) + sizeof(struct sof_abi_hdr))) {
2238                 dev_err(sdev->dev, "insufficient size for a bytes control %s: %zu.\n",
2239                         scontrol->name, scontrol->max_size);
2240                 return -EINVAL;
2241         }
2242
2243         if (scontrol->priv_size > scontrol->max_size - sizeof(*control_data)) {
2244                 dev_err(sdev->dev, "scontrol %s bytes data size %zu exceeds max %zu.\n",
2245                         scontrol->name, scontrol->priv_size,
2246                         scontrol->max_size - sizeof(*control_data));
2247                 return -EINVAL;
2248         }
2249
2250         scontrol->size = sizeof(struct sof_ipc4_control_data) + scontrol->priv_size;
2251
2252         scontrol->ipc_control_data = kzalloc(scontrol->max_size, GFP_KERNEL);
2253         if (!scontrol->ipc_control_data)
2254                 return -ENOMEM;
2255
2256         control_data = scontrol->ipc_control_data;
2257         control_data->index = scontrol->index;
2258         if (scontrol->priv_size > 0) {
2259                 memcpy(control_data->data, scontrol->priv, scontrol->priv_size);
2260                 kfree(scontrol->priv);
2261                 scontrol->priv = NULL;
2262
2263                 if (control_data->data->magic != SOF_IPC4_ABI_MAGIC) {
2264                         dev_err(sdev->dev, "Wrong ABI magic (%#x) for control: %s\n",
2265                                 control_data->data->magic, scontrol->name);
2266                         ret = -EINVAL;
2267                         goto err;
2268                 }
2269
2270                 /* TODO: check the ABI version */
2271
2272                 if (control_data->data->size + sizeof(struct sof_abi_hdr) !=
2273                     scontrol->priv_size) {
2274                         dev_err(sdev->dev, "Control %s conflict in bytes %zu vs. priv size %zu.\n",
2275                                 scontrol->name,
2276                                 control_data->data->size + sizeof(struct sof_abi_hdr),
2277                                 scontrol->priv_size);
2278                         ret = -EINVAL;
2279                         goto err;
2280                 }
2281         }
2282
2283         msg = &control_data->msg;
2284         msg->primary = SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_LARGE_CONFIG_SET);
2285         msg->primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2286         msg->primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2287
2288         return 0;
2289
2290 err:
2291         kfree(scontrol->ipc_control_data);
2292         scontrol->ipc_control_data = NULL;
2293         return ret;
2294 }
2295
2296 static int sof_ipc4_control_setup(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol)
2297 {
2298         switch (scontrol->info_type) {
2299         case SND_SOC_TPLG_CTL_VOLSW:
2300         case SND_SOC_TPLG_CTL_VOLSW_SX:
2301         case SND_SOC_TPLG_CTL_VOLSW_XR_SX:
2302                 return sof_ipc4_control_load_volume(sdev, scontrol);
2303         case SND_SOC_TPLG_CTL_BYTES:
2304                 return sof_ipc4_control_load_bytes(sdev, scontrol);
2305         case SND_SOC_TPLG_CTL_ENUM:
2306         case SND_SOC_TPLG_CTL_ENUM_VALUE:
2307                 return sof_ipc4_control_load_enum(sdev, scontrol);
2308         default:
2309                 break;
2310         }
2311
2312         return 0;
2313 }
2314
2315 static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget)
2316 {
2317         struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
2318         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
2319         struct sof_ipc4_pipeline *pipeline;
2320         struct sof_ipc4_msg *msg;
2321         void *ipc_data = NULL;
2322         u32 ipc_size = 0;
2323         int ret;
2324
2325         switch (swidget->id) {
2326         case snd_soc_dapm_scheduler:
2327                 pipeline = swidget->private;
2328
2329                 if (pipeline->use_chain_dma) {
2330                         dev_warn(sdev->dev, "use_chain_dma set for scheduler %s",
2331                                  swidget->widget->name);
2332                         return 0;
2333                 }
2334
2335                 dev_dbg(sdev->dev, "pipeline: %d memory pages: %d\n", swidget->pipeline_id,
2336                         pipeline->mem_usage);
2337
2338                 msg = &pipeline->msg;
2339                 msg->primary |= pipeline->mem_usage;
2340
2341                 swidget->instance_id = ida_alloc_max(&pipeline_ida, ipc4_data->max_num_pipelines,
2342                                                      GFP_KERNEL);
2343                 if (swidget->instance_id < 0) {
2344                         dev_err(sdev->dev, "failed to assign pipeline id for %s: %d\n",
2345                                 swidget->widget->name, swidget->instance_id);
2346                         return swidget->instance_id;
2347                 }
2348                 msg->primary &= ~SOF_IPC4_GLB_PIPE_INSTANCE_MASK;
2349                 msg->primary |= SOF_IPC4_GLB_PIPE_INSTANCE_ID(swidget->instance_id);
2350                 break;
2351         case snd_soc_dapm_aif_in:
2352         case snd_soc_dapm_aif_out:
2353         case snd_soc_dapm_buffer:
2354         {
2355                 struct sof_ipc4_copier *ipc4_copier = swidget->private;
2356
2357                 pipeline = pipe_widget->private;
2358                 if (pipeline->use_chain_dma)
2359                         return 0;
2360
2361                 ipc_size = ipc4_copier->ipc_config_size;
2362                 ipc_data = ipc4_copier->ipc_config_data;
2363
2364                 msg = &ipc4_copier->msg;
2365                 break;
2366         }
2367         case snd_soc_dapm_dai_in:
2368         case snd_soc_dapm_dai_out:
2369         {
2370                 struct snd_sof_dai *dai = swidget->private;
2371                 struct sof_ipc4_copier *ipc4_copier = dai->private;
2372
2373                 pipeline = pipe_widget->private;
2374                 if (pipeline->use_chain_dma)
2375                         return 0;
2376
2377                 ipc_size = ipc4_copier->ipc_config_size;
2378                 ipc_data = ipc4_copier->ipc_config_data;
2379
2380                 msg = &ipc4_copier->msg;
2381                 break;
2382         }
2383         case snd_soc_dapm_pga:
2384         {
2385                 struct sof_ipc4_gain *gain = swidget->private;
2386
2387                 ipc_size = sizeof(gain->data);
2388                 ipc_data = &gain->data;
2389
2390                 msg = &gain->msg;
2391                 break;
2392         }
2393         case snd_soc_dapm_mixer:
2394         {
2395                 struct sof_ipc4_mixer *mixer = swidget->private;
2396
2397                 ipc_size = sizeof(mixer->base_config);
2398                 ipc_data = &mixer->base_config;
2399
2400                 msg = &mixer->msg;
2401                 break;
2402         }
2403         case snd_soc_dapm_src:
2404         {
2405                 struct sof_ipc4_src *src = swidget->private;
2406
2407                 ipc_size = sizeof(src->data);
2408                 ipc_data = &src->data;
2409
2410                 msg = &src->msg;
2411                 break;
2412         }
2413         case snd_soc_dapm_effect:
2414         {
2415                 struct sof_ipc4_process *process = swidget->private;
2416
2417                 if (!process->ipc_config_size) {
2418                         dev_err(sdev->dev, "module %s has no config data!\n",
2419                                 swidget->widget->name);
2420                         return -EINVAL;
2421                 }
2422
2423                 ipc_size = process->ipc_config_size;
2424                 ipc_data = process->ipc_config_data;
2425
2426                 msg = &process->msg;
2427                 break;
2428         }
2429         default:
2430                 dev_err(sdev->dev, "widget type %d not supported", swidget->id);
2431                 return -EINVAL;
2432         }
2433
2434         if (swidget->id != snd_soc_dapm_scheduler) {
2435                 int module_id = msg->primary & SOF_IPC4_MOD_ID_MASK;
2436
2437                 ret = sof_ipc4_widget_assign_instance_id(sdev, swidget);
2438                 if (ret < 0) {
2439                         dev_err(sdev->dev, "failed to assign instance id for %s\n",
2440                                 swidget->widget->name);
2441                         return ret;
2442                 }
2443
2444                 msg->primary &= ~SOF_IPC4_MOD_INSTANCE_MASK;
2445                 msg->primary |= SOF_IPC4_MOD_INSTANCE(swidget->instance_id);
2446
2447                 msg->extension &= ~SOF_IPC4_MOD_EXT_PARAM_SIZE_MASK;
2448                 msg->extension |= ipc_size >> 2;
2449
2450                 msg->extension &= ~SOF_IPC4_MOD_EXT_PPL_ID_MASK;
2451                 msg->extension |= SOF_IPC4_MOD_EXT_PPL_ID(pipe_widget->instance_id);
2452
2453                 dev_dbg(sdev->dev, "Create widget %s (pipe %d) - ID %d, instance %d, core %d\n",
2454                         swidget->widget->name, swidget->pipeline_id, module_id,
2455                         swidget->instance_id, swidget->core);
2456         } else {
2457                 dev_dbg(sdev->dev, "Create pipeline %s (pipe %d) - instance %d, core %d\n",
2458                         swidget->widget->name, swidget->pipeline_id,
2459                         swidget->instance_id, swidget->core);
2460         }
2461
2462         msg->data_size = ipc_size;
2463         msg->data_ptr = ipc_data;
2464
2465         ret = sof_ipc_tx_message_no_reply(sdev->ipc, msg, ipc_size);
2466         if (ret < 0) {
2467                 dev_err(sdev->dev, "failed to create module %s\n", swidget->widget->name);
2468
2469                 if (swidget->id != snd_soc_dapm_scheduler) {
2470                         struct sof_ipc4_fw_module *fw_module = swidget->module_info;
2471
2472                         ida_free(&fw_module->m_ida, swidget->instance_id);
2473                 } else {
2474                         ida_free(&pipeline_ida, swidget->instance_id);
2475                 }
2476         }
2477
2478         return ret;
2479 }
2480
2481 static int sof_ipc4_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget)
2482 {
2483         struct sof_ipc4_fw_module *fw_module = swidget->module_info;
2484         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
2485         int ret = 0;
2486
2487         mutex_lock(&ipc4_data->pipeline_state_mutex);
2488
2489         /* freeing a pipeline frees all the widgets associated with it */
2490         if (swidget->id == snd_soc_dapm_scheduler) {
2491                 struct sof_ipc4_pipeline *pipeline = swidget->private;
2492                 struct sof_ipc4_msg msg = {{ 0 }};
2493                 u32 header;
2494
2495                 if (pipeline->use_chain_dma) {
2496                         dev_warn(sdev->dev, "use_chain_dma set for scheduler %s",
2497                                  swidget->widget->name);
2498                         mutex_unlock(&ipc4_data->pipeline_state_mutex);
2499                         return 0;
2500                 }
2501
2502                 header = SOF_IPC4_GLB_PIPE_INSTANCE_ID(swidget->instance_id);
2503                 header |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_GLB_DELETE_PIPELINE);
2504                 header |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2505                 header |= SOF_IPC4_MSG_TARGET(SOF_IPC4_FW_GEN_MSG);
2506
2507                 msg.primary = header;
2508
2509                 ret = sof_ipc_tx_message_no_reply(sdev->ipc, &msg, 0);
2510                 if (ret < 0)
2511                         dev_err(sdev->dev, "failed to free pipeline widget %s\n",
2512                                 swidget->widget->name);
2513
2514                 pipeline->mem_usage = 0;
2515                 pipeline->state = SOF_IPC4_PIPE_UNINITIALIZED;
2516                 ida_free(&pipeline_ida, swidget->instance_id);
2517                 swidget->instance_id = -EINVAL;
2518         } else {
2519                 struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
2520                 struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
2521
2522                 if (!pipeline->use_chain_dma)
2523                         ida_free(&fw_module->m_ida, swidget->instance_id);
2524         }
2525
2526         mutex_unlock(&ipc4_data->pipeline_state_mutex);
2527
2528         return ret;
2529 }
2530
2531 static int sof_ipc4_get_queue_id(struct snd_sof_widget *src_widget,
2532                                  struct snd_sof_widget *sink_widget, bool pin_type)
2533 {
2534         struct snd_sof_widget *current_swidget;
2535         struct snd_soc_component *scomp;
2536         struct ida *queue_ida;
2537         const char *buddy_name;
2538         char **pin_binding;
2539         u32 num_pins;
2540         int i;
2541
2542         if (pin_type == SOF_PIN_TYPE_OUTPUT) {
2543                 current_swidget = src_widget;
2544                 pin_binding = src_widget->output_pin_binding;
2545                 queue_ida = &src_widget->output_queue_ida;
2546                 num_pins = src_widget->num_output_pins;
2547                 buddy_name = sink_widget->widget->name;
2548         } else {
2549                 current_swidget = sink_widget;
2550                 pin_binding = sink_widget->input_pin_binding;
2551                 queue_ida = &sink_widget->input_queue_ida;
2552                 num_pins = sink_widget->num_input_pins;
2553                 buddy_name = src_widget->widget->name;
2554         }
2555
2556         scomp = current_swidget->scomp;
2557
2558         if (num_pins < 1) {
2559                 dev_err(scomp->dev, "invalid %s num_pins: %d for queue allocation for %s\n",
2560                         (pin_type == SOF_PIN_TYPE_OUTPUT ? "output" : "input"),
2561                         num_pins, current_swidget->widget->name);
2562                 return -EINVAL;
2563         }
2564
2565         /* If there is only one input/output pin, queue id must be 0 */
2566         if (num_pins == 1)
2567                 return 0;
2568
2569         /* Allocate queue ID from pin binding array if it is defined in topology. */
2570         if (pin_binding) {
2571                 for (i = 0; i < num_pins; i++) {
2572                         if (!strcmp(pin_binding[i], buddy_name))
2573                                 return i;
2574                 }
2575                 /*
2576                  * Fail if no queue ID found from pin binding array, so that we don't
2577                  * mixed use pin binding array and ida for queue ID allocation.
2578                  */
2579                 dev_err(scomp->dev, "no %s queue id found from pin binding array for %s\n",
2580                         (pin_type == SOF_PIN_TYPE_OUTPUT ? "output" : "input"),
2581                         current_swidget->widget->name);
2582                 return -EINVAL;
2583         }
2584
2585         /* If no pin binding array specified in topology, use ida to allocate one */
2586         return ida_alloc_max(queue_ida, num_pins, GFP_KERNEL);
2587 }
2588
2589 static void sof_ipc4_put_queue_id(struct snd_sof_widget *swidget, int queue_id,
2590                                   bool pin_type)
2591 {
2592         struct ida *queue_ida;
2593         char **pin_binding;
2594         int num_pins;
2595
2596         if (pin_type == SOF_PIN_TYPE_OUTPUT) {
2597                 pin_binding = swidget->output_pin_binding;
2598                 queue_ida = &swidget->output_queue_ida;
2599                 num_pins = swidget->num_output_pins;
2600         } else {
2601                 pin_binding = swidget->input_pin_binding;
2602                 queue_ida = &swidget->input_queue_ida;
2603                 num_pins = swidget->num_input_pins;
2604         }
2605
2606         /* Nothing to free if queue ID is not allocated with ida. */
2607         if (num_pins == 1 || pin_binding)
2608                 return;
2609
2610         ida_free(queue_ida, queue_id);
2611 }
2612
2613 static int sof_ipc4_set_copier_sink_format(struct snd_sof_dev *sdev,
2614                                            struct snd_sof_widget *src_widget,
2615                                            struct snd_sof_widget *sink_widget,
2616                                            int sink_id)
2617 {
2618         struct sof_ipc4_copier_config_set_sink_format format;
2619         const struct sof_ipc_ops *iops = sdev->ipc->ops;
2620         struct sof_ipc4_base_module_cfg *src_config;
2621         const struct sof_ipc4_audio_format *pin_fmt;
2622         struct sof_ipc4_fw_module *fw_module;
2623         struct sof_ipc4_msg msg = {{ 0 }};
2624
2625         dev_dbg(sdev->dev, "%s set copier sink %d format\n",
2626                 src_widget->widget->name, sink_id);
2627
2628         if (WIDGET_IS_DAI(src_widget->id)) {
2629                 struct snd_sof_dai *dai = src_widget->private;
2630
2631                 src_config = dai->private;
2632         } else {
2633                 src_config = src_widget->private;
2634         }
2635
2636         fw_module = src_widget->module_info;
2637
2638         format.sink_id = sink_id;
2639         memcpy(&format.source_fmt, &src_config->audio_fmt, sizeof(format.source_fmt));
2640
2641         pin_fmt = sof_ipc4_get_input_pin_audio_fmt(sink_widget, sink_id);
2642         if (!pin_fmt) {
2643                 dev_err(sdev->dev, "Unable to get pin %d format for %s",
2644                         sink_id, sink_widget->widget->name);
2645                 return -EINVAL;
2646         }
2647
2648         memcpy(&format.sink_fmt, pin_fmt, sizeof(format.sink_fmt));
2649
2650         msg.data_size = sizeof(format);
2651         msg.data_ptr = &format;
2652
2653         msg.primary = fw_module->man4_module_entry.id;
2654         msg.primary |= SOF_IPC4_MOD_INSTANCE(src_widget->instance_id);
2655         msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2656         msg.primary |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2657
2658         msg.extension =
2659                 SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_COPIER_MODULE_CFG_PARAM_SET_SINK_FORMAT);
2660
2661         return iops->set_get_data(sdev, &msg, msg.data_size, true);
2662 }
2663
2664 static int sof_ipc4_route_setup(struct snd_sof_dev *sdev, struct snd_sof_route *sroute)
2665 {
2666         struct snd_sof_widget *src_widget = sroute->src_widget;
2667         struct snd_sof_widget *sink_widget = sroute->sink_widget;
2668         struct snd_sof_widget *src_pipe_widget = src_widget->spipe->pipe_widget;
2669         struct snd_sof_widget *sink_pipe_widget = sink_widget->spipe->pipe_widget;
2670         struct sof_ipc4_fw_module *src_fw_module = src_widget->module_info;
2671         struct sof_ipc4_fw_module *sink_fw_module = sink_widget->module_info;
2672         struct sof_ipc4_pipeline *src_pipeline = src_pipe_widget->private;
2673         struct sof_ipc4_pipeline *sink_pipeline = sink_pipe_widget->private;
2674         struct sof_ipc4_msg msg = {{ 0 }};
2675         u32 header, extension;
2676         int ret;
2677
2678         /* no route set up if chain DMA is used */
2679         if (src_pipeline->use_chain_dma || sink_pipeline->use_chain_dma) {
2680                 if (!src_pipeline->use_chain_dma || !sink_pipeline->use_chain_dma) {
2681                         dev_err(sdev->dev,
2682                                 "use_chain_dma must be set for both src %s and sink %s pipelines\n",
2683                                 src_widget->widget->name, sink_widget->widget->name);
2684                         return -EINVAL;
2685                 }
2686                 return 0;
2687         }
2688
2689         if (!src_fw_module || !sink_fw_module) {
2690                 dev_err(sdev->dev,
2691                         "cannot bind %s -> %s, no firmware module for: %s%s\n",
2692                         src_widget->widget->name, sink_widget->widget->name,
2693                         src_fw_module ? "" : " source",
2694                         sink_fw_module ? "" : " sink");
2695
2696                 return -ENODEV;
2697         }
2698
2699         sroute->src_queue_id = sof_ipc4_get_queue_id(src_widget, sink_widget,
2700                                                      SOF_PIN_TYPE_OUTPUT);
2701         if (sroute->src_queue_id < 0) {
2702                 dev_err(sdev->dev, "failed to get queue ID for source widget: %s\n",
2703                         src_widget->widget->name);
2704                 return sroute->src_queue_id;
2705         }
2706
2707         sroute->dst_queue_id = sof_ipc4_get_queue_id(src_widget, sink_widget,
2708                                                      SOF_PIN_TYPE_INPUT);
2709         if (sroute->dst_queue_id < 0) {
2710                 dev_err(sdev->dev, "failed to get queue ID for sink widget: %s\n",
2711                         sink_widget->widget->name);
2712                 sof_ipc4_put_queue_id(src_widget, sroute->src_queue_id,
2713                                       SOF_PIN_TYPE_OUTPUT);
2714                 return sroute->dst_queue_id;
2715         }
2716
2717         /* Pin 0 format is already set during copier module init */
2718         if (sroute->src_queue_id > 0 && WIDGET_IS_COPIER(src_widget->id)) {
2719                 ret = sof_ipc4_set_copier_sink_format(sdev, src_widget, sink_widget,
2720                                                       sroute->src_queue_id);
2721                 if (ret < 0) {
2722                         dev_err(sdev->dev, "failed to set sink format for %s source queue ID %d\n",
2723                                 src_widget->widget->name, sroute->src_queue_id);
2724                         goto out;
2725                 }
2726         }
2727
2728         dev_dbg(sdev->dev, "bind %s:%d -> %s:%d\n",
2729                 src_widget->widget->name, sroute->src_queue_id,
2730                 sink_widget->widget->name, sroute->dst_queue_id);
2731
2732         header = src_fw_module->man4_module_entry.id;
2733         header |= SOF_IPC4_MOD_INSTANCE(src_widget->instance_id);
2734         header |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_BIND);
2735         header |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2736         header |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2737
2738         extension = sink_fw_module->man4_module_entry.id;
2739         extension |= SOF_IPC4_MOD_EXT_DST_MOD_INSTANCE(sink_widget->instance_id);
2740         extension |= SOF_IPC4_MOD_EXT_DST_MOD_QUEUE_ID(sroute->dst_queue_id);
2741         extension |= SOF_IPC4_MOD_EXT_SRC_MOD_QUEUE_ID(sroute->src_queue_id);
2742
2743         msg.primary = header;
2744         msg.extension = extension;
2745
2746         ret = sof_ipc_tx_message_no_reply(sdev->ipc, &msg, 0);
2747         if (ret < 0) {
2748                 dev_err(sdev->dev, "failed to bind modules %s:%d -> %s:%d\n",
2749                         src_widget->widget->name, sroute->src_queue_id,
2750                         sink_widget->widget->name, sroute->dst_queue_id);
2751                 goto out;
2752         }
2753
2754         return ret;
2755
2756 out:
2757         sof_ipc4_put_queue_id(src_widget, sroute->src_queue_id, SOF_PIN_TYPE_OUTPUT);
2758         sof_ipc4_put_queue_id(sink_widget, sroute->dst_queue_id, SOF_PIN_TYPE_INPUT);
2759         return ret;
2760 }
2761
2762 static int sof_ipc4_route_free(struct snd_sof_dev *sdev, struct snd_sof_route *sroute)
2763 {
2764         struct snd_sof_widget *src_widget = sroute->src_widget;
2765         struct snd_sof_widget *sink_widget = sroute->sink_widget;
2766         struct sof_ipc4_fw_module *src_fw_module = src_widget->module_info;
2767         struct sof_ipc4_fw_module *sink_fw_module = sink_widget->module_info;
2768         struct sof_ipc4_msg msg = {{ 0 }};
2769         struct snd_sof_widget *src_pipe_widget = src_widget->spipe->pipe_widget;
2770         struct snd_sof_widget *sink_pipe_widget = sink_widget->spipe->pipe_widget;
2771         struct sof_ipc4_pipeline *src_pipeline = src_pipe_widget->private;
2772         struct sof_ipc4_pipeline *sink_pipeline = sink_pipe_widget->private;
2773         u32 header, extension;
2774         int ret = 0;
2775
2776         /* no route is set up if chain DMA is used */
2777         if (src_pipeline->use_chain_dma || sink_pipeline->use_chain_dma)
2778                 return 0;
2779
2780         dev_dbg(sdev->dev, "unbind modules %s:%d -> %s:%d\n",
2781                 src_widget->widget->name, sroute->src_queue_id,
2782                 sink_widget->widget->name, sroute->dst_queue_id);
2783
2784         /*
2785          * routes belonging to the same pipeline will be disconnected by the FW when the pipeline
2786          * is freed. So avoid sending this IPC which will be ignored by the FW anyway.
2787          */
2788         if (src_widget->spipe->pipe_widget == sink_widget->spipe->pipe_widget)
2789                 goto out;
2790
2791         header = src_fw_module->man4_module_entry.id;
2792         header |= SOF_IPC4_MOD_INSTANCE(src_widget->instance_id);
2793         header |= SOF_IPC4_MSG_TYPE_SET(SOF_IPC4_MOD_UNBIND);
2794         header |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
2795         header |= SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
2796
2797         extension = sink_fw_module->man4_module_entry.id;
2798         extension |= SOF_IPC4_MOD_EXT_DST_MOD_INSTANCE(sink_widget->instance_id);
2799         extension |= SOF_IPC4_MOD_EXT_DST_MOD_QUEUE_ID(sroute->dst_queue_id);
2800         extension |= SOF_IPC4_MOD_EXT_SRC_MOD_QUEUE_ID(sroute->src_queue_id);
2801
2802         msg.primary = header;
2803         msg.extension = extension;
2804
2805         ret = sof_ipc_tx_message_no_reply(sdev->ipc, &msg, 0);
2806         if (ret < 0)
2807                 dev_err(sdev->dev, "failed to unbind modules %s:%d -> %s:%d\n",
2808                         src_widget->widget->name, sroute->src_queue_id,
2809                         sink_widget->widget->name, sroute->dst_queue_id);
2810 out:
2811         sof_ipc4_put_queue_id(sink_widget, sroute->dst_queue_id, SOF_PIN_TYPE_INPUT);
2812         sof_ipc4_put_queue_id(src_widget, sroute->src_queue_id, SOF_PIN_TYPE_OUTPUT);
2813
2814         return ret;
2815 }
2816
2817 static int sof_ipc4_dai_config(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget,
2818                                unsigned int flags, struct snd_sof_dai_config_data *data)
2819 {
2820         struct snd_sof_widget *pipe_widget = swidget->spipe->pipe_widget;
2821         struct sof_ipc4_pipeline *pipeline = pipe_widget->private;
2822         struct snd_sof_dai *dai = swidget->private;
2823         struct sof_ipc4_gtw_attributes *gtw_attr;
2824         struct sof_ipc4_copier_data *copier_data;
2825         struct sof_ipc4_copier *ipc4_copier;
2826
2827         if (!dai || !dai->private) {
2828                 dev_err(sdev->dev, "Invalid DAI or DAI private data for %s\n",
2829                         swidget->widget->name);
2830                 return -EINVAL;
2831         }
2832
2833         ipc4_copier = (struct sof_ipc4_copier *)dai->private;
2834         copier_data = &ipc4_copier->data;
2835
2836         if (!data)
2837                 return 0;
2838
2839         if (pipeline->use_chain_dma) {
2840                 pipeline->msg.primary &= ~SOF_IPC4_GLB_CHAIN_DMA_LINK_ID_MASK;
2841                 pipeline->msg.primary |= SOF_IPC4_GLB_CHAIN_DMA_LINK_ID(data->dai_data);
2842                 return 0;
2843         }
2844
2845         switch (ipc4_copier->dai_type) {
2846         case SOF_DAI_INTEL_HDA:
2847                 gtw_attr = ipc4_copier->gtw_attr;
2848                 gtw_attr->lp_buffer_alloc = pipeline->lp_mode;
2849                 fallthrough;
2850         case SOF_DAI_INTEL_ALH:
2851                 /*
2852                  * Do not clear the node ID when this op is invoked with
2853                  * SOF_DAI_CONFIG_FLAGS_HW_FREE. It is needed to free the group_ida during
2854                  * unprepare.
2855                  */
2856                 if (flags & SOF_DAI_CONFIG_FLAGS_HW_PARAMS) {
2857                         copier_data->gtw_cfg.node_id &= ~SOF_IPC4_NODE_INDEX_MASK;
2858                         copier_data->gtw_cfg.node_id |= SOF_IPC4_NODE_INDEX(data->dai_data);
2859                 }
2860                 break;
2861         case SOF_DAI_INTEL_DMIC:
2862         case SOF_DAI_INTEL_SSP:
2863                 /* nothing to do for SSP/DMIC */
2864                 break;
2865         default:
2866                 dev_err(sdev->dev, "%s: unsupported dai type %d\n", __func__,
2867                         ipc4_copier->dai_type);
2868                 return -EINVAL;
2869         }
2870
2871         return 0;
2872 }
2873
2874 static int sof_ipc4_parse_manifest(struct snd_soc_component *scomp, int index,
2875                                    struct snd_soc_tplg_manifest *man)
2876 {
2877         struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
2878         struct sof_ipc4_fw_data *ipc4_data = sdev->private;
2879         struct sof_manifest_tlv *manifest_tlv;
2880         struct sof_manifest *manifest;
2881         u32 size = le32_to_cpu(man->priv.size);
2882         u8 *man_ptr = man->priv.data;
2883         u32 len_check;
2884         int i;
2885
2886         if (!size || size < SOF_IPC4_TPLG_ABI_SIZE) {
2887                 dev_err(scomp->dev, "%s: Invalid topology ABI size: %u\n",
2888                         __func__, size);
2889                 return -EINVAL;
2890         }
2891
2892         manifest = (struct sof_manifest *)man_ptr;
2893
2894         dev_info(scomp->dev,
2895                  "Topology: ABI %d:%d:%d Kernel ABI %u:%u:%u\n",
2896                   le16_to_cpu(manifest->abi_major), le16_to_cpu(manifest->abi_minor),
2897                   le16_to_cpu(manifest->abi_patch),
2898                   SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH);
2899
2900         /* TODO: Add ABI compatibility check */
2901
2902         /* no more data after the ABI version */
2903         if (size <= SOF_IPC4_TPLG_ABI_SIZE)
2904                 return 0;
2905
2906         manifest_tlv = manifest->items;
2907         len_check = sizeof(struct sof_manifest);
2908         for (i = 0; i < le16_to_cpu(manifest->count); i++) {
2909                 len_check += sizeof(struct sof_manifest_tlv) + le32_to_cpu(manifest_tlv->size);
2910                 if (len_check > size)
2911                         return -EINVAL;
2912
2913                 switch (le32_to_cpu(manifest_tlv->type)) {
2914                 case SOF_MANIFEST_DATA_TYPE_NHLT:
2915                         /* no NHLT in BIOS, so use the one from topology manifest */
2916                         if (ipc4_data->nhlt)
2917                                 break;
2918                         ipc4_data->nhlt = devm_kmemdup(sdev->dev, manifest_tlv->data,
2919                                                        le32_to_cpu(manifest_tlv->size), GFP_KERNEL);
2920                         if (!ipc4_data->nhlt)
2921                                 return -ENOMEM;
2922                         break;
2923                 default:
2924                         dev_warn(scomp->dev, "Skipping unknown manifest data type %d\n",
2925                                  manifest_tlv->type);
2926                         break;
2927                 }
2928                 man_ptr += sizeof(struct sof_manifest_tlv) + le32_to_cpu(manifest_tlv->size);
2929                 manifest_tlv = (struct sof_manifest_tlv *)man_ptr;
2930         }
2931
2932         return 0;
2933 }
2934
2935 static int sof_ipc4_dai_get_clk(struct snd_sof_dev *sdev, struct snd_sof_dai *dai, int clk_type)
2936 {
2937         struct sof_ipc4_copier *ipc4_copier = dai->private;
2938         struct snd_soc_tplg_hw_config *hw_config;
2939         struct snd_sof_dai_link *slink;
2940         bool dai_link_found = false;
2941         bool hw_cfg_found = false;
2942         int i;
2943
2944         if (!ipc4_copier)
2945                 return 0;
2946
2947         list_for_each_entry(slink, &sdev->dai_link_list, list) {
2948                 if (!strcmp(slink->link->name, dai->name)) {
2949                         dai_link_found = true;
2950                         break;
2951                 }
2952         }
2953
2954         if (!dai_link_found) {
2955                 dev_err(sdev->dev, "no DAI link found for DAI %s\n", dai->name);
2956                 return -EINVAL;
2957         }
2958
2959         for (i = 0; i < slink->num_hw_configs; i++) {
2960                 hw_config = &slink->hw_configs[i];
2961                 if (dai->current_config == le32_to_cpu(hw_config->id)) {
2962                         hw_cfg_found = true;
2963                         break;
2964                 }
2965         }
2966
2967         if (!hw_cfg_found) {
2968                 dev_err(sdev->dev, "no matching hw_config found for DAI %s\n", dai->name);
2969                 return -EINVAL;
2970         }
2971
2972         switch (ipc4_copier->dai_type) {
2973         case SOF_DAI_INTEL_SSP:
2974                 switch (clk_type) {
2975                 case SOF_DAI_CLK_INTEL_SSP_MCLK:
2976                         return le32_to_cpu(hw_config->mclk_rate);
2977                 case SOF_DAI_CLK_INTEL_SSP_BCLK:
2978                         return le32_to_cpu(hw_config->bclk_rate);
2979                 default:
2980                         dev_err(sdev->dev, "Invalid clk type for SSP %d\n", clk_type);
2981                         break;
2982                 }
2983                 break;
2984         default:
2985                 dev_err(sdev->dev, "DAI type %d not supported yet!\n", ipc4_copier->dai_type);
2986                 break;
2987         }
2988
2989         return -EINVAL;
2990 }
2991
2992 static int sof_ipc4_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verify)
2993 {
2994         struct snd_sof_pcm *spcm;
2995         int dir, ret;
2996
2997         /*
2998          * This function is called during system suspend, we need to make sure
2999          * that all streams have been freed up.
3000          * Freeing might have been skipped when xrun happened just at the start
3001          * of the suspend and it sent a SNDRV_PCM_TRIGGER_STOP to the active
3002          * stream. This will call sof_pcm_stream_free() with
3003          * free_widget_list = false which will leave the kernel and firmware out
3004          * of sync during suspend/resume.
3005          *
3006          * This will also make sure that paused streams handled correctly.
3007          */
3008         list_for_each_entry(spcm, &sdev->pcm_list, list) {
3009                 for_each_pcm_streams(dir) {
3010                         struct snd_pcm_substream *substream = spcm->stream[dir].substream;
3011
3012                         if (!substream || !substream->runtime || spcm->stream[dir].suspend_ignored)
3013                                 continue;
3014
3015                         if (spcm->stream[dir].list) {
3016                                 ret = sof_pcm_stream_free(sdev, substream, spcm, dir, true);
3017                                 if (ret < 0)
3018                                         return ret;
3019                         }
3020                 }
3021         }
3022         return 0;
3023 }
3024
3025 static int sof_ipc4_link_setup(struct snd_sof_dev *sdev, struct snd_soc_dai_link *link)
3026 {
3027         if (link->no_pcm)
3028                 return 0;
3029
3030         /*
3031          * set default trigger order for all links. Exceptions to
3032          * the rule will be handled in sof_pcm_dai_link_fixup()
3033          * For playback, the sequence is the following: start BE,
3034          * start FE, stop FE, stop BE; for Capture the sequence is
3035          * inverted start FE, start BE, stop BE, stop FE
3036          */
3037         link->trigger[SNDRV_PCM_STREAM_PLAYBACK] = SND_SOC_DPCM_TRIGGER_POST;
3038         link->trigger[SNDRV_PCM_STREAM_CAPTURE] = SND_SOC_DPCM_TRIGGER_PRE;
3039
3040         return 0;
3041 }
3042
3043 static enum sof_tokens common_copier_token_list[] = {
3044         SOF_COMP_TOKENS,
3045         SOF_AUDIO_FMT_NUM_TOKENS,
3046         SOF_IN_AUDIO_FORMAT_TOKENS,
3047         SOF_OUT_AUDIO_FORMAT_TOKENS,
3048         SOF_COPIER_DEEP_BUFFER_TOKENS,
3049         SOF_COPIER_TOKENS,
3050         SOF_COMP_EXT_TOKENS,
3051 };
3052
3053 static enum sof_tokens pipeline_token_list[] = {
3054         SOF_SCHED_TOKENS,
3055         SOF_PIPELINE_TOKENS,
3056 };
3057
3058 static enum sof_tokens dai_token_list[] = {
3059         SOF_COMP_TOKENS,
3060         SOF_AUDIO_FMT_NUM_TOKENS,
3061         SOF_IN_AUDIO_FORMAT_TOKENS,
3062         SOF_OUT_AUDIO_FORMAT_TOKENS,
3063         SOF_COPIER_TOKENS,
3064         SOF_DAI_TOKENS,
3065         SOF_COMP_EXT_TOKENS,
3066 };
3067
3068 static enum sof_tokens pga_token_list[] = {
3069         SOF_COMP_TOKENS,
3070         SOF_GAIN_TOKENS,
3071         SOF_AUDIO_FMT_NUM_TOKENS,
3072         SOF_IN_AUDIO_FORMAT_TOKENS,
3073         SOF_OUT_AUDIO_FORMAT_TOKENS,
3074         SOF_COMP_EXT_TOKENS,
3075 };
3076
3077 static enum sof_tokens mixer_token_list[] = {
3078         SOF_COMP_TOKENS,
3079         SOF_AUDIO_FMT_NUM_TOKENS,
3080         SOF_IN_AUDIO_FORMAT_TOKENS,
3081         SOF_OUT_AUDIO_FORMAT_TOKENS,
3082         SOF_COMP_EXT_TOKENS,
3083 };
3084
3085 static enum sof_tokens src_token_list[] = {
3086         SOF_COMP_TOKENS,
3087         SOF_SRC_TOKENS,
3088         SOF_AUDIO_FMT_NUM_TOKENS,
3089         SOF_IN_AUDIO_FORMAT_TOKENS,
3090         SOF_OUT_AUDIO_FORMAT_TOKENS,
3091         SOF_COMP_EXT_TOKENS,
3092 };
3093
3094 static enum sof_tokens process_token_list[] = {
3095         SOF_COMP_TOKENS,
3096         SOF_AUDIO_FMT_NUM_TOKENS,
3097         SOF_IN_AUDIO_FORMAT_TOKENS,
3098         SOF_OUT_AUDIO_FORMAT_TOKENS,
3099         SOF_COMP_EXT_TOKENS,
3100 };
3101
3102 static const struct sof_ipc_tplg_widget_ops tplg_ipc4_widget_ops[SND_SOC_DAPM_TYPE_COUNT] = {
3103         [snd_soc_dapm_aif_in] =  {sof_ipc4_widget_setup_pcm, sof_ipc4_widget_free_comp_pcm,
3104                                   common_copier_token_list, ARRAY_SIZE(common_copier_token_list),
3105                                   NULL, sof_ipc4_prepare_copier_module,
3106                                   sof_ipc4_unprepare_copier_module},
3107         [snd_soc_dapm_aif_out] = {sof_ipc4_widget_setup_pcm, sof_ipc4_widget_free_comp_pcm,
3108                                   common_copier_token_list, ARRAY_SIZE(common_copier_token_list),
3109                                   NULL, sof_ipc4_prepare_copier_module,
3110                                   sof_ipc4_unprepare_copier_module},
3111         [snd_soc_dapm_dai_in] = {sof_ipc4_widget_setup_comp_dai, sof_ipc4_widget_free_comp_dai,
3112                                  dai_token_list, ARRAY_SIZE(dai_token_list), NULL,
3113                                  sof_ipc4_prepare_copier_module,
3114                                  sof_ipc4_unprepare_copier_module},
3115         [snd_soc_dapm_dai_out] = {sof_ipc4_widget_setup_comp_dai, sof_ipc4_widget_free_comp_dai,
3116                                   dai_token_list, ARRAY_SIZE(dai_token_list), NULL,
3117                                   sof_ipc4_prepare_copier_module,
3118                                   sof_ipc4_unprepare_copier_module},
3119         [snd_soc_dapm_buffer] = {sof_ipc4_widget_setup_pcm, sof_ipc4_widget_free_comp_pcm,
3120                                  common_copier_token_list, ARRAY_SIZE(common_copier_token_list),
3121                                  NULL, sof_ipc4_prepare_copier_module,
3122                                  sof_ipc4_unprepare_copier_module},
3123         [snd_soc_dapm_scheduler] = {sof_ipc4_widget_setup_comp_pipeline,
3124                                     sof_ipc4_widget_free_comp_pipeline,
3125                                     pipeline_token_list, ARRAY_SIZE(pipeline_token_list), NULL,
3126                                     NULL, NULL},
3127         [snd_soc_dapm_pga] = {sof_ipc4_widget_setup_comp_pga, sof_ipc4_widget_free_comp_pga,
3128                               pga_token_list, ARRAY_SIZE(pga_token_list), NULL,
3129                               sof_ipc4_prepare_gain_module,
3130                               NULL},
3131         [snd_soc_dapm_mixer] = {sof_ipc4_widget_setup_comp_mixer, sof_ipc4_widget_free_comp_mixer,
3132                                 mixer_token_list, ARRAY_SIZE(mixer_token_list),
3133                                 NULL, sof_ipc4_prepare_mixer_module,
3134                                 NULL},
3135         [snd_soc_dapm_src] = {sof_ipc4_widget_setup_comp_src, sof_ipc4_widget_free_comp_src,
3136                                 src_token_list, ARRAY_SIZE(src_token_list),
3137                                 NULL, sof_ipc4_prepare_src_module,
3138                                 NULL},
3139         [snd_soc_dapm_effect] = {sof_ipc4_widget_setup_comp_process,
3140                                 sof_ipc4_widget_free_comp_process,
3141                                 process_token_list, ARRAY_SIZE(process_token_list),
3142                                 NULL, sof_ipc4_prepare_process_module,
3143                                 NULL},
3144 };
3145
3146 const struct sof_ipc_tplg_ops ipc4_tplg_ops = {
3147         .widget = tplg_ipc4_widget_ops,
3148         .token_list = ipc4_token_list,
3149         .control_setup = sof_ipc4_control_setup,
3150         .control = &tplg_ipc4_control_ops,
3151         .widget_setup = sof_ipc4_widget_setup,
3152         .widget_free = sof_ipc4_widget_free,
3153         .route_setup = sof_ipc4_route_setup,
3154         .route_free = sof_ipc4_route_free,
3155         .dai_config = sof_ipc4_dai_config,
3156         .parse_manifest = sof_ipc4_parse_manifest,
3157         .dai_get_clk = sof_ipc4_dai_get_clk,
3158         .tear_down_all_pipelines = sof_ipc4_tear_down_all_pipelines,
3159         .link_setup = sof_ipc4_link_setup,
3160 };