sound/soc/intel/sst-haswell-ipc.c

   1 /*
   2  *  Intel SST Haswell/Broadwell IPC Support
   3  *
   4  * Copyright (C) 2013, Intel Corporation. All rights reserved.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License version
   8  * 2 as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  */
  16
  17 #include <linux/types.h>
  18 #include <linux/kernel.h>
  19 #include <linux/list.h>
  20 #include <linux/device.h>
  21 #include <linux/wait.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/workqueue.h>
  24 #include <linux/export.h>
  25 #include <linux/slab.h>
  26 #include <linux/delay.h>
  27 #include <linux/sched.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/kthread.h>
  30 #include <linux/firmware.h>
  31 #include <linux/dma-mapping.h>
  32 #include <linux/debugfs.h>
  33 #include <linux/pm_runtime.h>
  34 #include <sound/asound.h>
  35
  36 #include "sst-haswell-ipc.h"
  37 #include "sst-dsp.h"
  38 #include "sst-dsp-priv.h"
  39
  40 /* Global Message - Generic */
  41 #define IPC_GLB_TYPE_SHIFT      24
  42 #define IPC_GLB_TYPE_MASK       (0x1f << IPC_GLB_TYPE_SHIFT)
  43 #define IPC_GLB_TYPE(x)         (x << IPC_GLB_TYPE_SHIFT)
  44
  45 /* Global Message - Reply */
  46 #define IPC_GLB_REPLY_SHIFT     0
  47 #define IPC_GLB_REPLY_MASK      (0x1f << IPC_GLB_REPLY_SHIFT)
  48 #define IPC_GLB_REPLY_TYPE(x)   (x << IPC_GLB_REPLY_TYPE_SHIFT)
  49
  50 /* Stream Message - Generic */
  51 #define IPC_STR_TYPE_SHIFT      20
  52 #define IPC_STR_TYPE_MASK       (0xf << IPC_STR_TYPE_SHIFT)
  53 #define IPC_STR_TYPE(x)         (x << IPC_STR_TYPE_SHIFT)
  54 #define IPC_STR_ID_SHIFT        16
  55 #define IPC_STR_ID_MASK         (0xf << IPC_STR_ID_SHIFT)
  56 #define IPC_STR_ID(x)           (x << IPC_STR_ID_SHIFT)
  57
  58 /* Stream Message - Reply */
  59 #define IPC_STR_REPLY_SHIFT     0
  60 #define IPC_STR_REPLY_MASK      (0x1f << IPC_STR_REPLY_SHIFT)
  61
  62 /* Stream Stage Message - Generic */
  63 #define IPC_STG_TYPE_SHIFT      12
  64 #define IPC_STG_TYPE_MASK       (0xf << IPC_STG_TYPE_SHIFT)
  65 #define IPC_STG_TYPE(x)         (x << IPC_STG_TYPE_SHIFT)
  66 #define IPC_STG_ID_SHIFT        10
  67 #define IPC_STG_ID_MASK         (0x3 << IPC_STG_ID_SHIFT)
  68 #define IPC_STG_ID(x)           (x << IPC_STG_ID_SHIFT)
  69
  70 /* Stream Stage Message - Reply */
  71 #define IPC_STG_REPLY_SHIFT     0
  72 #define IPC_STG_REPLY_MASK      (0x1f << IPC_STG_REPLY_SHIFT)
  73
  74 /* Debug Log Message - Generic */
  75 #define IPC_LOG_OP_SHIFT        20
  76 #define IPC_LOG_OP_MASK         (0xf << IPC_LOG_OP_SHIFT)
  77 #define IPC_LOG_OP_TYPE(x)      (x << IPC_LOG_OP_SHIFT)
  78 #define IPC_LOG_ID_SHIFT        16
  79 #define IPC_LOG_ID_MASK         (0xf << IPC_LOG_ID_SHIFT)
  80 #define IPC_LOG_ID(x)           (x << IPC_LOG_ID_SHIFT)
  81
  82 /* IPC message timeout (msecs) */
  83 #define IPC_TIMEOUT_MSECS       300
  84 #define IPC_BOOT_MSECS          200
  85 #define IPC_MSG_WAIT            0
  86 #define IPC_MSG_NOWAIT          1
  87
  88 /* Firmware Ready Message */
  89 #define IPC_FW_READY            (0x1 << 29)
  90 #define IPC_STATUS_MASK         (0x3 << 30)
  91
  92 #define IPC_EMPTY_LIST_SIZE     8
  93 #define IPC_MAX_STREAMS         4
  94
  95 /* Mailbox */
  96 #define IPC_MAX_MAILBOX_BYTES   256
  97
  98 #define INVALID_STREAM_HW_ID    0xffffffff
  99
 100 /* Global Message - Types and Replies */
 101 enum ipc_glb_type {
 102         IPC_GLB_GET_FW_VERSION = 0,             /* Retrieves firmware version */
 103         IPC_GLB_PERFORMANCE_MONITOR = 1,        /* Performance monitoring actions */
 104         IPC_GLB_ALLOCATE_STREAM = 3,            /* Request to allocate new stream */
 105         IPC_GLB_FREE_STREAM = 4,                /* Request to free stream */
 106         IPC_GLB_GET_FW_CAPABILITIES = 5,        /* Retrieves firmware capabilities */
 107         IPC_GLB_STREAM_MESSAGE = 6,             /* Message directed to stream or its stages */
 108         /* Request to store firmware context during D0->D3 transition */
 109         IPC_GLB_REQUEST_DUMP = 7,
 110         /* Request to restore firmware context during D3->D0 transition */
 111         IPC_GLB_RESTORE_CONTEXT = 8,
 112         IPC_GLB_GET_DEVICE_FORMATS = 9,         /* Set device format */
 113         IPC_GLB_SET_DEVICE_FORMATS = 10,        /* Get device format */
 114         IPC_GLB_SHORT_REPLY = 11,
 115         IPC_GLB_ENTER_DX_STATE = 12,
 116         IPC_GLB_GET_MIXER_STREAM_INFO = 13,     /* Request mixer stream params */
 117         IPC_GLB_DEBUG_LOG_MESSAGE = 14,         /* Message to or from the debug logger. */
 118         IPC_GLB_REQUEST_TRANSFER = 16,          /* < Request Transfer for host */
 119         IPC_GLB_MAX_IPC_MESSAGE_TYPE = 17,      /* Maximum message number */
 120 };
 121
 122 enum ipc_glb_reply {
 123         IPC_GLB_REPLY_SUCCESS = 0,              /* The operation was successful. */
 124         IPC_GLB_REPLY_ERROR_INVALID_PARAM = 1,  /* Invalid parameter was passed. */
 125         IPC_GLB_REPLY_UNKNOWN_MESSAGE_TYPE = 2, /* Uknown message type was resceived. */
 126         IPC_GLB_REPLY_OUT_OF_RESOURCES = 3,     /* No resources to satisfy the request. */
 127         IPC_GLB_REPLY_BUSY = 4,                 /* The system or resource is busy. */
 128         IPC_GLB_REPLY_PENDING = 5,              /* The action was scheduled for processing.  */
 129         IPC_GLB_REPLY_FAILURE = 6,              /* Critical error happened. */
 130         IPC_GLB_REPLY_INVALID_REQUEST = 7,      /* Request can not be completed. */
 131         IPC_GLB_REPLY_STAGE_UNINITIALIZED = 8,  /* Processing stage was uninitialized. */
 132         IPC_GLB_REPLY_NOT_FOUND = 9,            /* Required resource can not be found. */
 133         IPC_GLB_REPLY_SOURCE_NOT_STARTED = 10,  /* Source was not started. */
 134 };
 135
 136 /* Stream Message - Types */
 137 enum ipc_str_operation {
 138         IPC_STR_RESET = 0,
 139         IPC_STR_PAUSE = 1,
 140         IPC_STR_RESUME = 2,
 141         IPC_STR_STAGE_MESSAGE = 3,
 142         IPC_STR_NOTIFICATION = 4,
 143         IPC_STR_MAX_MESSAGE
 144 };
 145
 146 /* Stream Stage Message Types */
 147 enum ipc_stg_operation {
 148         IPC_STG_GET_VOLUME = 0,
 149         IPC_STG_SET_VOLUME,
 150         IPC_STG_SET_WRITE_POSITION,
 151         IPC_STG_SET_FX_ENABLE,
 152         IPC_STG_SET_FX_DISABLE,
 153         IPC_STG_SET_FX_GET_PARAM,
 154         IPC_STG_SET_FX_SET_PARAM,
 155         IPC_STG_SET_FX_GET_INFO,
 156         IPC_STG_MUTE_LOOPBACK,
 157         IPC_STG_MAX_MESSAGE
 158 };
 159
 160 /* Stream Stage Message Types For Notification*/
 161 enum ipc_stg_operation_notify {
 162         IPC_POSITION_CHANGED = 0,
 163         IPC_STG_GLITCH,
 164         IPC_STG_MAX_NOTIFY
 165 };
 166
 167 enum ipc_glitch_type {
 168         IPC_GLITCH_UNDERRUN = 1,
 169         IPC_GLITCH_DECODER_ERROR,
 170         IPC_GLITCH_DOUBLED_WRITE_POS,
 171         IPC_GLITCH_MAX
 172 };
 173
 174 /* Debug Control */
 175 enum ipc_debug_operation {
 176         IPC_DEBUG_ENABLE_LOG = 0,
 177         IPC_DEBUG_DISABLE_LOG = 1,
 178         IPC_DEBUG_REQUEST_LOG_DUMP = 2,
 179         IPC_DEBUG_NOTIFY_LOG_DUMP = 3,
 180         IPC_DEBUG_MAX_DEBUG_LOG
 181 };
 182
 183 /* Firmware Ready */
 184 struct sst_hsw_ipc_fw_ready {
 185         u32 inbox_offset;
 186         u32 outbox_offset;
 187         u32 inbox_size;
 188         u32 outbox_size;
 189         u32 fw_info_size;
 190         u8 fw_info[IPC_MAX_MAILBOX_BYTES - 5 * sizeof(u32)];
 191 } __attribute__((packed));
 192
 193 struct ipc_message {
 194         struct list_head list;
 195         u32 header;
 196
 197         /* direction wrt host CPU */
 198         char tx_data[IPC_MAX_MAILBOX_BYTES];
 199         size_t tx_size;
 200         char rx_data[IPC_MAX_MAILBOX_BYTES];
 201         size_t rx_size;
 202
 203         wait_queue_head_t waitq;
 204         bool pending;
 205         bool complete;
 206         bool wait;
 207         int errno;
 208 };
 209
 210 struct sst_hsw_stream;
 211 struct sst_hsw;
 212
 213 /* Stream infomation */
 214 struct sst_hsw_stream {
 215         /* configuration */
 216         struct sst_hsw_ipc_stream_alloc_req request;
 217         struct sst_hsw_ipc_stream_alloc_reply reply;
 218         struct sst_hsw_ipc_stream_free_req free_req;
 219
 220         /* Mixer info */
 221         u32 mute_volume[SST_HSW_NO_CHANNELS];
 222         u32 mute[SST_HSW_NO_CHANNELS];
 223
 224         /* runtime info */
 225         struct sst_hsw *hsw;
 226         int host_id;
 227         bool commited;
 228         bool running;
 229
 230         /* Notification work */
 231         struct work_struct notify_work;
 232         u32 header;
 233
 234         /* Position info from DSP */
 235         struct sst_hsw_ipc_stream_set_position wpos;
 236         struct sst_hsw_ipc_stream_get_position rpos;
 237         struct sst_hsw_ipc_stream_glitch_position glitch;
 238
 239         /* Volume info */
 240         struct sst_hsw_ipc_volume_req vol_req;
 241
 242         /* driver callback */
 243         u32 (*notify_position)(struct sst_hsw_stream *stream, void *data);
 244         void *pdata;
 245
 246         /* record the fw read position when playback */
 247         snd_pcm_uframes_t old_position;
 248         bool play_silence;
 249         struct list_head node;
 250 };
 251
 252 /* FW log ring information */
 253 struct sst_hsw_log_stream {
 254         dma_addr_t dma_addr;
 255         unsigned char *dma_area;
 256         unsigned char *ring_descr;
 257         int pages;
 258         int size;
 259
 260         /* Notification work */
 261         struct work_struct notify_work;
 262         wait_queue_head_t readers_wait_q;
 263         struct mutex rw_mutex;
 264
 265         u32 last_pos;
 266         u32 curr_pos;
 267         u32 reader_pos;
 268
 269         /* fw log config */
 270         u32 config[SST_HSW_FW_LOG_CONFIG_DWORDS];
 271
 272         struct sst_hsw *hsw;
 273 };
 274
 275 /* SST Haswell IPC data */
 276 struct sst_hsw {
 277         struct device *dev;
 278         struct sst_dsp *dsp;
 279         struct platform_device *pdev_pcm;
 280
 281         /* FW config */
 282         struct sst_hsw_ipc_fw_ready fw_ready;
 283         struct sst_hsw_ipc_fw_version version;
 284         bool fw_done;
 285         struct sst_fw *sst_fw;
 286
 287         /* stream */
 288         struct list_head stream_list;
 289
 290         /* global mixer */
 291         struct sst_hsw_ipc_stream_info_reply mixer_info;
 292         enum sst_hsw_volume_curve curve_type;
 293         u32 curve_duration;
 294         u32 mute[SST_HSW_NO_CHANNELS];
 295         u32 mute_volume[SST_HSW_NO_CHANNELS];
 296
 297         /* DX */
 298         struct sst_hsw_ipc_dx_reply dx;
 299         void *dx_context;
 300         dma_addr_t dx_context_paddr;
 301
 302         /* boot */
 303         wait_queue_head_t boot_wait;
 304         bool boot_complete;
 305         bool shutdown;
 306
 307         /* IPC messaging */
 308         struct list_head tx_list;
 309         struct list_head rx_list;
 310         struct list_head empty_list;
 311         wait_queue_head_t wait_txq;
 312         struct task_struct *tx_thread;
 313         struct kthread_worker kworker;
 314         struct kthread_work kwork;
 315         bool pending;
 316         struct ipc_message *msg;
 317
 318         /* FW log stream */
 319         struct sst_hsw_log_stream log_stream;
 320 };
 321
 322 #define CREATE_TRACE_POINTS
 323 #include <trace/events/hswadsp.h>
 324
 325 static inline u32 msg_get_global_type(u32 msg)
 326 {
 327         return (msg & IPC_GLB_TYPE_MASK) >> IPC_GLB_TYPE_SHIFT;
 328 }
 329
 330 static inline u32 msg_get_global_reply(u32 msg)
 331 {
 332         return (msg & IPC_GLB_REPLY_MASK) >> IPC_GLB_REPLY_SHIFT;
 333 }
 334
 335 static inline u32 msg_get_stream_type(u32 msg)
 336 {
 337         return (msg & IPC_STR_TYPE_MASK) >>  IPC_STR_TYPE_SHIFT;
 338 }
 339
 340 static inline u32 msg_get_stage_type(u32 msg)
 341 {
 342         return (msg & IPC_STG_TYPE_MASK) >>  IPC_STG_TYPE_SHIFT;
 343 }
 344
 345 static inline u32 msg_get_stream_id(u32 msg)
 346 {
 347         return (msg & IPC_STR_ID_MASK) >>  IPC_STR_ID_SHIFT;
 348 }
 349
 350 static inline u32 msg_get_notify_reason(u32 msg)
 351 {
 352         return (msg & IPC_STG_TYPE_MASK) >> IPC_STG_TYPE_SHIFT;
 353 }
 354
 355 u32 create_channel_map(enum sst_hsw_channel_config config)
 356 {
 357         switch (config) {
 358         case SST_HSW_CHANNEL_CONFIG_MONO:
 359                 return (0xFFFFFFF0 | SST_HSW_CHANNEL_CENTER);
 360         case SST_HSW_CHANNEL_CONFIG_STEREO:
 361                 return (0xFFFFFF00 | SST_HSW_CHANNEL_LEFT
 362                         | (SST_HSW_CHANNEL_RIGHT << 4));
 363         case SST_HSW_CHANNEL_CONFIG_2_POINT_1:
 364                 return (0xFFFFF000 | SST_HSW_CHANNEL_LEFT
 365                         | (SST_HSW_CHANNEL_RIGHT << 4)
 366                         | (SST_HSW_CHANNEL_LFE << 8 ));
 367         case SST_HSW_CHANNEL_CONFIG_3_POINT_0:
 368                 return (0xFFFFF000 | SST_HSW_CHANNEL_LEFT
 369                         | (SST_HSW_CHANNEL_CENTER << 4)
 370                         | (SST_HSW_CHANNEL_RIGHT << 8));
 371         case SST_HSW_CHANNEL_CONFIG_3_POINT_1:
 372                 return (0xFFFF0000 | SST_HSW_CHANNEL_LEFT
 373                         | (SST_HSW_CHANNEL_CENTER << 4)
 374                         | (SST_HSW_CHANNEL_RIGHT << 8)
 375                         | (SST_HSW_CHANNEL_LFE << 12));
 376         case SST_HSW_CHANNEL_CONFIG_QUATRO:
 377                 return (0xFFFF0000 | SST_HSW_CHANNEL_LEFT
 378                         | (SST_HSW_CHANNEL_RIGHT << 4)
 379                         | (SST_HSW_CHANNEL_LEFT_SURROUND << 8)
 380                         | (SST_HSW_CHANNEL_RIGHT_SURROUND << 12));
 381         case SST_HSW_CHANNEL_CONFIG_4_POINT_0:
 382                 return (0xFFFF0000 | SST_HSW_CHANNEL_LEFT
 383                         | (SST_HSW_CHANNEL_CENTER << 4)
 384                         | (SST_HSW_CHANNEL_RIGHT << 8)
 385                         | (SST_HSW_CHANNEL_CENTER_SURROUND << 12));
 386         case SST_HSW_CHANNEL_CONFIG_5_POINT_0:
 387                 return (0xFFF00000 | SST_HSW_CHANNEL_LEFT
 388                         | (SST_HSW_CHANNEL_CENTER << 4)
 389                         | (SST_HSW_CHANNEL_RIGHT << 8)
 390                         | (SST_HSW_CHANNEL_LEFT_SURROUND << 12)
 391                         | (SST_HSW_CHANNEL_RIGHT_SURROUND << 16));
 392         case SST_HSW_CHANNEL_CONFIG_5_POINT_1:
 393                 return (0xFF000000 | SST_HSW_CHANNEL_CENTER
 394                         | (SST_HSW_CHANNEL_LEFT << 4)
 395                         | (SST_HSW_CHANNEL_RIGHT << 8)
 396                         | (SST_HSW_CHANNEL_LEFT_SURROUND << 12)
 397                         | (SST_HSW_CHANNEL_RIGHT_SURROUND << 16)
 398                         | (SST_HSW_CHANNEL_LFE << 20));
 399         case SST_HSW_CHANNEL_CONFIG_DUAL_MONO:
 400                 return (0xFFFFFF00 | SST_HSW_CHANNEL_LEFT
 401                         | (SST_HSW_CHANNEL_LEFT << 4));
 402         default:
 403                 return 0xFFFFFFFF;
 404         }
 405 }
 406
 407 static struct sst_hsw_stream *get_stream_by_id(struct sst_hsw *hsw,
 408         int stream_id)
 409 {
 410         struct sst_hsw_stream *stream;
 411
 412         list_for_each_entry(stream, &hsw->stream_list, node) {
 413                 if (stream->reply.stream_hw_id == stream_id)
 414                         return stream;
 415         }
 416
 417         return NULL;
 418 }
 419
 420 static void ipc_shim_dbg(struct sst_hsw *hsw, const char *text)
 421 {
 422         struct sst_dsp *sst = hsw->dsp;
 423         u32 isr, ipcd, imrx, ipcx;
 424
 425         ipcx = sst_dsp_shim_read_unlocked(sst, SST_IPCX);
 426         isr = sst_dsp_shim_read_unlocked(sst, SST_ISRX);
 427         ipcd = sst_dsp_shim_read_unlocked(sst, SST_IPCD);
 428         imrx = sst_dsp_shim_read_unlocked(sst, SST_IMRX);
 429
 430         dev_err(hsw->dev, "ipc: --%s-- ipcx 0x%8.8x isr 0x%8.8x ipcd 0x%8.8x imrx 0x%8.8x\n",
 431                 text, ipcx, isr, ipcd, imrx);
 432 }
 433
 434 /* locks held by caller */
 435 static struct ipc_message *msg_get_empty(struct sst_hsw *hsw)
 436 {
 437         struct ipc_message *msg = NULL;
 438
 439         if (!list_empty(&hsw->empty_list)) {
 440                 msg = list_first_entry(&hsw->empty_list, struct ipc_message,
 441                         list);
 442                 list_del(&msg->list);
 443         }
 444
 445         return msg;
 446 }
 447
 448 static void ipc_tx_msgs(struct kthread_work *work)
 449 {
 450         struct sst_hsw *hsw =
 451                 container_of(work, struct sst_hsw, kwork);
 452         struct ipc_message *msg;
 453         unsigned long flags;
 454         u32 ipcx;
 455
 456         spin_lock_irqsave(&hsw->dsp->spinlock, flags);
 457
 458         if (list_empty(&hsw->tx_list) || hsw->pending) {
 459                 spin_unlock_irqrestore(&hsw->dsp->spinlock, flags);
 460                 return;
 461         }
 462
 463         /* if the DSP is busy, we will TX messages after IRQ.
 464          * also postpone if we are in the middle of procesing completion irq*/
 465         ipcx = sst_dsp_shim_read_unlocked(hsw->dsp, SST_IPCX);
 466         if (ipcx & (SST_IPCX_BUSY | SST_IPCX_DONE)) {
 467                 spin_unlock_irqrestore(&hsw->dsp->spinlock, flags);
 468                 return;
 469         }
 470
 471         msg = list_first_entry(&hsw->tx_list, struct ipc_message, list);
 472
 473         list_move(&msg->list, &hsw->rx_list);
 474
 475         /* send the message */
 476         sst_dsp_outbox_write(hsw->dsp, msg->tx_data, msg->tx_size);
 477         sst_dsp_ipc_msg_tx(hsw->dsp, msg->header | SST_IPCX_BUSY);
 478
 479         spin_unlock_irqrestore(&hsw->dsp->spinlock, flags);
 480 }
 481
 482 /* locks held by caller */
 483 static void tx_msg_reply_complete(struct sst_hsw *hsw, struct ipc_message *msg)
 484 {
 485         msg->complete = true;
 486         trace_ipc_reply("completed", msg->header);
 487
 488         if (!msg->wait)
 489                 list_add_tail(&msg->list, &hsw->empty_list);
 490         else
 491                 wake_up(&msg->waitq);
 492 }
 493
 494 static int tx_wait_done(struct sst_hsw *hsw, struct ipc_message *msg,
 495         void *rx_data)
 496 {
 497         unsigned long flags;
 498         int ret;
 499
 500         /* wait for DSP completion (in all cases atm inc pending) */
 501         ret = wait_event_timeout(msg->waitq, msg->complete,
 502                 msecs_to_jiffies(IPC_TIMEOUT_MSECS));
 503
 504         spin_lock_irqsave(&hsw->dsp->spinlock, flags);
 505         if (ret == 0) {
 506                 ipc_shim_dbg(hsw, "message timeout");
 507
 508                 trace_ipc_error("error message timeout for", msg->header);
 509                 list_del(&msg->list);
 510                 ret = -ETIMEDOUT;
 511         } else {
 512
 513                 /* copy the data returned from DSP */
 514                 if (msg->rx_size)
 515                         memcpy(rx_data, msg->rx_data, msg->rx_size);
 516                 ret = msg->errno;
 517         }
 518
 519         list_add_tail(&msg->list, &hsw->empty_list);
 520         spin_unlock_irqrestore(&hsw->dsp->spinlock, flags);
 521         return ret;
 522 }
 523
 524 static int ipc_tx_message(struct sst_hsw *hsw, u32 header, void *tx_data,
 525         size_t tx_bytes, void *rx_data, size_t rx_bytes, int wait)
 526 {
 527         struct ipc_message *msg;
 528         unsigned long flags;
 529
 530         spin_lock_irqsave(&hsw->dsp->spinlock, flags);
 531
 532         msg = msg_get_empty(hsw);
 533         if (msg == NULL) {
 534                 spin_unlock_irqrestore(&hsw->dsp->spinlock, flags);
 535                 return -EBUSY;
 536         }
 537
 538         if (tx_bytes)
 539                 memcpy(msg->tx_data, tx_data, tx_bytes);
 540
 541         msg->header = header;
 542         msg->tx_size = tx_bytes;
 543         msg->rx_size = rx_bytes;
 544         msg->wait = wait;
 545         msg->errno = 0;
 546         msg->pending = false;
 547         msg->complete = false;
 548
 549         list_add_tail(&msg->list, &hsw->tx_list);
 550         spin_unlock_irqrestore(&hsw->dsp->spinlock, flags);
 551
 552         queue_kthread_work(&hsw->kworker, &hsw->kwork);
 553
 554         if (wait)
 555                 return tx_wait_done(hsw, msg, rx_data);
 556         else
 557                 return 0;
 558 }
 559
 560 static inline int ipc_tx_message_wait(struct sst_hsw *hsw, u32 header,
 561         void *tx_data, size_t tx_bytes, void *rx_data, size_t rx_bytes)
 562 {
 563         return ipc_tx_message(hsw, header, tx_data, tx_bytes, rx_data,
 564                 rx_bytes, 1);
 565 }
 566
 567 static inline int ipc_tx_message_nowait(struct sst_hsw *hsw, u32 header,
 568         void *tx_data, size_t tx_bytes)
 569 {
 570         return ipc_tx_message(hsw, header, tx_data, tx_bytes, NULL, 0, 0);
 571 }
 572
 573 static void hsw_fw_ready(struct sst_hsw *hsw, u32 header)
 574 {
 575         struct sst_hsw_ipc_fw_ready fw_ready;
 576         u32 offset;
 577         u8 fw_info[IPC_MAX_MAILBOX_BYTES - 5 * sizeof(u32)];
 578         char *tmp[5], *pinfo;
 579         int i = 0;
 580
 581         offset = (header & 0x1FFFFFFF) << 3;
 582
 583         dev_dbg(hsw->dev, "ipc: DSP is ready 0x%8.8x offset %d\n",
 584                 header, offset);
 585
 586         /* copy data from the DSP FW ready offset */
 587         sst_dsp_read(hsw->dsp, &fw_ready, offset, sizeof(fw_ready));
 588
 589         sst_dsp_mailbox_init(hsw->dsp, fw_ready.inbox_offset,
 590                 fw_ready.inbox_size, fw_ready.outbox_offset,
 591                 fw_ready.outbox_size);
 592
 593         hsw->boot_complete = true;
 594         wake_up(&hsw->boot_wait);
 595
 596         dev_dbg(hsw->dev, " mailbox upstream 0x%x - size 0x%x\n",
 597                 fw_ready.inbox_offset, fw_ready.inbox_size);
 598         dev_dbg(hsw->dev, " mailbox downstream 0x%x - size 0x%x\n",
 599                 fw_ready.outbox_offset, fw_ready.outbox_size);
 600         if (fw_ready.fw_info_size < sizeof(fw_ready.fw_info)) {
 601                 fw_ready.fw_info[fw_ready.fw_info_size] = 0;
 602                 dev_dbg(hsw->dev, " Firmware info: %s \n", fw_ready.fw_info);
 603
 604                 /* log the FW version info got from the mailbox here. */
 605                 memcpy(fw_info, fw_ready.fw_info, fw_ready.fw_info_size);
 606                 pinfo = &fw_info[0];
 607                 for (i = 0; i < sizeof(tmp) / sizeof(char *); i++)
 608                         tmp[i] = strsep(&pinfo, " ");
 609                 dev_info(hsw->dev, "FW loaded, mailbox readback FW info: type %s, - "
 610                         "version: %s.%s, build %s, source commit id: %s\n",
 611                         tmp[0], tmp[1], tmp[2], tmp[3], tmp[4]);
 612         }
 613 }
 614
 615 static void hsw_notification_work(struct work_struct *work)
 616 {
 617         struct sst_hsw_stream *stream = container_of(work,
 618                         struct sst_hsw_stream, notify_work);
 619         struct sst_hsw_ipc_stream_glitch_position *glitch = &stream->glitch;
 620         struct sst_hsw_ipc_stream_get_position *pos = &stream->rpos;
 621         struct sst_hsw *hsw = stream->hsw;
 622         u32 reason;
 623
 624         reason = msg_get_notify_reason(stream->header);
 625
 626         switch (reason) {
 627         case IPC_STG_GLITCH:
 628                 trace_ipc_notification("DSP stream under/overrun",
 629                         stream->reply.stream_hw_id);
 630                 sst_dsp_inbox_read(hsw->dsp, glitch, sizeof(*glitch));
 631
 632                 dev_err(hsw->dev, "glitch %d pos 0x%x write pos 0x%x\n",
 633                         glitch->glitch_type, glitch->present_pos,
 634                         glitch->write_pos);
 635                 break;
 636
 637         case IPC_POSITION_CHANGED:
 638                 trace_ipc_notification("DSP stream position changed for",
 639                         stream->reply.stream_hw_id);
 640                 sst_dsp_inbox_read(hsw->dsp, pos, sizeof(*pos));
 641
 642                 if (stream->notify_position)
 643                         stream->notify_position(stream, stream->pdata);
 644
 645                 break;
 646         default:
 647                 dev_err(hsw->dev, "error: unknown notification 0x%x\n",
 648                         stream->header);
 649                 break;
 650         }
 651
 652         /* tell DSP that notification has been handled */
 653         sst_dsp_shim_update_bits(hsw->dsp, SST_IPCD,
 654                 SST_IPCD_BUSY | SST_IPCD_DONE, SST_IPCD_DONE);
 655
 656         /* unmask busy interrupt */
 657         sst_dsp_shim_update_bits(hsw->dsp, SST_IMRX, SST_IMRX_BUSY, 0);
 658 }
 659
 660 static struct ipc_message *reply_find_msg(struct sst_hsw *hsw, u32 header)
 661 {
 662         struct ipc_message *msg;
 663
 664         /* clear reply bits & status bits */
 665         header &= ~(IPC_STATUS_MASK | IPC_GLB_REPLY_MASK);
 666
 667         if (list_empty(&hsw->rx_list)) {
 668                 dev_err(hsw->dev, "error: rx list empty but received 0x%x\n",
 669                         header);
 670                 return NULL;
 671         }
 672
 673         list_for_each_entry(msg, &hsw->rx_list, list) {
 674                 if (msg->header == header)
 675                         return msg;
 676         }
 677
 678         return NULL;
 679 }
 680
 681 static void hsw_stream_update(struct sst_hsw *hsw, struct ipc_message *msg)
 682 {
 683         struct sst_hsw_stream *stream;
 684         u32 header = msg->header & ~(IPC_STATUS_MASK | IPC_GLB_REPLY_MASK);
 685         u32 stream_id = msg_get_stream_id(header);
 686         u32 stream_msg = msg_get_stream_type(header);
 687
 688         stream = get_stream_by_id(hsw, stream_id);
 689         if (stream == NULL)
 690                 return;
 691
 692         switch (stream_msg) {
 693         case IPC_STR_STAGE_MESSAGE:
 694         case IPC_STR_NOTIFICATION:
 695                 break;
 696         case IPC_STR_RESET:
 697                 trace_ipc_notification("stream reset", stream->reply.stream_hw_id);
 698                 break;
 699         case IPC_STR_PAUSE:
 700                 stream->running = false;
 701                 trace_ipc_notification("stream paused",
 702                         stream->reply.stream_hw_id);
 703                 break;
 704         case IPC_STR_RESUME:
 705                 stream->running = true;
 706                 trace_ipc_notification("stream running",
 707                         stream->reply.stream_hw_id);
 708                 break;
 709         }
 710 }
 711
 712 static int hsw_process_reply(struct sst_hsw *hsw, u32 header)
 713 {
 714         struct ipc_message *msg;
 715         u32 reply = msg_get_global_reply(header);
 716
 717         trace_ipc_reply("processing -->", header);
 718
 719         msg = reply_find_msg(hsw, header);
 720         if (msg == NULL) {
 721                 trace_ipc_error("error: can't find message header", header);
 722                 return -EIO;
 723         }
 724
 725         /* first process the header */
 726         switch (reply) {
 727         case IPC_GLB_REPLY_PENDING:
 728                 trace_ipc_pending_reply("received", header);
 729                 msg->pending = true;
 730                 hsw->pending = true;
 731                 return 1;
 732         case IPC_GLB_REPLY_SUCCESS:
 733                 if (msg->pending) {
 734                         trace_ipc_pending_reply("completed", header);
 735                         sst_dsp_inbox_read(hsw->dsp, msg->rx_data,
 736                                 msg->rx_size);
 737                         hsw->pending = false;
 738                 } else {
 739                         /* copy data from the DSP */
 740                         sst_dsp_outbox_read(hsw->dsp, msg->rx_data,
 741                                 msg->rx_size);
 742                 }
 743                 break;
 744         /* these will be rare - but useful for debug */
 745         case IPC_GLB_REPLY_UNKNOWN_MESSAGE_TYPE:
 746                 trace_ipc_error("error: unknown message type", header);
 747                 msg->errno = -EBADMSG;
 748                 break;
 749         case IPC_GLB_REPLY_OUT_OF_RESOURCES:
 750                 trace_ipc_error("error: out of resources", header);
 751                 msg->errno = -ENOMEM;
 752                 break;
 753         case IPC_GLB_REPLY_BUSY:
 754                 trace_ipc_error("error: reply busy", header);
 755                 msg->errno = -EBUSY;
 756                 break;
 757         case IPC_GLB_REPLY_FAILURE:
 758                 trace_ipc_error("error: reply failure", header);
 759                 msg->errno = -EINVAL;
 760                 break;
 761         case IPC_GLB_REPLY_STAGE_UNINITIALIZED:
 762                 trace_ipc_error("error: stage uninitialized", header);
 763                 msg->errno = -EINVAL;
 764                 break;
 765         case IPC_GLB_REPLY_NOT_FOUND:
 766                 trace_ipc_error("error: reply not found", header);
 767                 msg->errno = -EINVAL;
 768                 break;
 769         case IPC_GLB_REPLY_SOURCE_NOT_STARTED:
 770                 trace_ipc_error("error: source not started", header);
 771                 msg->errno = -EINVAL;
 772                 break;
 773         case IPC_GLB_REPLY_INVALID_REQUEST:
 774                 trace_ipc_error("error: invalid request", header);
 775                 msg->errno = -EINVAL;
 776                 break;
 777         case IPC_GLB_REPLY_ERROR_INVALID_PARAM:
 778                 trace_ipc_error("error: invalid parameter", header);
 779                 msg->errno = -EINVAL;
 780                 break;
 781         default:
 782                 trace_ipc_error("error: unknown reply", header);
 783                 msg->errno = -EINVAL;
 784                 break;
 785         }
 786
 787         /* update any stream states */
 788         if (msg_get_global_type(header) == IPC_GLB_STREAM_MESSAGE)
 789                 hsw_stream_update(hsw, msg);
 790
 791         /* wake up and return the error if we have waiters on this message ? */
 792         list_del(&msg->list);
 793         tx_msg_reply_complete(hsw, msg);
 794
 795         return 1;
 796 }
 797
 798 static int hsw_stream_message(struct sst_hsw *hsw, u32 header)
 799 {
 800         u32 stream_msg, stream_id, stage_type;
 801         struct sst_hsw_stream *stream;
 802         int handled = 0;
 803
 804         stream_msg = msg_get_stream_type(header);
 805         stream_id = msg_get_stream_id(header);
 806         stage_type = msg_get_stage_type(header);
 807
 808         stream = get_stream_by_id(hsw, stream_id);
 809         if (stream == NULL)
 810                 return handled;
 811
 812         stream->header = header;
 813
 814         switch (stream_msg) {
 815         case IPC_STR_STAGE_MESSAGE:
 816                 dev_err(hsw->dev, "error: stage msg not implemented 0x%8.8x\n",
 817                         header);
 818                 break;
 819         case IPC_STR_NOTIFICATION:
 820                 schedule_work(&stream->notify_work);
 821                 break;
 822         default:
 823                 /* handle pending message complete request */
 824                 handled = hsw_process_reply(hsw, header);
 825                 break;
 826         }
 827
 828         return handled;
 829 }
 830
 831 static int hsw_log_message(struct sst_hsw *hsw, u32 header)
 832 {
 833         u32 operation = (header & IPC_LOG_OP_MASK) >>  IPC_LOG_OP_SHIFT;
 834         struct sst_hsw_log_stream *stream = &hsw->log_stream;
 835         int ret = 1;
 836
 837         if (operation != IPC_DEBUG_REQUEST_LOG_DUMP) {
 838                 dev_err(hsw->dev,
 839                         "error: log msg not implemented 0x%8.8x\n", header);
 840                 return 0;
 841         }
 842
 843         mutex_lock(&stream->rw_mutex);
 844         stream->last_pos = stream->curr_pos;
 845         sst_dsp_inbox_read(
 846                 hsw->dsp, &stream->curr_pos, sizeof(stream->curr_pos));
 847         mutex_unlock(&stream->rw_mutex);
 848
 849         schedule_work(&stream->notify_work);
 850
 851         return ret;
 852 }
 853
 854 static int hsw_process_notification(struct sst_hsw *hsw)
 855 {
 856         struct sst_dsp *sst = hsw->dsp;
 857         u32 type, header;
 858         int handled = 1;
 859
 860         header = sst_dsp_shim_read_unlocked(sst, SST_IPCD);
 861         type = msg_get_global_type(header);
 862
 863         trace_ipc_request("processing -->", header);
 864
 865         /* FW Ready is a special case */
 866         if (!hsw->boot_complete && header & IPC_FW_READY) {
 867                 hsw_fw_ready(hsw, header);
 868                 return handled;
 869         }
 870
 871         switch (type) {
 872         case IPC_GLB_GET_FW_VERSION:
 873         case IPC_GLB_ALLOCATE_STREAM:
 874         case IPC_GLB_FREE_STREAM:
 875         case IPC_GLB_GET_FW_CAPABILITIES:
 876         case IPC_GLB_REQUEST_DUMP:
 877         case IPC_GLB_GET_DEVICE_FORMATS:
 878         case IPC_GLB_SET_DEVICE_FORMATS:
 879         case IPC_GLB_ENTER_DX_STATE:
 880         case IPC_GLB_GET_MIXER_STREAM_INFO:
 881         case IPC_GLB_MAX_IPC_MESSAGE_TYPE:
 882         case IPC_GLB_RESTORE_CONTEXT:
 883         case IPC_GLB_SHORT_REPLY:
 884                 dev_err(hsw->dev, "error: message type %d header 0x%x\n",
 885                         type, header);
 886                 break;
 887         case IPC_GLB_STREAM_MESSAGE:
 888                 handled = hsw_stream_message(hsw, header);
 889                 break;
 890         case IPC_GLB_DEBUG_LOG_MESSAGE:
 891                 handled = hsw_log_message(hsw, header);
 892                 break;
 893         default:
 894                 dev_err(hsw->dev, "error: unexpected type %d hdr 0x%8.8x\n",
 895                         type, header);
 896                 break;
 897         }
 898
 899         return handled;
 900 }
 901
 902 static irqreturn_t hsw_irq_thread(int irq, void *context)
 903 {
 904         struct sst_dsp *sst = (struct sst_dsp *) context;
 905         struct sst_hsw *hsw = sst_dsp_get_thread_context(sst);
 906         u32 ipcx, ipcd;
 907         int handled;
 908         unsigned long flags;
 909
 910         spin_lock_irqsave(&sst->spinlock, flags);
 911
 912         ipcx = sst_dsp_ipc_msg_rx(hsw->dsp);
 913         ipcd = sst_dsp_shim_read_unlocked(sst, SST_IPCD);
 914
 915         /* reply message from DSP */
 916         if (ipcx & SST_IPCX_DONE) {
 917
 918                 /* Handle Immediate reply from DSP Core */
 919                 handled = hsw_process_reply(hsw, ipcx);
 920
 921                 if (handled > 0) {
 922                         /* clear DONE bit - tell DSP we have completed */
 923                         sst_dsp_shim_update_bits_unlocked(sst, SST_IPCX,
 924                                 SST_IPCX_DONE, 0);
 925
 926                         /* unmask Done interrupt */
 927                         sst_dsp_shim_update_bits_unlocked(sst, SST_IMRX,
 928                                 SST_IMRX_DONE, 0);
 929                 }
 930         }
 931
 932         /* new message from DSP */
 933         if (ipcd & SST_IPCD_BUSY) {
 934
 935                 /* Handle Notification and Delayed reply from DSP Core */
 936                 handled = hsw_process_notification(hsw);
 937
 938                 /* clear BUSY bit and set DONE bit - accept new messages */
 939                 if (handled > 0) {
 940                         sst_dsp_shim_update_bits_unlocked(sst, SST_IPCD,
 941                                 SST_IPCD_BUSY | SST_IPCD_DONE, SST_IPCD_DONE);
 942
 943                         /* unmask busy interrupt */
 944                         sst_dsp_shim_update_bits_unlocked(sst, SST_IMRX,
 945                                 SST_IMRX_BUSY, 0);
 946                 }
 947         }
 948
 949         spin_unlock_irqrestore(&sst->spinlock, flags);
 950
 951         /* continue to send any remaining messages... */
 952         queue_kthread_work(&hsw->kworker, &hsw->kwork);
 953
 954         return IRQ_HANDLED;
 955 }
 956
 957 int sst_hsw_fw_get_version(struct sst_hsw *hsw,
 958         struct sst_hsw_ipc_fw_version *version)
 959 {
 960         int ret;
 961
 962         ret = ipc_tx_message_wait(hsw, IPC_GLB_TYPE(IPC_GLB_GET_FW_VERSION),
 963                 NULL, 0, version, sizeof(*version));
 964         if (ret < 0)
 965                 dev_err(hsw->dev, "error: get version failed\n");
 966
 967         return ret;
 968 }
 969
 970 /* Mixer Controls */
 971 int sst_hsw_stream_get_volume(struct sst_hsw *hsw, struct sst_hsw_stream *stream,
 972         u32 stage_id, u32 channel, u32 *volume)
 973 {
 974         if (channel > 1)
 975                 return -EINVAL;
 976
 977         sst_dsp_read(hsw->dsp, volume,
 978                 stream->reply.volume_register_address[channel],
 979                 sizeof(*volume));
 980
 981         return 0;
 982 }
 983
 984 /* stream volume */
 985 int sst_hsw_stream_set_volume(struct sst_hsw *hsw,
 986         struct sst_hsw_stream *stream, u32 stage_id, u32 channel, u32 volume)
 987 {
 988         struct sst_hsw_ipc_volume_req *req;
 989         u32 header;
 990         int ret;
 991
 992         trace_ipc_request("set stream volume", stream->reply.stream_hw_id);
 993
 994         if (channel >= 2 && channel != SST_HSW_CHANNELS_ALL)
 995                 return -EINVAL;
 996
 997         header = IPC_GLB_TYPE(IPC_GLB_STREAM_MESSAGE) |
 998                 IPC_STR_TYPE(IPC_STR_STAGE_MESSAGE);
 999         header |= (stream->reply.stream_hw_id << IPC_STR_ID_SHIFT);
1000         header |= (IPC_STG_SET_VOLUME << IPC_STG_TYPE_SHIFT);
1001         header |= (stage_id << IPC_STG_ID_SHIFT);
1002
1003         req = &stream->vol_req;
1004         req->target_volume = volume;
1005
1006         /* set both at same time ? */
1007         if (channel == SST_HSW_CHANNELS_ALL) {
1008                 if (hsw->mute[0] && hsw->mute[1]) {
1009                         hsw->mute_volume[0] = hsw->mute_volume[1] = volume;
1010                         return 0;
1011                 } else if (hsw->mute[0])
1012                         req->channel = 1;
1013                 else if (hsw->mute[1])
1014                         req->channel = 0;
1015                 else
1016                         req->channel = SST_HSW_CHANNELS_ALL;
1017         } else {
1018                 /* set only 1 channel */
1019                 if (hsw->mute[channel]) {
1020                         hsw->mute_volume[channel] = volume;
1021                         return 0;
1022                 }
1023                 req->channel = channel;
1024         }
1025
1026         ret = ipc_tx_message_wait(hsw, header, req, sizeof(*req), NULL, 0);
1027         if (ret < 0) {
1028                 dev_err(hsw->dev, "error: set stream volume failed\n");
1029                 return ret;
1030         }
1031
1032         return 0;
1033 }
1034
1035 int sst_hsw_mixer_get_volume(struct sst_hsw *hsw, u32 stage_id, u32 channel,
1036         u32 *volume)
1037 {
1038         if (channel > 1)
1039                 return -EINVAL;
1040
1041         sst_dsp_read(hsw->dsp, volume,
1042                 hsw->mixer_info.volume_register_address[channel],
1043                 sizeof(*volume));
1044
1045         return 0;
1046 }
1047
1048 /* global mixer volume */
1049 int sst_hsw_mixer_set_volume(struct sst_hsw *hsw, u32 stage_id, u32 channel,
1050         u32 volume)
1051 {
1052         struct sst_hsw_ipc_volume_req req;
1053         u32 header;
1054         int ret;
1055
1056         trace_ipc_request("set mixer volume", volume);
1057
1058         if (channel >= 2 && channel != SST_HSW_CHANNELS_ALL)
1059                 return -EINVAL;
1060
1061         /* set both at same time ? */
1062         if (channel == SST_HSW_CHANNELS_ALL) {
1063                 if (hsw->mute[0] && hsw->mute[1]) {
1064                         hsw->mute_volume[0] = hsw->mute_volume[1] = volume;
1065                         return 0;
1066                 } else if (hsw->mute[0])
1067                         req.channel = 1;
1068                 else if (hsw->mute[1])
1069                         req.channel = 0;
1070                 else
1071                         req.channel = SST_HSW_CHANNELS_ALL;
1072         } else {
1073                 /* set only 1 channel */
1074                 if (hsw->mute[channel]) {
1075                         hsw->mute_volume[channel] = volume;
1076                         return 0;
1077                 }
1078                 req.channel = channel;
1079         }
1080
1081         header = IPC_GLB_TYPE(IPC_GLB_STREAM_MESSAGE) |
1082                 IPC_STR_TYPE(IPC_STR_STAGE_MESSAGE);
1083         header |= (hsw->mixer_info.mixer_hw_id << IPC_STR_ID_SHIFT);
1084         header |= (IPC_STG_SET_VOLUME << IPC_STG_TYPE_SHIFT);
1085         header |= (stage_id << IPC_STG_ID_SHIFT);
1086
1087         req.curve_duration = hsw->curve_duration;
1088         req.curve_type = hsw->curve_type;
1089         req.target_volume = volume;
1090
1091         ret = ipc_tx_message_wait(hsw, header, &req, sizeof(req), NULL, 0);
1092         if (ret < 0) {
1093                 dev_err(hsw->dev, "error: set mixer volume failed\n");
1094                 return ret;
1095         }
1096
1097         return 0;
1098 }
1099
1100 /* Stream API */
1101 struct sst_hsw_stream *sst_hsw_stream_new(struct sst_hsw *hsw, int id,
1102         u32 (*notify_position)(struct sst_hsw_stream *stream, void *data),
1103         void *data)
1104 {
1105         struct sst_hsw_stream *stream;
1106         struct sst_dsp *sst = hsw->dsp;
1107         unsigned long flags;
1108
1109         stream = kzalloc(sizeof(*stream), GFP_KERNEL);
1110         if (stream == NULL)
1111                 return NULL;
1112
1113         spin_lock_irqsave(&sst->spinlock, flags);
1114         stream->reply.stream_hw_id = INVALID_STREAM_HW_ID;
1115         list_add(&stream->node, &hsw->stream_list);
1116         stream->notify_position = notify_position;
1117         stream->pdata = data;
1118         stream->hsw = hsw;
1119         stream->host_id = id;
1120
1121         /* work to process notification messages */
1122         INIT_WORK(&stream->notify_work, hsw_notification_work);
1123         spin_unlock_irqrestore(&sst->spinlock, flags);
1124
1125         return stream;
1126 }
1127
1128 int sst_hsw_stream_free(struct sst_hsw *hsw, struct sst_hsw_stream *stream)
1129 {
1130         u32 header;
1131         int ret = 0;
1132         struct sst_dsp *sst = hsw->dsp;
1133         unsigned long flags;
1134
1135         if (!stream) {
1136                 dev_warn(hsw->dev, "warning: stream is NULL, no stream to free, ignore it.\n");
1137                 return 0;
1138         }
1139
1140         /* dont free DSP streams that are not commited */
1141         if (!stream->commited)
1142                 goto out;
1143
1144         trace_ipc_request("stream free", stream->host_id);
1145
1146         stream->free_req.stream_id = stream->reply.stream_hw_id;
1147         header = IPC_GLB_TYPE(IPC_GLB_FREE_STREAM);
1148
1149         ret = ipc_tx_message_wait(hsw, header, &stream->free_req,
1150                 sizeof(stream->free_req), NULL, 0);
1151         if (ret < 0) {
1152                 dev_err(hsw->dev, "error: free stream %d failed\n",
1153                         stream->free_req.stream_id);
1154                 return -EAGAIN;
1155         }
1156
1157         trace_hsw_stream_free_req(stream, &stream->free_req);
1158
1159 out:
1160         cancel_work_sync(&stream->notify_work);
1161         spin_lock_irqsave(&sst->spinlock, flags);
1162         list_del(&stream->node);
1163         kfree(stream);
1164         spin_unlock_irqrestore(&sst->spinlock, flags);
1165
1166         return ret;
1167 }
1168
1169 int sst_hsw_stream_set_bits(struct sst_hsw *hsw,
1170         struct sst_hsw_stream *stream, enum sst_hsw_bitdepth bits)
1171 {
1172         if (stream->commited) {
1173                 dev_err(hsw->dev, "error: stream committed for set bits\n");
1174                 return -EINVAL;
1175         }
1176
1177         stream->request.format.bitdepth = bits;
1178         return 0;
1179 }
1180
1181 int sst_hsw_stream_set_channels(struct sst_hsw *hsw,
1182         struct sst_hsw_stream *stream, int channels)
1183 {
1184         if (stream->commited) {
1185                 dev_err(hsw->dev, "error: stream committed for set channels\n");
1186                 return -EINVAL;
1187         }
1188
1189         stream->request.format.ch_num = channels;
1190         return 0;
1191 }
1192
1193 int sst_hsw_stream_set_rate(struct sst_hsw *hsw,
1194         struct sst_hsw_stream *stream, int rate)
1195 {
1196         if (stream->commited) {
1197                 dev_err(hsw->dev, "error: stream committed for set rate\n");
1198                 return -EINVAL;
1199         }
1200
1201         stream->request.format.frequency = rate;
1202         return 0;
1203 }
1204
1205 int sst_hsw_stream_set_map_config(struct sst_hsw *hsw,
1206         struct sst_hsw_stream *stream, u32 map,
1207         enum sst_hsw_channel_config config)
1208 {
1209         if (stream->commited) {
1210                 dev_err(hsw->dev, "error: stream committed for set map\n");
1211                 return -EINVAL;
1212         }
1213
1214         stream->request.format.map = map;
1215         stream->request.format.config = config;
1216         return 0;
1217 }
1218
1219 int sst_hsw_stream_set_style(struct sst_hsw *hsw,
1220         struct sst_hsw_stream *stream, enum sst_hsw_interleaving style)
1221 {
1222         if (stream->commited) {
1223                 dev_err(hsw->dev, "error: stream committed for set style\n");
1224                 return -EINVAL;
1225         }
1226
1227         stream->request.format.style = style;
1228         return 0;
1229 }
1230
1231 int sst_hsw_stream_set_valid(struct sst_hsw *hsw,
1232         struct sst_hsw_stream *stream, u32 bits)
1233 {
1234         if (stream->commited) {
1235                 dev_err(hsw->dev, "error: stream committed for set valid bits\n");
1236                 return -EINVAL;
1237         }
1238
1239         stream->request.format.valid_bit = bits;
1240         return 0;
1241 }
1242
1243 /* Stream Configuration */
1244 int sst_hsw_stream_format(struct sst_hsw *hsw, struct sst_hsw_stream *stream,
1245         enum sst_hsw_stream_path_id path_id,
1246         enum sst_hsw_stream_type stream_type,
1247         enum sst_hsw_stream_format format_id)
1248 {
1249         if (stream->commited) {
1250                 dev_err(hsw->dev, "error: stream committed for set format\n");
1251                 return -EINVAL;
1252         }
1253
1254         stream->request.path_id = path_id;
1255         stream->request.stream_type = stream_type;
1256         stream->request.format_id = format_id;
1257
1258         trace_hsw_stream_alloc_request(stream, &stream->request);
1259
1260         return 0;
1261 }
1262
1263 int sst_hsw_stream_buffer(struct sst_hsw *hsw, struct sst_hsw_stream *stream,
1264         u32 ring_pt_address, u32 num_pages,
1265         u32 ring_size, u32 ring_offset, u32 ring_first_pfn)
1266 {
1267         if (stream->commited) {
1268                 dev_err(hsw->dev, "error: stream committed for buffer\n");
1269                 return -EINVAL;
1270         }
1271
1272         stream->request.ringinfo.ring_pt_address = ring_pt_address;
1273         stream->request.ringinfo.num_pages = num_pages;
1274         stream->request.ringinfo.ring_size = ring_size;
1275         stream->request.ringinfo.ring_offset = ring_offset;
1276         stream->request.ringinfo.ring_first_pfn = ring_first_pfn;
1277
1278         trace_hsw_stream_buffer(stream);
1279
1280         return 0;
1281 }
1282
1283 int sst_hsw_stream_set_module_info(struct sst_hsw *hsw,
1284         struct sst_hsw_stream *stream, struct sst_module_runtime *runtime)
1285 {
1286         struct sst_hsw_module_map *map = &stream->request.map;
1287         struct sst_dsp *dsp = sst_hsw_get_dsp(hsw);
1288         struct sst_module *module = runtime->module;
1289
1290         if (stream->commited) {
1291                 dev_err(hsw->dev, "error: stream committed for set module\n");
1292                 return -EINVAL;
1293         }
1294
1295         /* only support initial module atm */
1296         map->module_entries_count = 1;
1297         map->module_entries[0].module_id = module->id;
1298         map->module_entries[0].entry_point = module->entry;
1299
1300         stream->request.persistent_mem.offset =
1301                 sst_dsp_get_offset(dsp, runtime->persistent_offset, SST_MEM_DRAM);
1302         stream->request.persistent_mem.size = module->persistent_size;
1303
1304         stream->request.scratch_mem.offset =
1305                 sst_dsp_get_offset(dsp, dsp->scratch_offset, SST_MEM_DRAM);
1306         stream->request.scratch_mem.size = dsp->scratch_size;
1307
1308         dev_dbg(hsw->dev, "module %d runtime %d using:\n", module->id,
1309                 runtime->id);
1310         dev_dbg(hsw->dev, " persistent offset 0x%x bytes 0x%x\n",
1311                 stream->request.persistent_mem.offset,
1312                 stream->request.persistent_mem.size);
1313         dev_dbg(hsw->dev, " scratch offset 0x%x bytes 0x%x\n",
1314                 stream->request.scratch_mem.offset,
1315                 stream->request.scratch_mem.size);
1316
1317         return 0;
1318 }
1319
1320 int sst_hsw_stream_commit(struct sst_hsw *hsw, struct sst_hsw_stream *stream)
1321 {
1322         struct sst_hsw_ipc_stream_alloc_req *str_req = &stream->request;
1323         struct sst_hsw_ipc_stream_alloc_reply *reply = &stream->reply;
1324         u32 header;
1325         int ret;
1326
1327         if (!stream) {
1328                 dev_warn(hsw->dev, "warning: stream is NULL, no stream to commit, ignore it.\n");
1329                 return 0;
1330         }
1331
1332         if (stream->commited) {
1333                 dev_warn(hsw->dev, "warning: stream is already committed, ignore it.\n");
1334                 return 0;
1335         }
1336
1337         trace_ipc_request("stream alloc", stream->host_id);
1338
1339         header = IPC_GLB_TYPE(IPC_GLB_ALLOCATE_STREAM);
1340
1341         ret = ipc_tx_message_wait(hsw, header, str_req, sizeof(*str_req),
1342                 reply, sizeof(*reply));
1343         if (ret < 0) {
1344                 dev_err(hsw->dev, "error: stream commit failed\n");
1345                 return ret;
1346         }
1347
1348         stream->commited = 1;
1349         trace_hsw_stream_alloc_reply(stream);
1350
1351         return 0;
1352 }
1353
1354 snd_pcm_uframes_t sst_hsw_stream_get_old_position(struct sst_hsw *hsw,
1355         struct sst_hsw_stream *stream)
1356 {
1357         return stream->old_position;
1358 }
1359
1360 void sst_hsw_stream_set_old_position(struct sst_hsw *hsw,
1361         struct sst_hsw_stream *stream, snd_pcm_uframes_t val)
1362 {
1363         stream->old_position = val;
1364 }
1365
1366 bool sst_hsw_stream_get_silence_start(struct sst_hsw *hsw,
1367         struct sst_hsw_stream *stream)
1368 {
1369         return stream->play_silence;
1370 }
1371
1372 void sst_hsw_stream_set_silence_start(struct sst_hsw *hsw,
1373         struct sst_hsw_stream *stream, bool val)
1374 {
1375         stream->play_silence = val;
1376 }
1377
1378 /* Stream Information - these calls could be inline but we want the IPC
1379  ABI to be opaque to client PCM drivers to cope with any future ABI changes */
1380 int sst_hsw_mixer_get_info(struct sst_hsw *hsw)
1381 {
1382         struct sst_hsw_ipc_stream_info_reply *reply;
1383         u32 header;
1384         int ret;
1385
1386         reply = &hsw->mixer_info;
1387         header = IPC_GLB_TYPE(IPC_GLB_GET_MIXER_STREAM_INFO);
1388
1389         trace_ipc_request("get global mixer info", 0);
1390
1391         ret = ipc_tx_message_wait(hsw, header, NULL, 0, reply, sizeof(*reply));
1392         if (ret < 0) {
1393                 dev_err(hsw->dev, "error: get stream info failed\n");
1394                 return ret;
1395         }
1396
1397         trace_hsw_mixer_info_reply(reply);
1398
1399         return 0;
1400 }
1401
1402 /* Send stream command */
1403 static int sst_hsw_stream_operations(struct sst_hsw *hsw, int type,
1404         int stream_id, int wait)
1405 {
1406         u32 header;
1407
1408         header = IPC_GLB_TYPE(IPC_GLB_STREAM_MESSAGE) | IPC_STR_TYPE(type);
1409         header |= (stream_id << IPC_STR_ID_SHIFT);
1410
1411         if (wait)
1412                 return ipc_tx_message_wait(hsw, header, NULL, 0, NULL, 0);
1413         else
1414                 return ipc_tx_message_nowait(hsw, header, NULL, 0);
1415 }
1416
1417 /* Stream ALSA trigger operations */
1418 int sst_hsw_stream_pause(struct sst_hsw *hsw, struct sst_hsw_stream *stream,
1419         int wait)
1420 {
1421         int ret;
1422
1423         if (!stream) {
1424                 dev_warn(hsw->dev, "warning: stream is NULL, no stream to pause, ignore it.\n");
1425                 return 0;
1426         }
1427
1428         trace_ipc_request("stream pause", stream->reply.stream_hw_id);
1429
1430         ret = sst_hsw_stream_operations(hsw, IPC_STR_PAUSE,
1431                 stream->reply.stream_hw_id, wait);
1432         if (ret < 0)
1433                 dev_err(hsw->dev, "error: failed to pause stream %d\n",
1434                         stream->reply.stream_hw_id);
1435
1436         return ret;
1437 }
1438
1439 int sst_hsw_stream_resume(struct sst_hsw *hsw, struct sst_hsw_stream *stream,
1440         int wait)
1441 {
1442         int ret;
1443
1444         if (!stream) {
1445                 dev_warn(hsw->dev, "warning: stream is NULL, no stream to resume, ignore it.\n");
1446                 return 0;
1447         }
1448
1449         trace_ipc_request("stream resume", stream->reply.stream_hw_id);
1450
1451         ret = sst_hsw_stream_operations(hsw, IPC_STR_RESUME,
1452                 stream->reply.stream_hw_id, wait);
1453         if (ret < 0)
1454                 dev_err(hsw->dev, "error: failed to resume stream %d\n",
1455                         stream->reply.stream_hw_id);
1456
1457         return ret;
1458 }
1459
1460 int sst_hsw_stream_reset(struct sst_hsw *hsw, struct sst_hsw_stream *stream)
1461 {
1462         int ret, tries = 10;
1463
1464         if (!stream) {
1465                 dev_warn(hsw->dev, "warning: stream is NULL, no stream to reset, ignore it.\n");
1466                 return 0;
1467         }
1468
1469         /* dont reset streams that are not commited */
1470         if (!stream->commited)
1471                 return 0;
1472
1473         /* wait for pause to complete before we reset the stream */
1474         while (stream->running && tries--)
1475                 msleep(1);
1476         if (!tries) {
1477                 dev_err(hsw->dev, "error: reset stream %d still running\n",
1478                         stream->reply.stream_hw_id);
1479                 return -EINVAL;
1480         }
1481
1482         trace_ipc_request("stream reset", stream->reply.stream_hw_id);
1483
1484         ret = sst_hsw_stream_operations(hsw, IPC_STR_RESET,
1485                 stream->reply.stream_hw_id, 1);
1486         if (ret < 0)
1487                 dev_err(hsw->dev, "error: failed to reset stream %d\n",
1488                         stream->reply.stream_hw_id);
1489         return ret;
1490 }
1491
1492 /* Stream pointer positions */
1493 u32 sst_hsw_get_dsp_position(struct sst_hsw *hsw,
1494         struct sst_hsw_stream *stream)
1495 {
1496         u32 rpos;
1497
1498         sst_dsp_read(hsw->dsp, &rpos,
1499                 stream->reply.read_position_register_address, sizeof(rpos));
1500
1501         return rpos;
1502 }
1503
1504 /* Stream presentation (monotonic) positions */
1505 u64 sst_hsw_get_dsp_presentation_position(struct sst_hsw *hsw,
1506         struct sst_hsw_stream *stream)
1507 {
1508         u64 ppos;
1509
1510         sst_dsp_read(hsw->dsp, &ppos,
1511                 stream->reply.presentation_position_register_address,
1512                 sizeof(ppos));
1513
1514         return ppos;
1515 }
1516
1517 /* physical BE config */
1518 int sst_hsw_device_set_config(struct sst_hsw *hsw,
1519         enum sst_hsw_device_id dev, enum sst_hsw_device_mclk mclk,
1520         enum sst_hsw_device_mode mode, u32 clock_divider)
1521 {
1522         struct sst_hsw_ipc_device_config_req config;
1523         u32 header;
1524         int ret;
1525
1526         trace_ipc_request("set device config", dev);
1527
1528         config.ssp_interface = dev;
1529         config.clock_frequency = mclk;
1530         config.mode = mode;
1531         config.clock_divider = clock_divider;
1532         if (mode == SST_HSW_DEVICE_TDM_CLOCK_MASTER)
1533                 config.channels = 4;
1534         else
1535                 config.channels = 2;
1536
1537         trace_hsw_device_config_req(&config);
1538
1539         header = IPC_GLB_TYPE(IPC_GLB_SET_DEVICE_FORMATS);
1540
1541         ret = ipc_tx_message_wait(hsw, header, &config, sizeof(config),
1542                 NULL, 0);
1543         if (ret < 0)
1544                 dev_err(hsw->dev, "error: set device formats failed\n");
1545
1546         return ret;
1547 }
1548 EXPORT_SYMBOL_GPL(sst_hsw_device_set_config);
1549
1550 /* DX Config */
1551 int sst_hsw_dx_set_state(struct sst_hsw *hsw,
1552         enum sst_hsw_dx_state state, struct sst_hsw_ipc_dx_reply *dx)
1553 {
1554         u32 header, state_;
1555         int ret, item;
1556
1557         header = IPC_GLB_TYPE(IPC_GLB_ENTER_DX_STATE);
1558         state_ = state;
1559
1560         trace_ipc_request("PM enter Dx state", state);
1561
1562         ret = ipc_tx_message_wait(hsw, header, &state_, sizeof(state_),
1563                 dx, sizeof(*dx));
1564         if (ret < 0) {
1565                 dev_err(hsw->dev, "ipc: error set dx state %d failed\n", state);
1566                 return ret;
1567         }
1568
1569         for (item = 0; item < dx->entries_no; item++) {
1570                 dev_dbg(hsw->dev,
1571                         "Item[%d] offset[%x] - size[%x] - source[%x]\n",
1572                         item, dx->mem_info[item].offset,
1573                         dx->mem_info[item].size,
1574                         dx->mem_info[item].source);
1575         }
1576         dev_dbg(hsw->dev, "ipc: got %d entry numbers for state %d\n",
1577                 dx->entries_no, state);
1578
1579         return ret;
1580 }
1581
1582 struct sst_module_runtime *sst_hsw_runtime_module_create(struct sst_hsw *hsw,
1583         int mod_id, int offset)
1584 {
1585         struct sst_dsp *dsp = hsw->dsp;
1586         struct sst_module *module;
1587         struct sst_module_runtime *runtime;
1588         int err;
1589
1590         module = sst_module_get_from_id(dsp, mod_id);
1591         if (module == NULL) {
1592                 dev_err(dsp->dev, "error: failed to get module %d for pcm\n",
1593                         mod_id);
1594                 return NULL;
1595         }
1596
1597         runtime = sst_module_runtime_new(module, mod_id, NULL);
1598         if (runtime == NULL) {
1599                 dev_err(dsp->dev, "error: failed to create module %d runtime\n",
1600                         mod_id);
1601                 return NULL;
1602         }
1603
1604         err = sst_module_runtime_alloc_blocks(runtime, offset);
1605         if (err < 0) {
1606                 dev_err(dsp->dev, "error: failed to alloc blocks for module %d runtime\n",
1607                         mod_id);
1608                 sst_module_runtime_free(runtime);
1609                 return NULL;
1610         }
1611
1612         dev_dbg(dsp->dev, "runtime id %d created for module %d\n", runtime->id,
1613                 mod_id);
1614         return runtime;
1615 }
1616
1617 void sst_hsw_runtime_module_free(struct sst_module_runtime *runtime)
1618 {
1619         sst_module_runtime_free_blocks(runtime);
1620         sst_module_runtime_free(runtime);
1621 }
1622
1623 #ifdef CONFIG_PM
1624 static int sst_hsw_dx_state_dump(struct sst_hsw *hsw)
1625 {
1626         struct sst_dsp *sst = hsw->dsp;
1627         u32 item, offset, size;
1628         int ret = 0;
1629
1630         trace_ipc_request("PM state dump. Items #", SST_HSW_MAX_DX_REGIONS);
1631
1632         if (hsw->dx.entries_no > SST_HSW_MAX_DX_REGIONS) {
1633                 dev_err(hsw->dev,
1634                         "error: number of FW context regions greater than %d\n",
1635                         SST_HSW_MAX_DX_REGIONS);
1636                 memset(&hsw->dx, 0, sizeof(hsw->dx));
1637                 return -EINVAL;
1638         }
1639
1640         ret = sst_dsp_dma_get_channel(sst, 0);
1641         if (ret < 0) {
1642                 dev_err(hsw->dev, "error: cant allocate dma channel %d\n", ret);
1643                 return ret;
1644         }
1645
1646         /* set on-demond mode on engine 0 channel 3 */
1647         sst_dsp_shim_update_bits(sst, SST_HMDC,
1648                         SST_HMDC_HDDA_E0_ALLCH | SST_HMDC_HDDA_E1_ALLCH,
1649                         SST_HMDC_HDDA_E0_ALLCH | SST_HMDC_HDDA_E1_ALLCH);
1650
1651         for (item = 0; item < hsw->dx.entries_no; item++) {
1652                 if (hsw->dx.mem_info[item].source == SST_HSW_DX_TYPE_MEMORY_DUMP
1653                         && hsw->dx.mem_info[item].offset > DSP_DRAM_ADDR_OFFSET
1654                         && hsw->dx.mem_info[item].offset <
1655                         DSP_DRAM_ADDR_OFFSET + SST_HSW_DX_CONTEXT_SIZE) {
1656
1657                         offset = hsw->dx.mem_info[item].offset
1658                                         - DSP_DRAM_ADDR_OFFSET;
1659                         size = (hsw->dx.mem_info[item].size + 3) & (~3);
1660
1661                         ret = sst_dsp_dma_copyfrom(sst, hsw->dx_context_paddr + offset,
1662                                 sst->addr.lpe_base + offset, size);
1663                         if (ret < 0) {
1664                                 dev_err(hsw->dev,
1665                                         "error: FW context dump failed\n");
1666                                 memset(&hsw->dx, 0, sizeof(hsw->dx));
1667                                 goto out;
1668                         }
1669                 }
1670         }
1671
1672 out:
1673         sst_dsp_dma_put_channel(sst);
1674         return ret;
1675 }
1676
1677 static int sst_hsw_dx_state_restore(struct sst_hsw *hsw)
1678 {
1679         struct sst_dsp *sst = hsw->dsp;
1680         u32 item, offset, size;
1681         int ret;
1682
1683         for (item = 0; item < hsw->dx.entries_no; item++) {
1684                 if (hsw->dx.mem_info[item].source == SST_HSW_DX_TYPE_MEMORY_DUMP
1685                         && hsw->dx.mem_info[item].offset > DSP_DRAM_ADDR_OFFSET
1686                         && hsw->dx.mem_info[item].offset <
1687                         DSP_DRAM_ADDR_OFFSET + SST_HSW_DX_CONTEXT_SIZE) {
1688
1689                         offset = hsw->dx.mem_info[item].offset
1690                                         - DSP_DRAM_ADDR_OFFSET;
1691                         size = (hsw->dx.mem_info[item].size + 3) & (~3);
1692
1693                         ret = sst_dsp_dma_copyto(sst, sst->addr.lpe_base + offset,
1694                                 hsw->dx_context_paddr + offset, size);
1695                         if (ret < 0) {
1696                                 dev_err(hsw->dev,
1697                                         "error: FW context restore failed\n");
1698                                 return ret;
1699                         }
1700                 }
1701         }
1702
1703         return 0;
1704 }
1705
1706 static void sst_hsw_drop_all(struct sst_hsw *hsw)
1707 {
1708         struct ipc_message *msg, *tmp;
1709         unsigned long flags;
1710         int tx_drop_cnt = 0, rx_drop_cnt = 0;
1711
1712         /* drop all TX and Rx messages before we stall + reset DSP */
1713         spin_lock_irqsave(&hsw->dsp->spinlock, flags);
1714
1715         list_for_each_entry_safe(msg, tmp, &hsw->tx_list, list) {
1716                 list_move(&msg->list, &hsw->empty_list);
1717                 tx_drop_cnt++;
1718         }
1719
1720         list_for_each_entry_safe(msg, tmp, &hsw->rx_list, list) {
1721                 list_move(&msg->list, &hsw->empty_list);
1722                 rx_drop_cnt++;
1723         }
1724
1725         spin_unlock_irqrestore(&hsw->dsp->spinlock, flags);
1726
1727         if (tx_drop_cnt || rx_drop_cnt)
1728                 dev_err(hsw->dev, "dropped IPC msg RX=%d, TX=%d\n",
1729                         tx_drop_cnt, rx_drop_cnt);
1730 }
1731
1732 int sst_hsw_dsp_load(struct sst_hsw *hsw)
1733 {
1734         struct sst_dsp *dsp = hsw->dsp;
1735         struct sst_fw *sst_fw, *t;
1736         int ret;
1737
1738         dev_dbg(hsw->dev, "loading audio DSP....");
1739
1740         ret = sst_dsp_wake(dsp);
1741         if (ret < 0) {
1742                 dev_err(hsw->dev, "error: failed to wake audio DSP\n");
1743                 return -ENODEV;
1744         }
1745
1746         ret = sst_dsp_dma_get_channel(dsp, 0);
1747         if (ret < 0) {
1748                 dev_err(hsw->dev, "error: cant allocate dma channel %d\n", ret);
1749                 return ret;
1750         }
1751
1752         list_for_each_entry_safe_reverse(sst_fw, t, &dsp->fw_list, list) {
1753                 ret = sst_fw_reload(sst_fw);
1754                 if (ret < 0) {
1755                         dev_err(hsw->dev, "error: SST FW reload failed\n");
1756                         sst_dsp_dma_put_channel(dsp);
1757                         return -ENOMEM;
1758                 }
1759         }
1760         ret = sst_block_alloc_scratch(hsw->dsp);
1761         if (ret < 0)
1762                 return -EINVAL;
1763
1764         sst_dsp_dma_put_channel(dsp);
1765         return 0;
1766 }
1767
1768 static int sst_hsw_dsp_restore(struct sst_hsw *hsw)
1769 {
1770         struct sst_dsp *dsp = hsw->dsp;
1771         int ret;
1772
1773         dev_dbg(hsw->dev, "restoring audio DSP....");
1774
1775         ret = sst_dsp_dma_get_channel(dsp, 0);
1776         if (ret < 0) {
1777                 dev_err(hsw->dev, "error: cant allocate dma channel %d\n", ret);
1778                 return ret;
1779         }
1780
1781         ret = sst_hsw_dx_state_restore(hsw);
1782         if (ret < 0) {
1783                 dev_err(hsw->dev, "error: SST FW context restore failed\n");
1784                 sst_dsp_dma_put_channel(dsp);
1785                 return -ENOMEM;
1786         }
1787         sst_dsp_dma_put_channel(dsp);
1788
1789         /* wait for DSP boot completion */
1790         sst_dsp_boot(dsp);
1791
1792         return ret;
1793 }
1794
1795 int sst_hsw_dsp_runtime_suspend(struct sst_hsw *hsw)
1796 {
1797         int ret;
1798
1799         dev_dbg(hsw->dev, "audio dsp runtime suspend\n");
1800
1801         ret = sst_hsw_dx_set_state(hsw, SST_HSW_DX_STATE_D3, &hsw->dx);
1802         if (ret < 0)
1803                 return ret;
1804
1805         sst_dsp_stall(hsw->dsp);
1806
1807         ret = sst_hsw_dx_state_dump(hsw);
1808         if (ret < 0)
1809                 return ret;
1810
1811         sst_hsw_drop_all(hsw);
1812
1813         return 0;
1814 }
1815
1816 int sst_hsw_dsp_runtime_sleep(struct sst_hsw *hsw)
1817 {
1818         struct sst_fw *sst_fw, *t;
1819         struct sst_dsp *dsp = hsw->dsp;
1820
1821         list_for_each_entry_safe(sst_fw, t, &dsp->fw_list, list) {
1822                 sst_fw_unload(sst_fw);
1823         }
1824         sst_block_free_scratch(dsp);
1825
1826         hsw->boot_complete = false;
1827
1828         sst_dsp_sleep(dsp);
1829
1830         return 0;
1831 }
1832
1833 int sst_hsw_dsp_runtime_resume(struct sst_hsw *hsw)
1834 {
1835         struct device *dev = hsw->dev;
1836         int ret;
1837
1838         dev_dbg(dev, "audio dsp runtime resume\n");
1839
1840         if (hsw->boot_complete)
1841                 return 1; /* tell caller no action is required */
1842
1843         ret = sst_hsw_dsp_restore(hsw);
1844         if (ret < 0)
1845                 dev_err(dev, "error: audio DSP boot failure\n");
1846
1847         ret = wait_event_timeout(hsw->boot_wait, hsw->boot_complete,
1848                 msecs_to_jiffies(IPC_BOOT_MSECS));
1849         if (ret == 0) {
1850                 dev_err(hsw->dev, "error: audio DSP boot timeout IPCD 0x%x IPCX 0x%x\n",
1851                         sst_dsp_shim_read_unlocked(hsw->dsp, SST_IPCD),
1852                         sst_dsp_shim_read_unlocked(hsw->dsp, SST_IPCX));
1853                 return -EIO;
1854         }
1855
1856         /* Set ADSP SSP port settings */
1857         ret = sst_hsw_device_set_config(hsw, SST_HSW_DEVICE_SSP_0,
1858                                         SST_HSW_DEVICE_MCLK_FREQ_24_MHZ,
1859                                         SST_HSW_DEVICE_CLOCK_MASTER, 9);
1860         if (ret < 0)
1861                 dev_err(dev, "error: SSP re-initialization failed\n");
1862
1863         return ret;
1864 }
1865 #endif
1866
1867 static int msg_empty_list_init(struct sst_hsw *hsw)
1868 {
1869         int i;
1870
1871         hsw->msg = kzalloc(sizeof(struct ipc_message) *
1872                 IPC_EMPTY_LIST_SIZE, GFP_KERNEL);
1873         if (hsw->msg == NULL)
1874                 return -ENOMEM;
1875
1876         for (i = 0; i < IPC_EMPTY_LIST_SIZE; i++) {
1877                 init_waitqueue_head(&hsw->msg[i].waitq);
1878                 list_add(&hsw->msg[i].list, &hsw->empty_list);
1879         }
1880
1881         return 0;
1882 }
1883
1884 struct sst_dsp *sst_hsw_get_dsp(struct sst_hsw *hsw)
1885 {
1886         return hsw->dsp;
1887 }
1888
1889 static struct sst_dsp_device hsw_dev = {
1890         .thread = hsw_irq_thread,
1891         .ops = &haswell_ops,
1892 };
1893
1894 int sst_hsw_dsp_init(struct device *dev, struct sst_pdata *pdata)
1895 {
1896         struct sst_hsw_ipc_fw_version version;
1897         struct sst_hsw *hsw;
1898         int ret;
1899
1900         dev_dbg(dev, "initialising Audio DSP IPC\n");
1901
1902         hsw = devm_kzalloc(dev, sizeof(*hsw), GFP_KERNEL);
1903         if (hsw == NULL)
1904                 return -ENOMEM;
1905
1906         hsw->dev = dev;
1907         INIT_LIST_HEAD(&hsw->stream_list);
1908         INIT_LIST_HEAD(&hsw->tx_list);
1909         INIT_LIST_HEAD(&hsw->rx_list);
1910         INIT_LIST_HEAD(&hsw->empty_list);
1911         init_waitqueue_head(&hsw->boot_wait);
1912         init_waitqueue_head(&hsw->wait_txq);
1913
1914         ret = msg_empty_list_init(hsw);
1915         if (ret < 0)
1916                 return -ENOMEM;
1917
1918         /* start the IPC message thread */
1919         init_kthread_worker(&hsw->kworker);
1920         hsw->tx_thread = kthread_run(kthread_worker_fn,
1921                                            &hsw->kworker, "%s",
1922                                            dev_name(hsw->dev));
1923         if (IS_ERR(hsw->tx_thread)) {
1924                 ret = PTR_ERR(hsw->tx_thread);
1925                 dev_err(hsw->dev, "error: failed to create message TX task\n");
1926                 goto err_free_msg;
1927         }
1928         init_kthread_work(&hsw->kwork, ipc_tx_msgs);
1929
1930         hsw_dev.thread_context = hsw;
1931
1932         /* init SST shim */
1933         hsw->dsp = sst_dsp_new(dev, &hsw_dev, pdata);
1934         if (hsw->dsp == NULL) {
1935                 ret = -ENODEV;
1936                 goto dsp_err;
1937         }
1938
1939         /* allocate DMA buffer for context storage */
1940         hsw->dx_context = dma_alloc_coherent(hsw->dsp->dma_dev,
1941                 SST_HSW_DX_CONTEXT_SIZE, &hsw->dx_context_paddr, GFP_KERNEL);
1942         if (hsw->dx_context == NULL) {
1943                 ret = -ENOMEM;
1944                 goto dma_err;
1945         }
1946
1947         /* keep the DSP in reset state for base FW loading */
1948         sst_dsp_reset(hsw->dsp);
1949
1950         hsw->sst_fw = sst_fw_new(hsw->dsp, pdata->fw, hsw);
1951         if (hsw->sst_fw == NULL) {
1952                 ret = -ENODEV;
1953                 dev_err(dev, "error: failed to load firmware\n");
1954                 goto fw_err;
1955         }
1956
1957         /* allocate scratch mem regions */
1958         ret = sst_block_alloc_scratch(hsw->dsp);
1959         if (ret < 0)
1960                 goto boot_err;
1961
1962         /* wait for DSP boot completion */
1963         sst_dsp_boot(hsw->dsp);
1964         ret = wait_event_timeout(hsw->boot_wait, hsw->boot_complete,
1965                 msecs_to_jiffies(IPC_BOOT_MSECS));
1966         if (ret == 0) {
1967                 ret = -EIO;
1968                 dev_err(hsw->dev, "error: audio DSP boot timeout IPCD 0x%x IPCX 0x%x\n",
1969                         sst_dsp_shim_read_unlocked(hsw->dsp, SST_IPCD),
1970                         sst_dsp_shim_read_unlocked(hsw->dsp, SST_IPCX));
1971                 goto boot_err;
1972         }
1973
1974         /* get the FW version */
1975         sst_hsw_fw_get_version(hsw, &version);
1976
1977         /* get the globalmixer */
1978         ret = sst_hsw_mixer_get_info(hsw);
1979         if (ret < 0) {
1980                 dev_err(hsw->dev, "error: failed to get stream info\n");
1981                 goto boot_err;
1982         }
1983
1984         pdata->dsp = hsw;
1985         return 0;
1986
1987 boot_err:
1988         sst_dsp_reset(hsw->dsp);
1989         sst_fw_free(hsw->sst_fw);
1990 fw_err:
1991         dma_free_coherent(hsw->dsp->dma_dev, SST_HSW_DX_CONTEXT_SIZE,
1992                         hsw->dx_context, hsw->dx_context_paddr);
1993 dma_err:
1994         sst_dsp_free(hsw->dsp);
1995 dsp_err:
1996         kthread_stop(hsw->tx_thread);
1997 err_free_msg:
1998         kfree(hsw->msg);
1999
2000         return ret;
2001 }
2002 EXPORT_SYMBOL_GPL(sst_hsw_dsp_init);
2003
2004 void sst_hsw_dsp_free(struct device *dev, struct sst_pdata *pdata)
2005 {
2006         struct sst_hsw *hsw = pdata->dsp;
2007
2008         sst_dsp_reset(hsw->dsp);
2009         sst_fw_free_all(hsw->dsp);
2010         dma_free_coherent(hsw->dsp->dma_dev, SST_HSW_DX_CONTEXT_SIZE,
2011                         hsw->dx_context, hsw->dx_context_paddr);
2012         sst_dsp_free(hsw->dsp);
2013         kthread_stop(hsw->tx_thread);
2014         kfree(hsw->msg);
2015 }
2016 EXPORT_SYMBOL_GPL(sst_hsw_dsp_free);