drivers/soc/ti/knav_qmss_acc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Keystone accumulator queue manager
   4  *
   5  * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
   6  * Author:      Sandeep Nair <sandeep_n@ti.com>
   7  *              Cyril Chemparathy <cyril@ti.com>
   8  *              Santosh Shilimkar <santosh.shilimkar@ti.com>
   9  */
  10
  11 #include <linux/dma-mapping.h>
  12 #include <linux/io.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/module.h>
  15 #include <linux/of_address.h>
  16 #include <linux/soc/ti/knav_qmss.h>
  17
  18 #include "knav_qmss.h"
  19
  20 #define knav_range_offset_to_inst(kdev, range, q)       \
  21         (range->queue_base_inst + (q << kdev->inst_shift))
  22
  23 static void __knav_acc_notify(struct knav_range_info *range,
  24                                 struct knav_acc_channel *acc)
  25 {
  26         struct knav_device *kdev = range->kdev;
  27         struct knav_queue_inst *inst;
  28         int range_base, queue;
  29
  30         range_base = kdev->base_id + range->queue_base;
  31
  32         if (range->flags & RANGE_MULTI_QUEUE) {
  33                 for (queue = 0; queue < range->num_queues; queue++) {
  34                         inst = knav_range_offset_to_inst(kdev, range,
  35                                                                 queue);
  36                         if (inst->notify_needed) {
  37                                 inst->notify_needed = 0;
  38                                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  39                                         range_base + queue);
  40                                 knav_queue_notify(inst);
  41                         }
  42                 }
  43         } else {
  44                 queue = acc->channel - range->acc_info.start_channel;
  45                 inst = knav_range_offset_to_inst(kdev, range, queue);
  46                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  47                         range_base + queue);
  48                 knav_queue_notify(inst);
  49         }
  50 }
  51
  52 static int knav_acc_set_notify(struct knav_range_info *range,
  53                                 struct knav_queue_inst *kq,
  54                                 bool enabled)
  55 {
  56         struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
  57         struct knav_device *kdev = range->kdev;
  58         u32 mask, offset;
  59
  60         /*
  61          * when enabling, we need to re-trigger an interrupt if we
  62          * have descriptors pending
  63          */
  64         if (!enabled || atomic_read(&kq->desc_count) <= 0)
  65                 return 0;
  66
  67         kq->notify_needed = 1;
  68         atomic_inc(&kq->acc->retrigger_count);
  69         mask = BIT(kq->acc->channel % 32);
  70         offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
  71         dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
  72                 kq->acc->name);
  73         writel_relaxed(mask, pdsp->intd + offset);
  74         return 0;
  75 }
  76
  77 static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
  78 {
  79         struct knav_acc_channel *acc;
  80         struct knav_queue_inst *kq = NULL;
  81         struct knav_range_info *range;
  82         struct knav_pdsp_info *pdsp;
  83         struct knav_acc_info *info;
  84         struct knav_device *kdev;
  85
  86         u32 *list, *list_cpu, val, idx, notifies;
  87         int range_base, channel, queue = 0;
  88         dma_addr_t list_dma;
  89
  90         range = _instdata;
  91         info  = &range->acc_info;
  92         kdev  = range->kdev;
  93         pdsp  = range->acc_info.pdsp;
  94         acc   = range->acc;
  95
  96         range_base = kdev->base_id + range->queue_base;
  97         if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
  98                 for (queue = 0; queue < range->num_irqs; queue++)
  99                         if (range->irqs[queue].irq == irq)
 100                                 break;
 101                 kq = knav_range_offset_to_inst(kdev, range, queue);
 102                 acc += queue;
 103         }
 104
 105         channel = acc->channel;
 106         list_dma = acc->list_dma[acc->list_index];
 107         list_cpu = acc->list_cpu[acc->list_index];
 108         dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",
 109                 channel, acc->list_index, list_cpu, &list_dma);
 110         if (atomic_read(&acc->retrigger_count)) {
 111                 atomic_dec(&acc->retrigger_count);
 112                 __knav_acc_notify(range, acc);
 113                 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 114                 /* ack the interrupt */
 115                 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 116                                pdsp->intd + ACC_INTD_OFFSET_EOI);
 117
 118                 return IRQ_HANDLED;
 119         }
 120
 121         notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 122         WARN_ON(!notifies);
 123         dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
 124                                 DMA_FROM_DEVICE);
 125
 126         for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
 127              list += ACC_LIST_ENTRY_WORDS) {
 128                 if (ACC_LIST_ENTRY_WORDS == 1) {
 129                         dev_dbg(kdev->dev,
 130                                 "acc-irq: list %d, entry @%p, %08x\n",
 131                                 acc->list_index, list, list[0]);
 132                 } else if (ACC_LIST_ENTRY_WORDS == 2) {
 133                         dev_dbg(kdev->dev,
 134                                 "acc-irq: list %d, entry @%p, %08x %08x\n",
 135                                 acc->list_index, list, list[0], list[1]);
 136                 } else if (ACC_LIST_ENTRY_WORDS == 4) {
 137                         dev_dbg(kdev->dev,
 138                                 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
 139                                 acc->list_index, list, list[0], list[1],
 140                                 list[2], list[3]);
 141                 }
 142
 143                 val = list[ACC_LIST_ENTRY_DESC_IDX];
 144                 if (!val)
 145                         break;
 146
 147                 if (range->flags & RANGE_MULTI_QUEUE) {
 148                         queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
 149                         if (queue < range_base ||
 150                             queue >= range_base + range->num_queues) {
 151                                 dev_err(kdev->dev,
 152                                         "bad queue %d, expecting %d-%d\n",
 153                                         queue, range_base,
 154                                         range_base + range->num_queues);
 155                                 break;
 156                         }
 157                         queue -= range_base;
 158                         kq = knav_range_offset_to_inst(kdev, range,
 159                                                                 queue);
 160                 }
 161
 162                 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
 163                         atomic_dec(&kq->desc_count);
 164                         dev_err(kdev->dev,
 165                                 "acc-irq: queue %d full, entry dropped\n",
 166                                 queue + range_base);
 167                         continue;
 168                 }
 169
 170                 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
 171                 kq->descs[idx] = val;
 172                 kq->notify_needed = 1;
 173                 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
 174                         val, idx, queue + range_base);
 175         }
 176
 177         __knav_acc_notify(range, acc);
 178         memset(list_cpu, 0, info->list_size);
 179         dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
 180                                    DMA_TO_DEVICE);
 181
 182         /* flip to the other list */
 183         acc->list_index ^= 1;
 184
 185         /* reset the interrupt counter */
 186         writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 187
 188         /* ack the interrupt */
 189         writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 190                        pdsp->intd + ACC_INTD_OFFSET_EOI);
 191
 192         return IRQ_HANDLED;
 193 }
 194
 195 static int knav_range_setup_acc_irq(struct knav_range_info *range,
 196                                 int queue, bool enabled)
 197 {
 198         struct knav_device *kdev = range->kdev;
 199         struct knav_acc_channel *acc;
 200         struct cpumask *cpu_mask;
 201         int ret = 0, irq;
 202         u32 old, new;
 203
 204         if (range->flags & RANGE_MULTI_QUEUE) {
 205                 acc = range->acc;
 206                 irq = range->irqs[0].irq;
 207                 cpu_mask = range->irqs[0].cpu_mask;
 208         } else {
 209                 acc = range->acc + queue;
 210                 irq = range->irqs[queue].irq;
 211                 cpu_mask = range->irqs[queue].cpu_mask;
 212         }
 213
 214         old = acc->open_mask;
 215         if (enabled)
 216                 new = old | BIT(queue);
 217         else
 218                 new = old & ~BIT(queue);
 219         acc->open_mask = new;
 220
 221         dev_dbg(kdev->dev,
 222                 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
 223                 old, new, acc->name);
 224
 225         if (likely(new == old))
 226                 return 0;
 227
 228         if (new && !old) {
 229                 dev_dbg(kdev->dev,
 230                         "setup-acc-irq: requesting %s for channel %s\n",
 231                         acc->name, acc->name);
 232                 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
 233                                   range);
 234                 if (!ret && cpu_mask) {
 235                         ret = irq_set_affinity_hint(irq, cpu_mask);
 236                         if (ret) {
 237                                 dev_warn(range->kdev->dev,
 238                                          "Failed to set IRQ affinity\n");
 239                                 return ret;
 240                         }
 241                 }
 242         }
 243
 244         if (old && !new) {
 245                 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
 246                         acc->name, acc->name);
 247                 ret = irq_set_affinity_hint(irq, NULL);
 248                 if (ret)
 249                         dev_warn(range->kdev->dev,
 250                                  "Failed to set IRQ affinity\n");
 251                 free_irq(irq, range);
 252         }
 253
 254         return ret;
 255 }
 256
 257 static const char *knav_acc_result_str(enum knav_acc_result result)
 258 {
 259         static const char * const result_str[] = {
 260                 [ACC_RET_IDLE]                  = "idle",
 261                 [ACC_RET_SUCCESS]               = "success",
 262                 [ACC_RET_INVALID_COMMAND]       = "invalid command",
 263                 [ACC_RET_INVALID_CHANNEL]       = "invalid channel",
 264                 [ACC_RET_INACTIVE_CHANNEL]      = "inactive channel",
 265                 [ACC_RET_ACTIVE_CHANNEL]        = "active channel",
 266                 [ACC_RET_INVALID_QUEUE]         = "invalid queue",
 267                 [ACC_RET_INVALID_RET]           = "invalid return code",
 268         };
 269
 270         if (result >= ARRAY_SIZE(result_str))
 271                 return result_str[ACC_RET_INVALID_RET];
 272         else
 273                 return result_str[result];
 274 }
 275
 276 static enum knav_acc_result
 277 knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
 278                 struct knav_reg_acc_command *cmd)
 279 {
 280         u32 result;
 281
 282         dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
 283                 cmd->command, cmd->queue_mask, cmd->list_dma,
 284                 cmd->queue_num, cmd->timer_config);
 285
 286         writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
 287         writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
 288         writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);
 289         writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
 290         writel_relaxed(cmd->command, &pdsp->acc_command->command);
 291
 292         /* wait for the command to clear */
 293         do {
 294                 result = readl_relaxed(&pdsp->acc_command->command);
 295         } while ((result >> 8) & 0xff);
 296
 297         return (result >> 24) & 0xff;
 298 }
 299
 300 static void knav_acc_setup_cmd(struct knav_device *kdev,
 301                                 struct knav_range_info *range,
 302                                 struct knav_reg_acc_command *cmd,
 303                                 int queue)
 304 {
 305         struct knav_acc_info *info = &range->acc_info;
 306         struct knav_acc_channel *acc;
 307         int queue_base;
 308         u32 queue_mask;
 309
 310         if (range->flags & RANGE_MULTI_QUEUE) {
 311                 acc = range->acc;
 312                 queue_base = range->queue_base;
 313                 queue_mask = BIT(range->num_queues) - 1;
 314         } else {
 315                 acc = range->acc + queue;
 316                 queue_base = range->queue_base + queue;
 317                 queue_mask = 0;
 318         }
 319
 320         memset(cmd, 0, sizeof(*cmd));
 321         cmd->command    = acc->channel;
 322         cmd->queue_mask = queue_mask;
 323         cmd->list_dma   = (u32)acc->list_dma[0];
 324         cmd->queue_num  = info->list_entries << 16;
 325         cmd->queue_num |= queue_base;
 326
 327         cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
 328         if (range->flags & RANGE_MULTI_QUEUE)
 329                 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
 330         cmd->timer_config |= info->pacing_mode << 16;
 331         cmd->timer_config |= info->timer_count;
 332 }
 333
 334 static void knav_acc_stop(struct knav_device *kdev,
 335                                 struct knav_range_info *range,
 336                                 int queue)
 337 {
 338         struct knav_reg_acc_command cmd;
 339         struct knav_acc_channel *acc;
 340         enum knav_acc_result result;
 341
 342         acc = range->acc + queue;
 343
 344         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 345         cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
 346         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 347
 348         dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
 349                 acc->name, knav_acc_result_str(result));
 350 }
 351
 352 static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
 353                                                 struct knav_range_info *range,
 354                                                 int queue)
 355 {
 356         struct knav_reg_acc_command cmd;
 357         struct knav_acc_channel *acc;
 358         enum knav_acc_result result;
 359
 360         acc = range->acc + queue;
 361
 362         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 363         cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
 364         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 365
 366         dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
 367                 acc->name, knav_acc_result_str(result));
 368
 369         return result;
 370 }
 371
 372 static int knav_acc_init_range(struct knav_range_info *range)
 373 {
 374         struct knav_device *kdev = range->kdev;
 375         struct knav_acc_channel *acc;
 376         enum knav_acc_result result;
 377         int queue;
 378
 379         for (queue = 0; queue < range->num_queues; queue++) {
 380                 acc = range->acc + queue;
 381
 382                 knav_acc_stop(kdev, range, queue);
 383                 acc->list_index = 0;
 384                 result = knav_acc_start(kdev, range, queue);
 385
 386                 if (result != ACC_RET_SUCCESS)
 387                         return -EIO;
 388
 389                 if (range->flags & RANGE_MULTI_QUEUE)
 390                         return 0;
 391         }
 392         return 0;
 393 }
 394
 395 static int knav_acc_init_queue(struct knav_range_info *range,
 396                                 struct knav_queue_inst *kq)
 397 {
 398         unsigned id = kq->id - range->queue_base;
 399
 400         kq->descs = devm_kcalloc(range->kdev->dev,
 401                                  ACC_DESCS_MAX, sizeof(u32), GFP_KERNEL);
 402         if (!kq->descs)
 403                 return -ENOMEM;
 404
 405         kq->acc = range->acc;
 406         if ((range->flags & RANGE_MULTI_QUEUE) == 0)
 407                 kq->acc += id;
 408         return 0;
 409 }
 410
 411 static int knav_acc_open_queue(struct knav_range_info *range,
 412                                 struct knav_queue_inst *inst, unsigned flags)
 413 {
 414         unsigned id = inst->id - range->queue_base;
 415
 416         return knav_range_setup_acc_irq(range, id, true);
 417 }
 418
 419 static int knav_acc_close_queue(struct knav_range_info *range,
 420                                         struct knav_queue_inst *inst)
 421 {
 422         unsigned id = inst->id - range->queue_base;
 423
 424         return knav_range_setup_acc_irq(range, id, false);
 425 }
 426
 427 static int knav_acc_free_range(struct knav_range_info *range)
 428 {
 429         struct knav_device *kdev = range->kdev;
 430         struct knav_acc_channel *acc;
 431         struct knav_acc_info *info;
 432         int channel, channels;
 433
 434         info = &range->acc_info;
 435
 436         if (range->flags & RANGE_MULTI_QUEUE)
 437                 channels = 1;
 438         else
 439                 channels = range->num_queues;
 440
 441         for (channel = 0; channel < channels; channel++) {
 442                 acc = range->acc + channel;
 443                 if (!acc->list_cpu[0])
 444                         continue;
 445                 dma_unmap_single(kdev->dev, acc->list_dma[0],
 446                                  info->mem_size, DMA_BIDIRECTIONAL);
 447                 free_pages_exact(acc->list_cpu[0], info->mem_size);
 448         }
 449         devm_kfree(range->kdev->dev, range->acc);
 450         return 0;
 451 }
 452
 453 struct knav_range_ops knav_acc_range_ops = {
 454         .set_notify     = knav_acc_set_notify,
 455         .init_queue     = knav_acc_init_queue,
 456         .open_queue     = knav_acc_open_queue,
 457         .close_queue    = knav_acc_close_queue,
 458         .init_range     = knav_acc_init_range,
 459         .free_range     = knav_acc_free_range,
 460 };
 461
 462 /**
 463  * knav_init_acc_range: Initialise accumulator ranges
 464  *
 465  * @kdev:               qmss device
 466  * @node:               device node
 467  * @range:              qmms range information
 468  *
 469  * Return 0 on success or error
 470  */
 471 int knav_init_acc_range(struct knav_device *kdev,
 472                         struct device_node *node,
 473                         struct knav_range_info *range)
 474 {
 475         struct knav_acc_channel *acc;
 476         struct knav_pdsp_info *pdsp;
 477         struct knav_acc_info *info;
 478         int ret, channel, channels;
 479         int list_size, mem_size;
 480         dma_addr_t list_dma;
 481         void *list_mem;
 482         u32 config[5];
 483
 484         range->flags |= RANGE_HAS_ACCUMULATOR;
 485         info = &range->acc_info;
 486
 487         ret = of_property_read_u32_array(node, "accumulator", config, 5);
 488         if (ret)
 489                 return ret;
 490
 491         info->pdsp_id           = config[0];
 492         info->start_channel     = config[1];
 493         info->list_entries      = config[2];
 494         info->pacing_mode       = config[3];
 495         info->timer_count       = config[4] / ACC_DEFAULT_PERIOD;
 496
 497         if (info->start_channel > ACC_MAX_CHANNEL) {
 498                 dev_err(kdev->dev, "channel %d invalid for range %s\n",
 499                         info->start_channel, range->name);
 500                 return -EINVAL;
 501         }
 502
 503         if (info->pacing_mode > 3) {
 504                 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
 505                         info->pacing_mode, range->name);
 506                 return -EINVAL;
 507         }
 508
 509         pdsp = knav_find_pdsp(kdev, info->pdsp_id);
 510         if (!pdsp) {
 511                 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
 512                         info->pdsp_id, range->name);
 513                 return -EINVAL;
 514         }
 515
 516         if (!pdsp->started) {
 517                 dev_err(kdev->dev, "pdsp id %d not started for range %s\n",
 518                         info->pdsp_id, range->name);
 519                 return -ENODEV;
 520         }
 521
 522         info->pdsp = pdsp;
 523         channels = range->num_queues;
 524         if (of_get_property(node, "multi-queue", NULL)) {
 525                 range->flags |= RANGE_MULTI_QUEUE;
 526                 channels = 1;
 527                 if (range->queue_base & (32 - 1)) {
 528                         dev_err(kdev->dev,
 529                                 "misaligned multi-queue accumulator range %s\n",
 530                                 range->name);
 531                         return -EINVAL;
 532                 }
 533                 if (range->num_queues > 32) {
 534                         dev_err(kdev->dev,
 535                                 "too many queues in accumulator range %s\n",
 536                                 range->name);
 537                         return -EINVAL;
 538                 }
 539         }
 540
 541         /* figure out list size */
 542         list_size  = info->list_entries;
 543         list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
 544         info->list_size = list_size;
 545         mem_size   = PAGE_ALIGN(list_size * 2);
 546         info->mem_size  = mem_size;
 547         range->acc = devm_kcalloc(kdev->dev, channels, sizeof(*range->acc),
 548                                   GFP_KERNEL);
 549         if (!range->acc)
 550                 return -ENOMEM;
 551
 552         for (channel = 0; channel < channels; channel++) {
 553                 acc = range->acc + channel;
 554                 acc->channel = info->start_channel + channel;
 555
 556                 /* allocate memory for the two lists */
 557                 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
 558                 if (!list_mem)
 559                         return -ENOMEM;
 560
 561                 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
 562                                           DMA_BIDIRECTIONAL);
 563                 if (dma_mapping_error(kdev->dev, list_dma)) {
 564                         free_pages_exact(list_mem, mem_size);
 565                         return -ENOMEM;
 566                 }
 567
 568                 memset(list_mem, 0, mem_size);
 569                 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
 570                                            DMA_TO_DEVICE);
 571                 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
 572                           acc->channel);
 573                 acc->list_cpu[0] = list_mem;
 574                 acc->list_cpu[1] = list_mem + list_size;
 575                 acc->list_dma[0] = list_dma;
 576                 acc->list_dma[1] = list_dma + list_size;
 577                 dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",
 578                         acc->name, acc->channel, &list_dma, list_mem);
 579         }
 580
 581         range->ops = &knav_acc_range_ops;
 582         return 0;
 583 }
 584 EXPORT_SYMBOL_GPL(knav_init_acc_range);