de1bd1fa04
Enable Zephyr device runtime power management mechanisms in Intel GP DMA driver. This allows Zephyr to track usage reference for power domain gating. Signed-off-by: Tomasz Leman <tomasz.m.leman@intel.com>
422 lines
10 KiB
C
422 lines
10 KiB
C
/*
|
|
* Copyright (c) 2022 Intel Corporation.
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
#include <zephyr/drivers/dma.h>
|
|
#include <zephyr/cache.h>
|
|
|
|
#define DT_DRV_COMPAT intel_adsp_gpdma
|
|
|
|
#define GPDMA_CTL_OFFSET 0x0004
|
|
#define GPDMA_CTL_FDCGB BIT(0)
|
|
#define GPDMA_CTL_DGCD BIT(30)
|
|
|
|
/* TODO make device tree defined? */
|
|
#define GPDMA_CHLLPC_OFFSET(channel) (0x0010 + channel*0x10)
|
|
#define GPDMA_CHLLPC_EN BIT(7)
|
|
#define GPDMA_CHLLPC_DHRS(x) SET_BITS(6, 0, x)
|
|
|
|
/* TODO make device tree defined? */
|
|
#define GPDMA_CHLLPL(channel) (0x0018 + channel*0x10)
|
|
#define GPDMA_CHLLPU(channel) (0x001c + channel*0x10)
|
|
|
|
#define GPDMA_OSEL(x) SET_BITS(25, 24, x)
|
|
#define SHIM_CLKCTL_LPGPDMA_SPA BIT(0)
|
|
#define SHIM_CLKCTL_LPGPDMA_CPA BIT(8)
|
|
|
|
# define DSP_INIT_LPGPDMA(x) (0x71A60 + (2*x))
|
|
# define LPGPDMA_CTLOSEL_FLAG BIT(15)
|
|
# define LPGPDMA_CHOSEL_FLAG 0xFF
|
|
|
|
#include "dma_dw_common.h"
|
|
#include <zephyr/pm/device.h>
|
|
#include <zephyr/pm/device_runtime.h>
|
|
|
|
#define LOG_LEVEL CONFIG_DMA_LOG_LEVEL
|
|
#include <zephyr/logging/log.h>
|
|
#include <zephyr/irq.h>
|
|
LOG_MODULE_REGISTER(dma_intel_adsp_gpdma);
|
|
|
|
|
|
/* Device run time data */
|
|
struct intel_adsp_gpdma_data {
|
|
struct dw_dma_dev_data dw_data;
|
|
};
|
|
|
|
/* Device constant configuration parameters */
|
|
struct intel_adsp_gpdma_cfg {
|
|
struct dw_dma_dev_cfg dw_cfg;
|
|
uint32_t shim;
|
|
};
|
|
|
|
static void intel_adsp_gpdma_llp_config(const struct device *dev,
|
|
uint32_t channel, uint32_t dma_slot)
|
|
{
|
|
#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA_HAS_LLP
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
|
|
dw_write(dev_cfg->shim, GPDMA_CHLLPC_OFFSET(channel),
|
|
GPDMA_CHLLPC_DHRS(dma_slot));
|
|
#endif
|
|
}
|
|
|
|
static inline void intel_adsp_gpdma_llp_enable(const struct device *dev,
|
|
uint32_t channel)
|
|
{
|
|
#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA_HAS_LLP
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
uint32_t val;
|
|
|
|
val = dw_read(dev_cfg->shim, GPDMA_CHLLPC_OFFSET(channel));
|
|
if (!(val & GPDMA_CHLLPC_EN)) {
|
|
dw_write(dev_cfg->shim, GPDMA_CHLLPC_OFFSET(channel),
|
|
val | GPDMA_CHLLPC_EN);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
static inline void intel_adsp_gpdma_llp_disable(const struct device *dev,
|
|
uint32_t channel)
|
|
{
|
|
#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA_HAS_LLP
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
uint32_t val;
|
|
|
|
val = dw_read(dev_cfg->shim, GPDMA_CHLLPC_OFFSET(channel));
|
|
dw_write(dev_cfg->shim, GPDMA_CHLLPC_OFFSET(channel),
|
|
val | GPDMA_CHLLPC_EN);
|
|
#endif
|
|
}
|
|
|
|
static inline void intel_adsp_gpdma_llp_read(const struct device *dev,
|
|
uint32_t channel, uint32_t *llp_l,
|
|
uint32_t *llp_u)
|
|
{
|
|
#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA_HAS_LLP
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
|
|
*llp_l = dw_read(dev_cfg->shim, GPDMA_CHLLPL(channel));
|
|
*llp_u = dw_read(dev_cfg->shim, GPDMA_CHLLPU(channel));
|
|
#endif
|
|
}
|
|
|
|
|
|
static int intel_adsp_gpdma_config(const struct device *dev, uint32_t channel,
|
|
struct dma_config *cfg)
|
|
{
|
|
int res = dw_dma_config(dev, channel, cfg);
|
|
|
|
if (res != 0) {
|
|
return res;
|
|
}
|
|
|
|
/* Assume all scatter/gathers are for the same device? */
|
|
switch (cfg->channel_direction) {
|
|
case MEMORY_TO_PERIPHERAL:
|
|
case PERIPHERAL_TO_MEMORY:
|
|
LOG_DBG("%s: dma %s configuring llp for %x",
|
|
__func__, dev->name, cfg->dma_slot);
|
|
intel_adsp_gpdma_llp_config(dev, channel, cfg->dma_slot);
|
|
break;
|
|
default:
|
|
break;
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
static int intel_adsp_gpdma_start(const struct device *dev, uint32_t channel)
|
|
{
|
|
int ret;
|
|
|
|
intel_adsp_gpdma_llp_enable(dev, channel);
|
|
ret = dw_dma_start(dev, channel);
|
|
if (ret != 0) {
|
|
intel_adsp_gpdma_llp_disable(dev, channel);
|
|
}
|
|
|
|
if (ret == 0) {
|
|
ret = pm_device_runtime_get(dev);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int intel_adsp_gpdma_stop(const struct device *dev, uint32_t channel)
|
|
{
|
|
int ret;
|
|
|
|
ret = dw_dma_stop(dev, channel);
|
|
if (ret == 0) {
|
|
intel_adsp_gpdma_llp_disable(dev, channel);
|
|
ret = pm_device_runtime_put(dev);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int intel_adsp_gpdma_copy(const struct device *dev, uint32_t channel,
|
|
uint32_t src, uint32_t dst, size_t size)
|
|
{
|
|
struct dw_dma_dev_data *const dev_data = dev->data;
|
|
struct dw_dma_chan_data *chan_data;
|
|
|
|
if (channel >= DW_MAX_CHAN) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
chan_data = &dev_data->chan[channel];
|
|
|
|
/* default action is to clear the DONE bit for all LLI making
|
|
* sure the cache is coherent between DSP and DMAC.
|
|
*/
|
|
for (int i = 0; i < chan_data->lli_count; i++) {
|
|
chan_data->lli[i].ctrl_hi &= ~DW_CTLH_DONE(1);
|
|
}
|
|
|
|
chan_data->ptr_data.current_ptr += size;
|
|
if (chan_data->ptr_data.current_ptr >= chan_data->ptr_data.end_ptr) {
|
|
chan_data->ptr_data.current_ptr = chan_data->ptr_data.start_ptr +
|
|
(chan_data->ptr_data.current_ptr - chan_data->ptr_data.end_ptr);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* Disables automatic clock gating (force disable clock gate) */
|
|
static void intel_adsp_gpdma_clock_enable(const struct device *dev)
|
|
{
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
uint32_t reg = dev_cfg->shim + GPDMA_CTL_OFFSET;
|
|
uint32_t val;
|
|
|
|
if (IS_ENABLED(CONFIG_SOC_SERIES_INTEL_ACE)) {
|
|
val = sys_read32(reg) | GPDMA_CTL_DGCD;
|
|
} else {
|
|
val = GPDMA_CTL_FDCGB;
|
|
}
|
|
|
|
sys_write32(val, reg);
|
|
}
|
|
|
|
static void intel_adsp_gpdma_select_owner(const struct device *dev)
|
|
{
|
|
#ifdef CONFIG_DMA_INTEL_ADSP_GPDMA_NEED_CONTROLLER_OWNERSHIP
|
|
#ifdef CONFIG_SOC_SERIES_INTEL_ACE
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
uint32_t reg = dev_cfg->shim + GPDMA_CTL_OFFSET;
|
|
uint32_t val = sys_read32(reg) | GPDMA_OSEL(0x3);
|
|
|
|
sys_write32(val, reg);
|
|
#else
|
|
sys_write32(LPGPDMA_CHOSEL_FLAG | LPGPDMA_CTLOSEL_FLAG, DSP_INIT_LPGPDMA(0));
|
|
sys_write32(LPGPDMA_CHOSEL_FLAG | LPGPDMA_CTLOSEL_FLAG, DSP_INIT_LPGPDMA(1));
|
|
ARG_UNUSED(dev);
|
|
#endif /* CONFIG_SOC_SERIES_INTEL_ACE */
|
|
#endif /* CONFIG_DMA_INTEL_ADSP_GPDMA_NEED_CONTROLLER_OWNERSHIP */
|
|
}
|
|
|
|
#ifdef CONFIG_SOC_SERIES_INTEL_ACE
|
|
static int intel_adsp_gpdma_enable(const struct device *dev)
|
|
{
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
uint32_t reg = dev_cfg->shim + GPDMA_CTL_OFFSET;
|
|
|
|
sys_write32(SHIM_CLKCTL_LPGPDMA_SPA, reg);
|
|
|
|
if (!WAIT_FOR((sys_read32(reg) & SHIM_CLKCTL_LPGPDMA_CPA), 10000,
|
|
k_busy_wait(1))) {
|
|
return -1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
int intel_adsp_gpdma_init(const struct device *dev)
|
|
{
|
|
const struct intel_adsp_gpdma_cfg *const dev_cfg = dev->config;
|
|
int ret;
|
|
|
|
#ifdef CONFIG_SOC_SERIES_INTEL_ACE
|
|
/* Power up */
|
|
ret = intel_adsp_gpdma_enable(dev);
|
|
|
|
if (ret == 0) {
|
|
pm_device_init_suspended(dev);
|
|
ret = pm_device_runtime_enable(dev);
|
|
}
|
|
|
|
if (ret != 0) {
|
|
LOG_ERR("%s: dma %s failed to initialize", __func__,
|
|
dev->name);
|
|
goto out;
|
|
}
|
|
#endif
|
|
|
|
/* DW DMA Owner Select to DSP */
|
|
intel_adsp_gpdma_select_owner(dev);
|
|
|
|
/* Disable dynamic clock gating appropriately before initializing */
|
|
intel_adsp_gpdma_clock_enable(dev);
|
|
|
|
/* Disable all channels and Channel interrupts */
|
|
ret = dw_dma_setup(dev);
|
|
if (ret != 0) {
|
|
LOG_ERR("%s: dma %s failed to initialize", __func__,
|
|
dev->name);
|
|
goto out;
|
|
}
|
|
|
|
/* Configure interrupts */
|
|
dev_cfg->dw_cfg.irq_config();
|
|
|
|
LOG_INF("%s: dma %s initialized", __func__,
|
|
dev->name);
|
|
|
|
out:
|
|
return 0;
|
|
}
|
|
|
|
int intel_adsp_gpdma_get_status(const struct device *dev, uint32_t channel, struct dma_status *stat)
|
|
{
|
|
uint32_t llp_l = 0;
|
|
uint32_t llp_u = 0;
|
|
|
|
if (channel >= DW_MAX_CHAN) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
intel_adsp_gpdma_llp_read(dev, channel, &llp_l, &llp_u);
|
|
stat->total_copied = ((uint64_t)llp_u << 32) | llp_l;
|
|
|
|
return dw_dma_get_status(dev, channel, stat);
|
|
}
|
|
|
|
int intel_adsp_gpdma_get_attribute(const struct device *dev, uint32_t type, uint32_t *value)
|
|
{
|
|
switch (type) {
|
|
case DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT:
|
|
*value = sys_cache_data_line_size_get();
|
|
break;
|
|
case DMA_ATTR_BUFFER_SIZE_ALIGNMENT:
|
|
*value = DMA_BUF_SIZE_ALIGNMENT(DT_COMPAT_GET_ANY_STATUS_OKAY(intel_adsp_gpdma));
|
|
break;
|
|
case DMA_ATTR_COPY_ALIGNMENT:
|
|
*value = DMA_COPY_ALIGNMENT(DT_COMPAT_GET_ANY_STATUS_OKAY(intel_adsp_gpdma));
|
|
break;
|
|
case DMA_ATTR_MAX_BLOCK_COUNT:
|
|
*value = CONFIG_DMA_DW_LLI_POOL_SIZE;
|
|
break;
|
|
default:
|
|
return -EINVAL;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
#ifdef CONFIG_PM_DEVICE
|
|
static int gpdma_pm_action(const struct device *dev, enum pm_device_action action)
|
|
{
|
|
switch (action) {
|
|
case PM_DEVICE_ACTION_SUSPEND:
|
|
case PM_DEVICE_ACTION_RESUME:
|
|
case PM_DEVICE_ACTION_TURN_ON:
|
|
case PM_DEVICE_ACTION_TURN_OFF:
|
|
break;
|
|
default:
|
|
return -ENOTSUP;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
static const struct dma_driver_api intel_adsp_gpdma_driver_api = {
|
|
.config = intel_adsp_gpdma_config,
|
|
.reload = intel_adsp_gpdma_copy,
|
|
.start = intel_adsp_gpdma_start,
|
|
.stop = intel_adsp_gpdma_stop,
|
|
.suspend = dw_dma_suspend,
|
|
.resume = dw_dma_resume,
|
|
.get_status = intel_adsp_gpdma_get_status,
|
|
.get_attribute = intel_adsp_gpdma_get_attribute,
|
|
};
|
|
|
|
#define INTEL_ADSP_GPDMA_CHAN_ARB_DATA(inst) \
|
|
static struct dw_drv_plat_data dmac##inst = { \
|
|
.chan[0] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
.chan[1] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
.chan[2] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
.chan[3] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
.chan[4] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
.chan[5] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
.chan[6] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
.chan[7] = { \
|
|
.class = 6, \
|
|
.weight = 0, \
|
|
}, \
|
|
}
|
|
|
|
#define INTEL_ADSP_GPDMA_INIT(inst) \
|
|
INTEL_ADSP_GPDMA_CHAN_ARB_DATA(inst); \
|
|
static void intel_adsp_gpdma##inst##_irq_config(void); \
|
|
\
|
|
static const struct intel_adsp_gpdma_cfg intel_adsp_gpdma##inst##_config = {\
|
|
.dw_cfg = { \
|
|
.base = DT_INST_REG_ADDR(inst), \
|
|
.irq_config = intel_adsp_gpdma##inst##_irq_config,\
|
|
}, \
|
|
.shim = DT_INST_PROP_BY_IDX(inst, shim, 0), \
|
|
}; \
|
|
\
|
|
static struct intel_adsp_gpdma_data intel_adsp_gpdma##inst##_data = {\
|
|
.dw_data = { \
|
|
.channel_data = &dmac##inst, \
|
|
}, \
|
|
}; \
|
|
\
|
|
PM_DEVICE_DT_INST_DEFINE(inst, gpdma_pm_action); \
|
|
\
|
|
DEVICE_DT_INST_DEFINE(inst, \
|
|
&intel_adsp_gpdma_init, \
|
|
PM_DEVICE_DT_INST_GET(inst), \
|
|
&intel_adsp_gpdma##inst##_data, \
|
|
&intel_adsp_gpdma##inst##_config, POST_KERNEL,\
|
|
CONFIG_DMA_INIT_PRIORITY, \
|
|
&intel_adsp_gpdma_driver_api); \
|
|
\
|
|
static void intel_adsp_gpdma##inst##_irq_config(void) \
|
|
{ \
|
|
IRQ_CONNECT(DT_INST_IRQN(inst), \
|
|
DT_INST_IRQ(inst, priority), dw_dma_isr, \
|
|
DEVICE_DT_INST_GET(inst), \
|
|
DT_INST_IRQ(inst, sense)); \
|
|
irq_enable(DT_INST_IRQN(inst)); \
|
|
}
|
|
|
|
DT_INST_FOREACH_STATUS_OKAY(INTEL_ADSP_GPDMA_INIT)
|