diff options
Diffstat (limited to 'target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch')
| -rw-r--r-- | target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch | 4380 |
1 files changed, 4380 insertions, 0 deletions
diff --git a/target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch b/target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch new file mode 100644 index 0000000000..aee4ae2946 --- /dev/null +++ b/target/linux/layerscape/patches-4.14/802-dma-support-layerscape.patch @@ -0,0 +1,4380 @@ +From 731adfb43892a1d7fe00e2036200f33a9b61a589 Mon Sep 17 00:00:00 2001 +From: Biwen Li <biwen.li@nxp.com> +Date: Tue, 30 Oct 2018 18:26:02 +0800 +Subject: [PATCH 19/40] dma: support layerscape +This is an integrated patch of dma for layerscape + +Signed-off-by: Catalin Horghidan <catalin.horghidan@nxp.com> +Signed-off-by: Changming Huang <jerry.huang@nxp.com> +Signed-off-by: Horia Geantă <horia.geanta@nxp.com> +Signed-off-by: jiaheng.fan <jiaheng.fan@nxp.com> +Signed-off-by: Peng Ma <peng.ma@nxp.com> +Signed-off-by: Radu Alexe <radu.alexe@nxp.com> +Signed-off-by: Rajiv Vishwakarma <rajiv.vishwakarma@nxp.com> +Signed-off-by: Tudor Ambarus <tudor-dan.ambarus@nxp.com> +Signed-off-by: Wen He <wen.he_1@nxp.com> +Signed-off-by: Yuan Yao <yao.yuan@nxp.com> +Signed-off-by: Biwen Li <biwen.li@nxp.com> +--- + .../devicetree/bindings/dma/fsl-qdma.txt | 51 + + drivers/dma/Kconfig | 33 +- + drivers/dma/Makefile | 3 + + drivers/dma/caam_dma.c | 462 ++++++ + drivers/dma/dpaa2-qdma/Kconfig | 8 + + drivers/dma/dpaa2-qdma/Makefile | 8 + + drivers/dma/dpaa2-qdma/dpaa2-qdma.c | 940 ++++++++++++ + drivers/dma/dpaa2-qdma/dpaa2-qdma.h | 227 +++ + drivers/dma/dpaa2-qdma/dpdmai.c | 515 +++++++ + drivers/dma/dpaa2-qdma/fsl_dpdmai.h | 521 +++++++ + drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h | 222 +++ + drivers/dma/fsl-qdma.c | 1278 +++++++++++++++++ + 12 files changed, 4267 insertions(+), 1 deletion(-) + create mode 100644 Documentation/devicetree/bindings/dma/fsl-qdma.txt + create mode 100644 drivers/dma/caam_dma.c + create mode 100644 drivers/dma/dpaa2-qdma/Kconfig + create mode 100644 drivers/dma/dpaa2-qdma/Makefile + create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.c + create mode 100644 drivers/dma/dpaa2-qdma/dpaa2-qdma.h + create mode 100644 drivers/dma/dpaa2-qdma/dpdmai.c + create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai.h + create mode 100644 drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h + create mode 100644 drivers/dma/fsl-qdma.c + +--- /dev/null ++++ b/Documentation/devicetree/bindings/dma/fsl-qdma.txt +@@ -0,0 +1,51 @@ ++* Freescale queue Direct Memory Access(qDMA) Controller ++ ++The qDMA supports channel virtualization by allowing DMA jobs to be enqueued into ++different command queues. Core can initiate a DMA transaction by preparing a command ++descriptor for each DMA job and enqueuing this job to a command queue. ++ ++* qDMA Controller ++Required properties: ++- compatible : ++ should be "fsl,ls1021a-qdma". ++- reg : Specifies base physical address(s) and size of the qDMA registers. ++ The 1st region is qDMA control register's address and size. ++ The 2nd region is status queue control register's address and size. ++ The 3rd region is virtual block control register's address and size. ++- interrupts : A list of interrupt-specifiers, one for each entry in ++ interrupt-names. ++- interrupt-names : Should contain: ++ "qdma-queue0" - the block0 interrupt ++ "qdma-queue1" - the block1 interrupt ++ "qdma-queue2" - the block2 interrupt ++ "qdma-queue3" - the block3 interrupt ++ "qdma-error" - the error interrupt ++- channels : Number of DMA channels supported ++- block-number : the virtual block number ++- block-offset : the offset of different virtual block ++- queues : the number of command queue per virtual block ++- status-sizes : status queue size of per virtual block ++- queue-sizes : command queue size of per virtual block, the size number based on queues ++- big-endian: If present registers and hardware scatter/gather descriptors ++ of the qDMA are implemented in big endian mode, otherwise in little ++ mode. ++ ++Examples: ++ qdma: qdma@8390000 { ++ compatible = "fsl,ls1021a-qdma"; ++ reg = <0x0 0x8388000 0x0 0x1000>, /* Controller regs */ ++ <0x0 0x8389000 0x0 0x1000>, /* Status regs */ ++ <0x0 0x838a000 0x0 0x2000>; /* Block regs */ ++ interrupts = <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>, ++ <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>, ++ <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>; ++ interrupt-names = "qdma-error", ++ "qdma-queue0", "qdma-queue1"; ++ channels = <8>; ++ block-number = <2>; ++ block-offset = <0x1000>; ++ queues = <2>; ++ status-sizes = <64>; ++ queue-sizes = <64 64>; ++ big-endian; ++ }; +--- a/drivers/dma/Kconfig ++++ b/drivers/dma/Kconfig +@@ -129,6 +129,24 @@ config COH901318 + help + Enable support for ST-Ericsson COH 901 318 DMA. + ++config CRYPTO_DEV_FSL_CAAM_DMA ++ tristate "CAAM DMA engine support" ++ depends on CRYPTO_DEV_FSL_CAAM_JR ++ default n ++ select DMA_ENGINE ++ select ASYNC_CORE ++ select ASYNC_TX_ENABLE_CHANNEL_SWITCH ++ help ++ Selecting this will offload the DMA operations for users of ++ the scatter gather memcopy API to the CAAM via job rings. The ++ CAAM is a hardware module that provides hardware acceleration to ++ cryptographic operations. It has a built-in DMA controller that can ++ be programmed to read/write cryptographic data. This module defines ++ a DMA driver that uses the DMA capabilities of the CAAM. ++ ++ To compile this as a module, choose M here: the module ++ will be called caam_dma. ++ + config DMA_BCM2835 + tristate "BCM2835 DMA engine support" + depends on ARCH_BCM2835 +@@ -215,6 +233,20 @@ config FSL_EDMA + multiplexing capability for DMA request sources(slot). + This module can be found on Freescale Vybrid and LS-1 SoCs. + ++config FSL_QDMA ++ tristate "NXP Layerscape qDMA engine support" ++ select DMA_ENGINE ++ select DMA_VIRTUAL_CHANNELS ++ select DMA_ENGINE_RAID ++ select ASYNC_TX_ENABLE_CHANNEL_SWITCH ++ help ++ Support the NXP Layerscape qDMA engine with command queue and legacy mode. ++ Channel virtualization is supported through enqueuing of DMA jobs to, ++ or dequeuing DMA jobs from, different work queues. ++ This module can be found on NXP Layerscape SoCs. ++ ++source drivers/dma/dpaa2-qdma/Kconfig ++ + config FSL_RAID + tristate "Freescale RAID engine Support" + depends on FSL_SOC && !ASYNC_TX_ENABLE_CHANNEL_SWITCH +@@ -600,7 +632,6 @@ config ZX_DMA + help + Support the DMA engine for ZTE ZX family platform devices. + +- + # driver files + source "drivers/dma/bestcomm/Kconfig" + +--- a/drivers/dma/Makefile ++++ b/drivers/dma/Makefile +@@ -31,7 +31,9 @@ obj-$(CONFIG_DMA_SUN6I) += sun6i-dma.o + obj-$(CONFIG_DW_DMAC_CORE) += dw/ + obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o + obj-$(CONFIG_FSL_DMA) += fsldma.o ++obj-$(CONFIG_FSL_DPAA2_QDMA) += dpaa2-qdma/ + obj-$(CONFIG_FSL_EDMA) += fsl-edma.o ++obj-$(CONFIG_FSL_QDMA) += fsl-qdma.o + obj-$(CONFIG_FSL_RAID) += fsl_raid.o + obj-$(CONFIG_HSU_DMA) += hsu/ + obj-$(CONFIG_IMG_MDC_DMA) += img-mdc-dma.o +@@ -71,6 +73,7 @@ obj-$(CONFIG_TI_EDMA) += edma.o + obj-$(CONFIG_XGENE_DMA) += xgene-dma.o + obj-$(CONFIG_ZX_DMA) += zx_dma.o + obj-$(CONFIG_ST_FDMA) += st_fdma.o ++obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_DMA) += caam_dma.o + + obj-y += qcom/ + obj-y += xilinx/ +--- /dev/null ++++ b/drivers/dma/caam_dma.c +@@ -0,0 +1,462 @@ ++/* ++ * caam support for SG DMA ++ * ++ * Copyright 2016 Freescale Semiconductor, Inc ++ * Copyright 2017 NXP ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the names of the above-listed copyright holders nor the ++ * names of any contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" ++ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ++ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ++ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE ++ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR ++ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF ++ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS ++ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN ++ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ++ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include <linux/dma-mapping.h> ++#include <linux/dmaengine.h> ++#include <linux/module.h> ++#include <linux/platform_device.h> ++#include <linux/slab.h> ++ ++#include "dmaengine.h" ++ ++#include "../crypto/caam/regs.h" ++#include "../crypto/caam/jr.h" ++#include "../crypto/caam/error.h" ++#include "../crypto/caam/desc_constr.h" ++ ++#define DESC_DMA_MEMCPY_LEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / \ ++ CAAM_CMD_SZ) ++ ++/* ++ * This is max chunk size of a DMA transfer. If a buffer is larger than this ++ * value it is internally broken into chunks of max CAAM_DMA_CHUNK_SIZE bytes ++ * and for each chunk a DMA transfer request is issued. ++ * This value is the largest number on 16 bits that is a multiple of 256 bytes ++ * (the largest configurable CAAM DMA burst size). ++ */ ++#define CAAM_DMA_CHUNK_SIZE 65280 ++ ++struct caam_dma_sh_desc { ++ u32 desc[DESC_DMA_MEMCPY_LEN] ____cacheline_aligned; ++ dma_addr_t desc_dma; ++}; ++ ++/* caam dma extended descriptor */ ++struct caam_dma_edesc { ++ struct dma_async_tx_descriptor async_tx; ++ struct list_head node; ++ struct caam_dma_ctx *ctx; ++ dma_addr_t src_dma; ++ dma_addr_t dst_dma; ++ unsigned int src_len; ++ unsigned int dst_len; ++ u32 jd[] ____cacheline_aligned; ++}; ++ ++/* ++ * caam_dma_ctx - per jr/channel context ++ * @chan: dma channel used by async_tx API ++ * @node: list_head used to attach to the global dma_ctx_list ++ * @jrdev: Job Ring device ++ * @pending_q: queue of pending (submitted, but not enqueued) jobs ++ * @done_not_acked: jobs that have been completed by jr, but maybe not acked ++ * @edesc_lock: protects extended descriptor ++ */ ++struct caam_dma_ctx { ++ struct dma_chan chan; ++ struct list_head node; ++ struct device *jrdev; ++ struct list_head pending_q; ++ struct list_head done_not_acked; ++ spinlock_t edesc_lock; ++}; ++ ++static struct dma_device *dma_dev; ++static struct caam_dma_sh_desc *dma_sh_desc; ++static LIST_HEAD(dma_ctx_list); ++ ++static dma_cookie_t caam_dma_tx_submit(struct dma_async_tx_descriptor *tx) ++{ ++ struct caam_dma_edesc *edesc = NULL; ++ struct caam_dma_ctx *ctx = NULL; ++ dma_cookie_t cookie; ++ ++ edesc = container_of(tx, struct caam_dma_edesc, async_tx); ++ ctx = container_of(tx->chan, struct caam_dma_ctx, chan); ++ ++ spin_lock_bh(&ctx->edesc_lock); ++ ++ cookie = dma_cookie_assign(tx); ++ list_add_tail(&edesc->node, &ctx->pending_q); ++ ++ spin_unlock_bh(&ctx->edesc_lock); ++ ++ return cookie; ++} ++ ++static void caam_jr_chan_free_edesc(struct caam_dma_edesc *edesc) ++{ ++ struct caam_dma_ctx *ctx = edesc->ctx; ++ struct caam_dma_edesc *_edesc = NULL; ++ ++ spin_lock_bh(&ctx->edesc_lock); ++ ++ list_add_tail(&edesc->node, &ctx->done_not_acked); ++ list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) { ++ if (async_tx_test_ack(&edesc->async_tx)) { ++ list_del(&edesc->node); ++ kfree(edesc); ++ } ++ } ++ ++ spin_unlock_bh(&ctx->edesc_lock); ++} ++ ++static void caam_dma_done(struct device *dev, u32 *hwdesc, u32 err, ++ void *context) ++{ ++ struct caam_dma_edesc *edesc = context; ++ struct caam_dma_ctx *ctx = edesc->ctx; ++ dma_async_tx_callback callback; ++ void *callback_param; ++ ++ if (err) ++ caam_jr_strstatus(ctx->jrdev, err); ++ ++ dma_run_dependencies(&edesc->async_tx); ++ ++ spin_lock_bh(&ctx->edesc_lock); ++ dma_cookie_complete(&edesc->async_tx); ++ spin_unlock_bh(&ctx->edesc_lock); ++ ++ callback = edesc->async_tx.callback; ++ callback_param = edesc->async_tx.callback_param; ++ ++ dma_descriptor_unmap(&edesc->async_tx); ++ ++ caam_jr_chan_free_edesc(edesc); ++ ++ if (callback) ++ callback(callback_param); ++} ++ ++static void caam_dma_memcpy_init_job_desc(struct caam_dma_edesc *edesc) ++{ ++ u32 *jd = edesc->jd; ++ u32 *sh_desc = dma_sh_desc->desc; ++ dma_addr_t desc_dma = dma_sh_desc->desc_dma; ++ ++ /* init the job descriptor */ ++ init_job_desc_shared(jd, desc_dma, desc_len(sh_desc), HDR_REVERSE); ++ ++ /* set SEQIN PTR */ ++ append_seq_in_ptr(jd, edesc->src_dma, edesc->src_len, 0); ++ ++ /* set SEQOUT PTR */ ++ append_seq_out_ptr(jd, edesc->dst_dma, edesc->dst_len, 0); ++ ++ print_hex_dump_debug("caam dma desc@" __stringify(__LINE__) ": ", ++ DUMP_PREFIX_ADDRESS, 16, 4, jd, desc_bytes(jd), 1); ++} ++ ++static struct dma_async_tx_descriptor * ++caam_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, ++ size_t len, unsigned long flags) ++{ ++ struct caam_dma_edesc *edesc; ++ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, ++ chan); ++ ++ edesc = kzalloc(sizeof(*edesc) + DESC_JOB_IO_LEN, GFP_DMA | GFP_NOWAIT); ++ if (!edesc) ++ return ERR_PTR(-ENOMEM); ++ ++ dma_async_tx_descriptor_init(&edesc->async_tx, chan); ++ edesc->async_tx.tx_submit = caam_dma_tx_submit; ++ edesc->async_tx.flags = flags; ++ edesc->async_tx.cookie = -EBUSY; ++ ++ edesc->src_dma = src; ++ edesc->src_len = len; ++ edesc->dst_dma = dst; ++ edesc->dst_len = len; ++ edesc->ctx = ctx; ++ ++ caam_dma_memcpy_init_job_desc(edesc); ++ ++ return &edesc->async_tx; ++} ++ ++/* This function can be called in an interrupt context */ ++static void caam_dma_issue_pending(struct dma_chan *chan) ++{ ++ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, ++ chan); ++ struct caam_dma_edesc *edesc, *_edesc; ++ ++ spin_lock_bh(&ctx->edesc_lock); ++ list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) { ++ if (caam_jr_enqueue(ctx->jrdev, edesc->jd, ++ caam_dma_done, edesc) < 0) ++ break; ++ list_del(&edesc->node); ++ } ++ spin_unlock_bh(&ctx->edesc_lock); ++} ++ ++static void caam_dma_free_chan_resources(struct dma_chan *chan) ++{ ++ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, ++ chan); ++ struct caam_dma_edesc *edesc, *_edesc; ++ ++ spin_lock_bh(&ctx->edesc_lock); ++ list_for_each_entry_safe(edesc, _edesc, &ctx->pending_q, node) { ++ list_del(&edesc->node); ++ kfree(edesc); ++ } ++ list_for_each_entry_safe(edesc, _edesc, &ctx->done_not_acked, node) { ++ list_del(&edesc->node); ++ kfree(edesc); ++ } ++ spin_unlock_bh(&ctx->edesc_lock); ++} ++ ++static int caam_dma_jr_chan_bind(void) ++{ ++ struct device *jrdev; ++ struct caam_dma_ctx *ctx; ++ int bonds = 0; ++ int i; ++ ++ for (i = 0; i < caam_jr_driver_probed(); i++) { ++ jrdev = caam_jridx_alloc(i); ++ if (IS_ERR(jrdev)) { ++ pr_err("job ring device %d allocation failed\n", i); ++ continue; ++ } ++ ++ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); ++ if (!ctx) { ++ caam_jr_free(jrdev); ++ continue; ++ } ++ ++ ctx->chan.device = dma_dev; ++ ctx->chan.private = ctx; ++ ++ ctx->jrdev = jrdev; ++ ++ INIT_LIST_HEAD(&ctx->pending_q); ++ INIT_LIST_HEAD(&ctx->done_not_acked); ++ INIT_LIST_HEAD(&ctx->node); ++ spin_lock_init(&ctx->edesc_lock); ++ ++ dma_cookie_init(&ctx->chan); ++ ++ /* add the context of this channel to the context list */ ++ list_add_tail(&ctx->node, &dma_ctx_list); ++ ++ /* add this channel to the device chan list */ ++ list_add_tail(&ctx->chan.device_node, &dma_dev->channels); ++ ++ bonds++; ++ } ++ ++ return bonds; ++} ++ ++static inline void caam_jr_dma_free(struct dma_chan *chan) ++{ ++ struct caam_dma_ctx *ctx = container_of(chan, struct caam_dma_ctx, ++ chan); ++ ++ list_del(&ctx->node); ++ list_del(&chan->device_node); ++ caam_jr_free(ctx->jrdev); ++ kfree(ctx); ++} ++ ++static void set_caam_dma_desc(u32 *desc) ++{ ++ u32 *jmp_cmd; ++ ++ /* dma shared descriptor */ ++ init_sh_desc(desc, HDR_SHARE_NEVER | (1 << HDR_START_IDX_SHIFT)); ++ ++ /* REG1 = CAAM_DMA_CHUNK_SIZE */ ++ append_math_add_imm_u32(desc, REG1, ZERO, IMM, CAAM_DMA_CHUNK_SIZE); ++ ++ /* REG0 = SEQINLEN - CAAM_DMA_CHUNK_SIZE */ ++ append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, CAAM_DMA_CHUNK_SIZE); ++ ++ /* ++ * if (REG0 > 0) ++ * jmp to LABEL1 ++ */ ++ jmp_cmd = append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N | ++ JUMP_COND_MATH_Z); ++ ++ /* REG1 = SEQINLEN */ ++ append_math_sub(desc, REG1, SEQINLEN, ZERO, CAAM_CMD_SZ); ++ ++ /* LABEL1 */ ++ set_jump_tgt_here(desc, jmp_cmd); ++ ++ /* VARSEQINLEN = REG1 */ ++ append_math_add(desc, VARSEQINLEN, REG1, ZERO, CAAM_CMD_SZ); ++ ++ /* VARSEQOUTLEN = REG1 */ ++ append_math_add(desc, VARSEQOUTLEN, REG1, ZERO, CAAM_CMD_SZ); ++ ++ /* do FIFO STORE */ ++ append_seq_fifo_store(desc, 0, FIFOST_TYPE_METADATA | LDST_VLF); ++ ++ /* do FIFO LOAD */ ++ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | ++ FIFOLD_TYPE_IFIFO | LDST_VLF); ++ ++ /* ++ * if (REG0 > 0) ++ * jmp 0xF8 (after shared desc header) ++ */ ++ append_jump(desc, JUMP_TEST_INVALL | JUMP_COND_MATH_N | ++ JUMP_COND_MATH_Z | 0xF8); ++ ++ print_hex_dump_debug("caam dma shdesc@" __stringify(__LINE__) ": ", ++ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), ++ 1); ++} ++ ++static int __init caam_dma_probe(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct device *ctrldev = dev->parent; ++ struct dma_chan *chan, *_chan; ++ u32 *sh_desc; ++ int err = -ENOMEM; ++ int bonds; ++ ++ if (!caam_jr_driver_probed()) { ++ dev_info(dev, "Defer probing after JR driver probing\n"); ++ return -EPROBE_DEFER; ++ } ++ ++ dma_dev = kzalloc(sizeof(*dma_dev), GFP_KERNEL); ++ if (!dma_dev) ++ return -ENOMEM; ++ ++ dma_sh_desc = kzalloc(sizeof(*dma_sh_desc), GFP_KERNEL | GFP_DMA); ++ if (!dma_sh_desc) ++ goto desc_err; ++ ++ sh_desc = dma_sh_desc->desc; ++ set_caam_dma_desc(sh_desc); ++ dma_sh_desc->desc_dma = dma_map_single(ctrldev, sh_desc, ++ desc_bytes(sh_desc), ++ DMA_TO_DEVICE); ++ if (dma_mapping_error(ctrldev, dma_sh_desc->desc_dma)) { ++ dev_err(dev, "unable to map dma descriptor\n"); ++ goto map_err; ++ } ++ ++ INIT_LIST_HEAD(&dma_dev->channels); ++ ++ bonds = caam_dma_jr_chan_bind(); ++ if (!bonds) { ++ err = -ENODEV; ++ goto jr_bind_err; ++ } ++ ++ dma_dev->dev = dev; ++ dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; ++ dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); ++ dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); ++ dma_dev->device_tx_status = dma_cookie_status; ++ dma_dev->device_issue_pending = caam_dma_issue_pending; ++ dma_dev->device_prep_dma_memcpy = caam_dma_prep_memcpy; ++ dma_dev->device_free_chan_resources = caam_dma_free_chan_resources; ++ ++ err = dma_async_device_register(dma_dev); ++ if (err) { ++ dev_err(dev, "Failed to register CAAM DMA engine\n"); ++ goto jr_bind_err; ++ } ++ ++ dev_info(dev, "caam dma support with %d job rings\n", bonds); ++ ++ return err; ++ ++jr_bind_err: ++ list_for_each_entry_safe(chan, _chan, &dma_dev->channels, device_node) ++ caam_jr_dma_free(chan); ++ ++ dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, desc_bytes(sh_desc), ++ DMA_TO_DEVICE); ++map_err: ++ kfree(dma_sh_desc); ++desc_err: ++ kfree(dma_dev); ++ return err; ++} ++ ++static int caam_dma_remove(struct platform_device *pdev) ++{ ++ struct device *dev = &pdev->dev; ++ struct device *ctrldev = dev->parent; ++ struct caam_dma_ctx *ctx, *_ctx; ++ ++ dma_async_device_unregister(dma_dev); ++ ++ list_for_each_entry_safe(ctx, _ctx, &dma_ctx_list, node) { ++ list_del(&ctx->node); ++ caam_jr_free(ctx->jrdev); ++ kfree(ctx); ++ } ++ ++ dma_unmap_single(ctrldev, dma_sh_desc->desc_dma, ++ desc_bytes(dma_sh_desc->desc), DMA_TO_DEVICE); ++ ++ kfree(dma_sh_desc); ++ kfree(dma_dev); ++ ++ dev_info(dev, "caam dma support disabled\n"); ++ return 0; ++} ++ ++static struct platform_driver caam_dma_driver = { ++ .driver = { ++ .name = "caam-dma", ++ }, ++ .probe = caam_dma_probe, ++ .remove = caam_dma_remove, ++}; ++module_platform_driver(caam_dma_driver); ++ ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_DESCRIPTION("NXP CAAM support for DMA engine"); ++MODULE_AUTHOR("NXP Semiconductors"); ++MODULE_ALIAS("platform:caam-dma"); +--- /dev/null ++++ b/drivers/dma/dpaa2-qdma/Kconfig +@@ -0,0 +1,8 @@ ++menuconfig FSL_DPAA2_QDMA ++ tristate "NXP DPAA2 QDMA" ++ depends on FSL_MC_BUS && FSL_MC_DPIO ++ select DMA_ENGINE ++ select DMA_VIRTUAL_CHANNELS ++ ---help--- ++ NXP Data Path Acceleration Architecture 2 QDMA driver, ++ using the NXP MC bus driver. +--- /dev/null ++++ b/drivers/dma/dpaa2-qdma/Makefile +@@ -0,0 +1,8 @@ ++# ++# Makefile for the NXP DPAA2 CAAM controllers ++# ++ccflags-y += -DVERSION=\"\" ++ ++obj-$(CONFIG_FSL_DPAA2_QDMA) += fsl-dpaa2-qdma.o ++ ++fsl-dpaa2-qdma-objs := dpaa2-qdma.o dpdmai.o +--- /dev/null ++++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.c +@@ -0,0 +1,940 @@ ++/* ++ * drivers/dma/dpaa2-qdma/dpaa2-qdma.c ++ * ++ * Copyright 2015-2017 NXP Semiconductor, Inc. ++ * Author: Changming Huang <jerry.huang@nxp.com> ++ * ++ * Driver for the NXP QDMA engine with QMan mode. ++ * Channel virtualization is supported through enqueuing of DMA jobs to, ++ * or dequeuing DMA jobs from different work queues with QMan portal. ++ * This module can be found on NXP LS2 SoCs. ++ * ++ * This program is free software; you can redistribute it and/or modify it ++ * under the terms of the GNU General Public License as published by the ++ * Free Software Foundation; either version 2 of the License, or (at your ++ * option) any later version. ++ */ ++ ++#include <linux/init.h> ++#include <linux/module.h> ++#include <linux/interrupt.h> ++#include <linux/clk.h> ++#include <linux/dma-mapping.h> ++#include <linux/dmapool.h> ++#include <linux/slab.h> ++#include <linux/spinlock.h> ++#include <linux/of.h> ++#include <linux/of_device.h> ++#include <linux/of_address.h> ++#include <linux/of_irq.h> ++#include <linux/of_dma.h> ++#include <linux/types.h> ++#include <linux/delay.h> ++#include <linux/iommu.h> ++ ++#include "../virt-dma.h" ++ ++#include <linux/fsl/mc.h> ++#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h" ++#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" ++#include "fsl_dpdmai_cmd.h" ++#include "fsl_dpdmai.h" ++#include "dpaa2-qdma.h" ++ ++static bool smmu_disable = true; ++ ++static struct dpaa2_qdma_chan *to_dpaa2_qdma_chan(struct dma_chan *chan) ++{ ++ return container_of(chan, struct dpaa2_qdma_chan, vchan.chan); ++} ++ ++static struct dpaa2_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) ++{ ++ return container_of(vd, struct dpaa2_qdma_comp, vdesc); ++} ++ ++static int dpaa2_qdma_alloc_chan_resources(struct dma_chan *chan) ++{ ++ return 0; ++} ++ ++static void dpaa2_qdma_free_chan_resources(struct dma_chan *chan) ++{ ++ struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); ++ unsigned long flags; ++ LIST_HEAD(head); ++ ++ spin_lock_irqsave(&dpaa2_chan->vchan.lock, flags); ++ vchan_get_all_descriptors(&dpaa2_chan->vchan, &head); ++ spin_unlock_irqrestore(&dpaa2_chan->vchan.lock, flags); ++ ++ vchan_dma_desc_free_list(&dpaa2_chan->vchan, &head); ++} ++ ++/* ++ * Request a command descriptor for enqueue. ++ */ ++static struct dpaa2_qdma_comp * ++dpaa2_qdma_request_desc(struct dpaa2_qdma_chan *dpaa2_chan) ++{ ++ struct dpaa2_qdma_comp *comp_temp = NULL; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); ++ if (list_empty(&dpaa2_chan->comp_free)) { ++ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); ++ comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL); ++ if (!comp_temp) ++ goto err; ++ comp_temp->fd_virt_addr = dma_pool_alloc(dpaa2_chan->fd_pool, ++ GFP_NOWAIT, &comp_temp->fd_bus_addr); ++ if (!comp_temp->fd_virt_addr) ++ goto err; ++ ++ comp_temp->fl_virt_addr = ++ (void *)((struct dpaa2_fd *) ++ comp_temp->fd_virt_addr + 1); ++ comp_temp->fl_bus_addr = comp_temp->fd_bus_addr + ++ sizeof(struct dpaa2_fd); ++ comp_temp->desc_virt_addr = ++ (void *)((struct dpaa2_fl_entry *) ++ comp_temp->fl_virt_addr + 3); ++ comp_temp->desc_bus_addr = comp_temp->fl_bus_addr + ++ sizeof(struct dpaa2_fl_entry) * 3; ++ ++ comp_temp->qchan = dpaa2_chan; ++ comp_temp->sg_blk_num = 0; ++ INIT_LIST_HEAD(&comp_temp->sg_src_head); ++ INIT_LIST_HEAD(&comp_temp->sg_dst_head); ++ return comp_temp; ++ } ++ comp_temp = list_first_entry(&dpaa2_chan->comp_free, ++ struct dpaa2_qdma_comp, list); ++ list_del(&comp_temp->list); ++ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); ++ ++ comp_temp->qchan = dpaa2_chan; ++err: ++ return comp_temp; ++} ++ ++static void dpaa2_qdma_populate_fd(uint32_t format, ++ struct dpaa2_qdma_comp *dpaa2_comp) ++{ ++ struct dpaa2_fd *fd; ++ ++ fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr; ++ memset(fd, 0, sizeof(struct dpaa2_fd)); ++ ++ /* fd populated */ ++ dpaa2_fd_set_addr(fd, dpaa2_comp->fl_bus_addr); ++ /* Bypass memory translation, Frame list format, short length disable */ ++ /* we need to disable BMT if fsl-mc use iova addr */ ++ if (smmu_disable) ++ dpaa2_fd_set_bpid(fd, QMAN_FD_BMT_ENABLE); ++ dpaa2_fd_set_format(fd, QMAN_FD_FMT_ENABLE | QMAN_FD_SL_DISABLE); ++ ++ dpaa2_fd_set_frc(fd, format | QDMA_SER_CTX); ++} ++ ++/* first frame list for descriptor buffer */ ++static void dpaa2_qdma_populate_first_framel( ++ struct dpaa2_fl_entry *f_list, ++ struct dpaa2_qdma_comp *dpaa2_comp) ++{ ++ struct dpaa2_qdma_sd_d *sdd; ++ ++ sdd = (struct dpaa2_qdma_sd_d *)dpaa2_comp->desc_virt_addr; ++ memset(sdd, 0, 2 * (sizeof(*sdd))); ++ /* source and destination descriptor */ ++ sdd->cmd = cpu_to_le32(QDMA_SD_CMD_RDTTYPE_COHERENT); /* source descriptor CMD */ ++ sdd++; ++ sdd->cmd = cpu_to_le32(QDMA_DD_CMD_WRTTYPE_COHERENT); /* dest descriptor CMD */ ++ ++ memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); ++ /* first frame list to source descriptor */ ++ ++ dpaa2_fl_set_addr(f_list, dpaa2_comp->desc_bus_addr); ++ dpaa2_fl_set_len(f_list, 0x20); ++ dpaa2_fl_set_format(f_list, QDMA_FL_FMT_SBF | QDMA_FL_SL_LONG); ++ ++ if (smmu_disable) ++ f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */ ++} ++ ++/* source and destination frame list */ ++static void dpaa2_qdma_populate_frames(struct dpaa2_fl_entry *f_list, ++ dma_addr_t dst, dma_addr_t src, size_t len, uint8_t fmt) ++{ ++ /* source frame list to source buffer */ ++ memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); ++ ++ ++ dpaa2_fl_set_addr(f_list, src); ++ dpaa2_fl_set_len(f_list, len); ++ dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); /* single buffer frame or scatter gather frame */ ++ if (smmu_disable) ++ f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */ ++ ++ f_list++; ++ /* destination frame list to destination buffer */ ++ memset(f_list, 0, sizeof(struct dpaa2_fl_entry)); ++ ++ dpaa2_fl_set_addr(f_list, dst); ++ dpaa2_fl_set_len(f_list, len); ++ dpaa2_fl_set_format(f_list, (fmt | QDMA_FL_SL_LONG)); ++ dpaa2_fl_set_final(f_list, QDMA_FL_F); /* single buffer frame or scatter gather frame */ ++ if (smmu_disable) ++ f_list->bpid = cpu_to_le16(QDMA_FL_BMT_ENABLE); /* bypass memory translation */ ++} ++ ++static struct dma_async_tx_descriptor *dpaa2_qdma_prep_memcpy( ++ struct dma_chan *chan, dma_addr_t dst, ++ dma_addr_t src, size_t len, unsigned long flags) ++{ ++ struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); ++ struct dpaa2_qdma_comp *dpaa2_comp; ++ struct dpaa2_fl_entry *f_list; ++ uint32_t format; ++ ++ dpaa2_comp = dpaa2_qdma_request_desc(dpaa2_chan); ++ ++#ifdef LONG_FORMAT ++ format = QDMA_FD_LONG_FORMAT; ++#else ++ format = QDMA_FD_SHORT_FORMAT; ++#endif ++ /* populate Frame descriptor */ ++ dpaa2_qdma_populate_fd(format, dpaa2_comp); ++ ++ f_list = (struct dpaa2_fl_entry *)dpaa2_comp->fl_virt_addr; ++ ++#ifdef LONG_FORMAT ++ /* first frame list for descriptor buffer (logn format) */ ++ dpaa2_qdma_populate_first_framel(f_list, dpaa2_comp); ++ ++ f_list++; ++#endif ++ ++ dpaa2_qdma_populate_frames(f_list, dst, src, len, QDMA_FL_FMT_SBF); ++ ++ return vchan_tx_prep(&dpaa2_chan->vchan, &dpaa2_comp->vdesc, flags); ++} ++ ++static struct qdma_sg_blk *dpaa2_qdma_get_sg_blk( ++ struct dpaa2_qdma_comp *dpaa2_comp, ++ struct dpaa2_qdma_chan *dpaa2_chan) ++{ ++ struct qdma_sg_blk *sg_blk = NULL; ++ dma_addr_t phy_sgb; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); ++ if (list_empty(&dpaa2_chan->sgb_free)) { ++ sg_blk = (struct qdma_sg_blk *)dma_pool_alloc( ++ dpaa2_chan->sg_blk_pool, ++ GFP_NOWAIT, &phy_sgb); ++ if (!sg_blk) { ++ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); ++ return sg_blk; ++ } ++ sg_blk->blk_virt_addr = (void *)(sg_blk + 1); ++ sg_blk->blk_bus_addr = phy_sgb + sizeof(*sg_blk); ++ } else { ++ sg_blk = list_first_entry(&dpaa2_chan->sgb_free, ++ struct qdma_sg_blk, list); ++ list_del(&sg_blk->list); ++ } ++ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); ++ ++ return sg_blk; ++} ++ ++static uint32_t dpaa2_qdma_populate_sg(struct device *dev, ++ struct dpaa2_qdma_chan *dpaa2_chan, ++ struct dpaa2_qdma_comp *dpaa2_comp, ++ struct scatterlist *dst_sg, u32 dst_nents, ++ struct scatterlist *src_sg, u32 src_nents) ++{ ++ struct dpaa2_qdma_sg *src_sge; ++ struct dpaa2_qdma_sg *dst_sge; ++ struct qdma_sg_blk *sg_blk; ++ struct qdma_sg_blk *sg_blk_dst; ++ dma_addr_t src; ++ dma_addr_t dst; ++ uint32_t num; ++ uint32_t blocks; ++ uint32_t len = 0; ++ uint32_t total_len = 0; ++ int i, j = 0; ++ ++ num = min(dst_nents, src_nents); ++ blocks = num / (NUM_SG_PER_BLK - 1); ++ if (num % (NUM_SG_PER_BLK - 1)) ++ blocks += 1; ++ if (dpaa2_comp->sg_blk_num < blocks) { ++ len = blocks - dpaa2_comp->sg_blk_num; ++ for (i = 0; i < len; i++) { ++ /* source sg blocks */ ++ sg_blk = dpaa2_qdma_get_sg_blk(dpaa2_comp, dpaa2_chan); ++ if (!sg_blk) ++ return 0; ++ list_add_tail(&sg_blk->list, &dpaa2_comp->sg_src_head); ++ /* destination sg blocks */ ++ sg_blk = dpaa2_qdma_get_sg_blk(dpaa2_comp, dpaa2_chan); ++ if (!sg_blk) ++ return 0; ++ list_add_tail(&sg_blk->list, &dpaa2_comp->sg_dst_head); ++ } ++ } else { ++ len = dpaa2_comp->sg_blk_num - blocks; ++ for (i = 0; i < len; i++) { ++ spin_lock(&dpaa2_chan->queue_lock); ++ /* handle source sg blocks */ ++ sg_blk = list_first_entry(&dpaa2_comp->sg_src_head, ++ struct qdma_sg_blk, list); ++ list_del(&sg_blk->list); ++ list_add_tail(&sg_blk->list, &dpaa2_chan->sgb_free); ++ /* handle destination sg blocks */ ++ sg_blk = list_first_entry(&dpaa2_comp->sg_dst_head, ++ struct qdma_sg_blk, list); ++ list_del(&sg_blk->list); ++ list_add_tail(&sg_blk->list, &dpaa2_chan->sgb_free); ++ spin_unlock(&dpaa2_chan->queue_lock); ++ } ++ } ++ dpaa2_comp->sg_blk_num = blocks; ++ ++ /* get the first source sg phy address */ ++ sg_blk = list_first_entry(&dpaa2_comp->sg_src_head, ++ struct qdma_sg_blk, list); ++ dpaa2_comp->sge_src_bus_addr = sg_blk->blk_bus_addr; ++ /* get the first destinaiton sg phy address */ ++ sg_blk_dst = list_first_entry(&dpaa2_comp->sg_dst_head, ++ struct qdma_sg_blk, list); ++ dpaa2_comp->sge_dst_bus_addr = sg_blk_dst->blk_bus_addr; ++ ++ for (i = 0; i < blocks; i++) { ++ src_sge = (struct dpaa2_qdma_sg *)sg_blk->blk_virt_addr; ++ dst_sge = (struct dpaa2_qdma_sg *)sg_blk_dst->blk_virt_addr; ++ ++ for (j = 0; j < (NUM_SG_PER_BLK - 1); j++) { ++ len = min(sg_dma_len(dst_sg), sg_dma_len(src_sg)); ++ if (0 == len) ++ goto fetch; ++ total_len += len; ++ src = sg_dma_address(src_sg); ++ dst = sg_dma_address(dst_sg); ++ ++ /* source SG */ ++ src_sge->addr_lo = src; ++ src_sge->addr_hi = (src >> 32); ++ src_sge->data_len.data_len_sl0 = len; ++ src_sge->ctrl.sl = QDMA_SG_SL_LONG; ++ src_sge->ctrl.fmt = QDMA_SG_FMT_SDB; ++ /* destination SG */ ++ dst_sge->addr_lo = dst; ++ dst_sge->addr_hi = (dst >> 32); ++ dst_sge->data_len.data_len_sl0 = len; ++ dst_sge->ctrl.sl = QDMA_SG_SL_LONG; ++ dst_sge->ctrl.fmt = QDMA_SG_FMT_SDB; ++fetch: ++ num--; ++ if (0 == num) { ++ src_sge->ctrl.f = QDMA_SG_F; ++ dst_sge->ctrl.f = QDMA_SG_F; ++ goto end; ++ } ++ dst_sg = sg_next(dst_sg); ++ src_sg = sg_next(src_sg); ++ src_sge++; ++ dst_sge++; ++ if (j == (NUM_SG_PER_BLK - 2)) { ++ /* for next blocks, extension */ ++ sg_blk = list_next_entry(sg_blk, list); ++ sg_blk_dst = list_next_entry(sg_blk_dst, list); ++ src_sge->addr_lo = sg_blk->blk_bus_addr; ++ src_sge->addr_hi = sg_blk->blk_bus_addr >> 32; ++ src_sge->ctrl.sl = QDMA_SG_SL_LONG; ++ src_sge->ctrl.fmt = QDMA_SG_FMT_SGTE; ++ dst_sge->addr_lo = sg_blk_dst->blk_bus_addr; ++ dst_sge->addr_hi = ++ sg_blk_dst->blk_bus_addr >> 32; ++ dst_sge->ctrl.sl = QDMA_SG_SL_LONG; ++ dst_sge->ctrl.fmt = QDMA_SG_FMT_SGTE; ++ } ++ } ++ } ++ ++end: ++ return total_len; ++} ++ ++static enum dma_status dpaa2_qdma_tx_status(struct dma_chan *chan, ++ dma_cookie_t cookie, struct dma_tx_state *txstate) ++{ ++ return dma_cookie_status(chan, cookie, txstate); ++} ++ ++static void dpaa2_qdma_free_desc(struct virt_dma_desc *vdesc) ++{ ++} ++ ++static void dpaa2_qdma_issue_pending(struct dma_chan *chan) ++{ ++ struct dpaa2_qdma_comp *dpaa2_comp; ++ struct dpaa2_qdma_chan *dpaa2_chan = to_dpaa2_qdma_chan(chan); ++ struct dpaa2_qdma_engine *dpaa2_qdma = dpaa2_chan->qdma; ++ struct dpaa2_qdma_priv *priv = dpaa2_qdma->priv; ++ struct virt_dma_desc *vdesc; ++ struct dpaa2_fd *fd; ++ int err; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&dpaa2_chan->queue_lock, flags); ++ spin_lock(&dpaa2_chan->vchan.lock); ++ if (vchan_issue_pending(&dpaa2_chan->vchan)) { ++ vdesc = vchan_next_desc(&dpaa2_chan->vchan); ++ if (!vdesc) ++ goto err_enqueue; ++ dpaa2_comp = to_fsl_qdma_comp(vdesc); ++ ++ fd = (struct dpaa2_fd *)dpaa2_comp->fd_virt_addr; ++ ++ list_del(&vdesc->node); ++ list_add_tail(&dpaa2_comp->list, &dpaa2_chan->comp_used); ++ ++ /* TOBO: priority hard-coded to zero */ ++ err = dpaa2_io_service_enqueue_fq(NULL, ++ priv->tx_queue_attr[0].fqid, fd); ++ if (err) { ++ list_del(&dpaa2_comp->list); ++ list_add_tail(&dpaa2_comp->list, ++ &dpaa2_chan->comp_free); ++ } ++ ++ } ++err_enqueue: ++ spin_unlock(&dpaa2_chan->vchan.lock); ++ spin_unlock_irqrestore(&dpaa2_chan->queue_lock, flags); ++} ++ ++static int __cold dpaa2_qdma_setup(struct fsl_mc_device *ls_dev) ++{ ++ struct device *dev = &ls_dev->dev; ++ struct dpaa2_qdma_priv *priv; ++ struct dpaa2_qdma_priv_per_prio *ppriv; ++ uint8_t prio_def = DPDMAI_PRIO_NUM; ++ int err; ++ int i; ++ ++ priv = dev_get_drvdata(dev); ++ ++ priv->dev = dev; ++ priv->dpqdma_id = ls_dev->obj_desc.id; ++ ++ /*Get the handle for the DPDMAI this interface is associate with */ ++ err = dpdmai_open(priv->mc_io, 0, priv->dpqdma_id, &ls_dev->mc_handle); ++ if (err) { ++ dev_err(dev, "dpdmai_open() failed\n"); ++ return err; ++ } ++ dev_info(dev, "Opened dpdmai object successfully\n"); ++ ++ err = dpdmai_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, ++ &priv->dpdmai_attr); ++ if (err) { ++ dev_err(dev, "dpdmai_get_attributes() failed\n"); ++ return err; ++ } ++ ++ if (priv->dpdmai_attr.version.major > DPDMAI_VER_MAJOR) { ++ dev_err(dev, "DPDMAI major version mismatch\n" ++ "Found %u.%u, supported version is %u.%u\n", ++ priv->dpdmai_attr.version.major, ++ priv->dpdmai_attr.version.minor, ++ DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); ++ } ++ ++ if (priv->dpdmai_attr.version.minor > DPDMAI_VER_MINOR) { ++ dev_err(dev, "DPDMAI minor version mismatch\n" ++ "Found %u.%u, supported version is %u.%u\n", ++ priv->dpdmai_attr.version.major, ++ priv->dpdmai_attr.version.minor, ++ DPDMAI_VER_MAJOR, DPDMAI_VER_MINOR); ++ } ++ ++ priv->num_pairs = min(priv->dpdmai_attr.num_of_priorities, prio_def); ++ ppriv = kcalloc(priv->num_pairs, sizeof(*ppriv), GFP_KERNEL); ++ if (!ppriv) { ++ dev_err(dev, "kzalloc for ppriv failed\n"); ++ return -1; ++ } ++ priv->ppriv = ppriv; ++ ++ for (i = 0; i < priv->num_pairs; i++) { ++ err = dpdmai_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, ++ i, &priv->rx_queue_attr[i]); ++ if (err) { ++ dev_err(dev, "dpdmai_get_rx_queue() failed\n"); ++ return err; ++ } ++ ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; ++ ++ err = dpdmai_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, ++ i, &priv->tx_queue_attr[i]); ++ if (err) { ++ dev_err(dev, "dpdmai_get_tx_queue() failed\n"); ++ return err; ++ } ++ ppriv->req_fqid = priv->tx_queue_attr[i].fqid; ++ ppriv->prio = i; ++ ppriv->priv = priv; ++ ppriv++; ++ } ++ ++ return 0; ++} ++ ++static void dpaa2_qdma_fqdan_cb(struct dpaa2_io_notification_ctx *ctx) ++{ ++ struct dpaa2_qdma_priv_per_prio *ppriv = container_of(ctx, ++ struct dpaa2_qdma_priv_per_prio, nctx); ++ struct dpaa2_qdma_priv *priv = ppriv->priv; ++ struct dpaa2_qdma_comp *dpaa2_comp, *_comp_tmp; ++ struct dpaa2_qdma_chan *qchan; ++ const struct dpaa2_fd *fd; ++ const struct dpaa2_fd *fd_eq; ++ struct dpaa2_dq *dq; ++ int err; ++ int is_last = 0; ++ uint8_t status; ++ int i; ++ int found; ++ uint32_t n_chans = priv->dpaa2_qdma->n_chans; ++ ++ do { ++ err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid, ++ ppriv->store); ++ } while (err); ++ ++ while (!is_last) { ++ do { ++ dq = dpaa2_io_store_next(ppriv->store, &is_last); ++ } while (!is_last && !dq); ++ if (!dq) { ++ dev_err(priv->dev, "FQID returned no valid frames!\n"); ++ continue; ++ } ++ ++ /* obtain FD and process the error */ ++ fd = dpaa2_dq_fd(dq); ++ ++ status = dpaa2_fd_get_ctrl(fd) & 0xff; ++ if (status) ++ dev_err(priv->dev, "FD error occurred\n"); ++ found = 0; ++ for (i = 0; i < n_chans; i++) { ++ qchan = &priv->dpaa2_qdma->chans[i]; ++ spin_lock(&qchan->queue_lock); ++ if (list_empty(&qchan->comp_used)) { ++ spin_unlock(&qchan->queue_lock); ++ continue; ++ } ++ list_for_each_entry_safe(dpaa2_comp, _comp_tmp, ++ &qchan->comp_used, list) { ++ fd_eq = (struct dpaa2_fd *) ++ dpaa2_comp->fd_virt_addr; ++ ++ if (le64_to_cpu(fd_eq->simple.addr) == ++ le64_to_cpu(fd->simple.addr)) { ++ ++ list_del(&dpaa2_comp->list); ++ list_add_tail(&dpaa2_comp->list, ++ &qchan->comp_free); ++ ++ spin_lock(&qchan->vchan.lock); ++ vchan_cookie_complete( ++ &dpaa2_comp->vdesc); ++ spin_unlock(&qchan->vchan.lock); ++ found = 1; ++ break; ++ } ++ } ++ spin_unlock(&qchan->queue_lock); ++ if (found) ++ break; ++ } ++ } ++ ++ dpaa2_io_service_rearm(NULL, ctx); ++} ++ ++static int __cold dpaa2_qdma_dpio_setup(struct dpaa2_qdma_priv *priv) ++{ ++ int err, i, num; ++ struct device *dev = priv->dev; ++ struct dpaa2_qdma_priv_per_prio *ppriv; ++ ++ num = priv->num_pairs; ++ ppriv = priv->ppriv; ++ for (i = 0; i < num; i++) { ++ ppriv->nctx.is_cdan = 0; ++ ppriv->nctx.desired_cpu = 1; ++ ppriv->nctx.id = ppriv->rsp_fqid; ++ ppriv->nctx.cb = dpaa2_qdma_fqdan_cb; ++ err = dpaa2_io_service_register(NULL, &ppriv->nctx); ++ if (err) { ++ dev_err(dev, "Notification register failed\n"); ++ goto err_service; ++ } ++ ++ ppriv->store = dpaa2_io_store_create(DPAA2_QDMA_STORE_SIZE, ++ dev); ++ if (!ppriv->store) { ++ dev_err(dev, "dpaa2_io_store_create() failed\n"); ++ goto err_store; ++ } ++ ++ ppriv++; ++ } ++ return 0; ++ ++err_store: ++ dpaa2_io_service_deregister(NULL, &ppriv->nctx); ++err_service: ++ ppriv--; ++ while (ppriv >= priv->ppriv) { ++ dpaa2_io_service_deregister(NULL, &ppriv->nctx); ++ dpaa2_io_store_destroy(ppriv->store); ++ ppriv--; ++ } ++ return -1; ++} ++ ++static void __cold dpaa2_dpmai_store_free(struct dpaa2_qdma_priv *priv) ++{ ++ struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; ++ int i; ++ ++ for (i = 0; i < priv->num_pairs; i++) { ++ dpaa2_io_store_destroy(ppriv->store); ++ ppriv++; ++ } ++} ++ ++static void __cold dpaa2_dpdmai_dpio_free(struct dpaa2_qdma_priv *priv) ++{ ++ struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; ++ int i; ++ ++ for (i = 0; i < priv->num_pairs; i++) { ++ dpaa2_io_service_deregister(NULL, &ppriv->nctx); ++ ppriv++; ++ } ++} ++ ++static int __cold dpaa2_dpdmai_bind(struct dpaa2_qdma_priv *priv) ++{ ++ int err; ++ struct dpdmai_rx_queue_cfg rx_queue_cfg; ++ struct device *dev = priv->dev; ++ struct dpaa2_qdma_priv_per_prio *ppriv; ++ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); ++ int i, num; ++ ++ num = priv->num_pairs; ++ ppriv = priv->ppriv; ++ for (i = 0; i < num; i++) { ++ rx_queue_cfg.options = DPDMAI_QUEUE_OPT_USER_CTX | ++ DPDMAI_QUEUE_OPT_DEST; ++ rx_queue_cfg.user_ctx = ppriv->nctx.qman64; ++ rx_queue_cfg.dest_cfg.dest_type = DPDMAI_DEST_DPIO; ++ rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; ++ rx_queue_cfg.dest_cfg.priority = ppriv->prio; ++ err = dpdmai_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, ++ rx_queue_cfg.dest_cfg.priority, &rx_queue_cfg); ++ if (err) { ++ dev_err(dev, "dpdmai_set_rx_queue() failed\n"); ++ return err; ++ } ++ ++ ppriv++; ++ } ++ ++ return 0; ++} ++ ++static int __cold dpaa2_dpdmai_dpio_unbind(struct dpaa2_qdma_priv *priv) ++{ ++ int err = 0; ++ struct device *dev = priv->dev; ++ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); ++ struct dpaa2_qdma_priv_per_prio *ppriv = priv->ppriv; ++ int i; ++ ++ for (i = 0; i < priv->num_pairs; i++) { ++ ppriv->nctx.qman64 = 0; ++ ppriv->nctx.dpio_id = 0; ++ ppriv++; ++ } ++ ++ err = dpdmai_reset(priv->mc_io, 0, ls_dev->mc_handle); ++ if (err) ++ dev_err(dev, "dpdmai_reset() failed\n"); ++ ++ return err; ++} ++ ++static void __cold dpaa2_dpdmai_free_pool(struct dpaa2_qdma_chan *qchan, ++ struct list_head *head) ++{ ++ struct qdma_sg_blk *sgb_tmp, *_sgb_tmp; ++ /* free the QDMA SG pool block */ ++ list_for_each_entry_safe(sgb_tmp, _sgb_tmp, head, list) { ++ sgb_tmp->blk_virt_addr = (void *)((struct qdma_sg_blk *) ++ sgb_tmp->blk_virt_addr - 1); ++ sgb_tmp->blk_bus_addr = sgb_tmp->blk_bus_addr ++ - sizeof(*sgb_tmp); ++ dma_pool_free(qchan->sg_blk_pool, sgb_tmp->blk_virt_addr, ++ sgb_tmp->blk_bus_addr); ++ } ++ ++} ++ ++static void __cold dpaa2_dpdmai_free_comp(struct dpaa2_qdma_chan *qchan, ++ struct list_head *head) ++{ ++ struct dpaa2_qdma_comp *comp_tmp, *_comp_tmp; ++ /* free the QDMA comp resource */ ++ list_for_each_entry_safe(comp_tmp, _comp_tmp, ++ head, list) { ++ dma_pool_free(qchan->fd_pool, ++ comp_tmp->fd_virt_addr, ++ comp_tmp->fd_bus_addr); ++ /* free the SG source block on comp */ ++ dpaa2_dpdmai_free_pool(qchan, &comp_tmp->sg_src_head); ++ /* free the SG destination block on comp */ ++ dpaa2_dpdmai_free_pool(qchan, &comp_tmp->sg_dst_head); ++ list_del(&comp_tmp->list); ++ kfree(comp_tmp); ++ } ++ ++} ++ ++static void __cold dpaa2_dpdmai_free_channels( ++ struct dpaa2_qdma_engine *dpaa2_qdma) ++{ ++ struct dpaa2_qdma_chan *qchan; ++ int num, i; ++ ++ num = dpaa2_qdma->n_chans; ++ for (i = 0; i < num; i++) { ++ qchan = &dpaa2_qdma->chans[i]; ++ dpaa2_dpdmai_free_comp(qchan, &qchan->comp_used); ++ dpaa2_dpdmai_free_comp(qchan, &qchan->comp_free); ++ dpaa2_dpdmai_free_pool(qchan, &qchan->sgb_free); ++ dma_pool_destroy(qchan->fd_pool); ++ dma_pool_destroy(qchan->sg_blk_pool); ++ } ++} ++ ++static int dpaa2_dpdmai_alloc_channels(struct dpaa2_qdma_engine *dpaa2_qdma) ++{ ++ struct dpaa2_qdma_chan *dpaa2_chan; ++ struct device *dev = &dpaa2_qdma->priv->dpdmai_dev->dev; ++ int i; ++ ++ INIT_LIST_HEAD(&dpaa2_qdma->dma_dev.channels); ++ for (i = 0; i < dpaa2_qdma->n_chans; i++) { ++ dpaa2_chan = &dpaa2_qdma->chans[i]; ++ dpaa2_chan->qdma = dpaa2_qdma; ++ dpaa2_chan->vchan.desc_free = dpaa2_qdma_free_desc; ++ vchan_init(&dpaa2_chan->vchan, &dpaa2_qdma->dma_dev); ++ ++ dpaa2_chan->fd_pool = dma_pool_create("fd_pool", ++ dev, FD_POOL_SIZE, 32, 0); ++ if (!dpaa2_chan->fd_pool) ++ return -1; ++ dpaa2_chan->sg_blk_pool = dma_pool_create("sg_blk_pool", ++ dev, SG_POOL_SIZE, 32, 0); ++ if (!dpaa2_chan->sg_blk_pool) ++ return -1; ++ ++ spin_lock_init(&dpaa2_chan->queue_lock); ++ INIT_LIST_HEAD(&dpaa2_chan->comp_used); ++ INIT_LIST_HEAD(&dpaa2_chan->comp_free); ++ INIT_LIST_HEAD(&dpaa2_chan->sgb_free); ++ } ++ return 0; ++} ++ ++static int dpaa2_qdma_probe(struct fsl_mc_device *dpdmai_dev) ++{ ++ struct dpaa2_qdma_priv *priv; ++ struct device *dev = &dpdmai_dev->dev; ++ struct dpaa2_qdma_engine *dpaa2_qdma; ++ int err; ++ ++ priv = kzalloc(sizeof(*priv), GFP_KERNEL); ++ if (!priv) ++ return -ENOMEM; ++ dev_set_drvdata(dev, priv); ++ priv->dpdmai_dev = dpdmai_dev; ++ ++ priv->iommu_domain = iommu_get_domain_for_dev(dev); ++ if (priv->iommu_domain) ++ smmu_disable = false; ++ ++ /* obtain a MC portal */ ++ err = fsl_mc_portal_allocate(dpdmai_dev, 0, &priv->mc_io); ++ if (err) { ++ dev_err(dev, "MC portal allocation failed\n"); ++ goto err_mcportal; ++ } ++ ++ /* DPDMAI initialization */ ++ err = dpaa2_qdma_setup(dpdmai_dev); ++ if (err) { ++ dev_err(dev, "dpaa2_dpdmai_setup() failed\n"); ++ goto err_dpdmai_setup; ++ } ++ ++ /* DPIO */ ++ err = dpaa2_qdma_dpio_setup(priv); ++ if (err) { ++ dev_err(dev, "dpaa2_dpdmai_dpio_setup() failed\n"); ++ goto err_dpio_setup; ++ } ++ ++ /* DPDMAI binding to DPIO */ ++ err = dpaa2_dpdmai_bind(priv); ++ if (err) { ++ dev_err(dev, "dpaa2_dpdmai_bind() failed\n"); ++ goto err_bind; ++ } ++ ++ /* DPDMAI enable */ ++ err = dpdmai_enable(priv->mc_io, 0, dpdmai_dev->mc_handle); ++ if (err) { ++ dev_err(dev, "dpdmai_enable() faile\n"); ++ goto err_enable; ++ } ++ ++ dpaa2_qdma = kzalloc(sizeof(*dpaa2_qdma), GFP_KERNEL); ++ if (!dpaa2_qdma) { ++ err = -ENOMEM; ++ goto err_eng; ++ } ++ ++ priv->dpaa2_qdma = dpaa2_qdma; ++ dpaa2_qdma->priv = priv; ++ ++ dpaa2_qdma->n_chans = NUM_CH; ++ ++ err = dpaa2_dpdmai_alloc_channels(dpaa2_qdma); ++ if (err) { ++ dev_err(dev, "QDMA alloc channels faile\n"); ++ goto err_reg; ++ } ++ ++ dma_cap_set(DMA_PRIVATE, dpaa2_qdma->dma_dev.cap_mask); ++ dma_cap_set(DMA_SLAVE, dpaa2_qdma->dma_dev.cap_mask); ++ dma_cap_set(DMA_MEMCPY, dpaa2_qdma->dma_dev.cap_mask); ++ ++ dpaa2_qdma->dma_dev.dev = dev; ++ dpaa2_qdma->dma_dev.device_alloc_chan_resources ++ = dpaa2_qdma_alloc_chan_resources; ++ dpaa2_qdma->dma_dev.device_free_chan_resources ++ = dpaa2_qdma_free_chan_resources; ++ dpaa2_qdma->dma_dev.device_tx_status = dpaa2_qdma_tx_status; ++ dpaa2_qdma->dma_dev.device_prep_dma_memcpy = dpaa2_qdma_prep_memcpy; ++ dpaa2_qdma->dma_dev.device_issue_pending = dpaa2_qdma_issue_pending; ++ ++ err = dma_async_device_register(&dpaa2_qdma->dma_dev); ++ if (err) { ++ dev_err(dev, "Can't register NXP QDMA engine.\n"); ++ goto err_reg; ++ } ++ ++ return 0; ++ ++err_reg: ++ dpaa2_dpdmai_free_channels(dpaa2_qdma); ++ kfree(dpaa2_qdma); ++err_eng: ++ dpdmai_disable(priv->mc_io, 0, dpdmai_dev->mc_handle); ++err_enable: ++ dpaa2_dpdmai_dpio_unbind(priv); ++err_bind: ++ dpaa2_dpmai_store_free(priv); ++ dpaa2_dpdmai_dpio_free(priv); ++err_dpio_setup: ++ dpdmai_close(priv->mc_io, 0, dpdmai_dev->mc_handle); ++err_dpdmai_setup: ++ fsl_mc_portal_free(priv->mc_io); ++err_mcportal: ++ kfree(priv->ppriv); ++ kfree(priv); ++ dev_set_drvdata(dev, NULL); ++ return err; ++} ++ ++static int dpaa2_qdma_remove(struct fsl_mc_device *ls_dev) ++{ ++ struct device *dev; ++ struct dpaa2_qdma_priv *priv; ++ struct dpaa2_qdma_engine *dpaa2_qdma; ++ ++ dev = &ls_dev->dev; ++ priv = dev_get_drvdata(dev); ++ dpaa2_qdma = priv->dpaa2_qdma; ++ ++ dpdmai_disable(priv->mc_io, 0, ls_dev->mc_handle); ++ dpaa2_dpdmai_dpio_unbind(priv); ++ dpaa2_dpmai_store_free(priv); ++ dpaa2_dpdmai_dpio_free(priv); ++ dpdmai_close(priv->mc_io, 0, ls_dev->mc_handle); ++ fsl_mc_portal_free(priv->mc_io); ++ dev_set_drvdata(dev, NULL); ++ dpaa2_dpdmai_free_channels(dpaa2_qdma); ++ ++ dma_async_device_unregister(&dpaa2_qdma->dma_dev); ++ kfree(priv); ++ kfree(dpaa2_qdma); ++ ++ return 0; ++} ++ ++static const struct fsl_mc_device_id dpaa2_qdma_id_table[] = { ++ { ++ .vendor = FSL_MC_VENDOR_FREESCALE, ++ .obj_type = "dpdmai", ++ }, ++ { .vendor = 0x0 } ++}; ++ ++static struct fsl_mc_driver dpaa2_qdma_driver = { ++ .driver = { ++ .name = "dpaa2-qdma", ++ .owner = THIS_MODULE, ++ }, ++ .probe = dpaa2_qdma_probe, ++ .remove = dpaa2_qdma_remove, ++ .match_id_table = dpaa2_qdma_id_table ++}; ++ ++static int __init dpaa2_qdma_driver_init(void) ++{ ++ return fsl_mc_driver_register(&(dpaa2_qdma_driver)); ++} ++late_initcall(dpaa2_qdma_driver_init); ++ ++static void __exit fsl_qdma_exit(void) ++{ ++ fsl_mc_driver_unregister(&(dpaa2_qdma_driver)); ++} ++module_exit(fsl_qdma_exit); ++ ++MODULE_DESCRIPTION("NXP DPAA2 qDMA driver"); ++MODULE_LICENSE("Dual BSD/GPL"); +--- /dev/null ++++ b/drivers/dma/dpaa2-qdma/dpaa2-qdma.h +@@ -0,0 +1,227 @@ ++/* Copyright 2015 NXP Semiconductor Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of NXP Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY NXP Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL NXP Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef __DPAA2_QDMA_H ++#define __DPAA2_QDMA_H ++ ++#define LONG_FORMAT 1 ++ ++#define DPAA2_QDMA_STORE_SIZE 16 ++#define NUM_CH 8 ++#define NUM_SG_PER_BLK 16 ++ ++#define QDMA_DMR_OFFSET 0x0 ++#define QDMA_DQ_EN (0 << 30) ++#define QDMA_DQ_DIS (1 << 30) ++ ++#define QDMA_DSR_M_OFFSET 0x10004 ++ ++struct dpaa2_qdma_sd_d { ++ uint32_t rsv:32; ++ union { ++ struct { ++ uint32_t ssd:12; /* souce stride distance */ ++ uint32_t sss:12; /* souce stride size */ ++ uint32_t rsv1:8; ++ } sdf; ++ struct { ++ uint32_t dsd:12; /* Destination stride distance */ ++ uint32_t dss:12; /* Destination stride size */ ++ uint32_t rsv2:8; ++ } ddf; ++ } df; ++ uint32_t rbpcmd; /* Route-by-port command */ ++ uint32_t cmd; ++} __attribute__((__packed__)); ++/* Source descriptor command read transaction type for RBP=0: ++ coherent copy of cacheable memory */ ++#define QDMA_SD_CMD_RDTTYPE_COHERENT (0xb << 28) ++/* Destination descriptor command write transaction type for RBP=0: ++ coherent copy of cacheable memory */ ++#define QDMA_DD_CMD_WRTTYPE_COHERENT (0x6 << 28) ++ ++#define QDMA_SG_FMT_SDB 0x0 /* single data buffer */ ++#define QDMA_SG_FMT_FDS 0x1 /* frame data section */ ++#define QDMA_SG_FMT_SGTE 0x2 /* SGT extension */ ++#define QDMA_SG_SL_SHORT 0x1 /* short length */ ++#define QDMA_SG_SL_LONG 0x0 /* short length */ ++#define QDMA_SG_F 0x1 /* last sg entry */ ++struct dpaa2_qdma_sg { ++ uint32_t addr_lo; /* address 0:31 */ ++ uint32_t addr_hi:17; /* address 32:48 */ ++ uint32_t rsv:15; ++ union { ++ uint32_t data_len_sl0; /* SL=0, the long format */ ++ struct { ++ uint32_t len:17; /* SL=1, the short format */ ++ uint32_t reserve:3; ++ uint32_t sf:1; ++ uint32_t sr:1; ++ uint32_t size:10; /* buff size */ ++ } data_len_sl1; ++ } data_len; /* AVAIL_LENGTH */ ++ struct { ++ uint32_t bpid:14; ++ uint32_t ivp:1; ++ uint32_t mbt:1; ++ uint32_t offset:12; ++ uint32_t fmt:2; ++ uint32_t sl:1; ++ uint32_t f:1; ++ } ctrl; ++} __attribute__((__packed__)); ++ ++#define QMAN_FD_FMT_ENABLE (1) /* frame list table enable */ ++#define QMAN_FD_BMT_ENABLE (1 << 15) /* bypass memory translation */ ++#define QMAN_FD_BMT_DISABLE (0 << 15) /* bypass memory translation */ ++#define QMAN_FD_SL_DISABLE (0 << 14) /* short lengthe disabled */ ++#define QMAN_FD_SL_ENABLE (1 << 14) /* short lengthe enabled */ ++ ++#define QDMA_SB_FRAME (0 << 28) /* single frame */ ++#define QDMA_SG_FRAME (2 << 28) /* scatter gather frames */ ++#define QDMA_FINAL_BIT_DISABLE (0 << 31) /* final bit disable */ ++#define QDMA_FINAL_BIT_ENABLE (1 << 31) /* final bit enable */ ++ ++#define QDMA_FD_SHORT_FORMAT (1 << 11) /* short format */ ++#define QDMA_FD_LONG_FORMAT (0 << 11) /* long format */ ++#define QDMA_SER_DISABLE (0 << 8) /* no notification */ ++#define QDMA_SER_CTX (1 << 8) /* notification by FQD_CTX[fqid] */ ++#define QDMA_SER_DEST (2 << 8) /* notification by destination desc */ ++#define QDMA_SER_BOTH (3 << 8) /* soruce and dest notification */ ++#define QDMA_FD_SPF_ENALBE (1 << 30) /* source prefetch enable */ ++ ++#define QMAN_FD_VA_ENABLE (1 << 14) /* Address used is virtual address */ ++#define QMAN_FD_VA_DISABLE (0 << 14)/* Address used is a real address */ ++#define QMAN_FD_CBMT_ENABLE (1 << 15) /* Flow Context: 49bit physical address */ ++#define QMAN_FD_CBMT_DISABLE (0 << 15) /* Flow Context: 64bit virtual address */ ++#define QMAN_FD_SC_DISABLE (0 << 27) /* stashing control */ ++ ++#define QDMA_FL_FMT_SBF (0x0) /* Single buffer frame */ ++#define QDMA_FL_FMT_SGE 0x2 /* Scatter gather frame */ ++#define QDMA_FL_BMT_ENABLE (0x1 << 15)/* enable bypass memory translation */ ++#define QDMA_FL_BMT_DISABLE 0x0 /* enable bypass memory translation */ ++#define QDMA_FL_SL_LONG (0x0 << 2)/* long length */ ++#define QDMA_FL_SL_SHORT 0x1 /* short length */ ++#define QDMA_FL_F (0x1)/* last frame list bit */ ++/*Description of Frame list table structure*/ ++ ++struct dpaa2_qdma_chan { ++ struct virt_dma_chan vchan; ++ struct virt_dma_desc vdesc; ++ enum dma_status status; ++ struct dpaa2_qdma_engine *qdma; ++ ++ struct mutex dpaa2_queue_mutex; ++ spinlock_t queue_lock; ++ struct dma_pool *fd_pool; ++ struct dma_pool *sg_blk_pool; ++ ++ struct list_head comp_used; ++ struct list_head comp_free; ++ ++ struct list_head sgb_free; ++}; ++ ++struct qdma_sg_blk { ++ dma_addr_t blk_bus_addr; ++ void *blk_virt_addr; ++ struct list_head list; ++}; ++ ++struct dpaa2_qdma_comp { ++ dma_addr_t fd_bus_addr; ++ dma_addr_t fl_bus_addr; ++ dma_addr_t desc_bus_addr; ++ dma_addr_t sge_src_bus_addr; ++ dma_addr_t sge_dst_bus_addr; ++ void *fd_virt_addr; ++ void *fl_virt_addr; ++ void *desc_virt_addr; ++ void *sg_src_virt_addr; ++ void *sg_dst_virt_addr; ++ struct qdma_sg_blk *sg_blk; ++ uint32_t sg_blk_num; ++ struct list_head sg_src_head; ++ struct list_head sg_dst_head; ++ struct dpaa2_qdma_chan *qchan; ++ struct virt_dma_desc vdesc; ++ struct list_head list; ++}; ++ ++struct dpaa2_qdma_engine { ++ struct dma_device dma_dev; ++ u32 n_chans; ++ struct dpaa2_qdma_chan chans[NUM_CH]; ++ ++ struct dpaa2_qdma_priv *priv; ++}; ++ ++/* ++ * dpaa2_qdma_priv - driver private data ++ */ ++struct dpaa2_qdma_priv { ++ int dpqdma_id; ++ ++ struct iommu_domain *iommu_domain; ++ struct dpdmai_attr dpdmai_attr; ++ struct device *dev; ++ struct fsl_mc_io *mc_io; ++ struct fsl_mc_device *dpdmai_dev; ++ ++ struct dpdmai_rx_queue_attr rx_queue_attr[DPDMAI_PRIO_NUM]; ++ struct dpdmai_tx_queue_attr tx_queue_attr[DPDMAI_PRIO_NUM]; ++ ++ uint8_t num_pairs; ++ ++ struct dpaa2_qdma_engine *dpaa2_qdma; ++ struct dpaa2_qdma_priv_per_prio *ppriv; ++}; ++ ++struct dpaa2_qdma_priv_per_prio { ++ int req_fqid; ++ int rsp_fqid; ++ int prio; ++ ++ struct dpaa2_io_store *store; ++ struct dpaa2_io_notification_ctx nctx; ++ ++ struct dpaa2_qdma_priv *priv; ++}; ++ ++/* FD pool size: one FD + 3 Frame list + 2 source/destination descriptor */ ++#define FD_POOL_SIZE (sizeof(struct dpaa2_fd) + \ ++ sizeof(struct dpaa2_fl_entry) * 3 + \ ++ sizeof(struct dpaa2_qdma_sd_d) * 2) ++ ++/* qdma_sg_blk + 16 SGs */ ++#define SG_POOL_SIZE (sizeof(struct qdma_sg_blk) +\ ++ sizeof(struct dpaa2_qdma_sg) * NUM_SG_PER_BLK) ++#endif /* __DPAA2_QDMA_H */ +--- /dev/null ++++ b/drivers/dma/dpaa2-qdma/dpdmai.c +@@ -0,0 +1,515 @@ ++/* Copyright 2013-2015 Freescale Semiconductor Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of the above-listed copyright holders nor the ++ * names of any contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" ++ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ++ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ++ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE ++ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR ++ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF ++ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS ++ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN ++ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ++ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ */ ++#include <linux/types.h> ++#include <linux/io.h> ++#include "fsl_dpdmai.h" ++#include "fsl_dpdmai_cmd.h" ++#include <linux/fsl/mc.h> ++ ++struct dpdmai_cmd_open { ++ __le32 dpdmai_id; ++}; ++ ++struct dpdmai_rsp_get_attributes { ++ __le32 id; ++ u8 num_of_priorities; ++ u8 pad0[3]; ++ __le16 major; ++ __le16 minor; ++}; ++ ++ ++struct dpdmai_cmd_queue { ++ __le32 dest_id; ++ u8 priority; ++ u8 queue; ++ u8 dest_type; ++ u8 pad; ++ __le64 user_ctx; ++ union { ++ __le32 options; ++ __le32 fqid; ++ }; ++}; ++ ++struct dpdmai_rsp_get_tx_queue { ++ __le64 pad; ++ __le32 fqid; ++}; ++ ++ ++int dpdmai_open(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ int dpdmai_id, ++ uint16_t *token) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ struct dpdmai_cmd_open *cmd_params; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_OPEN, ++ cmd_flags, ++ 0); ++ ++ cmd_params = (struct dpdmai_cmd_open *)cmd.params; ++ cmd_params->dpdmai_id = cpu_to_le32(dpdmai_id); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ *token = mc_cmd_hdr_read_token(&cmd); ++ return 0; ++} ++ ++int dpdmai_close(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLOSE, ++ cmd_flags, token); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_create(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ const struct dpdmai_cfg *cfg, ++ uint16_t *token) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CREATE, ++ cmd_flags, ++ 0); ++ DPDMAI_CMD_CREATE(cmd, cfg); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ *token = MC_CMD_HDR_READ_TOKEN(cmd.header); ++ ++ return 0; ++} ++ ++int dpdmai_destroy(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DESTROY, ++ cmd_flags, ++ token); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_enable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_ENABLE, ++ cmd_flags, ++ token); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_disable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_DISABLE, ++ cmd_flags, ++ token); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_is_enabled(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ int *en) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ int err; ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_IS_ENABLED, ++ cmd_flags, ++ token); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ DPDMAI_RSP_IS_ENABLED(cmd, *en); ++ ++ return 0; ++} ++ ++int dpdmai_reset(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_RESET, ++ cmd_flags, ++ token); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_get_irq(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ int *type, ++ struct dpdmai_irq_cfg *irq_cfg) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_GET_IRQ(cmd, irq_index); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ DPDMAI_RSP_GET_IRQ(cmd, *type, irq_cfg); ++ ++ return 0; ++} ++ ++int dpdmai_set_irq(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ struct dpdmai_irq_cfg *irq_cfg) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint8_t *en) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_ENABLE, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ DPDMAI_RSP_GET_IRQ_ENABLE(cmd, *en); ++ ++ return 0; ++} ++ ++int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint8_t en) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_ENABLE, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, en); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t *mask) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_MASK, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ DPDMAI_RSP_GET_IRQ_MASK(cmd, *mask); ++ ++ return 0; ++} ++ ++int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t mask) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_IRQ_MASK, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_get_irq_status(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t *status) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_IRQ_STATUS, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, *status); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ DPDMAI_RSP_GET_IRQ_STATUS(cmd, *status); ++ ++ return 0; ++} ++ ++int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t status) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_CLEAR_IRQ_STATUS, ++ cmd_flags, ++ token); ++ DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status); ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_get_attributes(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ struct dpdmai_attr *attr) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ int err; ++ struct dpdmai_rsp_get_attributes *rsp_params; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_ATTR, ++ cmd_flags, ++ token); ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ rsp_params = (struct dpdmai_rsp_get_attributes *)cmd.params; ++ attr->id = le32_to_cpu(rsp_params->id); ++ attr->version.major = le16_to_cpu(rsp_params->major); ++ attr->version.minor = le16_to_cpu(rsp_params->minor); ++ attr->num_of_priorities = rsp_params->num_of_priorities; ++ ++ ++ return 0; ++} ++ ++int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t priority, ++ const struct dpdmai_rx_queue_cfg *cfg) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ struct dpdmai_cmd_queue *cmd_params; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_SET_RX_QUEUE, ++ cmd_flags, ++ token); ++ ++ cmd_params = (struct dpdmai_cmd_queue *)cmd.params; ++ cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); ++ cmd_params->priority = cfg->dest_cfg.priority; ++ cmd_params->queue = priority; ++ cmd_params->dest_type = cfg->dest_cfg.dest_type; ++ cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); ++ cmd_params->options = cpu_to_le32(cfg->options); ++ ++ ++ /* send command to mc*/ ++ return mc_send_command(mc_io, &cmd); ++} ++ ++int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t priority, struct dpdmai_rx_queue_attr *attr) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ struct dpdmai_cmd_queue *cmd_params; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_RX_QUEUE, ++ cmd_flags, ++ token); ++ ++ cmd_params = (struct dpdmai_cmd_queue *)cmd.params; ++ cmd_params->queue = priority; ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); ++ attr->dest_cfg.priority = cmd_params->priority; ++ attr->dest_cfg.dest_type = cmd_params->dest_type; ++ attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); ++ attr->fqid = le32_to_cpu(cmd_params->fqid); ++ ++ return 0; ++} ++ ++int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t priority, ++ struct dpdmai_tx_queue_attr *attr) ++{ ++ struct fsl_mc_command cmd = { 0 }; ++ struct dpdmai_cmd_queue *cmd_params; ++ struct dpdmai_rsp_get_tx_queue *rsp_params; ++ int err; ++ ++ /* prepare command */ ++ cmd.header = mc_encode_cmd_header(DPDMAI_CMDID_GET_TX_QUEUE, ++ cmd_flags, ++ token); ++ ++ cmd_params = (struct dpdmai_cmd_queue *)cmd.params; ++ cmd_params->queue = priority; ++ ++ /* send command to mc*/ ++ err = mc_send_command(mc_io, &cmd); ++ if (err) ++ return err; ++ ++ /* retrieve response parameters */ ++ ++ rsp_params = (struct dpdmai_rsp_get_tx_queue *)cmd.params; ++ attr->fqid = le32_to_cpu(rsp_params->fqid); ++ ++ return 0; ++} +--- /dev/null ++++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai.h +@@ -0,0 +1,521 @@ ++/* Copyright 2013-2015 Freescale Semiconductor Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of the above-listed copyright holders nor the ++ * names of any contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" ++ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ++ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ++ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE ++ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR ++ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF ++ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS ++ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN ++ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ++ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ */ ++#ifndef __FSL_DPDMAI_H ++#define __FSL_DPDMAI_H ++ ++struct fsl_mc_io; ++ ++/* Data Path DMA Interface API ++ * Contains initialization APIs and runtime control APIs for DPDMAI ++ */ ++ ++/* General DPDMAI macros */ ++ ++/** ++ * Maximum number of Tx/Rx priorities per DPDMAI object ++ */ ++#define DPDMAI_PRIO_NUM 2 ++ ++/** ++ * All queues considered; see dpdmai_set_rx_queue() ++ */ ++#define DPDMAI_ALL_QUEUES (uint8_t)(-1) ++ ++/** ++ * dpdmai_open() - Open a control session for the specified object ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @dpdmai_id: DPDMAI unique ID ++ * @token: Returned token; use in subsequent API calls ++ * ++ * This function can be used to open a control session for an ++ * already created object; an object may have been declared in ++ * the DPL or by calling the dpdmai_create() function. ++ * This function returns a unique authentication token, ++ * associated with the specific object ID and the specific MC ++ * portal; this token must be used in all subsequent commands for ++ * this specific object. ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_open(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ int dpdmai_id, ++ uint16_t *token); ++ ++/** ++ * dpdmai_close() - Close the control session of the object ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * ++ * After this function is called, no further operations are ++ * allowed on the object without opening a new control session. ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_close(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token); ++ ++/** ++ * struct dpdmai_cfg - Structure representing DPDMAI configuration ++ * @priorities: Priorities for the DMA hardware processing; valid priorities are ++ * configured with values 1-8; the entry following last valid entry ++ * should be configured with 0 ++ */ ++struct dpdmai_cfg { ++ uint8_t priorities[DPDMAI_PRIO_NUM]; ++}; ++ ++/** ++ * dpdmai_create() - Create the DPDMAI object ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @cfg: Configuration structure ++ * @token: Returned token; use in subsequent API calls ++ * ++ * Create the DPDMAI object, allocate required resources and ++ * perform required initialization. ++ * ++ * The object can be created either by declaring it in the ++ * DPL file, or by calling this function. ++ * ++ * This function returns a unique authentication token, ++ * associated with the specific object ID and the specific MC ++ * portal; this token must be used in all subsequent calls to ++ * this specific object. For objects that are created using the ++ * DPL file, call dpdmai_open() function to get an authentication ++ * token first. ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_create(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ const struct dpdmai_cfg *cfg, ++ uint16_t *token); ++ ++/** ++ * dpdmai_destroy() - Destroy the DPDMAI object and release all its resources. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * ++ * Return: '0' on Success; error code otherwise. ++ */ ++int dpdmai_destroy(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token); ++ ++/** ++ * dpdmai_enable() - Enable the DPDMAI, allow sending and receiving frames. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_enable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token); ++ ++/** ++ * dpdmai_disable() - Disable the DPDMAI, stop sending and receiving frames. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_disable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token); ++ ++/** ++ * dpdmai_is_enabled() - Check if the DPDMAI is enabled. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @en: Returns '1' if object is enabled; '0' otherwise ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_is_enabled(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ int *en); ++ ++/** ++ * dpdmai_reset() - Reset the DPDMAI, returns the object to initial state. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_reset(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token); ++ ++/** ++ * struct dpdmai_irq_cfg - IRQ configuration ++ * @addr: Address that must be written to signal a message-based interrupt ++ * @val: Value to write into irq_addr address ++ * @irq_num: A user defined number associated with this IRQ ++ */ ++struct dpdmai_irq_cfg { ++ uint64_t addr; ++ uint32_t val; ++ int irq_num; ++}; ++ ++/** ++ * dpdmai_set_irq() - Set IRQ information for the DPDMAI to trigger an interrupt. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: Identifies the interrupt index to configure ++ * @irq_cfg: IRQ configuration ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_set_irq(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ struct dpdmai_irq_cfg *irq_cfg); ++ ++/** ++ * dpdmai_get_irq() - Get IRQ information from the DPDMAI ++ * ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: The interrupt index to configure ++ * @type: Interrupt type: 0 represents message interrupt ++ * type (both irq_addr and irq_val are valid) ++ * @irq_cfg: IRQ attributes ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_get_irq(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ int *type, ++ struct dpdmai_irq_cfg *irq_cfg); ++ ++/** ++ * dpdmai_set_irq_enable() - Set overall interrupt state. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: The interrupt index to configure ++ * @en: Interrupt state - enable = 1, disable = 0 ++ * ++ * Allows GPP software to control when interrupts are generated. ++ * Each interrupt can have up to 32 causes. The enable/disable control's the ++ * overall interrupt state. if the interrupt is disabled no causes will cause ++ * an interrupt ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_set_irq_enable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint8_t en); ++ ++/** ++ * dpdmai_get_irq_enable() - Get overall interrupt state ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: The interrupt index to configure ++ * @en: Returned Interrupt state - enable = 1, disable = 0 ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_get_irq_enable(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint8_t *en); ++ ++/** ++ * dpdmai_set_irq_mask() - Set interrupt mask. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: The interrupt index to configure ++ * @mask: event mask to trigger interrupt; ++ * each bit: ++ * 0 = ignore event ++ * 1 = consider event for asserting IRQ ++ * ++ * Every interrupt can have up to 32 causes and the interrupt model supports ++ * masking/unmasking each cause independently ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_set_irq_mask(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t mask); ++ ++/** ++ * dpdmai_get_irq_mask() - Get interrupt mask. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: The interrupt index to configure ++ * @mask: Returned event mask to trigger interrupt ++ * ++ * Every interrupt can have up to 32 causes and the interrupt model supports ++ * masking/unmasking each cause independently ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_get_irq_mask(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t *mask); ++ ++/** ++ * dpdmai_get_irq_status() - Get the current status of any pending interrupts ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: The interrupt index to configure ++ * @status: Returned interrupts status - one bit per cause: ++ * 0 = no interrupt pending ++ * 1 = interrupt pending ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_get_irq_status(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t *status); ++ ++/** ++ * dpdmai_clear_irq_status() - Clear a pending interrupt's status ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @irq_index: The interrupt index to configure ++ * @status: bits to clear (W1C) - one bit per cause: ++ * 0 = don't change ++ * 1 = clear status bit ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_clear_irq_status(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t irq_index, ++ uint32_t status); ++ ++/** ++ * struct dpdmai_attr - Structure representing DPDMAI attributes ++ * @id: DPDMAI object ID ++ * @version: DPDMAI version ++ * @num_of_priorities: number of priorities ++ */ ++struct dpdmai_attr { ++ int id; ++ /** ++ * struct version - DPDMAI version ++ * @major: DPDMAI major version ++ * @minor: DPDMAI minor version ++ */ ++ struct { ++ uint16_t major; ++ uint16_t minor; ++ } version; ++ uint8_t num_of_priorities; ++}; ++ ++/** ++ * dpdmai_get_attributes() - Retrieve DPDMAI attributes. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @attr: Returned object's attributes ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_get_attributes(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ struct dpdmai_attr *attr); ++ ++/** ++ * enum dpdmai_dest - DPDMAI destination types ++ * @DPDMAI_DEST_NONE: Unassigned destination; The queue is set in parked mode ++ * and does not generate FQDAN notifications; user is expected to dequeue ++ * from the queue based on polling or other user-defined method ++ * @DPDMAI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN ++ * notifications to the specified DPIO; user is expected to dequeue ++ * from the queue only after notification is received ++ * @DPDMAI_DEST_DPCON: The queue is set in schedule mode and does not generate ++ * FQDAN notifications, but is connected to the specified DPCON object; ++ * user is expected to dequeue from the DPCON channel ++ */ ++enum dpdmai_dest { ++ DPDMAI_DEST_NONE = 0, ++ DPDMAI_DEST_DPIO = 1, ++ DPDMAI_DEST_DPCON = 2 ++}; ++ ++/** ++ * struct dpdmai_dest_cfg - Structure representing DPDMAI destination parameters ++ * @dest_type: Destination type ++ * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type ++ * @priority: Priority selection within the DPIO or DPCON channel; valid values ++ * are 0-1 or 0-7, depending on the number of priorities in that ++ * channel; not relevant for 'DPDMAI_DEST_NONE' option ++ */ ++struct dpdmai_dest_cfg { ++ enum dpdmai_dest dest_type; ++ int dest_id; ++ uint8_t priority; ++}; ++ ++/* DPDMAI queue modification options */ ++ ++/** ++ * Select to modify the user's context associated with the queue ++ */ ++#define DPDMAI_QUEUE_OPT_USER_CTX 0x00000001 ++ ++/** ++ * Select to modify the queue's destination ++ */ ++#define DPDMAI_QUEUE_OPT_DEST 0x00000002 ++ ++/** ++ * struct dpdmai_rx_queue_cfg - DPDMAI RX queue configuration ++ * @options: Flags representing the suggested modifications to the queue; ++ * Use any combination of 'DPDMAI_QUEUE_OPT_<X>' flags ++ * @user_ctx: User context value provided in the frame descriptor of each ++ * dequeued frame; ++ * valid only if 'DPDMAI_QUEUE_OPT_USER_CTX' is contained in 'options' ++ * @dest_cfg: Queue destination parameters; ++ * valid only if 'DPDMAI_QUEUE_OPT_DEST' is contained in 'options' ++ */ ++struct dpdmai_rx_queue_cfg { ++ uint32_t options; ++ uint64_t user_ctx; ++ struct dpdmai_dest_cfg dest_cfg; ++ ++}; ++ ++/** ++ * dpdmai_set_rx_queue() - Set Rx queue configuration ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @priority: Select the queue relative to number of ++ * priorities configured at DPDMAI creation; use ++ * DPDMAI_ALL_QUEUES to configure all Rx queues ++ * identically. ++ * @cfg: Rx queue configuration ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_set_rx_queue(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t priority, ++ const struct dpdmai_rx_queue_cfg *cfg); ++ ++/** ++ * struct dpdmai_rx_queue_attr - Structure representing attributes of Rx queues ++ * @user_ctx: User context value provided in the frame descriptor of each ++ * dequeued frame ++ * @dest_cfg: Queue destination configuration ++ * @fqid: Virtual FQID value to be used for dequeue operations ++ */ ++struct dpdmai_rx_queue_attr { ++ uint64_t user_ctx; ++ struct dpdmai_dest_cfg dest_cfg; ++ uint32_t fqid; ++}; ++ ++/** ++ * dpdmai_get_rx_queue() - Retrieve Rx queue attributes. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @priority: Select the queue relative to number of ++ * priorities configured at DPDMAI creation ++ * @attr: Returned Rx queue attributes ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_get_rx_queue(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t priority, ++ struct dpdmai_rx_queue_attr *attr); ++ ++/** ++ * struct dpdmai_tx_queue_attr - Structure representing attributes of Tx queues ++ * @fqid: Virtual FQID to be used for sending frames to DMA hardware ++ */ ++ ++struct dpdmai_tx_queue_attr { ++ uint32_t fqid; ++}; ++ ++/** ++ * dpdmai_get_tx_queue() - Retrieve Tx queue attributes. ++ * @mc_io: Pointer to MC portal's I/O object ++ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' ++ * @token: Token of DPDMAI object ++ * @priority: Select the queue relative to number of ++ * priorities configured at DPDMAI creation ++ * @attr: Returned Tx queue attributes ++ * ++ * Return: '0' on Success; Error code otherwise. ++ */ ++int dpdmai_get_tx_queue(struct fsl_mc_io *mc_io, ++ uint32_t cmd_flags, ++ uint16_t token, ++ uint8_t priority, ++ struct dpdmai_tx_queue_attr *attr); ++ ++#endif /* __FSL_DPDMAI_H */ +--- /dev/null ++++ b/drivers/dma/dpaa2-qdma/fsl_dpdmai_cmd.h +@@ -0,0 +1,222 @@ ++/* Copyright 2013-2016 Freescale Semiconductor Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of the above-listed copyright holders nor the ++ * names of any contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" ++ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE ++ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ++ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE ++ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR ++ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF ++ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS ++ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN ++ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ++ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE ++ * POSSIBILITY OF SUCH DAMAGE. ++ */ ++#ifndef _FSL_DPDMAI_CMD_H ++#define _FSL_DPDMAI_CMD_H ++ ++/* DPDMAI Version */ ++#define DPDMAI_VER_MAJOR 2 ++#define DPDMAI_VER_MINOR 2 ++ ++#define DPDMAI_CMD_BASE_VERSION 0 ++#define DPDMAI_CMD_ID_OFFSET 4 ++ ++/* Command IDs */ ++#define DPDMAI_CMDID_CLOSE ((0x800 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_OPEN ((0x80E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_CREATE ((0x90E << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_DESTROY ((0x900 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++ ++#define DPDMAI_CMDID_ENABLE ((0x002 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_DISABLE ((0x003 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_GET_ATTR ((0x004 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_RESET ((0x005 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_IS_ENABLED ((0x006 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++ ++#define DPDMAI_CMDID_SET_IRQ ((0x010 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_GET_IRQ ((0x011 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_SET_IRQ_ENABLE ((0x012 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_GET_IRQ_ENABLE ((0x013 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_SET_IRQ_MASK ((0x014 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_GET_IRQ_MASK ((0x015 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_GET_IRQ_STATUS ((0x016 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_CLEAR_IRQ_STATUS ((0x017 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++ ++#define DPDMAI_CMDID_SET_RX_QUEUE ((0x1A0 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_GET_RX_QUEUE ((0x1A1 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++#define DPDMAI_CMDID_GET_TX_QUEUE ((0x1A2 << DPDMAI_CMD_ID_OFFSET) | DPDMAI_CMD_BASE_VERSION) ++ ++ ++#define MC_CMD_HDR_TOKEN_O 32 /* Token field offset */ ++#define MC_CMD_HDR_TOKEN_S 16 /* Token field size */ ++ ++ ++#define MAKE_UMASK64(_width) \ ++ ((uint64_t)((_width) < 64 ? ((uint64_t)1 << (_width)) - 1 : \ ++ (uint64_t)-1)) ++ ++static inline uint64_t mc_enc(int lsoffset, int width, uint64_t val) ++{ ++ return (uint64_t)(((uint64_t)val & MAKE_UMASK64(width)) << lsoffset); ++} ++ ++static inline uint64_t mc_dec(uint64_t val, int lsoffset, int width) ++{ ++ return (uint64_t)((val >> lsoffset) & MAKE_UMASK64(width)); ++} ++ ++#define MC_CMD_OP(_cmd, _param, _offset, _width, _type, _arg) \ ++ ((_cmd).params[_param] |= mc_enc((_offset), (_width), _arg)) ++ ++#define MC_RSP_OP(_cmd, _param, _offset, _width, _type, _arg) \ ++ (_arg = (_type)mc_dec(_cmd.params[_param], (_offset), (_width))) ++ ++#define MC_CMD_HDR_READ_TOKEN(_hdr) \ ++ ((uint16_t)mc_dec((_hdr), MC_CMD_HDR_TOKEN_O, MC_CMD_HDR_TOKEN_S)) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_OPEN(cmd, dpdmai_id) \ ++ MC_CMD_OP(cmd, 0, 0, 32, int, dpdmai_id) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_CREATE(cmd, cfg) \ ++do { \ ++ MC_CMD_OP(cmd, 0, 8, 8, uint8_t, cfg->priorities[0]);\ ++ MC_CMD_OP(cmd, 0, 16, 8, uint8_t, cfg->priorities[1]);\ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_IS_ENABLED(cmd, en) \ ++ MC_RSP_OP(cmd, 0, 0, 1, int, en) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_SET_IRQ(cmd, irq_index, irq_cfg) \ ++do { \ ++ MC_CMD_OP(cmd, 0, 0, 8, uint8_t, irq_index);\ ++ MC_CMD_OP(cmd, 0, 32, 32, uint32_t, irq_cfg->val);\ ++ MC_CMD_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ ++ MC_CMD_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_GET_IRQ(cmd, irq_index) \ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_GET_IRQ(cmd, type, irq_cfg) \ ++do { \ ++ MC_RSP_OP(cmd, 0, 0, 32, uint32_t, irq_cfg->val); \ ++ MC_RSP_OP(cmd, 1, 0, 64, uint64_t, irq_cfg->addr);\ ++ MC_RSP_OP(cmd, 2, 0, 32, int, irq_cfg->irq_num); \ ++ MC_RSP_OP(cmd, 2, 32, 32, int, type); \ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_SET_IRQ_ENABLE(cmd, irq_index, enable_state) \ ++do { \ ++ MC_CMD_OP(cmd, 0, 0, 8, uint8_t, enable_state); \ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_GET_IRQ_ENABLE(cmd, irq_index) \ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_GET_IRQ_ENABLE(cmd, enable_state) \ ++ MC_RSP_OP(cmd, 0, 0, 8, uint8_t, enable_state) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_SET_IRQ_MASK(cmd, irq_index, mask) \ ++do { \ ++ MC_CMD_OP(cmd, 0, 0, 32, uint32_t, mask); \ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_GET_IRQ_MASK(cmd, irq_index) \ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_GET_IRQ_MASK(cmd, mask) \ ++ MC_RSP_OP(cmd, 0, 0, 32, uint32_t, mask) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_GET_IRQ_STATUS(cmd, irq_index, status) \ ++do { \ ++ MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status);\ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index);\ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_GET_IRQ_STATUS(cmd, status) \ ++ MC_RSP_OP(cmd, 0, 0, 32, uint32_t, status) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_CLEAR_IRQ_STATUS(cmd, irq_index, status) \ ++do { \ ++ MC_CMD_OP(cmd, 0, 0, 32, uint32_t, status); \ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, irq_index); \ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_GET_ATTR(cmd, attr) \ ++do { \ ++ MC_RSP_OP(cmd, 0, 0, 32, int, attr->id); \ ++ MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->num_of_priorities); \ ++ MC_RSP_OP(cmd, 1, 0, 16, uint16_t, attr->version.major);\ ++ MC_RSP_OP(cmd, 1, 16, 16, uint16_t, attr->version.minor);\ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_SET_RX_QUEUE(cmd, priority, cfg) \ ++do { \ ++ MC_CMD_OP(cmd, 0, 0, 32, int, cfg->dest_cfg.dest_id); \ ++ MC_CMD_OP(cmd, 0, 32, 8, uint8_t, cfg->dest_cfg.priority); \ ++ MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority); \ ++ MC_CMD_OP(cmd, 0, 48, 4, enum dpdmai_dest, cfg->dest_cfg.dest_type); \ ++ MC_CMD_OP(cmd, 1, 0, 64, uint64_t, cfg->user_ctx); \ ++ MC_CMD_OP(cmd, 2, 0, 32, uint32_t, cfg->options);\ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_GET_RX_QUEUE(cmd, priority) \ ++ MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_GET_RX_QUEUE(cmd, attr) \ ++do { \ ++ MC_RSP_OP(cmd, 0, 0, 32, int, attr->dest_cfg.dest_id);\ ++ MC_RSP_OP(cmd, 0, 32, 8, uint8_t, attr->dest_cfg.priority);\ ++ MC_RSP_OP(cmd, 0, 48, 4, enum dpdmai_dest, attr->dest_cfg.dest_type);\ ++ MC_RSP_OP(cmd, 1, 0, 64, uint64_t, attr->user_ctx);\ ++ MC_RSP_OP(cmd, 2, 0, 32, uint32_t, attr->fqid);\ ++} while (0) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_CMD_GET_TX_QUEUE(cmd, priority) \ ++ MC_CMD_OP(cmd, 0, 40, 8, uint8_t, priority) ++ ++/* cmd, param, offset, width, type, arg_name */ ++#define DPDMAI_RSP_GET_TX_QUEUE(cmd, attr) \ ++ MC_RSP_OP(cmd, 1, 0, 32, uint32_t, attr->fqid) ++ ++#endif /* _FSL_DPDMAI_CMD_H */ +--- /dev/null ++++ b/drivers/dma/fsl-qdma.c +@@ -0,0 +1,1278 @@ ++/* ++ * Driver for NXP Layerscape Queue direct memory access controller (qDMA) ++ * ++ * Copyright 2017 NXP ++ * ++ * Author: ++ * Jiaheng Fan <jiaheng.fan@nxp.com> ++ * Wen He <wen.he_1@nxp.com> ++ * ++ * SPDX-License-Identifier: GPL-2.0+ ++ */ ++ ++#include <linux/interrupt.h> ++#include <linux/module.h> ++#include <linux/delay.h> ++#include <linux/of_irq.h> ++#include <linux/of_address.h> ++#include <linux/of_platform.h> ++#include <linux/of_dma.h> ++#include <linux/dma-mapping.h> ++#include <linux/dmapool.h> ++#include <linux/dmaengine.h> ++#include <linux/slab.h> ++#include <linux/spinlock.h> ++ ++#include "virt-dma.h" ++ ++#define FSL_QDMA_DMR 0x0 ++#define FSL_QDMA_DSR 0x4 ++#define FSL_QDMA_DEIER 0xe00 ++#define FSL_QDMA_DEDR 0xe04 ++#define FSL_QDMA_DECFDW0R 0xe10 ++#define FSL_QDMA_DECFDW1R 0xe14 ++#define FSL_QDMA_DECFDW2R 0xe18 ++#define FSL_QDMA_DECFDW3R 0xe1c ++#define FSL_QDMA_DECFQIDR 0xe30 ++#define FSL_QDMA_DECBR 0xe34 ++ ++#define FSL_QDMA_BCQMR(x) (0xc0 + 0x100 * (x)) ++#define FSL_QDMA_BCQSR(x) (0xc4 + 0x100 * (x)) ++#define FSL_QDMA_BCQEDPA_SADDR(x) (0xc8 + 0x100 * (x)) ++#define FSL_QDMA_BCQDPA_SADDR(x) (0xcc + 0x100 * (x)) ++#define FSL_QDMA_BCQEEPA_SADDR(x) (0xd0 + 0x100 * (x)) ++#define FSL_QDMA_BCQEPA_SADDR(x) (0xd4 + 0x100 * (x)) ++#define FSL_QDMA_BCQIER(x) (0xe0 + 0x100 * (x)) ++#define FSL_QDMA_BCQIDR(x) (0xe4 + 0x100 * (x)) ++ ++#define FSL_QDMA_SQDPAR 0x80c ++#define FSL_QDMA_SQEPAR 0x814 ++#define FSL_QDMA_BSQMR 0x800 ++#define FSL_QDMA_BSQSR 0x804 ++#define FSL_QDMA_BSQICR 0x828 ++#define FSL_QDMA_CQMR 0xa00 ++#define FSL_QDMA_CQDSCR1 0xa08 ++#define FSL_QDMA_CQDSCR2 0xa0c ++#define FSL_QDMA_CQIER 0xa10 ++#define FSL_QDMA_CQEDR 0xa14 ++#define FSL_QDMA_SQCCMR 0xa20 ++ ++#define FSL_QDMA_SQICR_ICEN ++ ++#define FSL_QDMA_CQIDR_CQT 0xff000000 ++#define FSL_QDMA_CQIDR_SQPE 0x800000 ++#define FSL_QDMA_CQIDR_SQT 0x8000 ++ ++#define FSL_QDMA_BCQIER_CQTIE 0x8000 ++#define FSL_QDMA_BCQIER_CQPEIE 0x800000 ++#define FSL_QDMA_BSQICR_ICEN 0x80000000 ++#define FSL_QDMA_BSQICR_ICST(x) ((x) << 16) ++#define FSL_QDMA_CQIER_MEIE 0x80000000 ++#define FSL_QDMA_CQIER_TEIE 0x1 ++#define FSL_QDMA_SQCCMR_ENTER_WM 0x200000 ++ ++#define FSL_QDMA_QUEUE_MAX 8 ++ ++#define FSL_QDMA_BCQMR_EN 0x80000000 ++#define FSL_QDMA_BCQMR_EI 0x40000000 ++#define FSL_QDMA_BCQMR_CD_THLD(x) ((x) << 20) ++#define FSL_QDMA_BCQMR_CQ_SIZE(x) ((x) << 16) ++ ++#define FSL_QDMA_BCQSR_QF 0x10000 ++#define FSL_QDMA_BCQSR_XOFF 0x1 ++ ++#define FSL_QDMA_BSQMR_EN 0x80000000 ++#define FSL_QDMA_BSQMR_DI 0x40000000 ++#define FSL_QDMA_BSQMR_CQ_SIZE(x) ((x) << 16) ++ ++#define FSL_QDMA_BSQSR_QE 0x20000 ++ ++#define FSL_QDMA_DMR_DQD 0x40000000 ++#define FSL_QDMA_DSR_DB 0x80000000 ++ ++#define FSL_QDMA_COMMAND_BUFFER_SIZE 64 ++#define FSL_QDMA_DESCRIPTOR_BUFFER_SIZE 32 ++#define FSL_QDMA_CIRCULAR_DESC_SIZE_MIN 64 ++#define FSL_QDMA_CIRCULAR_DESC_SIZE_MAX 16384 ++#define FSL_QDMA_QUEUE_NUM_MAX 8 ++ ++#define FSL_QDMA_CMD_RWTTYPE 0x4 ++#define FSL_QDMA_CMD_LWC 0x2 ++ ++#define FSL_QDMA_CMD_RWTTYPE_OFFSET 28 ++#define FSL_QDMA_CMD_NS_OFFSET 27 ++#define FSL_QDMA_CMD_DQOS_OFFSET 24 ++#define FSL_QDMA_CMD_WTHROTL_OFFSET 20 ++#define FSL_QDMA_CMD_DSEN_OFFSET 19 ++#define FSL_QDMA_CMD_LWC_OFFSET 16 ++ ++#define QDMA_CCDF_STATUS 20 ++#define QDMA_CCDF_OFFSET 20 ++#define QDMA_CCDF_MASK GENMASK(28, 20) ++#define QDMA_CCDF_FOTMAT BIT(29) ++#define QDMA_CCDF_SER BIT(30) ++ ++#define QDMA_SG_FIN BIT(30) ++#define QDMA_SG_EXT BIT(31) ++#define QDMA_SG_LEN_MASK GENMASK(29, 0) ++ ++#define QDMA_BIG_ENDIAN 0x00000001 ++#define COMP_TIMEOUT 1000 ++#define COMMAND_QUEUE_OVERFLLOW 10 ++ ++#define QDMA_IN(fsl_qdma_engine, addr) \ ++ (((fsl_qdma_engine)->big_endian & QDMA_BIG_ENDIAN) ? \ ++ ioread32be(addr) : ioread32(addr)) ++#define QDMA_OUT(fsl_qdma_engine, addr, val) \ ++ (((fsl_qdma_engine)->big_endian & QDMA_BIG_ENDIAN) ? \ ++ iowrite32be(val, addr) : iowrite32(val, addr)) ++ ++#define FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma_engine, x) \ ++ (((fsl_qdma_engine)->block_offset) * (x)) ++ ++static DEFINE_PER_CPU(u64, pre_addr); ++static DEFINE_PER_CPU(u64, pre_queue); ++ ++/* qDMA Command Descriptor Fotmats */ ++ ++struct fsl_qdma_format { ++ __le32 status; /* ser, status */ ++ __le32 cfg; /* format, offset */ ++ union { ++ struct { ++ __le32 addr_lo; /* low 32-bits of 40-bit address */ ++ u8 addr_hi; /* high 8-bits of 40-bit address */ ++ u8 __reserved1[2]; ++ u8 cfg8b_w1; /* dd, queue */ ++ } __packed; ++ __le64 data; ++ }; ++} __packed; ++ ++static inline u64 ++qdma_ccdf_addr_get64(const struct fsl_qdma_format *ccdf) ++{ ++ return le64_to_cpu(ccdf->data) & 0xffffffffffLLU; ++} ++ ++static inline void ++qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr) ++{ ++ ccdf->addr_hi = upper_32_bits(addr); ++ ccdf->addr_lo = cpu_to_le32(lower_32_bits(addr)); ++} ++ ++static inline u64 ++qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf) ++{ ++ return ccdf->cfg8b_w1 & 0xff; ++} ++ ++static inline int ++qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf) ++{ ++ return (le32_to_cpu(ccdf->cfg) & QDMA_CCDF_MASK) >> QDMA_CCDF_OFFSET; ++} ++ ++static inline void ++qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset) ++{ ++ ccdf->cfg = cpu_to_le32(QDMA_CCDF_FOTMAT | offset); ++} ++ ++static inline int ++qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf) ++{ ++ return (le32_to_cpu(ccdf->status) & QDMA_CCDF_MASK) >> QDMA_CCDF_STATUS; ++} ++ ++static inline void ++qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status) ++{ ++ ccdf->status = cpu_to_le32(QDMA_CCDF_SER | status); ++} ++ ++static inline void qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len) ++{ ++ csgf->cfg = cpu_to_le32(len & QDMA_SG_LEN_MASK); ++} ++ ++static inline void qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len) ++{ ++ csgf->cfg = cpu_to_le32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK)); ++} ++ ++static inline void qdma_csgf_set_e(struct fsl_qdma_format *csgf, int len) ++{ ++ csgf->cfg = cpu_to_le32(QDMA_SG_EXT | (len & QDMA_SG_LEN_MASK)); ++} ++ ++/* qDMA Source Descriptor Format */ ++struct fsl_qdma_sdf { ++ __le32 rev3; ++ __le32 cfg; /* rev4, bit[0-11] - ssd, bit[12-23] sss */ ++ __le32 rev5; ++ __le32 cmd; ++} __packed; ++ ++/* qDMA Destination Descriptor Format */ ++struct fsl_qdma_ddf { ++ __le32 rev1; ++ __le32 cfg; /* rev2, bit[0-11] - dsd, bit[12-23] - dss */ ++ __le32 rev3; ++ __le32 cmd; ++} __packed; ++ ++struct fsl_qdma_chan { ++ struct virt_dma_chan vchan; ++ struct virt_dma_desc vdesc; ++ enum dma_status status; ++ struct fsl_qdma_engine *qdma; ++ struct fsl_qdma_queue *queue; ++}; ++ ++struct fsl_qdma_queue { ++ struct fsl_qdma_format *virt_head; ++ struct fsl_qdma_format *virt_tail; ++ struct list_head comp_used; ++ struct list_head comp_free; ++ struct dma_pool *comp_pool; ++ struct dma_pool *desc_pool; ++ spinlock_t queue_lock; ++ dma_addr_t bus_addr; ++ u32 n_cq; ++ u32 id; ++ struct fsl_qdma_format *cq; ++ void __iomem *block_base; ++}; ++ ++struct fsl_qdma_comp { ++ dma_addr_t bus_addr; ++ dma_addr_t desc_bus_addr; ++ void *virt_addr; ++ void *desc_virt_addr; ++ struct fsl_qdma_chan *qchan; ++ struct virt_dma_desc vdesc; ++ struct list_head list; ++}; ++ ++struct fsl_qdma_engine { ++ struct dma_device dma_dev; ++ void __iomem *ctrl_base; ++ void __iomem *status_base; ++ void __iomem *block_base; ++ u32 n_chans; ++ u32 n_queues; ++ struct mutex fsl_qdma_mutex; ++ int error_irq; ++ int *queue_irq; ++ bool big_endian; ++ struct fsl_qdma_queue *queue; ++ struct fsl_qdma_queue **status; ++ struct fsl_qdma_chan *chans; ++ int block_number; ++ int block_offset; ++ int irq_base; ++ int desc_allocated; ++ ++}; ++ ++static u32 qdma_readl(struct fsl_qdma_engine *qdma, void __iomem *addr) ++{ ++ return QDMA_IN(qdma, addr); ++} ++ ++static void qdma_writel(struct fsl_qdma_engine *qdma, u32 val, ++ void __iomem *addr) ++{ ++ QDMA_OUT(qdma, addr, val); ++} ++ ++static struct fsl_qdma_chan *to_fsl_qdma_chan(struct dma_chan *chan) ++{ ++ return container_of(chan, struct fsl_qdma_chan, vchan.chan); ++} ++ ++static struct fsl_qdma_comp *to_fsl_qdma_comp(struct virt_dma_desc *vd) ++{ ++ return container_of(vd, struct fsl_qdma_comp, vdesc); ++} ++ ++static void fsl_qdma_free_chan_resources(struct dma_chan *chan) ++{ ++ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); ++ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; ++ struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma; ++ struct fsl_qdma_comp *comp_temp, *_comp_temp; ++ unsigned long flags; ++ LIST_HEAD(head); ++ ++ spin_lock_irqsave(&fsl_chan->vchan.lock, flags); ++ vchan_get_all_descriptors(&fsl_chan->vchan, &head); ++ spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); ++ ++ vchan_dma_desc_free_list(&fsl_chan->vchan, &head); ++ ++ if (!fsl_queue->comp_pool && !fsl_queue->comp_pool) ++ return; ++ ++ list_for_each_entry_safe(comp_temp, _comp_temp, ++ &fsl_queue->comp_used, list) { ++ dma_pool_free(fsl_queue->comp_pool, ++ comp_temp->virt_addr, ++ comp_temp->bus_addr); ++ dma_pool_free(fsl_queue->desc_pool, ++ comp_temp->desc_virt_addr, ++ comp_temp->desc_bus_addr); ++ list_del(&comp_temp->list); ++ kfree(comp_temp); ++ } ++ ++ list_for_each_entry_safe(comp_temp, _comp_temp, ++ &fsl_queue->comp_free, list) { ++ dma_pool_free(fsl_queue->comp_pool, ++ comp_temp->virt_addr, ++ comp_temp->bus_addr); ++ dma_pool_free(fsl_queue->desc_pool, ++ comp_temp->desc_virt_addr, ++ comp_temp->desc_bus_addr); ++ list_del(&comp_temp->list); ++ kfree(comp_temp); ++ } ++ ++ dma_pool_destroy(fsl_queue->comp_pool); ++ dma_pool_destroy(fsl_queue->desc_pool); ++ ++ fsl_qdma->desc_allocated--; ++ fsl_queue->comp_pool = NULL; ++ fsl_queue->desc_pool = NULL; ++} ++ ++static void fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp, ++ dma_addr_t dst, dma_addr_t src, u32 len) ++{ ++ struct fsl_qdma_format *ccdf, *csgf_desc, *csgf_src, *csgf_dest; ++ struct fsl_qdma_sdf *sdf; ++ struct fsl_qdma_ddf *ddf; ++ ++ ccdf = (struct fsl_qdma_format *)fsl_comp->virt_addr; ++ csgf_desc = (struct fsl_qdma_format *)fsl_comp->virt_addr + 1; ++ csgf_src = (struct fsl_qdma_format *)fsl_comp->virt_addr + 2; ++ csgf_dest = (struct fsl_qdma_format *)fsl_comp->virt_addr + 3; ++ sdf = (struct fsl_qdma_sdf *)fsl_comp->desc_virt_addr; ++ ddf = (struct fsl_qdma_ddf *)fsl_comp->desc_virt_addr + 1; ++ ++ memset(fsl_comp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE); ++ memset(fsl_comp->desc_virt_addr, 0, FSL_QDMA_DESCRIPTOR_BUFFER_SIZE); ++ /* Head Command Descriptor(Frame Descriptor) */ ++ qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16); ++ qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(ccdf)); ++ qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(ccdf)); ++ /* Status notification is enqueued to status queue. */ ++ /* Compound Command Descriptor(Frame List Table) */ ++ qdma_desc_addr_set64(csgf_desc, fsl_comp->desc_bus_addr); ++ /* It must be 32 as Compound S/G Descriptor */ ++ qdma_csgf_set_len(csgf_desc, 32); ++ qdma_desc_addr_set64(csgf_src, src); ++ qdma_csgf_set_len(csgf_src, len); ++ qdma_desc_addr_set64(csgf_dest, dst); ++ qdma_csgf_set_len(csgf_dest, len); ++ /* This entry is the last entry. */ ++ qdma_csgf_set_f(csgf_dest, len); ++ /* Descriptor Buffer */ ++ sdf->cmd = cpu_to_le32( ++ FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET); ++ ddf->cmd = cpu_to_le32( ++ FSL_QDMA_CMD_RWTTYPE << FSL_QDMA_CMD_RWTTYPE_OFFSET); ++ ddf->cmd |= cpu_to_le32( ++ FSL_QDMA_CMD_LWC << FSL_QDMA_CMD_LWC_OFFSET); ++} ++ ++/* ++ * Pre-request command descriptor and compound S/G for enqueue. ++ */ ++static int fsl_qdma_pre_request_enqueue_comp_desc(struct fsl_qdma_queue *queue) ++{ ++ struct fsl_qdma_comp *comp_temp; ++ int i; ++ ++ for (i = 0; i < queue->n_cq + COMMAND_QUEUE_OVERFLLOW; i++) { ++ comp_temp = kzalloc(sizeof(*comp_temp), GFP_KERNEL); ++ if (!comp_temp) ++ return -ENOMEM; ++ comp_temp->virt_addr = dma_pool_alloc(queue->comp_pool, ++ GFP_KERNEL, ++ &comp_temp->bus_addr); ++ ++ if (!comp_temp->virt_addr) { ++ kfree(comp_temp); ++ return -ENOMEM; ++ } ++ ++ list_add_tail(&comp_temp->list, &queue->comp_free); ++ } ++ ++ return 0; ++} ++ ++/* ++ * Pre-request source and destination descriptor for enqueue. ++ */ ++static int fsl_qdma_pre_request_enqueue_sd_desc(struct fsl_qdma_queue *queue) ++{ ++ struct fsl_qdma_comp *comp_temp, *_comp_temp; ++ ++ list_for_each_entry_safe(comp_temp, _comp_temp, ++ &queue->comp_free, list) { ++ comp_temp->desc_virt_addr = dma_pool_alloc(queue->desc_pool, ++ GFP_KERNEL, ++ &comp_temp->desc_bus_addr); ++ if (!comp_temp->desc_virt_addr) ++ return -ENOMEM; ++ } ++ ++ return 0; ++} ++ ++/* ++ * Request a command descriptor for enqueue. ++ */ ++static struct fsl_qdma_comp *fsl_qdma_request_enqueue_desc( ++ struct fsl_qdma_chan *fsl_chan) ++{ ++ struct fsl_qdma_comp *comp_temp; ++ struct fsl_qdma_queue *queue = fsl_chan->queue; ++ unsigned long flags; ++ int timeout = COMP_TIMEOUT; ++ ++ while (timeout) { ++ spin_lock_irqsave(&queue->queue_lock, flags); ++ if (!list_empty(&queue->comp_free)) { ++ comp_temp = list_first_entry(&queue->comp_free, ++ struct fsl_qdma_comp, ++ list); ++ list_del(&comp_temp->list); ++ ++ spin_unlock_irqrestore(&queue->queue_lock, flags); ++ comp_temp->qchan = fsl_chan; ++ return comp_temp; ++ } ++ spin_unlock_irqrestore(&queue->queue_lock, flags); ++ udelay(1); ++ timeout--; ++ } ++ ++ return NULL; ++} ++ ++static struct fsl_qdma_queue *fsl_qdma_alloc_queue_resources( ++ struct platform_device *pdev, ++ struct fsl_qdma_engine *fsl_qdma) ++{ ++ struct fsl_qdma_queue *queue_head, *queue_temp; ++ int ret, len, i, j; ++ unsigned int queue_size[FSL_QDMA_QUEUE_MAX]; ++ int queue_num; ++ int block_number; ++ ++ queue_num = fsl_qdma->n_queues; ++ block_number = fsl_qdma->block_number; ++ ++ if (queue_num > FSL_QDMA_QUEUE_MAX) ++ queue_num = FSL_QDMA_QUEUE_MAX; ++ len = sizeof(*queue_head) * queue_num * block_number; ++ queue_head = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); ++ if (!queue_head) ++ return NULL; ++ ++ ret = device_property_read_u32_array(&pdev->dev, "queue-sizes", ++ queue_size, queue_num); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't get queue-sizes.\n"); ++ return NULL; ++ } ++ for (j = 0; j < block_number; j++) { ++ for (i = 0; i < queue_num; i++) { ++ if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX || ++ queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) { ++ dev_err(&pdev->dev, ++ "Get wrong queue-sizes.\n"); ++ return NULL; ++ } ++ queue_temp = queue_head + i + (j * queue_num); ++ ++ queue_temp->cq = ++ dma_alloc_coherent(&pdev->dev, ++ sizeof(struct fsl_qdma_format) * ++ queue_size[i], ++ &queue_temp->bus_addr, ++ GFP_KERNEL); ++ if (!queue_temp->cq) ++ return NULL; ++ queue_temp->block_base = fsl_qdma->block_base + ++ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); ++ queue_temp->n_cq = queue_size[i]; ++ queue_temp->id = i; ++ queue_temp->virt_head = queue_temp->cq; ++ queue_temp->virt_tail = queue_temp->cq; ++ /* ++ * List for queue command buffer ++ */ ++ INIT_LIST_HEAD(&queue_temp->comp_used); ++ spin_lock_init(&queue_temp->queue_lock); ++ } ++ } ++ return queue_head; ++} ++ ++static struct fsl_qdma_queue *fsl_qdma_prep_status_queue( ++ struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ struct fsl_qdma_queue *status_head; ++ unsigned int status_size; ++ int ret; ++ ++ ret = of_property_read_u32(np, "status-sizes", &status_size); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't get status-sizes.\n"); ++ return NULL; ++ } ++ if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ++ || status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) { ++ dev_err(&pdev->dev, "Get wrong status_size.\n"); ++ return NULL; ++ } ++ status_head = devm_kzalloc(&pdev->dev, sizeof(*status_head), ++ GFP_KERNEL); ++ if (!status_head) ++ return NULL; ++ ++ /* ++ * Buffer for queue command ++ */ ++ status_head->cq = dma_alloc_coherent(&pdev->dev, ++ sizeof(struct fsl_qdma_format) * ++ status_size, ++ &status_head->bus_addr, ++ GFP_KERNEL); ++ if (!status_head->cq) ++ return NULL; ++ status_head->n_cq = status_size; ++ status_head->virt_head = status_head->cq; ++ status_head->virt_tail = status_head->cq; ++ status_head->comp_pool = NULL; ++ ++ return status_head; ++} ++ ++static int fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma) ++{ ++ void __iomem *ctrl = fsl_qdma->ctrl_base; ++ void __iomem *block; ++ int i, count = 5; ++ int j; ++ u32 reg; ++ ++ /* Disable the command queue and wait for idle state. */ ++ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); ++ reg |= FSL_QDMA_DMR_DQD; ++ qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); ++ for (j = 0; j < fsl_qdma->block_number; j++) { ++ block = fsl_qdma->block_base + ++ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); ++ for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++) ++ qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQMR(i)); ++ } ++ while (1) { ++ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DSR); ++ if (!(reg & FSL_QDMA_DSR_DB)) ++ break; ++ if (count-- < 0) ++ return -EBUSY; ++ udelay(100); ++ } ++ ++ for (j = 0; j < fsl_qdma->block_number; j++) { ++ ++ block = fsl_qdma->block_base + ++ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); ++ ++ /* Disable status queue. */ ++ qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BSQMR); ++ ++ /* ++ * clear the command queue interrupt detect register for ++ * all queues. ++ */ ++ qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0)); ++ } ++ ++ return 0; ++} ++ ++static int fsl_qdma_queue_transfer_complete( ++ struct fsl_qdma_engine *fsl_qdma, ++ void *block, ++ int id) ++{ ++ struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue; ++ struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id]; ++ struct fsl_qdma_queue *temp_queue; ++ struct fsl_qdma_format *status_addr; ++ struct fsl_qdma_comp *fsl_comp = NULL; ++ u32 reg, i; ++ bool duplicate, duplicate_handle; ++ ++ while (1) { ++ duplicate = 0; ++ duplicate_handle = 0; ++ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQSR); ++ if (reg & FSL_QDMA_BSQSR_QE) ++ return 0; ++ ++ status_addr = fsl_status->virt_head; ++ ++ if (qdma_ccdf_get_queue(status_addr) == ++ __this_cpu_read(pre_queue) && ++ qdma_ccdf_addr_get64(status_addr) == ++ __this_cpu_read(pre_addr)) ++ duplicate = 1; ++ i = qdma_ccdf_get_queue(status_addr) + ++ id * fsl_qdma->n_queues; ++ __this_cpu_write(pre_addr, qdma_ccdf_addr_get64(status_addr)); ++ __this_cpu_write(pre_queue, qdma_ccdf_get_queue(status_addr)); ++ temp_queue = fsl_queue + i; ++ ++ spin_lock(&temp_queue->queue_lock); ++ if (list_empty(&temp_queue->comp_used)) { ++ if (duplicate) ++ duplicate_handle = 1; ++ else { ++ spin_unlock(&temp_queue->queue_lock); ++ return -1; ++ } ++ } else { ++ fsl_comp = list_first_entry(&temp_queue->comp_used, ++ struct fsl_qdma_comp, ++ list); ++ if (fsl_comp->bus_addr + 16 != ++ __this_cpu_read(pre_addr)) { ++ if (duplicate) ++ duplicate_handle = 1; ++ else { ++ spin_unlock(&temp_queue->queue_lock); ++ return -1; ++ } ++ } ++ ++ } ++ ++ if (duplicate_handle) { ++ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); ++ reg |= FSL_QDMA_BSQMR_DI; ++ qdma_desc_addr_set64(status_addr, 0x0); ++ fsl_status->virt_head++; ++ if (fsl_status->virt_head == fsl_status->cq ++ + fsl_status->n_cq) ++ fsl_status->virt_head = fsl_status->cq; ++ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); ++ spin_unlock(&temp_queue->queue_lock); ++ continue; ++ } ++ list_del(&fsl_comp->list); ++ ++ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); ++ reg |= FSL_QDMA_BSQMR_DI; ++ qdma_desc_addr_set64(status_addr, 0x0); ++ fsl_status->virt_head++; ++ if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq) ++ fsl_status->virt_head = fsl_status->cq; ++ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); ++ spin_unlock(&temp_queue->queue_lock); ++ ++ spin_lock(&fsl_comp->qchan->vchan.lock); ++ vchan_cookie_complete(&fsl_comp->vdesc); ++ fsl_comp->qchan->status = DMA_COMPLETE; ++ spin_unlock(&fsl_comp->qchan->vchan.lock); ++ } ++ return 0; ++} ++ ++static irqreturn_t fsl_qdma_error_handler(int irq, void *dev_id) ++{ ++ struct fsl_qdma_engine *fsl_qdma = dev_id; ++ unsigned int intr; ++ void __iomem *status = fsl_qdma->status_base; ++ ++ intr = qdma_readl(fsl_qdma, status + FSL_QDMA_DEDR); ++ ++ if (intr) ++ dev_err(fsl_qdma->dma_dev.dev, "DMA transaction error!\n"); ++ ++ qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR); ++ return IRQ_HANDLED; ++} ++ ++static irqreturn_t fsl_qdma_queue_handler(int irq, void *dev_id) ++{ ++ struct fsl_qdma_engine *fsl_qdma = dev_id; ++ unsigned int intr, reg; ++ void __iomem *ctrl = fsl_qdma->ctrl_base; ++ void __iomem *block; ++ int id; ++ ++ id = irq - fsl_qdma->irq_base; ++ if (id < 0 && id > fsl_qdma->block_number) { ++ dev_err(fsl_qdma->dma_dev.dev, ++ "irq %d is wrong irq_base is %d\n", ++ irq, fsl_qdma->irq_base); ++ } ++ ++ block = fsl_qdma->block_base + ++ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id); ++ ++ intr = qdma_readl(fsl_qdma, block + FSL_QDMA_BCQIDR(0)); ++ ++ if ((intr & FSL_QDMA_CQIDR_SQT) != 0) ++ intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id); ++ ++ if (intr != 0) { ++ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); ++ reg |= FSL_QDMA_DMR_DQD; ++ qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); ++ qdma_writel(fsl_qdma, 0, block + FSL_QDMA_BCQIER(0)); ++ dev_err(fsl_qdma->dma_dev.dev, "QDMA: status err!\n"); ++ } ++ ++ qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0)); ++ ++ return IRQ_HANDLED; ++} ++ ++static int ++fsl_qdma_irq_init(struct platform_device *pdev, ++ struct fsl_qdma_engine *fsl_qdma) ++{ ++ char irq_name[20]; ++ int i; ++ int cpu; ++ int ret; ++ ++ fsl_qdma->error_irq = platform_get_irq_byname(pdev, ++ "qdma-error"); ++ if (fsl_qdma->error_irq < 0) { ++ dev_err(&pdev->dev, "Can't get qdma controller irq.\n"); ++ return fsl_qdma->error_irq; ++ } ++ ++ ret = devm_request_irq(&pdev->dev, fsl_qdma->error_irq, ++ fsl_qdma_error_handler, 0, "qDMA error", fsl_qdma); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't register qDMA controller IRQ.\n"); ++ return ret; ++ } ++ ++ for (i = 0; i < fsl_qdma->block_number; i++) { ++ sprintf(irq_name, "qdma-queue%d", i); ++ fsl_qdma->queue_irq[i] = platform_get_irq_byname(pdev, ++ irq_name); ++ ++ if (fsl_qdma->queue_irq[i] < 0) { ++ dev_err(&pdev->dev, ++ "Can't get qdma queue %d irq.\n", ++ i); ++ return fsl_qdma->queue_irq[i]; ++ } ++ ++ ret = devm_request_irq(&pdev->dev, ++ fsl_qdma->queue_irq[i], ++ fsl_qdma_queue_handler, ++ 0, ++ "qDMA queue", ++ fsl_qdma); ++ if (ret) { ++ dev_err(&pdev->dev, ++ "Can't register qDMA queue IRQ.\n"); ++ return ret; ++ } ++ ++ cpu = i % num_online_cpus(); ++ ret = irq_set_affinity_hint(fsl_qdma->queue_irq[i], ++ get_cpu_mask(cpu)); ++ if (ret) { ++ dev_err(&pdev->dev, ++ "Can't set cpu %d affinity to IRQ %d.\n", ++ cpu, ++ fsl_qdma->queue_irq[i]); ++ return ret; ++ } ++ ++ } ++ ++ return 0; ++} ++ ++static void fsl_qdma_irq_exit( ++ struct platform_device *pdev, struct fsl_qdma_engine *fsl_qdma) ++{ ++ if (fsl_qdma->queue_irq[0] == fsl_qdma->error_irq) { ++ devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[0], fsl_qdma); ++ } else { ++ devm_free_irq(&pdev->dev, fsl_qdma->queue_irq[0], fsl_qdma); ++ devm_free_irq(&pdev->dev, fsl_qdma->error_irq, fsl_qdma); ++ } ++} ++ ++static int fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma) ++{ ++ struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue; ++ struct fsl_qdma_queue *temp; ++ void __iomem *ctrl = fsl_qdma->ctrl_base; ++ void __iomem *status = fsl_qdma->status_base; ++ void __iomem *block; ++ int i, j, ret; ++ u32 reg; ++ ++ /* Try to halt the qDMA engine first. */ ++ ret = fsl_qdma_halt(fsl_qdma); ++ if (ret) { ++ dev_err(fsl_qdma->dma_dev.dev, "DMA halt failed!"); ++ return ret; ++ } ++ ++ for (i = 0; i < fsl_qdma->block_number; i++) { ++ /* ++ * Clear the command queue interrupt detect register for ++ * all queues. ++ */ ++ ++ block = fsl_qdma->block_base + ++ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, i); ++ qdma_writel(fsl_qdma, 0xffffffff, block + FSL_QDMA_BCQIDR(0)); ++ } ++ ++ for (j = 0; j < fsl_qdma->block_number; j++) { ++ block = fsl_qdma->block_base + ++ FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j); ++ for (i = 0; i < fsl_qdma->n_queues; i++) { ++ temp = fsl_queue + i + (j * fsl_qdma->n_queues); ++ /* ++ * Initialize Command Queue registers to ++ * point to the first ++ * command descriptor in memory. ++ * Dequeue Pointer Address Registers ++ * Enqueue Pointer Address Registers ++ */ ++ ++ qdma_writel(fsl_qdma, temp->bus_addr, ++ block + FSL_QDMA_BCQDPA_SADDR(i)); ++ qdma_writel(fsl_qdma, temp->bus_addr, ++ block + FSL_QDMA_BCQEPA_SADDR(i)); ++ ++ /* Initialize the queue mode. */ ++ reg = FSL_QDMA_BCQMR_EN; ++ reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4); ++ reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6); ++ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BCQMR(i)); ++ } ++ ++ /* ++ * Workaround for erratum: ERR010812. ++ * We must enable XOFF to avoid the enqueue rejection occurs. ++ * Setting SQCCMR ENTER_WM to 0x20. ++ */ ++ ++ qdma_writel(fsl_qdma, FSL_QDMA_SQCCMR_ENTER_WM, ++ block + FSL_QDMA_SQCCMR); ++ ++ /* ++ * Initialize status queue registers to point to the first ++ * command descriptor in memory. ++ * Dequeue Pointer Address Registers ++ * Enqueue Pointer Address Registers ++ */ ++ ++ qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr, ++ block + FSL_QDMA_SQEPAR); ++ qdma_writel(fsl_qdma, fsl_qdma->status[j]->bus_addr, ++ block + FSL_QDMA_SQDPAR); ++ /* Initialize status queue interrupt. */ ++ qdma_writel(fsl_qdma, FSL_QDMA_BCQIER_CQTIE, ++ block + FSL_QDMA_BCQIER(0)); ++ qdma_writel(fsl_qdma, FSL_QDMA_BSQICR_ICEN | ++ FSL_QDMA_BSQICR_ICST(5) | 0x8000, ++ block + FSL_QDMA_BSQICR); ++ qdma_writel(fsl_qdma, FSL_QDMA_CQIER_MEIE | ++ FSL_QDMA_CQIER_TEIE, ++ block + FSL_QDMA_CQIER); ++ ++ /* Initialize the status queue mode. */ ++ reg = FSL_QDMA_BSQMR_EN; ++ reg |= FSL_QDMA_BSQMR_CQ_SIZE(ilog2( ++ fsl_qdma->status[j]->n_cq) - 6); ++ ++ qdma_writel(fsl_qdma, reg, block + FSL_QDMA_BSQMR); ++ reg = qdma_readl(fsl_qdma, block + FSL_QDMA_BSQMR); ++ ++ } ++ ++ /* Initialize controller interrupt register. */ ++ qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEDR); ++ qdma_writel(fsl_qdma, 0xffffffff, status + FSL_QDMA_DEIER); ++ ++ reg = qdma_readl(fsl_qdma, ctrl + FSL_QDMA_DMR); ++ reg &= ~FSL_QDMA_DMR_DQD; ++ qdma_writel(fsl_qdma, reg, ctrl + FSL_QDMA_DMR); ++ ++ return 0; ++} ++ ++static struct dma_async_tx_descriptor * ++fsl_qdma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, ++ dma_addr_t src, size_t len, unsigned long flags) ++{ ++ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); ++ struct fsl_qdma_comp *fsl_comp; ++ ++ fsl_comp = fsl_qdma_request_enqueue_desc(fsl_chan); ++ ++ if (!fsl_comp) ++ return NULL; ++ ++ fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len); ++ ++ return vchan_tx_prep(&fsl_chan->vchan, &fsl_comp->vdesc, flags); ++} ++ ++static void fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan) ++{ ++ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; ++ struct fsl_qdma_comp *fsl_comp; ++ struct virt_dma_desc *vdesc; ++ void __iomem *block = fsl_queue->block_base; ++ u32 reg; ++ ++ reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQSR(fsl_queue->id)); ++ if (reg & (FSL_QDMA_BCQSR_QF | FSL_QDMA_BCQSR_XOFF)) ++ return; ++ vdesc = vchan_next_desc(&fsl_chan->vchan); ++ if (!vdesc) ++ return; ++ list_del(&vdesc->node); ++ fsl_comp = to_fsl_qdma_comp(vdesc); ++ ++ memcpy(fsl_queue->virt_head++, fsl_comp->virt_addr, 16); ++ if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq) ++ fsl_queue->virt_head = fsl_queue->cq; ++ ++ list_add_tail(&fsl_comp->list, &fsl_queue->comp_used); ++ barrier(); ++ reg = qdma_readl(fsl_chan->qdma, block + FSL_QDMA_BCQMR(fsl_queue->id)); ++ reg |= FSL_QDMA_BCQMR_EI; ++ qdma_writel(fsl_chan->qdma, reg, block + FSL_QDMA_BCQMR(fsl_queue->id)); ++ fsl_chan->status = DMA_IN_PROGRESS; ++} ++ ++static enum dma_status fsl_qdma_tx_status(struct dma_chan *chan, ++ dma_cookie_t cookie, struct dma_tx_state *txstate) ++{ ++ return dma_cookie_status(chan, cookie, txstate); ++} ++ ++static void fsl_qdma_free_desc(struct virt_dma_desc *vdesc) ++{ ++ struct fsl_qdma_comp *fsl_comp; ++ struct fsl_qdma_queue *fsl_queue; ++ unsigned long flags; ++ ++ fsl_comp = to_fsl_qdma_comp(vdesc); ++ fsl_queue = fsl_comp->qchan->queue; ++ ++ spin_lock_irqsave(&fsl_queue->queue_lock, flags); ++ list_add_tail(&fsl_comp->list, &fsl_queue->comp_free); ++ spin_unlock_irqrestore(&fsl_queue->queue_lock, flags); ++} ++ ++static void fsl_qdma_issue_pending(struct dma_chan *chan) ++{ ++ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); ++ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&fsl_queue->queue_lock, flags); ++ spin_lock(&fsl_chan->vchan.lock); ++ if (vchan_issue_pending(&fsl_chan->vchan)) ++ fsl_qdma_enqueue_desc(fsl_chan); ++ spin_unlock(&fsl_chan->vchan.lock); ++ spin_unlock_irqrestore(&fsl_queue->queue_lock, flags); ++} ++ ++static void fsl_qdma_synchronize(struct dma_chan *chan) ++{ ++ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); ++ ++ vchan_synchronize(&fsl_chan->vchan); ++} ++ ++static int fsl_qdma_terminate_all(struct dma_chan *chan) ++{ ++ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); ++ unsigned long flags; ++ LIST_HEAD(head); ++ ++ spin_lock_irqsave(&fsl_chan->vchan.lock, flags); ++ vchan_get_all_descriptors(&fsl_chan->vchan, &head); ++ spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); ++ vchan_dma_desc_free_list(&fsl_chan->vchan, &head); ++ return 0; ++} ++ ++static int fsl_qdma_alloc_chan_resources(struct dma_chan *chan) ++{ ++ struct fsl_qdma_chan *fsl_chan = to_fsl_qdma_chan(chan); ++ struct fsl_qdma_queue *fsl_queue = fsl_chan->queue; ++ struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma; ++ int ret; ++ ++ if (fsl_queue->comp_pool && fsl_queue->desc_pool) ++ return fsl_qdma->desc_allocated; ++ ++ INIT_LIST_HEAD(&fsl_queue->comp_free); ++ ++ /* ++ * The dma pool for queue command buffer ++ */ ++ fsl_queue->comp_pool = ++ dma_pool_create("comp_pool", ++ chan->device->dev, ++ FSL_QDMA_COMMAND_BUFFER_SIZE, ++ 64, 0); ++ if (!fsl_queue->comp_pool) ++ return -ENOMEM; ++ ++ /* ++ * The dma pool for Descriptor(SD/DD) buffer ++ */ ++ fsl_queue->desc_pool = ++ dma_pool_create("desc_pool", ++ chan->device->dev, ++ FSL_QDMA_DESCRIPTOR_BUFFER_SIZE, ++ 32, 0); ++ if (!fsl_queue->desc_pool) ++ goto err_desc_pool; ++ ++ ret = fsl_qdma_pre_request_enqueue_comp_desc(fsl_queue); ++ if (ret) { ++ dev_err(chan->device->dev, "failed to alloc dma buffer for " ++ "comp S/G descriptor\n"); ++ goto err_mem; ++ } ++ ++ ret = fsl_qdma_pre_request_enqueue_sd_desc(fsl_queue); ++ if (ret) { ++ dev_err(chan->device->dev, "failed to alloc dma buffer for " ++ "S/D descriptor\n"); ++ goto err_mem; ++ } ++ ++ fsl_qdma->desc_allocated++; ++ return fsl_qdma->desc_allocated; ++ ++err_mem: ++ dma_pool_destroy(fsl_queue->desc_pool); ++err_desc_pool: ++ dma_pool_destroy(fsl_queue->comp_pool); ++ return -ENOMEM; ++} ++ ++static int fsl_qdma_probe(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ struct fsl_qdma_engine *fsl_qdma; ++ struct fsl_qdma_chan *fsl_chan; ++ struct resource *res; ++ unsigned int len, chans, queues; ++ int ret, i; ++ int blk_num; ++ int blk_off; ++ ++ ret = of_property_read_u32(np, "channels", &chans); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't get channels.\n"); ++ return ret; ++ } ++ ++ ret = of_property_read_u32(np, "block-offset", &blk_off); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't get block-offset.\n"); ++ return ret; ++ } ++ ++ ret = of_property_read_u32(np, "block-number", &blk_num); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't get block-number.\n"); ++ return ret; ++ } ++ ++ blk_num = min_t(int, blk_num, num_online_cpus()); ++ ++ len = sizeof(*fsl_qdma); ++ fsl_qdma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); ++ if (!fsl_qdma) ++ return -ENOMEM; ++ ++ len = sizeof(*fsl_chan) * chans; ++ fsl_qdma->chans = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); ++ if (!fsl_qdma->chans) ++ return -ENOMEM; ++ ++ len = sizeof(struct fsl_qdma_queue *) * blk_num; ++ fsl_qdma->status = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); ++ if (!fsl_qdma->status) ++ return -ENOMEM; ++ ++ len = sizeof(int) * blk_num; ++ fsl_qdma->queue_irq = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); ++ if (!fsl_qdma->queue_irq) ++ return -ENOMEM; ++ ++ ret = of_property_read_u32(np, "queues", &queues); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't get queues.\n"); ++ return ret; ++ } ++ ++ fsl_qdma->desc_allocated = 0; ++ fsl_qdma->n_chans = chans; ++ fsl_qdma->n_queues = queues; ++ fsl_qdma->block_number = blk_num; ++ fsl_qdma->block_offset = blk_off; ++ ++ mutex_init(&fsl_qdma->fsl_qdma_mutex); ++ ++ for (i = 0; i < fsl_qdma->block_number; i++) { ++ fsl_qdma->status[i] = fsl_qdma_prep_status_queue(pdev); ++ if (!fsl_qdma->status[i]) ++ return -ENOMEM; ++ } ++ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); ++ fsl_qdma->ctrl_base = devm_ioremap_resource(&pdev->dev, res); ++ if (IS_ERR(fsl_qdma->ctrl_base)) ++ return PTR_ERR(fsl_qdma->ctrl_base); ++ ++ res = platform_get_resource(pdev, IORESOURCE_MEM, 1); ++ fsl_qdma->status_base = devm_ioremap_resource(&pdev->dev, res); ++ if (IS_ERR(fsl_qdma->status_base)) ++ return PTR_ERR(fsl_qdma->status_base); ++ ++ res = platform_get_resource(pdev, IORESOURCE_MEM, 2); ++ fsl_qdma->block_base = devm_ioremap_resource(&pdev->dev, res); ++ if (IS_ERR(fsl_qdma->block_base)) ++ return PTR_ERR(fsl_qdma->block_base); ++ fsl_qdma->queue = fsl_qdma_alloc_queue_resources(pdev, fsl_qdma); ++ if (!fsl_qdma->queue) ++ return -ENOMEM; ++ ++ ret = fsl_qdma_irq_init(pdev, fsl_qdma); ++ if (ret) ++ return ret; ++ ++ fsl_qdma->irq_base = platform_get_irq_byname(pdev, "qdma-queue0"); ++ fsl_qdma->big_endian = of_property_read_bool(np, "big-endian"); ++ INIT_LIST_HEAD(&fsl_qdma->dma_dev.channels); ++ ++ for (i = 0; i < fsl_qdma->n_chans; i++) { ++ struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i]; ++ ++ fsl_chan->qdma = fsl_qdma; ++ fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues * ++ fsl_qdma->block_number); ++ fsl_chan->vchan.desc_free = fsl_qdma_free_desc; ++ vchan_init(&fsl_chan->vchan, &fsl_qdma->dma_dev); ++ } ++ ++ dma_cap_set(DMA_MEMCPY, fsl_qdma->dma_dev.cap_mask); ++ ++ fsl_qdma->dma_dev.dev = &pdev->dev; ++ fsl_qdma->dma_dev.device_free_chan_resources ++ = fsl_qdma_free_chan_resources; ++ fsl_qdma->dma_dev.device_alloc_chan_resources ++ = fsl_qdma_alloc_chan_resources; ++ fsl_qdma->dma_dev.device_tx_status = fsl_qdma_tx_status; ++ fsl_qdma->dma_dev.device_prep_dma_memcpy = fsl_qdma_prep_memcpy; ++ fsl_qdma->dma_dev.device_issue_pending = fsl_qdma_issue_pending; ++ fsl_qdma->dma_dev.device_synchronize = fsl_qdma_synchronize; ++ fsl_qdma->dma_dev.device_terminate_all = fsl_qdma_terminate_all; ++ ++ dma_set_mask(&pdev->dev, DMA_BIT_MASK(40)); ++ ++ platform_set_drvdata(pdev, fsl_qdma); ++ ++ ret = dma_async_device_register(&fsl_qdma->dma_dev); ++ if (ret) { ++ dev_err(&pdev->dev, ++ "Can't register NXP Layerscape qDMA engine.\n"); ++ return ret; ++ } ++ ++ ret = fsl_qdma_reg_init(fsl_qdma); ++ if (ret) { ++ dev_err(&pdev->dev, "Can't Initialize the qDMA engine.\n"); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static void fsl_qdma_cleanup_vchan(struct dma_device *dmadev) ++{ ++ struct fsl_qdma_chan *chan, *_chan; ++ ++ list_for_each_entry_safe(chan, _chan, ++ &dmadev->channels, vchan.chan.device_node) { ++ list_del(&chan->vchan.chan.device_node); ++ tasklet_kill(&chan->vchan.task); ++ } ++} ++ ++static int fsl_qdma_remove(struct platform_device *pdev) ++{ ++ struct device_node *np = pdev->dev.of_node; ++ struct fsl_qdma_engine *fsl_qdma = platform_get_drvdata(pdev); ++ struct fsl_qdma_queue *status; ++ int i; ++ ++ fsl_qdma_irq_exit(pdev, fsl_qdma); ++ fsl_qdma_cleanup_vchan(&fsl_qdma->dma_dev); ++ of_dma_controller_free(np); ++ dma_async_device_unregister(&fsl_qdma->dma_dev); ++ ++ for (i = 0; i < fsl_qdma->block_number; i++) { ++ status = fsl_qdma->status[i]; ++ dma_free_coherent(&pdev->dev, sizeof(struct fsl_qdma_format) * ++ status->n_cq, status->cq, status->bus_addr); ++ } ++ return 0; ++} ++ ++static const struct of_device_id fsl_qdma_dt_ids[] = { ++ { .compatible = "fsl,ls1021a-qdma", }, ++ { /* sentinel */ } ++}; ++MODULE_DEVICE_TABLE(of, fsl_qdma_dt_ids); ++ ++static struct platform_driver fsl_qdma_driver = { ++ .driver = { ++ .name = "fsl-qdma", ++ .of_match_table = fsl_qdma_dt_ids, ++ }, ++ .probe = fsl_qdma_probe, ++ .remove = fsl_qdma_remove, ++}; ++ ++module_platform_driver(fsl_qdma_driver); ++ ++MODULE_ALIAS("platform:fsl-qdma"); ++MODULE_DESCRIPTION("NXP Layerscape qDMA engine driver"); ++MODULE_LICENSE("GPL v2"); |