diff options
| author | Yutang Jiang <yutang.jiang@nxp.com> | 2016-10-29 00:14:32 +0800 |
|---|---|---|
| committer | John Crispin <john@phrozen.org> | 2016-10-31 17:00:10 +0100 |
| commit | c6c731fe311f7da42777ffd31804a4f6aa3f8e19 (patch) | |
| tree | d92c7296f82d46d1b2da30933a97595f6cb8ad66 /target/linux/layerscape/patches-4.4/7017-fsl_qbman-add-qbman-driver.patch | |
| parent | a34f96d6cf80c7c3c425076714d9c4caa67e3670 (diff) | |
| download | upstream-c6c731fe311f7da42777ffd31804a4f6aa3f8e19.tar.gz upstream-c6c731fe311f7da42777ffd31804a4f6aa3f8e19.tar.bz2 upstream-c6c731fe311f7da42777ffd31804a4f6aa3f8e19.zip | |
layerscape: add 64b/32b target for ls1043ardb device
Add support for NXP layerscape ls1043ardb 64b/32b Dev board.
LS1043a is an SoC with 4x64-bit up to 1.6 GHz ARMv8 A53 cores.
ls1043ardb support features as: 2GB DDR4, 128MB NOR/512MB NAND, USB3.0, eSDHC,
I2C, GPIO, PCIe/Mini-PCIe, 6x1G/1x10G network port, etc.
64b/32b ls1043ardb target is using 4.4 kernel, and rcw/u-boot/fman images from
NXP QorIQ SDK release.
All of 4.4 kernel patches porting from SDK release or upstream.
QorIQ SDK ISOs can be downloaded from this location:
http://www.nxp.com/products/software-and-tools/run-time-software/linux-sdk/linux-sdk-for-qoriq-processors:SDKLINUX
Signed-off-by: Yutang Jiang <yutang.jiang@nxp.com>
Diffstat (limited to 'target/linux/layerscape/patches-4.4/7017-fsl_qbman-add-qbman-driver.patch')
| -rw-r--r-- | target/linux/layerscape/patches-4.4/7017-fsl_qbman-add-qbman-driver.patch | 24828 |
1 files changed, 24828 insertions, 0 deletions
diff --git a/target/linux/layerscape/patches-4.4/7017-fsl_qbman-add-qbman-driver.patch b/target/linux/layerscape/patches-4.4/7017-fsl_qbman-add-qbman-driver.patch new file mode 100644 index 0000000000..48da3f2ba7 --- /dev/null +++ b/target/linux/layerscape/patches-4.4/7017-fsl_qbman-add-qbman-driver.patch @@ -0,0 +1,24828 @@ +From f6f8ed4784936724154832ff9e4c5afe8caa63e4 Mon Sep 17 00:00:00 2001 +From: Zhao Qiang <qiang.zhao@nxp.com> +Date: Mon, 11 Jul 2016 14:39:18 +0800 +Subject: [PATCH 17/70] fsl_qbman: add qbman driver + +The QMan and BMan are infrastructure components of dpaa, which are used +by both software and hardware for queuing and memory allocation/deallocation. + +Signed-off-by: Roy Pledge <Roy.Pledge@freescale.com> Signed-off-by: +Camelia Groza <camelia.groza@freescale.com> Signed-off-by: Geoff Thorpe +<Geoff.Thorpe@freescale.com> Signed-off-by: Ahmed Mansour +<Ahmed.Mansour@freescale.com> Signed-off-by: Alex Porosanu +<alexandru.porosanu@nxp.com> Signed-off-by: Pan Jiafei +<Jiafei.Pan@nxp.com> Signed-off-by: Haiying Wang +<Haiying.wang@freescale.com> +Signed-off-by: Xie Jianhua-B29408 <Jianhua.Xie@freescale.com> +Signed-off-by: Zhao Qiang <qiang.zhao@nxp.com> +--- + arch/arm/Kconfig | 5 + + arch/powerpc/Kconfig | 9 +- + drivers/misc/Kconfig | 17 + + drivers/staging/Kconfig | 2 + + drivers/staging/Makefile | 1 + + drivers/staging/fsl_qbman/Kconfig | 211 + + drivers/staging/fsl_qbman/Makefile | 28 + + drivers/staging/fsl_qbman/bman_config.c | 705 +++ + drivers/staging/fsl_qbman/bman_debugfs.c | 119 + + drivers/staging/fsl_qbman/bman_driver.c | 574 +++ + drivers/staging/fsl_qbman/bman_high.c | 1141 +++++ + drivers/staging/fsl_qbman/bman_low.h | 559 +++ + drivers/staging/fsl_qbman/bman_private.h | 166 + + drivers/staging/fsl_qbman/bman_test.c | 56 + + drivers/staging/fsl_qbman/bman_test.h | 44 + + drivers/staging/fsl_qbman/bman_test_high.c | 183 + + drivers/staging/fsl_qbman/bman_test_thresh.c | 196 + + drivers/staging/fsl_qbman/dpa_alloc.c | 706 +++ + drivers/staging/fsl_qbman/dpa_sys.h | 259 ++ + drivers/staging/fsl_qbman/dpa_sys_arm.h | 95 + + drivers/staging/fsl_qbman/dpa_sys_arm64.h | 102 + + drivers/staging/fsl_qbman/dpa_sys_ppc32.h | 70 + + drivers/staging/fsl_qbman/dpa_sys_ppc64.h | 79 + + drivers/staging/fsl_qbman/fsl_usdpaa.c | 1982 ++++++++ + drivers/staging/fsl_qbman/fsl_usdpaa_irq.c | 289 ++ + drivers/staging/fsl_qbman/qbman_driver.c | 88 + + drivers/staging/fsl_qbman/qman_config.c | 1199 +++++ + drivers/staging/fsl_qbman/qman_debugfs.c | 1594 +++++++ + drivers/staging/fsl_qbman/qman_driver.c | 980 ++++ + drivers/staging/fsl_qbman/qman_high.c | 5568 +++++++++++++++++++++++ + drivers/staging/fsl_qbman/qman_low.h | 1407 ++++++ + drivers/staging/fsl_qbman/qman_private.h | 398 ++ + drivers/staging/fsl_qbman/qman_test.c | 57 + + drivers/staging/fsl_qbman/qman_test.h | 45 + + drivers/staging/fsl_qbman/qman_test_high.c | 216 + + drivers/staging/fsl_qbman/qman_test_hotpotato.c | 499 ++ + drivers/staging/fsl_qbman/qman_utility.c | 129 + + include/linux/fsl_bman.h | 532 +++ + include/linux/fsl_qman.h | 3889 ++++++++++++++++ + include/linux/fsl_usdpaa.h | 372 ++ + 40 files changed, 24569 insertions(+), 2 deletions(-) + create mode 100644 drivers/staging/fsl_qbman/Kconfig + create mode 100644 drivers/staging/fsl_qbman/Makefile + create mode 100644 drivers/staging/fsl_qbman/bman_config.c + create mode 100644 drivers/staging/fsl_qbman/bman_debugfs.c + create mode 100644 drivers/staging/fsl_qbman/bman_driver.c + create mode 100644 drivers/staging/fsl_qbman/bman_high.c + create mode 100644 drivers/staging/fsl_qbman/bman_low.h + create mode 100644 drivers/staging/fsl_qbman/bman_private.h + create mode 100644 drivers/staging/fsl_qbman/bman_test.c + create mode 100644 drivers/staging/fsl_qbman/bman_test.h + create mode 100644 drivers/staging/fsl_qbman/bman_test_high.c + create mode 100644 drivers/staging/fsl_qbman/bman_test_thresh.c + create mode 100644 drivers/staging/fsl_qbman/dpa_alloc.c + create mode 100644 drivers/staging/fsl_qbman/dpa_sys.h + create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm.h + create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm64.h + create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc32.h + create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc64.h + create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa.c + create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa_irq.c + create mode 100644 drivers/staging/fsl_qbman/qbman_driver.c + create mode 100644 drivers/staging/fsl_qbman/qman_config.c + create mode 100644 drivers/staging/fsl_qbman/qman_debugfs.c + create mode 100644 drivers/staging/fsl_qbman/qman_driver.c + create mode 100644 drivers/staging/fsl_qbman/qman_high.c + create mode 100644 drivers/staging/fsl_qbman/qman_low.h + create mode 100644 drivers/staging/fsl_qbman/qman_private.h + create mode 100644 drivers/staging/fsl_qbman/qman_test.c + create mode 100644 drivers/staging/fsl_qbman/qman_test.h + create mode 100644 drivers/staging/fsl_qbman/qman_test_high.c + create mode 100644 drivers/staging/fsl_qbman/qman_test_hotpotato.c + create mode 100644 drivers/staging/fsl_qbman/qman_utility.c + create mode 100644 include/linux/fsl_bman.h + create mode 100644 include/linux/fsl_qman.h + create mode 100644 include/linux/fsl_usdpaa.h + +--- a/arch/arm/Kconfig ++++ b/arch/arm/Kconfig +@@ -1250,6 +1250,11 @@ source "arch/arm/common/Kconfig" + + menu "Bus support" + ++config HAS_FSL_QBMAN ++ bool "Datapath Acceleration Queue and Buffer management" ++ help ++ Datapath Acceleration Queue and Buffer management ++ + config ISA + bool + help +--- a/arch/powerpc/Kconfig ++++ b/arch/powerpc/Kconfig +@@ -786,6 +786,11 @@ config FSL_GTM + help + Freescale General-purpose Timers support + ++config HAS_FSL_QBMAN ++ bool "Datapath Acceleration Queue and Buffer management" ++ help ++ Datapath Acceleration Queue and Buffer management ++ + # Yes MCA RS/6000s exist but Linux-PPC does not currently support any + config MCA + bool +@@ -918,14 +923,14 @@ config DYNAMIC_MEMSTART + select NONSTATIC_KERNEL + help + This option enables the kernel to be loaded at any page aligned +- physical address. The kernel creates a mapping from KERNELBASE to ++ physical address. The kernel creates a mapping from KERNELBASE to + the address where the kernel is loaded. The page size here implies + the TLB page size of the mapping for kernel on the particular platform. + Please refer to the init code for finding the TLB page size. + + DYNAMIC_MEMSTART is an easy way of implementing pseudo-RELOCATABLE + kernel image, where the only restriction is the page aligned kernel +- load address. When this option is enabled, the compile time physical ++ load address. When this option is enabled, the compile time physical + address CONFIG_PHYSICAL_START is ignored. + + This option is overridden by CONFIG_RELOCATABLE +--- a/drivers/misc/Kconfig ++++ b/drivers/misc/Kconfig +@@ -224,6 +224,23 @@ config SGI_XP + this feature will allow for direct communication between SSIs + based on a network adapter and DMA messaging. + ++config FSL_USDPAA ++ bool "Freescale USDPAA process driver" ++ depends on FSL_DPA ++ default y ++ help ++ This driver provides user-space access to kernel-managed ++ resource interfaces for USDPAA applications, on the assumption ++ that each process will open this device once. Specifically, this ++ device exposes functionality that would be awkward if exposed ++ via the portal devices - ie. this device exposes functionality ++ that is inherently process-wide rather than portal-specific. ++ This device is necessary for obtaining access to DMA memory and ++ for allocation of Qman and Bman resources. In short, if you wish ++ to use USDPAA applications, you need this. ++ ++ If unsure, say Y. ++ + config CS5535_MFGPT + tristate "CS5535/CS5536 Geode Multi-Function General Purpose Timer (MFGPT) support" + depends on MFD_CS5535 +--- a/drivers/staging/Kconfig ++++ b/drivers/staging/Kconfig +@@ -106,6 +106,8 @@ source "drivers/staging/fbtft/Kconfig" + + source "drivers/staging/fsl-mc/Kconfig" + ++source "drivers/staging/fsl_qbman/Kconfig" ++ + source "drivers/staging/wilc1000/Kconfig" + + source "drivers/staging/most/Kconfig" +--- a/drivers/staging/Makefile ++++ b/drivers/staging/Makefile +@@ -45,5 +45,6 @@ obj-$(CONFIG_UNISYSSPAR) += unisys/ + obj-$(CONFIG_COMMON_CLK_XLNX_CLKWZRD) += clocking-wizard/ + obj-$(CONFIG_FB_TFT) += fbtft/ + obj-$(CONFIG_FSL_MC_BUS) += fsl-mc/ ++obj-$(CONFIG_FSL_DPA) += fsl_qbman/ + obj-$(CONFIG_WILC1000) += wilc1000/ + obj-$(CONFIG_MOST) += most/ +--- /dev/null ++++ b/drivers/staging/fsl_qbman/Kconfig +@@ -0,0 +1,211 @@ ++config FSL_DPA ++ bool "Freescale Datapath Queue and Buffer management" ++ depends on HAS_FSL_QBMAN ++ default y ++ select FSL_QMAN_FQ_LOOKUP if PPC64 ++ select FSL_QMAN_FQ_LOOKUP if ARM64 ++ ++ ++menu "Freescale Datapath QMan/BMan options" ++ depends on FSL_DPA ++ ++config FSL_DPA_CHECKING ++ bool "additional driver checking" ++ default n ++ ---help--- ++ Compiles in additional checks to sanity-check the drivers and any ++ use of it by other code. Not recommended for performance. ++ ++config FSL_DPA_CAN_WAIT ++ bool ++ default y ++ ++config FSL_DPA_CAN_WAIT_SYNC ++ bool ++ default y ++ ++config FSL_DPA_PIRQ_FAST ++ bool ++ default y ++ ++config FSL_DPA_PIRQ_SLOW ++ bool ++ default y ++ ++config FSL_DPA_PORTAL_SHARE ++ bool ++ default y ++ ++config FSL_BMAN ++ bool "Freescale Buffer Manager (BMan) support" ++ default y ++ ++if FSL_BMAN ++ ++config FSL_BMAN_CONFIG ++ bool "BMan device management" ++ default y ++ ---help--- ++ If this linux image is running natively, you need this option. If this ++ linux image is running as a guest OS under the hypervisor, only one ++ guest OS ("the control plane") needs this option. ++ ++config FSL_BMAN_TEST ++ tristate "BMan self-tests" ++ default n ++ ---help--- ++ This option compiles self-test code for BMan. ++ ++config FSL_BMAN_TEST_HIGH ++ bool "BMan high-level self-test" ++ depends on FSL_BMAN_TEST ++ default y ++ ---help--- ++ This requires the presence of cpu-affine portals, and performs ++ high-level API testing with them (whichever portal(s) are affine to ++ the cpu(s) the test executes on). ++ ++config FSL_BMAN_TEST_THRESH ++ bool "BMan threshold test" ++ depends on FSL_BMAN_TEST ++ default y ++ ---help--- ++ Multi-threaded (SMP) test of BMan pool depletion. A pool is seeded ++ before multiple threads (one per cpu) create pool objects to track ++ depletion state changes. The pool is then drained to empty by a ++ "drainer" thread, and the other threads that they observe exactly ++ the depletion state changes that are expected. ++ ++config FSL_BMAN_DEBUGFS ++ tristate "BMan debugfs interface" ++ depends on DEBUG_FS ++ default y ++ ---help--- ++ This option compiles debugfs code for BMan. ++ ++endif # FSL_BMAN ++ ++config FSL_QMAN ++ bool "Freescale Queue Manager (QMan) support" ++ default y ++ ++if FSL_QMAN ++ ++config FSL_QMAN_POLL_LIMIT ++ int ++ default 32 ++ ++config FSL_QMAN_CONFIG ++ bool "QMan device management" ++ default y ++ ---help--- ++ If this linux image is running natively, you need this option. If this ++ linux image is running as a guest OS under the hypervisor, only one ++ guest OS ("the control plane") needs this option. ++ ++config FSL_QMAN_TEST ++ tristate "QMan self-tests" ++ default n ++ ---help--- ++ This option compiles self-test code for QMan. ++ ++config FSL_QMAN_TEST_STASH_POTATO ++ bool "QMan 'hot potato' data-stashing self-test" ++ depends on FSL_QMAN_TEST ++ default y ++ ---help--- ++ This performs a "hot potato" style test enqueuing/dequeuing a frame ++ across a series of FQs scheduled to different portals (and cpus), with ++ DQRR, data and context stashing always on. ++ ++config FSL_QMAN_TEST_HIGH ++ bool "QMan high-level self-test" ++ depends on FSL_QMAN_TEST ++ default y ++ ---help--- ++ This requires the presence of cpu-affine portals, and performs ++ high-level API testing with them (whichever portal(s) are affine to ++ the cpu(s) the test executes on). ++ ++config FSL_QMAN_DEBUGFS ++ tristate "QMan debugfs interface" ++ depends on DEBUG_FS ++ default y ++ ---help--- ++ This option compiles debugfs code for QMan. ++ ++# H/w settings that can be hard-coded for now. ++config FSL_QMAN_FQD_SZ ++ int "size of Frame Queue Descriptor region" ++ default 10 ++ ---help--- ++ This is the size of the FQD region defined as: PAGE_SIZE * (2^value) ++ ex: 10 => PAGE_SIZE * (2^10) ++ Note: Default device-trees now require minimum Kconfig setting of 10. ++ ++config FSL_QMAN_PFDR_SZ ++ int "size of the PFDR pool" ++ default 13 ++ ---help--- ++ This is the size of the PFDR pool defined as: PAGE_SIZE * (2^value) ++ ex: 13 => PAGE_SIZE * (2^13) ++ ++# Corenet initiator settings. Stash request queues are 4-deep to match cores' ++# ability to snart. Stash priority is 3, other priorities are 2. ++config FSL_QMAN_CI_SCHED_CFG_SRCCIV ++ int ++ depends on FSL_QMAN_CONFIG ++ default 4 ++config FSL_QMAN_CI_SCHED_CFG_SRQ_W ++ int ++ depends on FSL_QMAN_CONFIG ++ default 3 ++config FSL_QMAN_CI_SCHED_CFG_RW_W ++ int ++ depends on FSL_QMAN_CONFIG ++ default 2 ++config FSL_QMAN_CI_SCHED_CFG_BMAN_W ++ int ++ depends on FSL_QMAN_CONFIG ++ default 2 ++ ++# portal interrupt settings ++config FSL_QMAN_PIRQ_DQRR_ITHRESH ++ int ++ default 12 ++config FSL_QMAN_PIRQ_MR_ITHRESH ++ int ++ default 4 ++config FSL_QMAN_PIRQ_IPERIOD ++ int ++ default 100 ++ ++# 64 bit kernel support ++config FSL_QMAN_FQ_LOOKUP ++ bool ++ default n ++ ++config QMAN_CEETM_UPDATE_PERIOD ++ int "Token update period for shaping, in nanoseconds" ++ default 1000 ++ ---help--- ++ Traffic shaping works by performing token calculations (using ++ credits) on shaper instances periodically. This update period ++ sets the granularity for how often those token rate credit ++ updates are performed, and thus determines the accuracy and ++ range of traffic rates that can be configured by users. The ++ reference manual recommends a 1 microsecond period as providing ++ a good balance between granularity and range. ++ ++ Unless you know what you are doing, leave this value at its default. ++ ++config FSL_QMAN_INIT_TIMEOUT ++ int "timeout for qman init stage, in seconds" ++ default 10 ++ ---help--- ++ The timeout setting to quit the initialization loop for non-control ++ partition in case the control partition fails to boot-up. ++ ++endif # FSL_QMAN ++ ++endmenu +--- /dev/null ++++ b/drivers/staging/fsl_qbman/Makefile +@@ -0,0 +1,28 @@ ++subdir-ccflags-y := -Werror ++ ++# Common ++obj-$(CONFIG_FSL_DPA) += dpa_alloc.o ++obj-$(CONFIG_HAS_FSL_QBMAN) += qbman_driver.o ++ ++# Bman ++obj-$(CONFIG_FSL_BMAN) += bman_high.o ++obj-$(CONFIG_FSL_BMAN_CONFIG) += bman_config.o bman_driver.o ++obj-$(CONFIG_FSL_BMAN_TEST) += bman_tester.o ++obj-$(CONFIG_FSL_BMAN_DEBUGFS) += bman_debugfs_interface.o ++bman_tester-y = bman_test.o ++bman_tester-$(CONFIG_FSL_BMAN_TEST_HIGH) += bman_test_high.o ++bman_tester-$(CONFIG_FSL_BMAN_TEST_THRESH) += bman_test_thresh.o ++bman_debugfs_interface-y = bman_debugfs.o ++ ++# Qman ++obj-$(CONFIG_FSL_QMAN) += qman_high.o qman_utility.o ++obj-$(CONFIG_FSL_QMAN_CONFIG) += qman_config.o qman_driver.o ++obj-$(CONFIG_FSL_QMAN_TEST) += qman_tester.o ++qman_tester-y = qman_test.o ++qman_tester-$(CONFIG_FSL_QMAN_TEST_STASH_POTATO) += qman_test_hotpotato.o ++qman_tester-$(CONFIG_FSL_QMAN_TEST_HIGH) += qman_test_high.o ++obj-$(CONFIG_FSL_QMAN_DEBUGFS) += qman_debugfs_interface.o ++qman_debugfs_interface-y = qman_debugfs.o ++ ++# USDPAA ++obj-$(CONFIG_FSL_USDPAA) += fsl_usdpaa.o fsl_usdpaa_irq.o +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_config.c +@@ -0,0 +1,705 @@ ++/* Copyright (c) 2009-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include <asm/cacheflush.h> ++#include "bman_private.h" ++#include <linux/of_reserved_mem.h> ++ ++/* Last updated for v00.79 of the BG */ ++ ++struct bman; ++ ++/* Register offsets */ ++#define REG_POOL_SWDET(n) (0x0000 + ((n) * 0x04)) ++#define REG_POOL_HWDET(n) (0x0100 + ((n) * 0x04)) ++#define REG_POOL_SWDXT(n) (0x0200 + ((n) * 0x04)) ++#define REG_POOL_HWDXT(n) (0x0300 + ((n) * 0x04)) ++#define REG_POOL_CONTENT(n) (0x0600 + ((n) * 0x04)) ++#define REG_FBPR_FPC 0x0800 ++#define REG_STATE_IDLE 0x960 ++#define REG_STATE_STOP 0x964 ++#define REG_ECSR 0x0a00 ++#define REG_ECIR 0x0a04 ++#define REG_EADR 0x0a08 ++#define REG_EDATA(n) (0x0a10 + ((n) * 0x04)) ++#define REG_SBEC(n) (0x0a80 + ((n) * 0x04)) ++#define REG_IP_REV_1 0x0bf8 ++#define REG_IP_REV_2 0x0bfc ++#define REG_FBPR_BARE 0x0c00 ++#define REG_FBPR_BAR 0x0c04 ++#define REG_FBPR_AR 0x0c10 ++#define REG_SRCIDR 0x0d04 ++#define REG_LIODNR 0x0d08 ++#define REG_ERR_ISR 0x0e00 /* + "enum bm_isr_reg" */ ++ ++/* Used by all error interrupt registers except 'inhibit' */ ++#define BM_EIRQ_IVCI 0x00000010 /* Invalid Command Verb */ ++#define BM_EIRQ_FLWI 0x00000008 /* FBPR Low Watermark */ ++#define BM_EIRQ_MBEI 0x00000004 /* Multi-bit ECC Error */ ++#define BM_EIRQ_SBEI 0x00000002 /* Single-bit ECC Error */ ++#define BM_EIRQ_BSCN 0x00000001 /* pool State Change Notification */ ++ ++/* BMAN_ECIR valid error bit */ ++#define PORTAL_ECSR_ERR (BM_EIRQ_IVCI) ++ ++union bman_ecir { ++ u32 ecir_raw; ++ struct { ++ u32 __reserved1:4; ++ u32 portal_num:4; ++ u32 __reserved2:12; ++ u32 numb:4; ++ u32 __reserved3:2; ++ u32 pid:6; ++ } __packed info; ++}; ++ ++union bman_eadr { ++ u32 eadr_raw; ++ struct { ++ u32 __reserved1:5; ++ u32 memid:3; ++ u32 __reserved2:14; ++ u32 eadr:10; ++ } __packed info; ++}; ++ ++struct bman_hwerr_txt { ++ u32 mask; ++ const char *txt; ++}; ++ ++#define BMAN_HWE_TXT(a, b) { .mask = BM_EIRQ_##a, .txt = b } ++ ++static const struct bman_hwerr_txt bman_hwerr_txts[] = { ++ BMAN_HWE_TXT(IVCI, "Invalid Command Verb"), ++ BMAN_HWE_TXT(FLWI, "FBPR Low Watermark"), ++ BMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"), ++ BMAN_HWE_TXT(SBEI, "Single-bit ECC Error"), ++ BMAN_HWE_TXT(BSCN, "Pool State Change Notification"), ++}; ++#define BMAN_HWE_COUNT (sizeof(bman_hwerr_txts)/sizeof(struct bman_hwerr_txt)) ++ ++struct bman_error_info_mdata { ++ u16 addr_mask; ++ u16 bits; ++ const char *txt; ++}; ++ ++#define BMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c} ++static const struct bman_error_info_mdata error_mdata[] = { ++ BMAN_ERR_MDATA(0x03FF, 192, "Stockpile memory"), ++ BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 1"), ++ BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 2"), ++}; ++#define BMAN_ERR_MDATA_COUNT \ ++ (sizeof(error_mdata)/sizeof(struct bman_error_info_mdata)) ++ ++/* Add this in Kconfig */ ++#define BMAN_ERRS_TO_UNENABLE (BM_EIRQ_FLWI) ++ ++/** ++ * bm_err_isr_<reg>_<verb> - Manipulate global interrupt registers ++ * @v: for accessors that write values, this is the 32-bit value ++ * ++ * Manipulates BMAN_ERR_ISR, BMAN_ERR_IER, BMAN_ERR_ISDR, BMAN_ERR_IIR. All ++ * manipulations except bm_err_isr_[un]inhibit() use 32-bit masks composed of ++ * the BM_EIRQ_*** definitions. Note that "bm_err_isr_enable_write" means ++ * "write the enable register" rather than "enable the write register"! ++ */ ++#define bm_err_isr_status_read(bm) \ ++ __bm_err_isr_read(bm, bm_isr_status) ++#define bm_err_isr_status_clear(bm, m) \ ++ __bm_err_isr_write(bm, bm_isr_status, m) ++#define bm_err_isr_enable_read(bm) \ ++ __bm_err_isr_read(bm, bm_isr_enable) ++#define bm_err_isr_enable_write(bm, v) \ ++ __bm_err_isr_write(bm, bm_isr_enable, v) ++#define bm_err_isr_disable_read(bm) \ ++ __bm_err_isr_read(bm, bm_isr_disable) ++#define bm_err_isr_disable_write(bm, v) \ ++ __bm_err_isr_write(bm, bm_isr_disable, v) ++#define bm_err_isr_inhibit(bm) \ ++ __bm_err_isr_write(bm, bm_isr_inhibit, 1) ++#define bm_err_isr_uninhibit(bm) \ ++ __bm_err_isr_write(bm, bm_isr_inhibit, 0) ++ ++/* ++ * TODO: unimplemented registers ++ * ++ * BMAN_POOLk_SDCNT, BMAN_POOLk_HDCNT, BMAN_FULT, ++ * BMAN_VLDPL, BMAN_EECC, BMAN_SBET, BMAN_EINJ ++ */ ++ ++/* Encapsulate "struct bman *" as a cast of the register space address. */ ++ ++static struct bman *bm_create(void *regs) ++{ ++ return (struct bman *)regs; ++} ++ ++static inline u32 __bm_in(struct bman *bm, u32 offset) ++{ ++ return in_be32((void *)bm + offset); ++} ++static inline void __bm_out(struct bman *bm, u32 offset, u32 val) ++{ ++ out_be32((void *)bm + offset, val); ++} ++#define bm_in(reg) __bm_in(bm, REG_##reg) ++#define bm_out(reg, val) __bm_out(bm, REG_##reg, val) ++ ++static u32 __bm_err_isr_read(struct bman *bm, enum bm_isr_reg n) ++{ ++ return __bm_in(bm, REG_ERR_ISR + (n << 2)); ++} ++ ++static void __bm_err_isr_write(struct bman *bm, enum bm_isr_reg n, u32 val) ++{ ++ __bm_out(bm, REG_ERR_ISR + (n << 2), val); ++} ++ ++static void bm_get_version(struct bman *bm, u16 *id, u8 *major, u8 *minor) ++{ ++ u32 v = bm_in(IP_REV_1); ++ *id = (v >> 16); ++ *major = (v >> 8) & 0xff; ++ *minor = v & 0xff; ++} ++ ++static u32 __generate_thresh(u32 val, int roundup) ++{ ++ u32 e = 0; /* co-efficient, exponent */ ++ int oddbit = 0; ++ while (val > 0xff) { ++ oddbit = val & 1; ++ val >>= 1; ++ e++; ++ if (roundup && oddbit) ++ val++; ++ } ++ DPA_ASSERT(e < 0x10); ++ return val | (e << 8); ++} ++ ++static void bm_set_pool(struct bman *bm, u8 pool, u32 swdet, u32 swdxt, ++ u32 hwdet, u32 hwdxt) ++{ ++ DPA_ASSERT(pool < bman_pool_max); ++ bm_out(POOL_SWDET(pool), __generate_thresh(swdet, 0)); ++ bm_out(POOL_SWDXT(pool), __generate_thresh(swdxt, 1)); ++ bm_out(POOL_HWDET(pool), __generate_thresh(hwdet, 0)); ++ bm_out(POOL_HWDXT(pool), __generate_thresh(hwdxt, 1)); ++} ++ ++static void bm_set_memory(struct bman *bm, u64 ba, int prio, u32 size) ++{ ++ u32 exp = ilog2(size); ++ /* choke if size isn't within range */ ++ DPA_ASSERT((size >= 4096) && (size <= 1073741824) && ++ is_power_of_2(size)); ++ /* choke if '[e]ba' has lower-alignment than 'size' */ ++ DPA_ASSERT(!(ba & (size - 1))); ++ bm_out(FBPR_BARE, upper_32_bits(ba)); ++ bm_out(FBPR_BAR, lower_32_bits(ba)); ++ bm_out(FBPR_AR, (prio ? 0x40000000 : 0) | (exp - 1)); ++} ++ ++/*****************/ ++/* Config driver */ ++/*****************/ ++ ++/* TODO: Kconfig these? */ ++#define DEFAULT_FBPR_SZ (PAGE_SIZE << 12) ++ ++/* We support only one of these. */ ++static struct bman *bm; ++static struct device_node *bm_node; ++ ++/* And this state belongs to 'bm'. It is set during fsl_bman_init(), but used ++ * during bman_init_ccsr(). */ ++static dma_addr_t fbpr_a; ++static size_t fbpr_sz = DEFAULT_FBPR_SZ; ++ ++static int bman_fbpr(struct reserved_mem *rmem) ++{ ++ fbpr_a = rmem->base; ++ fbpr_sz = rmem->size; ++ ++ WARN_ON(!(fbpr_a && fbpr_sz)); ++ ++ return 0; ++} ++RESERVEDMEM_OF_DECLARE(bman_fbpr, "fsl,bman-fbpr", bman_fbpr); ++ ++static int __init fsl_bman_init(struct device_node *node) ++{ ++ struct resource res; ++ u32 __iomem *regs; ++ const char *s; ++ int ret, standby = 0; ++ u16 id; ++ u8 major, minor; ++ ++ ret = of_address_to_resource(node, 0, &res); ++ if (ret) { ++ pr_err("Can't get %s property 'reg'\n", ++ node->full_name); ++ return ret; ++ } ++ s = of_get_property(node, "fsl,hv-claimable", &ret); ++ if (s && !strcmp(s, "standby")) ++ standby = 1; ++ /* Global configuration */ ++ regs = ioremap(res.start, res.end - res.start + 1); ++ bm = bm_create(regs); ++ BUG_ON(!bm); ++ bm_node = node; ++ bm_get_version(bm, &id, &major, &minor); ++ pr_info("Bman ver:%04x,%02x,%02x\n", id, major, minor); ++ if ((major == 1) && (minor == 0)) { ++ bman_ip_rev = BMAN_REV10; ++ bman_pool_max = 64; ++ } else if ((major == 2) && (minor == 0)) { ++ bman_ip_rev = BMAN_REV20; ++ bman_pool_max = 8; ++ } else if ((major == 2) && (minor == 1)) { ++ bman_ip_rev = BMAN_REV21; ++ bman_pool_max = 64; ++ } else { ++ pr_warn("unknown Bman version, default to rev1.0\n"); ++ } ++ ++ if (standby) { ++ pr_info(" -> in standby mode\n"); ++ return 0; ++ } ++ return 0; ++} ++ ++int bman_have_ccsr(void) ++{ ++ return bm ? 1 : 0; ++} ++ ++int bm_pool_set(u32 bpid, const u32 *thresholds) ++{ ++ if (!bm) ++ return -ENODEV; ++ bm_set_pool(bm, bpid, thresholds[0], ++ thresholds[1], thresholds[2], ++ thresholds[3]); ++ return 0; ++} ++EXPORT_SYMBOL(bm_pool_set); ++ ++__init int bman_init_early(void) ++{ ++ struct device_node *dn; ++ int ret; ++ ++ for_each_compatible_node(dn, NULL, "fsl,bman") { ++ if (bm) ++ pr_err("%s: only one 'fsl,bman' allowed\n", ++ dn->full_name); ++ else { ++ if (!of_device_is_available(dn)) ++ continue; ++ ++ ret = fsl_bman_init(dn); ++ BUG_ON(ret); ++ } ++ } ++ return 0; ++} ++postcore_initcall_sync(bman_init_early); ++ ++ ++static void log_edata_bits(u32 bit_count) ++{ ++ u32 i, j, mask = 0xffffffff; ++ ++ pr_warn("Bman ErrInt, EDATA:\n"); ++ i = bit_count/32; ++ if (bit_count%32) { ++ i++; ++ mask = ~(mask << bit_count%32); ++ } ++ j = 16-i; ++ pr_warn(" 0x%08x\n", bm_in(EDATA(j)) & mask); ++ j++; ++ for (; j < 16; j++) ++ pr_warn(" 0x%08x\n", bm_in(EDATA(j))); ++} ++ ++static void log_additional_error_info(u32 isr_val, u32 ecsr_val) ++{ ++ union bman_ecir ecir_val; ++ union bman_eadr eadr_val; ++ ++ ecir_val.ecir_raw = bm_in(ECIR); ++ /* Is portal info valid */ ++ if (ecsr_val & PORTAL_ECSR_ERR) { ++ pr_warn("Bman ErrInt: SWP id %d, numb %d, pid %d\n", ++ ecir_val.info.portal_num, ecir_val.info.numb, ++ ecir_val.info.pid); ++ } ++ if (ecsr_val & (BM_EIRQ_SBEI|BM_EIRQ_MBEI)) { ++ eadr_val.eadr_raw = bm_in(EADR); ++ pr_warn("Bman ErrInt: EADR Memory: %s, 0x%x\n", ++ error_mdata[eadr_val.info.memid].txt, ++ error_mdata[eadr_val.info.memid].addr_mask ++ & eadr_val.info.eadr); ++ log_edata_bits(error_mdata[eadr_val.info.memid].bits); ++ } ++} ++ ++/* Bman interrupt handler */ ++static irqreturn_t bman_isr(int irq, void *ptr) ++{ ++ u32 isr_val, ier_val, ecsr_val, isr_mask, i; ++ ++ ier_val = bm_err_isr_enable_read(bm); ++ isr_val = bm_err_isr_status_read(bm); ++ ecsr_val = bm_in(ECSR); ++ isr_mask = isr_val & ier_val; ++ ++ if (!isr_mask) ++ return IRQ_NONE; ++ for (i = 0; i < BMAN_HWE_COUNT; i++) { ++ if (bman_hwerr_txts[i].mask & isr_mask) { ++ pr_warn("Bman ErrInt: %s\n", bman_hwerr_txts[i].txt); ++ if (bman_hwerr_txts[i].mask & ecsr_val) { ++ log_additional_error_info(isr_mask, ecsr_val); ++ /* Re-arm error capture registers */ ++ bm_out(ECSR, ecsr_val); ++ } ++ if (bman_hwerr_txts[i].mask & BMAN_ERRS_TO_UNENABLE) { ++ pr_devel("Bman un-enabling error 0x%x\n", ++ bman_hwerr_txts[i].mask); ++ ier_val &= ~bman_hwerr_txts[i].mask; ++ bm_err_isr_enable_write(bm, ier_val); ++ } ++ } ++ } ++ bm_err_isr_status_clear(bm, isr_val); ++ return IRQ_HANDLED; ++} ++ ++static int __bind_irq(void) ++{ ++ int ret, err_irq; ++ ++ err_irq = of_irq_to_resource(bm_node, 0, NULL); ++ if (err_irq == 0) { ++ pr_info("Can't get %s property '%s'\n", bm_node->full_name, ++ "interrupts"); ++ return -ENODEV; ++ } ++ ret = request_irq(err_irq, bman_isr, IRQF_SHARED, "bman-err", bm_node); ++ if (ret) { ++ pr_err("request_irq() failed %d for '%s'\n", ret, ++ bm_node->full_name); ++ return -ENODEV; ++ } ++ /* Disable Buffer Pool State Change */ ++ bm_err_isr_disable_write(bm, BM_EIRQ_BSCN); ++ /* Write-to-clear any stale bits, (eg. starvation being asserted prior ++ * to resource allocation during driver init). */ ++ bm_err_isr_status_clear(bm, 0xffffffff); ++ /* Enable Error Interrupts */ ++ bm_err_isr_enable_write(bm, 0xffffffff); ++ return 0; ++} ++ ++int bman_init_ccsr(struct device_node *node) ++{ ++ int ret; ++ if (!bman_have_ccsr()) ++ return 0; ++ if (node != bm_node) ++ return -EINVAL; ++ /* FBPR memory */ ++ bm_set_memory(bm, fbpr_a, 0, fbpr_sz); ++ pr_info("bman-fbpr addr 0x%llx size 0x%zx\n", ++ (unsigned long long)fbpr_a, fbpr_sz); ++ ++ ret = __bind_irq(); ++ if (ret) ++ return ret; ++ return 0; ++} ++ ++u32 bm_pool_free_buffers(u32 bpid) ++{ ++ return bm_in(POOL_CONTENT(bpid)); ++} ++ ++#ifdef CONFIG_SYSFS ++ ++#define DRV_NAME "fsl-bman" ++#define SBEC_MAX_ID 1 ++#define SBEC_MIN_ID 0 ++ ++static ssize_t show_fbpr_fpc(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(FBPR_FPC)); ++}; ++ ++static ssize_t show_pool_count(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ u32 data; ++ int i; ++ ++ if (!sscanf(dev_attr->attr.name, "%d", &i) || (i >= bman_pool_max)) ++ return -EINVAL; ++ data = bm_in(POOL_CONTENT(i)); ++ return snprintf(buf, PAGE_SIZE, "%d\n", data); ++}; ++ ++static ssize_t show_err_isr(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "0x%08x\n", bm_in(ERR_ISR)); ++}; ++ ++static ssize_t show_sbec(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ int i; ++ ++ if (!sscanf(dev_attr->attr.name, "sbec_%d", &i)) ++ return -EINVAL; ++ if (i < SBEC_MIN_ID || i > SBEC_MAX_ID) ++ return -EINVAL; ++ return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(SBEC(i))); ++}; ++ ++static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL); ++static DEVICE_ATTR(fbpr_fpc, S_IRUSR, show_fbpr_fpc, NULL); ++ ++/* Didn't use DEVICE_ATTR as 64 of this would be required. ++ * Initialize them when needed. */ ++static char *name_attrs_pool_count; /* "xx" + null-terminator */ ++static struct device_attribute *dev_attr_buffer_pool_count; ++ ++static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL); ++ ++static struct attribute *bman_dev_attributes[] = { ++ &dev_attr_fbpr_fpc.attr, ++ &dev_attr_err_isr.attr, ++ NULL ++}; ++ ++static struct attribute *bman_dev_ecr_attributes[] = { ++ &dev_attr_sbec_0.attr, ++ &dev_attr_sbec_1.attr, ++ NULL ++}; ++ ++static struct attribute **bman_dev_pool_count_attributes; ++ ++ ++/* root level */ ++static const struct attribute_group bman_dev_attr_grp = { ++ .name = NULL, ++ .attrs = bman_dev_attributes ++}; ++static const struct attribute_group bman_dev_ecr_grp = { ++ .name = "error_capture", ++ .attrs = bman_dev_ecr_attributes ++}; ++static struct attribute_group bman_dev_pool_countent_grp = { ++ .name = "pool_count", ++}; ++ ++static int of_fsl_bman_remove(struct platform_device *ofdev) ++{ ++ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp); ++ return 0; ++}; ++ ++static int of_fsl_bman_probe(struct platform_device *ofdev) ++{ ++ int ret, i; ++ ++ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_attr_grp); ++ if (ret) ++ goto done; ++ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_ecr_grp); ++ if (ret) ++ goto del_group_0; ++ ++ name_attrs_pool_count = kmalloc(sizeof(char) * bman_pool_max * 3, ++ GFP_KERNEL); ++ if (!name_attrs_pool_count) { ++ pr_err("Can't alloc name_attrs_pool_count\n"); ++ goto del_group_1; ++ } ++ ++ dev_attr_buffer_pool_count = kmalloc(sizeof(struct device_attribute) * ++ bman_pool_max, GFP_KERNEL); ++ if (!dev_attr_buffer_pool_count) { ++ pr_err("Can't alloc dev_attr-buffer_pool_count\n"); ++ goto del_group_2; ++ } ++ ++ bman_dev_pool_count_attributes = kmalloc(sizeof(struct attribute *) * ++ (bman_pool_max + 1), GFP_KERNEL); ++ if (!bman_dev_pool_count_attributes) { ++ pr_err("can't alloc bman_dev_pool_count_attributes\n"); ++ goto del_group_3; ++ } ++ ++ for (i = 0; i < bman_pool_max; i++) { ++ ret = scnprintf((name_attrs_pool_count + i * 3), 3, "%d", i); ++ if (!ret) ++ goto del_group_4; ++ dev_attr_buffer_pool_count[i].attr.name = ++ (name_attrs_pool_count + i * 3); ++ dev_attr_buffer_pool_count[i].attr.mode = S_IRUSR; ++ dev_attr_buffer_pool_count[i].show = show_pool_count; ++ bman_dev_pool_count_attributes[i] = ++ &dev_attr_buffer_pool_count[i].attr; ++ sysfs_attr_init(bman_dev_pool_count_attributes[i]); ++ } ++ bman_dev_pool_count_attributes[bman_pool_max] = NULL; ++ ++ bman_dev_pool_countent_grp.attrs = bman_dev_pool_count_attributes; ++ ++ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_pool_countent_grp); ++ if (ret) ++ goto del_group_4; ++ ++ goto done; ++ ++del_group_4: ++ kfree(bman_dev_pool_count_attributes); ++del_group_3: ++ kfree(dev_attr_buffer_pool_count); ++del_group_2: ++ kfree(name_attrs_pool_count); ++del_group_1: ++ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_ecr_grp); ++del_group_0: ++ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp); ++done: ++ if (ret) ++ dev_err(&ofdev->dev, ++ "Cannot create dev attributes ret=%d\n", ret); ++ return ret; ++}; ++ ++static struct of_device_id of_fsl_bman_ids[] = { ++ { ++ .compatible = "fsl,bman", ++ }, ++ {} ++}; ++MODULE_DEVICE_TABLE(of, of_fsl_bman_ids); ++ ++#ifdef CONFIG_SUSPEND ++static u32 saved_isdr; ++ ++static int bman_pm_suspend_noirq(struct device *dev) ++{ ++ uint32_t idle_state; ++ ++ suspend_unused_bportal(); ++ /* save isdr, disable all, clear isr */ ++ saved_isdr = bm_err_isr_disable_read(bm); ++ bm_err_isr_disable_write(bm, 0xffffffff); ++ bm_err_isr_status_clear(bm, 0xffffffff); ++ ++ if (bman_ip_rev < BMAN_REV21) { ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Bman version doesn't have STATE_IDLE\n"); ++#endif ++ return 0; ++ } ++ idle_state = bm_in(STATE_IDLE); ++ if (!(idle_state & 0x1)) { ++ pr_err("Bman not idle 0x%x aborting\n", idle_state); ++ bm_err_isr_disable_write(bm, saved_isdr); ++ resume_unused_bportal(); ++ return -EBUSY; ++ } ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Bman suspend code, IDLE_STAT = 0x%x\n", idle_state); ++#endif ++ return 0; ++} ++ ++static int bman_pm_resume_noirq(struct device *dev) ++{ ++ /* restore isdr */ ++ bm_err_isr_disable_write(bm, saved_isdr); ++ resume_unused_bportal(); ++ return 0; ++} ++#else ++#define bman_pm_suspend_noirq NULL ++#define bman_pm_resume_noirq NULL ++#endif ++ ++static const struct dev_pm_ops bman_pm_ops = { ++ .suspend_noirq = bman_pm_suspend_noirq, ++ .resume_noirq = bman_pm_resume_noirq, ++}; ++ ++static struct platform_driver of_fsl_bman_driver = { ++ .driver = { ++ .owner = THIS_MODULE, ++ .name = DRV_NAME, ++ .of_match_table = of_fsl_bman_ids, ++ .pm = &bman_pm_ops, ++ }, ++ .probe = of_fsl_bman_probe, ++ .remove = of_fsl_bman_remove, ++}; ++ ++static int bman_ctrl_init(void) ++{ ++ return platform_driver_register(&of_fsl_bman_driver); ++} ++ ++static void bman_ctrl_exit(void) ++{ ++ platform_driver_unregister(&of_fsl_bman_driver); ++} ++ ++module_init(bman_ctrl_init); ++module_exit(bman_ctrl_exit); ++ ++#endif /* CONFIG_SYSFS */ +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_debugfs.c +@@ -0,0 +1,119 @@ ++/* Copyright 2010-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++#include <linux/module.h> ++#include <linux/fsl_bman.h> ++#include <linux/debugfs.h> ++#include <linux/seq_file.h> ++#include <linux/uaccess.h> ++ ++static struct dentry *dfs_root; /* debugfs root directory */ ++ ++/******************************************************************************* ++ * Query Buffer Pool State ++ ******************************************************************************/ ++static int query_bp_state_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct bm_pool_state state; ++ int i, j; ++ u32 mask; ++ ++ memset(&state, 0, sizeof(struct bm_pool_state)); ++ ret = bman_query_pools(&state); ++ if (ret) { ++ seq_printf(file, "Error %d\n", ret); ++ return 0; ++ } ++ seq_puts(file, "bp_id free_buffers_avail bp_depleted\n"); ++ for (i = 0; i < 2; i++) { ++ mask = 0x80000000; ++ for (j = 0; j < 32; j++) { ++ seq_printf(file, ++ " %-2u %-3s %-3s\n", ++ (i*32)+j, ++ (state.as.state.__state[i] & mask) ? "no" : "yes", ++ (state.ds.state.__state[i] & mask) ? "yes" : "no"); ++ mask >>= 1; ++ } ++ } ++ return 0; ++} ++ ++static int query_bp_state_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, query_bp_state_show, NULL); ++} ++ ++static const struct file_operations query_bp_state_fops = { ++ .owner = THIS_MODULE, ++ .open = query_bp_state_open, ++ .read = seq_read, ++ .release = single_release, ++}; ++ ++static int __init bman_debugfs_module_init(void) ++{ ++ int ret = 0; ++ struct dentry *d; ++ ++ dfs_root = debugfs_create_dir("bman", NULL); ++ ++ if (dfs_root == NULL) { ++ ret = -ENOMEM; ++ pr_err("Cannot create bman debugfs dir\n"); ++ goto _return; ++ } ++ d = debugfs_create_file("query_bp_state", ++ S_IRUGO, ++ dfs_root, ++ NULL, ++ &query_bp_state_fops); ++ if (d == NULL) { ++ ret = -ENOMEM; ++ pr_err("Cannot create query_bp_state\n"); ++ goto _return; ++ } ++ return 0; ++ ++_return: ++ debugfs_remove_recursive(dfs_root); ++ return ret; ++} ++ ++static void __exit bman_debugfs_module_exit(void) ++{ ++ debugfs_remove_recursive(dfs_root); ++} ++ ++ ++module_init(bman_debugfs_module_init); ++module_exit(bman_debugfs_module_exit); ++MODULE_LICENSE("Dual BSD/GPL"); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_driver.c +@@ -0,0 +1,574 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++#include "bman_low.h" ++#ifdef CONFIG_HOTPLUG_CPU ++#include <linux/cpu.h> ++#endif ++/* ++ * Global variables of the max portal/pool number this bman version supported ++ */ ++u16 bman_ip_rev; ++EXPORT_SYMBOL(bman_ip_rev); ++u16 bman_pool_max; ++EXPORT_SYMBOL(bman_pool_max); ++static u16 bman_portal_max; ++ ++/* After initialising cpus that own shared portal configs, we cache the ++ * resulting portals (ie. not just the configs) in this array. Then we ++ * initialise slave cpus that don't have their own portals, redirecting them to ++ * portals from this cache in a round-robin assignment. */ ++static struct bman_portal *shared_portals[NR_CPUS]; ++static int num_shared_portals; ++static int shared_portals_idx; ++static LIST_HEAD(unused_pcfgs); ++static DEFINE_SPINLOCK(unused_pcfgs_lock); ++static void *affine_bportals[NR_CPUS]; ++ ++static int __init fsl_bpool_init(struct device_node *node) ++{ ++ int ret; ++ u32 *thresh, *bpid = (u32 *)of_get_property(node, "fsl,bpid", &ret); ++ if (!bpid || (ret != 4)) { ++ pr_err("Can't get %s property 'fsl,bpid'\n", node->full_name); ++ return -ENODEV; ++ } ++ thresh = (u32 *)of_get_property(node, "fsl,bpool-thresholds", &ret); ++ if (thresh) { ++ if (ret != 16) { ++ pr_err("Invalid %s property '%s'\n", ++ node->full_name, "fsl,bpool-thresholds"); ++ return -ENODEV; ++ } ++ } ++ if (thresh) { ++#ifdef CONFIG_FSL_BMAN_CONFIG ++ ret = bm_pool_set(be32_to_cpu(*bpid), thresh); ++ if (ret) ++ pr_err("No CCSR node for %s property '%s'\n", ++ node->full_name, "fsl,bpool-thresholds"); ++ return ret; ++#else ++ pr_err("Ignoring %s property '%s', no CCSR support\n", ++ node->full_name, "fsl,bpool-thresholds"); ++#endif ++ } ++ return 0; ++} ++ ++static int __init fsl_bpid_range_init(struct device_node *node) ++{ ++ int ret; ++ u32 *range = (u32 *)of_get_property(node, "fsl,bpid-range", &ret); ++ if (!range) { ++ pr_err("No 'fsl,bpid-range' property in node %s\n", ++ node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err("'fsl,bpid-range' is not a 2-cell range in node %s\n", ++ node->full_name); ++ return -EINVAL; ++ } ++ bman_seed_bpid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ pr_info("Bman: BPID allocator includes range %d:%d\n", ++ be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ return 0; ++} ++ ++static struct bm_portal_config * __init parse_pcfg(struct device_node *node) ++{ ++ struct bm_portal_config *pcfg; ++ const u32 *index; ++ int irq, ret; ++ resource_size_t len; ++ ++ pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL); ++ if (!pcfg) { ++ pr_err("can't allocate portal config"); ++ return NULL; ++ } ++ ++ if (of_device_is_compatible(node, "fsl,bman-portal-1.0") || ++ of_device_is_compatible(node, "fsl,bman-portal-1.0.0")) { ++ bman_ip_rev = BMAN_REV10; ++ bman_pool_max = 64; ++ bman_portal_max = 10; ++ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.0") || ++ of_device_is_compatible(node, "fsl,bman-portal-2.0.8")) { ++ bman_ip_rev = BMAN_REV20; ++ bman_pool_max = 8; ++ bman_portal_max = 3; ++ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.0")) { ++ bman_ip_rev = BMAN_REV21; ++ bman_pool_max = 64; ++ bman_portal_max = 50; ++ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.1")) { ++ bman_ip_rev = BMAN_REV21; ++ bman_pool_max = 64; ++ bman_portal_max = 25; ++ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.2")) { ++ bman_ip_rev = BMAN_REV21; ++ bman_pool_max = 64; ++ bman_portal_max = 18; ++ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.3")) { ++ bman_ip_rev = BMAN_REV21; ++ bman_pool_max = 64; ++ bman_portal_max = 10; ++ } else { ++ pr_warn("unknown BMan version in portal node," ++ "default to rev1.0\n"); ++ bman_ip_rev = BMAN_REV10; ++ bman_pool_max = 64; ++ bman_portal_max = 10; ++ } ++ ++ ret = of_address_to_resource(node, DPA_PORTAL_CE, ++ &pcfg->addr_phys[DPA_PORTAL_CE]); ++ if (ret) { ++ pr_err("Can't get %s property 'reg::CE'\n", node->full_name); ++ goto err; ++ } ++ ret = of_address_to_resource(node, DPA_PORTAL_CI, ++ &pcfg->addr_phys[DPA_PORTAL_CI]); ++ if (ret) { ++ pr_err("Can't get %s property 'reg::CI'\n", node->full_name); ++ goto err; ++ } ++ ++ index = of_get_property(node, "cell-index", &ret); ++ if (!index || (ret != 4)) { ++ pr_err("Can't get %s property '%s'\n", node->full_name, ++ "cell-index"); ++ goto err; ++ } ++ if (be32_to_cpu(*index) >= bman_portal_max) { ++ pr_err("BMan portal cell index %d out of range, max %d\n", ++ be32_to_cpu(*index), bman_portal_max); ++ goto err; ++ } ++ ++ pcfg->public_cfg.cpu = -1; ++ ++ irq = irq_of_parse_and_map(node, 0); ++ if (irq == 0) { ++ pr_err("Can't get %s property 'interrupts'\n", node->full_name); ++ goto err; ++ } ++ pcfg->public_cfg.irq = irq; ++ pcfg->public_cfg.index = be32_to_cpu(*index); ++ bman_depletion_fill(&pcfg->public_cfg.mask); ++ ++ len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]); ++ if (len != (unsigned long)len) ++ goto err; ++ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns( ++ pcfg->addr_phys[DPA_PORTAL_CE].start, ++ resource_size(&pcfg->addr_phys[DPA_PORTAL_CE])); ++ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap( ++ pcfg->addr_phys[DPA_PORTAL_CI].start, ++ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI])); ++ ++#else ++ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot( ++ pcfg->addr_phys[DPA_PORTAL_CE].start, ++ (unsigned long)len, ++ 0); ++ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot( ++ pcfg->addr_phys[DPA_PORTAL_CI].start, ++ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]), ++ _PAGE_GUARDED | _PAGE_NO_CACHE); ++#endif ++ /* disable bp depletion */ ++ __raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(0)); ++ __raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(1)); ++ return pcfg; ++err: ++ kfree(pcfg); ++ return NULL; ++} ++ ++static struct bm_portal_config *get_pcfg(struct list_head *list) ++{ ++ struct bm_portal_config *pcfg; ++ if (list_empty(list)) ++ return NULL; ++ pcfg = list_entry(list->prev, struct bm_portal_config, list); ++ list_del(&pcfg->list); ++ return pcfg; ++} ++ ++static struct bm_portal_config *get_pcfg_idx(struct list_head *list, ++ uint32_t idx) ++{ ++ struct bm_portal_config *pcfg; ++ if (list_empty(list)) ++ return NULL; ++ list_for_each_entry(pcfg, list, list) { ++ if (pcfg->public_cfg.index == idx) { ++ list_del(&pcfg->list); ++ return pcfg; ++ } ++ } ++ return NULL; ++} ++ ++struct bm_portal_config *bm_get_unused_portal(void) ++{ ++ return bm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX); ++} ++ ++struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx) ++{ ++ struct bm_portal_config *ret; ++ spin_lock(&unused_pcfgs_lock); ++ if (idx == QBMAN_ANY_PORTAL_IDX) ++ ret = get_pcfg(&unused_pcfgs); ++ else ++ ret = get_pcfg_idx(&unused_pcfgs, idx); ++ spin_unlock(&unused_pcfgs_lock); ++ return ret; ++} ++ ++void bm_put_unused_portal(struct bm_portal_config *pcfg) ++{ ++ spin_lock(&unused_pcfgs_lock); ++ list_add(&pcfg->list, &unused_pcfgs); ++ spin_unlock(&unused_pcfgs_lock); ++} ++ ++static struct bman_portal *init_pcfg(struct bm_portal_config *pcfg) ++{ ++ struct bman_portal *p; ++ p = bman_create_affine_portal(pcfg); ++ if (p) { ++#ifdef CONFIG_FSL_DPA_PIRQ_SLOW ++ bman_p_irqsource_add(p, BM_PIRQ_RCRI | BM_PIRQ_BSCN); ++#endif ++ pr_info("Bman portal %sinitialised, cpu %d\n", ++ pcfg->public_cfg.is_shared ? "(shared) " : "", ++ pcfg->public_cfg.cpu); ++ affine_bportals[pcfg->public_cfg.cpu] = p; ++ } else ++ pr_crit("Bman portal failure on cpu %d\n", ++ pcfg->public_cfg.cpu); ++ return p; ++} ++ ++static void init_slave(int cpu) ++{ ++ struct bman_portal *p; ++ p = bman_create_affine_slave(shared_portals[shared_portals_idx++], cpu); ++ if (!p) ++ pr_err("Bman slave portal failure on cpu %d\n", cpu); ++ else ++ pr_info("Bman portal %sinitialised, cpu %d\n", "(slave) ", cpu); ++ if (shared_portals_idx >= num_shared_portals) ++ shared_portals_idx = 0; ++ affine_bportals[cpu] = p; ++} ++ ++/* Bootarg "bportals=[...]" has the same syntax as "qportals=", and so the ++ * parsing is in dpa_sys.h. The syntax is a comma-separated list of indexes ++ * and/or ranges of indexes, with each being optionally prefixed by "s" to ++ * explicitly mark it or them for sharing. ++ * Eg; ++ * bportals=s0,1-3,s4 ++ * means that cpus 1,2,3 get "unshared" portals, cpus 0 and 4 get "shared" ++ * portals, and any remaining cpus share the portals that are assigned to cpus 0 ++ * or 4, selected in a round-robin fashion. (In this example, cpu 5 would share ++ * cpu 0's portal, cpu 6 would share cpu4's portal, and cpu 7 would share cpu ++ * 0's portal.) */ ++static struct cpumask want_unshared __initdata; /* cpus requested without "s" */ ++static struct cpumask want_shared __initdata; /* cpus requested with "s" */ ++ ++static int __init parse_bportals(char *str) ++{ ++ return parse_portals_bootarg(str, &want_shared, &want_unshared, ++ "bportals"); ++} ++__setup("bportals=", parse_bportals); ++ ++static void bman_offline_cpu(unsigned int cpu) ++{ ++ struct bman_portal *p; ++ const struct bm_portal_config *pcfg; ++ p = (struct bman_portal *)affine_bportals[cpu]; ++ if (p) { ++ pcfg = bman_get_bm_portal_config(p); ++ if (pcfg) ++ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0)); ++ } ++} ++ ++#ifdef CONFIG_HOTPLUG_CPU ++static void bman_online_cpu(unsigned int cpu) ++{ ++ struct bman_portal *p; ++ const struct bm_portal_config *pcfg; ++ p = (struct bman_portal *)affine_bportals[cpu]; ++ if (p) { ++ pcfg = bman_get_bm_portal_config(p); ++ if (pcfg) ++ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu)); ++ } ++} ++ ++static int bman_hotplug_cpu_callback(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ unsigned int cpu = (unsigned long)hcpu; ++ ++ switch (action) { ++ case CPU_ONLINE: ++ case CPU_ONLINE_FROZEN: ++ bman_online_cpu(cpu); ++ break; ++ case CPU_DOWN_PREPARE: ++ case CPU_DOWN_PREPARE_FROZEN: ++ bman_offline_cpu(cpu); ++ default: ++ break; ++ } ++ return NOTIFY_OK; ++} ++ ++static struct notifier_block bman_hotplug_cpu_notifier = { ++ .notifier_call = bman_hotplug_cpu_callback, ++}; ++#endif /* CONFIG_HOTPLUG_CPU */ ++ ++/* Initialise the Bman driver. The meat of this function deals with portals. The ++ * following describes the flow of portal-handling, the code "steps" refer to ++ * this description; ++ * 1. Portal configs are parsed from the device-tree into 'unused_pcfgs', with ++ * ::cpu==-1. Regions and interrupts are mapped (but interrupts are not ++ * bound). ++ * 2. The "want_shared" and "want_unshared" lists (as filled by the ++ * "bportals=[...]" bootarg) are processed, allocating portals and assigning ++ * them to cpus, placing them in the relevant list and setting ::cpu as ++ * appropriate. If no "bportals" bootarg was present, the defaut is to try to ++ * assign portals to all online cpus at the time of driver initialisation. ++ * Any failure to allocate portals (when parsing the "want" lists or when ++ * using default behaviour) will be silently tolerated (the "fixup" logic in ++ * step 3 will determine what happens in this case). ++ * 3. Do fixups relative to cpu_online_mask(). If no portals are marked for ++ * sharing and sharing is required (because not all cpus have been assigned ++ * portals), then one portal will marked for sharing. Conversely if no ++ * sharing is required, any portals marked for sharing will not be shared. It ++ * may be that sharing occurs when it wasn't expected, if portal allocation ++ * failed to honour all the requested assignments (including the default ++ * assignments if no bootarg is present). ++ * 4. Unshared portals are initialised on their respective cpus. ++ * 5. Shared portals are initialised on their respective cpus. ++ * 6. Each remaining cpu is initialised to slave to one of the shared portals, ++ * which are selected in a round-robin fashion. ++ * Any portal configs left unused are available for USDPAA allocation. ++ */ ++__init int bman_init(void) ++{ ++ struct cpumask slave_cpus; ++ struct cpumask unshared_cpus = *cpu_none_mask; ++ struct cpumask shared_cpus = *cpu_none_mask; ++ LIST_HEAD(unshared_pcfgs); ++ LIST_HEAD(shared_pcfgs); ++ struct device_node *dn; ++ struct bm_portal_config *pcfg; ++ struct bman_portal *p; ++ int cpu, ret; ++ struct cpumask offline_cpus; ++ ++ /* Initialise the Bman (CCSR) device */ ++ for_each_compatible_node(dn, NULL, "fsl,bman") { ++ if (!bman_init_ccsr(dn)) ++ pr_info("Bman err interrupt handler present\n"); ++ else ++ pr_err("Bman CCSR setup failed\n"); ++ } ++ /* Initialise any declared buffer pools */ ++ for_each_compatible_node(dn, NULL, "fsl,bpool") { ++ ret = fsl_bpool_init(dn); ++ if (ret) ++ return ret; ++ } ++ /* Step 1. See comments at the beginning of the file. */ ++ for_each_compatible_node(dn, NULL, "fsl,bman-portal") { ++ if (!of_device_is_available(dn)) ++ continue; ++ pcfg = parse_pcfg(dn); ++ if (pcfg) ++ list_add_tail(&pcfg->list, &unused_pcfgs); ++ } ++ /* Step 2. */ ++ for_each_possible_cpu(cpu) { ++ if (cpumask_test_cpu(cpu, &want_shared)) { ++ pcfg = get_pcfg(&unused_pcfgs); ++ if (!pcfg) ++ break; ++ pcfg->public_cfg.cpu = cpu; ++ list_add_tail(&pcfg->list, &shared_pcfgs); ++ cpumask_set_cpu(cpu, &shared_cpus); ++ } ++ if (cpumask_test_cpu(cpu, &want_unshared)) { ++ if (cpumask_test_cpu(cpu, &shared_cpus)) ++ continue; ++ pcfg = get_pcfg(&unused_pcfgs); ++ if (!pcfg) ++ break; ++ pcfg->public_cfg.cpu = cpu; ++ list_add_tail(&pcfg->list, &unshared_pcfgs); ++ cpumask_set_cpu(cpu, &unshared_cpus); ++ } ++ } ++ if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) { ++ /* Default, give an unshared portal to each online cpu */ ++ for_each_online_cpu(cpu) { ++ pcfg = get_pcfg(&unused_pcfgs); ++ if (!pcfg) ++ break; ++ pcfg->public_cfg.cpu = cpu; ++ list_add_tail(&pcfg->list, &unshared_pcfgs); ++ cpumask_set_cpu(cpu, &unshared_cpus); ++ } ++ } ++ /* Step 3. */ ++ cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus); ++ cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus); ++ if (cpumask_empty(&slave_cpus)) { ++ /* No sharing required */ ++ if (!list_empty(&shared_pcfgs)) { ++ /* Migrate "shared" to "unshared" */ ++ cpumask_or(&unshared_cpus, &unshared_cpus, ++ &shared_cpus); ++ cpumask_clear(&shared_cpus); ++ list_splice_tail(&shared_pcfgs, &unshared_pcfgs); ++ INIT_LIST_HEAD(&shared_pcfgs); ++ } ++ } else { ++ /* Sharing required */ ++ if (list_empty(&shared_pcfgs)) { ++ /* Migrate one "unshared" to "shared" */ ++ pcfg = get_pcfg(&unshared_pcfgs); ++ if (!pcfg) { ++ pr_crit("No BMan portals available!\n"); ++ return 0; ++ } ++ cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus); ++ cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus); ++ list_add_tail(&pcfg->list, &shared_pcfgs); ++ } ++ } ++ /* Step 4. */ ++ list_for_each_entry(pcfg, &unshared_pcfgs, list) { ++ pcfg->public_cfg.is_shared = 0; ++ p = init_pcfg(pcfg); ++ if (!p) { ++ pr_crit("Unable to initialize bman portal\n"); ++ return 0; ++ } ++ } ++ /* Step 5. */ ++ list_for_each_entry(pcfg, &shared_pcfgs, list) { ++ pcfg->public_cfg.is_shared = 1; ++ p = init_pcfg(pcfg); ++ if (p) ++ shared_portals[num_shared_portals++] = p; ++ } ++ /* Step 6. */ ++ if (!cpumask_empty(&slave_cpus)) ++ for_each_cpu(cpu, &slave_cpus) ++ init_slave(cpu); ++ pr_info("Bman portals initialised\n"); ++ cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask); ++ for_each_cpu(cpu, &offline_cpus) ++ bman_offline_cpu(cpu); ++#ifdef CONFIG_HOTPLUG_CPU ++ register_hotcpu_notifier(&bman_hotplug_cpu_notifier); ++#endif ++ return 0; ++} ++ ++__init int bman_resource_init(void) ++{ ++ struct device_node *dn; ++ int ret; ++ ++ /* Initialise BPID allocation ranges */ ++ for_each_compatible_node(dn, NULL, "fsl,bpid-range") { ++ ret = fsl_bpid_range_init(dn); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++#ifdef CONFIG_SUSPEND ++void suspend_unused_bportal(void) ++{ ++ struct bm_portal_config *pcfg; ++ ++ if (list_empty(&unused_pcfgs)) ++ return; ++ ++ list_for_each_entry(pcfg, &unused_pcfgs, list) { ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Need to save bportal %d\n", pcfg->public_cfg.index); ++#endif ++ /* save isdr, disable all via isdr, clear isr */ ++ pcfg->saved_isdr = ++ __raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); ++ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + ++ 0xe08); ++ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + ++ 0xe00); ++ } ++ return; ++} ++ ++void resume_unused_bportal(void) ++{ ++ struct bm_portal_config *pcfg; ++ ++ if (list_empty(&unused_pcfgs)) ++ return; ++ ++ list_for_each_entry(pcfg, &unused_pcfgs, list) { ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Need to resume bportal %d\n", pcfg->public_cfg.index); ++#endif ++ /* restore isdr */ ++ __raw_writel(pcfg->saved_isdr, ++ pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); ++ } ++ return; ++} ++#endif +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_high.c +@@ -0,0 +1,1141 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "bman_low.h" ++ ++/* Compilation constants */ ++#define RCR_THRESH 2 /* reread h/w CI when running out of space */ ++#define IRQNAME "BMan portal %d" ++#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */ ++ ++struct bman_portal { ++ struct bm_portal p; ++ /* 2-element array. pools[0] is mask, pools[1] is snapshot. */ ++ struct bman_depletion *pools; ++ int thresh_set; ++ unsigned long irq_sources; ++ u32 slowpoll; /* only used when interrupts are off */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */ ++#endif ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ raw_spinlock_t sharing_lock; /* only used if is_shared */ ++ int is_shared; ++ struct bman_portal *sharing_redirect; ++#endif ++ /* When the cpu-affine portal is activated, this is non-NULL */ ++ const struct bm_portal_config *config; ++ /* This is needed for power management */ ++ struct platform_device *pdev; ++ /* 64-entry hash-table of pool objects that are tracking depletion ++ * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so ++ * we're not fussy about cache-misses and so forth - whereas the above ++ * members should all fit in one cacheline. ++ * BTW, with 64 entries in the hash table and 64 buffer pools to track, ++ * you'll never guess the hash-function ... */ ++ struct bman_pool *cb[64]; ++ char irqname[MAX_IRQNAME]; ++ /* Track if the portal was alloced by the driver */ ++ u8 alloced; ++ /* power management data */ ++ u32 save_isdr; ++}; ++ ++/* For an explanation of the locking, redirection, or affine-portal logic, ++ * please consult the Qman driver for details. This is the same, only simpler ++ * (no fiddly Qman-specific bits.) */ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++#define PORTAL_IRQ_LOCK(p, irqflags) \ ++ do { \ ++ if ((p)->is_shared) \ ++ raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \ ++ else \ ++ local_irq_save(irqflags); \ ++ } while (0) ++#define PORTAL_IRQ_UNLOCK(p, irqflags) \ ++ do { \ ++ if ((p)->is_shared) \ ++ raw_spin_unlock_irqrestore(&(p)->sharing_lock, \ ++ irqflags); \ ++ else \ ++ local_irq_restore(irqflags); \ ++ } while (0) ++#else ++#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags) ++#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags) ++#endif ++ ++static cpumask_t affine_mask; ++static DEFINE_SPINLOCK(affine_mask_lock); ++static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal); ++static inline struct bman_portal *get_raw_affine_portal(void) ++{ ++ return &get_cpu_var(bman_affine_portal); ++} ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++static inline struct bman_portal *get_affine_portal(void) ++{ ++ struct bman_portal *p = get_raw_affine_portal(); ++ if (p->sharing_redirect) ++ return p->sharing_redirect; ++ return p; ++} ++#else ++#define get_affine_portal() get_raw_affine_portal() ++#endif ++static inline void put_affine_portal(void) ++{ ++ put_cpu_var(bman_affine_portal); ++} ++static inline struct bman_portal *get_poll_portal(void) ++{ ++ return &get_cpu_var(bman_affine_portal); ++} ++#define put_poll_portal() ++ ++/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be ++ * more than one such object per Bman buffer pool, eg. if different users of the ++ * pool are operating via different portals. */ ++struct bman_pool { ++ struct bman_pool_params params; ++ /* Used for hash-table admin when using depletion notifications. */ ++ struct bman_portal *portal; ++ struct bman_pool *next; ++ /* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */ ++ struct bm_buffer *sp; ++ unsigned int sp_fill; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ atomic_t in_use; ++#endif ++}; ++ ++/* (De)Registration of depletion notification callbacks */ ++static void depletion_link(struct bman_portal *portal, struct bman_pool *pool) ++{ ++ __maybe_unused unsigned long irqflags; ++ pool->portal = portal; ++ PORTAL_IRQ_LOCK(portal, irqflags); ++ pool->next = portal->cb[pool->params.bpid]; ++ portal->cb[pool->params.bpid] = pool; ++ if (!pool->next) ++ /* First object for that bpid on this portal, enable the BSCN ++ * mask bit. */ ++ bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1); ++ PORTAL_IRQ_UNLOCK(portal, irqflags); ++} ++static void depletion_unlink(struct bman_pool *pool) ++{ ++ struct bman_pool *it, *last = NULL; ++ struct bman_pool **base = &pool->portal->cb[pool->params.bpid]; ++ __maybe_unused unsigned long irqflags; ++ PORTAL_IRQ_LOCK(pool->portal, irqflags); ++ it = *base; /* <-- gotcha, don't do this prior to the irq_save */ ++ while (it != pool) { ++ last = it; ++ it = it->next; ++ } ++ if (!last) ++ *base = pool->next; ++ else ++ last->next = pool->next; ++ if (!last && !pool->next) { ++ /* Last object for that bpid on this portal, disable the BSCN ++ * mask bit. */ ++ bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0); ++ /* And "forget" that we last saw this pool as depleted */ ++ bman_depletion_unset(&pool->portal->pools[1], ++ pool->params.bpid); ++ } ++ PORTAL_IRQ_UNLOCK(pool->portal, irqflags); ++} ++ ++/* In the case that the application's core loop calls qman_poll() and ++ * bman_poll(), we ought to balance how often we incur the overheads of the ++ * slow-path poll. We'll use two decrementer sources. The idle decrementer ++ * constant is used when the last slow-poll detected no work to do, and the busy ++ * decrementer constant when the last slow-poll had work to do. */ ++#define SLOW_POLL_IDLE 1000 ++#define SLOW_POLL_BUSY 10 ++static u32 __poll_portal_slow(struct bman_portal *p, u32 is); ++ ++/* Portal interrupt handler */ ++static irqreturn_t portal_isr(__always_unused int irq, void *ptr) ++{ ++ struct bman_portal *p = ptr; ++ u32 clear = p->irq_sources; ++ u32 is = bm_isr_status_read(&p->p) & p->irq_sources; ++ clear |= __poll_portal_slow(p, is); ++ bm_isr_status_clear(&p->p, clear); ++ return IRQ_HANDLED; ++} ++ ++#ifdef CONFIG_SUSPEND ++static int _bman_portal_suspend_noirq(struct device *dev) ++{ ++ struct bman_portal *p = (struct bman_portal *)dev->platform_data; ++#ifdef CONFIG_PM_DEBUG ++ struct platform_device *pdev = to_platform_device(dev); ++#endif ++ p->save_isdr = bm_isr_disable_read(&p->p); ++ bm_isr_disable_write(&p->p, 0xffffffff); ++ bm_isr_status_clear(&p->p, 0xffffffff); ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Suspend for %s\n", pdev->name); ++#endif ++ return 0; ++} ++ ++static int _bman_portal_resume_noirq(struct device *dev) ++{ ++ struct bman_portal *p = (struct bman_portal *)dev->platform_data; ++ ++ /* restore isdr */ ++ bm_isr_disable_write(&p->p, p->save_isdr); ++ return 0; ++} ++#else ++#define _bman_portal_suspend_noirq NULL ++#define _bman_portal_resume_noirq NULL ++#endif ++ ++struct dev_pm_domain bman_portal_device_pm_domain = { ++ .ops = { ++ USE_PLATFORM_PM_SLEEP_OPS ++ .suspend_noirq = _bman_portal_suspend_noirq, ++ .resume_noirq = _bman_portal_resume_noirq, ++ } ++}; ++ ++struct bman_portal *bman_create_portal( ++ struct bman_portal *portal, ++ const struct bm_portal_config *config) ++{ ++ struct bm_portal *__p; ++ const struct bman_depletion *pools = &config->public_cfg.mask; ++ int ret; ++ u8 bpid = 0; ++ char buf[16]; ++ ++ if (!portal) { ++ portal = kmalloc(sizeof(*portal), GFP_KERNEL); ++ if (!portal) ++ return portal; ++ portal->alloced = 1; ++ } else ++ portal->alloced = 0; ++ ++ __p = &portal->p; ++ ++ /* prep the low-level portal struct with the mapped addresses from the ++ * config, everything that follows depends on it and "config" is more ++ * for (de)reference... */ ++ __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; ++ __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; ++ if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) { ++ pr_err("Bman RCR initialisation failed\n"); ++ goto fail_rcr; ++ } ++ if (bm_mc_init(__p)) { ++ pr_err("Bman MC initialisation failed\n"); ++ goto fail_mc; ++ } ++ if (bm_isr_init(__p)) { ++ pr_err("Bman ISR initialisation failed\n"); ++ goto fail_isr; ++ } ++ portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL); ++ if (!portal->pools) ++ goto fail_pools; ++ portal->pools[0] = *pools; ++ bman_depletion_init(portal->pools + 1); ++ while (bpid < bman_pool_max) { ++ /* Default to all BPIDs disabled, we enable as required at ++ * run-time. */ ++ bm_isr_bscn_mask(__p, bpid, 0); ++ bpid++; ++ } ++ portal->slowpoll = 0; ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ portal->rcri_owned = NULL; ++#endif ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ raw_spin_lock_init(&portal->sharing_lock); ++ portal->is_shared = config->public_cfg.is_shared; ++ portal->sharing_redirect = NULL; ++#endif ++ sprintf(buf, "bportal-%u", config->public_cfg.index); ++ portal->pdev = platform_device_alloc(buf, -1); ++ if (!portal->pdev) ++ goto fail_devalloc; ++ portal->pdev->dev.pm_domain = &bman_portal_device_pm_domain; ++ portal->pdev->dev.platform_data = portal; ++ ret = platform_device_add(portal->pdev); ++ if (ret) ++ goto fail_devadd; ++ memset(&portal->cb, 0, sizeof(portal->cb)); ++ /* Write-to-clear any stale interrupt status bits */ ++ bm_isr_disable_write(__p, 0xffffffff); ++ portal->irq_sources = 0; ++ bm_isr_enable_write(__p, portal->irq_sources); ++ bm_isr_status_clear(__p, 0xffffffff); ++ snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu); ++ if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname, ++ portal)) { ++ pr_err("request_irq() failed\n"); ++ goto fail_irq; ++ } ++ if ((config->public_cfg.cpu != -1) && ++ irq_can_set_affinity(config->public_cfg.irq) && ++ irq_set_affinity(config->public_cfg.irq, ++ cpumask_of(config->public_cfg.cpu))) { ++ pr_err("irq_set_affinity() failed %s\n", portal->irqname); ++ goto fail_affinity; ++ } ++ ++ /* Need RCR to be empty before continuing */ ++ ret = bm_rcr_get_fill(__p); ++ if (ret) { ++ pr_err("Bman RCR unclean\n"); ++ goto fail_rcr_empty; ++ } ++ /* Success */ ++ portal->config = config; ++ ++ bm_isr_disable_write(__p, 0); ++ bm_isr_uninhibit(__p); ++ return portal; ++fail_rcr_empty: ++fail_affinity: ++ free_irq(config->public_cfg.irq, portal); ++fail_irq: ++ platform_device_del(portal->pdev); ++fail_devadd: ++ platform_device_put(portal->pdev); ++fail_devalloc: ++ kfree(portal->pools); ++fail_pools: ++ bm_isr_finish(__p); ++fail_isr: ++ bm_mc_finish(__p); ++fail_mc: ++ bm_rcr_finish(__p); ++fail_rcr: ++ if (portal->alloced) ++ kfree(portal); ++ return NULL; ++} ++ ++struct bman_portal *bman_create_affine_portal( ++ const struct bm_portal_config *config) ++{ ++ struct bman_portal *portal; ++ ++ portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu); ++ portal = bman_create_portal(portal, config); ++ if (portal) { ++ spin_lock(&affine_mask_lock); ++ cpumask_set_cpu(config->public_cfg.cpu, &affine_mask); ++ spin_unlock(&affine_mask_lock); ++ } ++ return portal; ++} ++ ++ ++struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect, ++ int cpu) ++{ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ struct bman_portal *p; ++ p = &per_cpu(bman_affine_portal, cpu); ++ BUG_ON(p->config); ++ BUG_ON(p->is_shared); ++ BUG_ON(!redirect->config->public_cfg.is_shared); ++ p->irq_sources = 0; ++ p->sharing_redirect = redirect; ++ return p; ++#else ++ BUG(); ++ return NULL; ++#endif ++} ++ ++void bman_destroy_portal(struct bman_portal *bm) ++{ ++ const struct bm_portal_config *pcfg; ++ pcfg = bm->config; ++ bm_rcr_cce_update(&bm->p); ++ bm_rcr_cce_update(&bm->p); ++ ++ free_irq(pcfg->public_cfg.irq, bm); ++ ++ kfree(bm->pools); ++ bm_isr_finish(&bm->p); ++ bm_mc_finish(&bm->p); ++ bm_rcr_finish(&bm->p); ++ bm->config = NULL; ++ if (bm->alloced) ++ kfree(bm); ++} ++ ++const struct bm_portal_config *bman_destroy_affine_portal(void) ++{ ++ struct bman_portal *bm = get_raw_affine_portal(); ++ const struct bm_portal_config *pcfg; ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (bm->sharing_redirect) { ++ bm->sharing_redirect = NULL; ++ put_affine_portal(); ++ return NULL; ++ } ++ bm->is_shared = 0; ++#endif ++ pcfg = bm->config; ++ bman_destroy_portal(bm); ++ spin_lock(&affine_mask_lock); ++ cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask); ++ spin_unlock(&affine_mask_lock); ++ put_affine_portal(); ++ return pcfg; ++} ++ ++/* When release logic waits on available RCR space, we need a global waitqueue ++ * in the case of "affine" use (as the waits wake on different cpus which means ++ * different portals - so we can't wait on any per-portal waitqueue). */ ++static DECLARE_WAIT_QUEUE_HEAD(affine_queue); ++ ++static u32 __poll_portal_slow(struct bman_portal *p, u32 is) ++{ ++ struct bman_depletion tmp; ++ u32 ret = is; ++ ++ /* There is a gotcha to be aware of. If we do the query before clearing ++ * the status register, we may miss state changes that occur between the ++ * two. If we write to clear the status register before the query, the ++ * cache-enabled query command may overtake the status register write ++ * unless we use a heavyweight sync (which we don't want). Instead, we ++ * write-to-clear the status register then *read it back* before doing ++ * the query, hence the odd while loop with the 'is' accumulation. */ ++ if (is & BM_PIRQ_BSCN) { ++ struct bm_mc_result *mcr; ++ __maybe_unused unsigned long irqflags; ++ unsigned int i, j; ++ u32 __is; ++ bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); ++ while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) { ++ is |= __is; ++ bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); ++ } ++ is &= ~BM_PIRQ_BSCN; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ bm_mc_start(&p->p); ++ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); ++ while (!(mcr = bm_mc_result(&p->p))) ++ cpu_relax(); ++ tmp = mcr->query.ds.state; ++ tmp.__state[0] = be32_to_cpu(tmp.__state[0]); ++ tmp.__state[1] = be32_to_cpu(tmp.__state[1]); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ for (i = 0; i < 2; i++) { ++ int idx = i * 32; ++ /* tmp is a mask of currently-depleted pools. ++ * pools[0] is mask of those we care about. ++ * pools[1] is our previous view (we only want to ++ * be told about changes). */ ++ tmp.__state[i] &= p->pools[0].__state[i]; ++ if (tmp.__state[i] == p->pools[1].__state[i]) ++ /* fast-path, nothing to see, move along */ ++ continue; ++ for (j = 0; j <= 31; j++, idx++) { ++ struct bman_pool *pool = p->cb[idx]; ++ int b4 = bman_depletion_get(&p->pools[1], idx); ++ int af = bman_depletion_get(&tmp, idx); ++ if (b4 == af) ++ continue; ++ while (pool) { ++ pool->params.cb(p, pool, ++ pool->params.cb_ctx, af); ++ pool = pool->next; ++ } ++ } ++ } ++ p->pools[1] = tmp; ++ } ++ ++ if (is & BM_PIRQ_RCRI) { ++ __maybe_unused unsigned long irqflags; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ bm_rcr_cce_update(&p->p); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ /* If waiting for sync, we only cancel the interrupt threshold ++ * when the ring utilisation hits zero. */ ++ if (p->rcri_owned) { ++ if (!bm_rcr_get_fill(&p->p)) { ++ p->rcri_owned = NULL; ++ bm_rcr_set_ithresh(&p->p, 0); ++ } ++ } else ++#endif ++ bm_rcr_set_ithresh(&p->p, 0); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ wake_up(&affine_queue); ++ bm_isr_status_clear(&p->p, BM_PIRQ_RCRI); ++ is &= ~BM_PIRQ_RCRI; ++ } ++ ++ /* There should be no status register bits left undefined */ ++ DPA_ASSERT(!is); ++ return ret; ++} ++ ++const struct bman_portal_config *bman_get_portal_config(void) ++{ ++ struct bman_portal *p = get_affine_portal(); ++ const struct bman_portal_config *ret = &p->config->public_cfg; ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(bman_get_portal_config); ++ ++u32 bman_irqsource_get(void) ++{ ++ struct bman_portal *p = get_raw_affine_portal(); ++ u32 ret = p->irq_sources & BM_PIRQ_VISIBLE; ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(bman_irqsource_get); ++ ++int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits) ++{ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (p->sharing_redirect) ++ return -EINVAL; ++ else ++#endif ++ { ++ __maybe_unused unsigned long irqflags; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources); ++ bm_isr_enable_write(&p->p, p->irq_sources); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ } ++ return 0; ++} ++EXPORT_SYMBOL(bman_p_irqsource_add); ++ ++int bman_irqsource_add(__maybe_unused u32 bits) ++{ ++ struct bman_portal *p = get_raw_affine_portal(); ++ int ret = 0; ++ ret = bman_p_irqsource_add(p, bits); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(bman_irqsource_add); ++ ++int bman_irqsource_remove(u32 bits) ++{ ++ struct bman_portal *p = get_raw_affine_portal(); ++ __maybe_unused unsigned long irqflags; ++ u32 ier; ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (p->sharing_redirect) { ++ put_affine_portal(); ++ return -EINVAL; ++ } ++#endif ++ /* Our interrupt handler only processes+clears status register bits that ++ * are in p->irq_sources. As we're trimming that mask, if one of them ++ * were to assert in the status register just before we remove it from ++ * the enable register, there would be an interrupt-storm when we ++ * release the IRQ lock. So we wait for the enable register update to ++ * take effect in h/w (by reading it back) and then clear all other bits ++ * in the status register. Ie. we clear them from ISR once it's certain ++ * IER won't allow them to reassert. */ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ bits &= BM_PIRQ_VISIBLE; ++ clear_bits(bits, &p->irq_sources); ++ bm_isr_enable_write(&p->p, p->irq_sources); ++ ier = bm_isr_enable_read(&p->p); ++ /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a ++ * data-dependency, ie. to protect against re-ordering. */ ++ bm_isr_status_clear(&p->p, ~ier); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return 0; ++} ++EXPORT_SYMBOL(bman_irqsource_remove); ++ ++const cpumask_t *bman_affine_cpus(void) ++{ ++ return &affine_mask; ++} ++EXPORT_SYMBOL(bman_affine_cpus); ++ ++u32 bman_poll_slow(void) ++{ ++ struct bman_portal *p = get_poll_portal(); ++ u32 ret; ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (unlikely(p->sharing_redirect)) ++ ret = (u32)-1; ++ else ++#endif ++ { ++ u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; ++ ret = __poll_portal_slow(p, is); ++ bm_isr_status_clear(&p->p, ret); ++ } ++ put_poll_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(bman_poll_slow); ++ ++/* Legacy wrapper */ ++void bman_poll(void) ++{ ++ struct bman_portal *p = get_poll_portal(); ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (unlikely(p->sharing_redirect)) ++ goto done; ++#endif ++ if (!(p->slowpoll--)) { ++ u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; ++ u32 active = __poll_portal_slow(p, is); ++ if (active) ++ p->slowpoll = SLOW_POLL_BUSY; ++ else ++ p->slowpoll = SLOW_POLL_IDLE; ++ } ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++done: ++#endif ++ put_poll_portal(); ++} ++EXPORT_SYMBOL(bman_poll); ++ ++static const u32 zero_thresholds[4] = {0, 0, 0, 0}; ++ ++struct bman_pool *bman_new_pool(const struct bman_pool_params *params) ++{ ++ struct bman_pool *pool = NULL; ++ u32 bpid; ++ ++ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) { ++ int ret = bman_alloc_bpid(&bpid); ++ if (ret) ++ return NULL; ++ } else { ++ if (params->bpid >= bman_pool_max) ++ return NULL; ++ bpid = params->bpid; ++ } ++#ifdef CONFIG_FSL_BMAN_CONFIG ++ if (params->flags & BMAN_POOL_FLAG_THRESH) { ++ int ret = bm_pool_set(bpid, params->thresholds); ++ if (ret) ++ goto err; ++ } ++#else ++ if (params->flags & BMAN_POOL_FLAG_THRESH) ++ goto err; ++#endif ++ pool = kmalloc(sizeof(*pool), GFP_KERNEL); ++ if (!pool) ++ goto err; ++ pool->sp = NULL; ++ pool->sp_fill = 0; ++ pool->params = *params; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ atomic_set(&pool->in_use, 1); ++#endif ++ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) ++ pool->params.bpid = bpid; ++ if (params->flags & BMAN_POOL_FLAG_STOCKPILE) { ++ pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ, ++ GFP_KERNEL); ++ if (!pool->sp) ++ goto err; ++ } ++ if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) { ++ struct bman_portal *p = get_affine_portal(); ++ if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) { ++ pr_err("Depletion events disabled for bpid %d\n", bpid); ++ goto err; ++ } ++ depletion_link(p, pool); ++ put_affine_portal(); ++ } ++ return pool; ++err: ++#ifdef CONFIG_FSL_BMAN_CONFIG ++ if (params->flags & BMAN_POOL_FLAG_THRESH) ++ bm_pool_set(bpid, zero_thresholds); ++#endif ++ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) ++ bman_release_bpid(bpid); ++ if (pool) { ++ kfree(pool->sp); ++ kfree(pool); ++ } ++ return NULL; ++} ++EXPORT_SYMBOL(bman_new_pool); ++ ++void bman_free_pool(struct bman_pool *pool) ++{ ++#ifdef CONFIG_FSL_BMAN_CONFIG ++ if (pool->params.flags & BMAN_POOL_FLAG_THRESH) ++ bm_pool_set(pool->params.bpid, zero_thresholds); ++#endif ++ if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) ++ depletion_unlink(pool); ++ if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) { ++ if (pool->sp_fill) ++ pr_err("Stockpile not flushed, has %u in bpid %u.\n", ++ pool->sp_fill, pool->params.bpid); ++ kfree(pool->sp); ++ pool->sp = NULL; ++ pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE; ++ } ++ if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID) ++ bman_release_bpid(pool->params.bpid); ++ kfree(pool); ++} ++EXPORT_SYMBOL(bman_free_pool); ++ ++const struct bman_pool_params *bman_get_params(const struct bman_pool *pool) ++{ ++ return &pool->params; ++} ++EXPORT_SYMBOL(bman_get_params); ++ ++static noinline void update_rcr_ci(struct bman_portal *p, u8 avail) ++{ ++ if (avail) ++ bm_rcr_cce_prefetch(&p->p); ++ else ++ bm_rcr_cce_update(&p->p); ++} ++ ++int bman_rcr_is_empty(void) ++{ ++ __maybe_unused unsigned long irqflags; ++ struct bman_portal *p = get_affine_portal(); ++ u8 avail; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ update_rcr_ci(p, 0); ++ avail = bm_rcr_get_fill(&p->p); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return avail == 0; ++} ++EXPORT_SYMBOL(bman_rcr_is_empty); ++ ++static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p, ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ __maybe_unused struct bman_pool *pool, ++#endif ++ __maybe_unused unsigned long *irqflags, ++ __maybe_unused u32 flags) ++{ ++ struct bm_rcr_entry *r; ++ u8 avail; ++ ++ *p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(*p, (*irqflags)); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && ++ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { ++ if ((*p)->rcri_owned) { ++ PORTAL_IRQ_UNLOCK(*p, (*irqflags)); ++ put_affine_portal(); ++ return NULL; ++ } ++ (*p)->rcri_owned = pool; ++ } ++#endif ++ avail = bm_rcr_get_avail(&(*p)->p); ++ if (avail < 2) ++ update_rcr_ci(*p, avail); ++ r = bm_rcr_start(&(*p)->p); ++ if (unlikely(!r)) { ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && ++ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) ++ (*p)->rcri_owned = NULL; ++#endif ++ PORTAL_IRQ_UNLOCK(*p, (*irqflags)); ++ put_affine_portal(); ++ } ++ return r; ++} ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p, ++ struct bman_pool *pool, ++ __maybe_unused unsigned long *irqflags, ++ u32 flags) ++{ ++ struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags); ++ if (!rcr) ++ bm_rcr_set_ithresh(&(*p)->p, 1); ++ return rcr; ++} ++ ++static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p, ++ struct bman_pool *pool, ++ __maybe_unused unsigned long *irqflags, ++ u32 flags) ++{ ++ struct bm_rcr_entry *rcr; ++#ifndef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ pool = NULL; ++#endif ++ if (flags & BMAN_RELEASE_FLAG_WAIT_INT) ++ /* NB: return NULL if signal occurs before completion. Signal ++ * can occur during return. Caller must check for signal */ ++ wait_event_interruptible(affine_queue, ++ (rcr = __wait_rel_start(p, pool, irqflags, flags))); ++ else ++ wait_event(affine_queue, ++ (rcr = __wait_rel_start(p, pool, irqflags, flags))); ++ return rcr; ++} ++#endif ++ ++static inline int __bman_release(struct bman_pool *pool, ++ const struct bm_buffer *bufs, u8 num, u32 flags) ++{ ++ struct bman_portal *p; ++ struct bm_rcr_entry *r; ++ __maybe_unused unsigned long irqflags; ++ u32 i = num - 1; ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & BMAN_RELEASE_FLAG_WAIT) ++ r = wait_rel_start(&p, pool, &irqflags, flags); ++ else ++ r = try_rel_start(&p, pool, &irqflags, flags); ++#else ++ r = try_rel_start(&p, &irqflags, flags); ++#endif ++ if (!r) ++ return -EBUSY; ++ /* We can copy all but the first entry, as this can trigger badness ++ * with the valid-bit. Use the overlay to mask the verb byte. */ ++ r->bufs[0].opaque = ++ ((cpu_to_be64((bufs[0].opaque | ++ ((u64)pool->params.bpid<<48)) ++ & 0x00ffffffffffffff))); ++ if (i) { ++ for (i = 1; i < num; i++) ++ r->bufs[i].opaque = ++ cpu_to_be64(bufs[i].opaque); ++ } ++ ++ bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE | ++ (num & BM_RCR_VERB_BUFCOUNT_MASK)); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ /* if we wish to sync we need to set the threshold after h/w sees the ++ * new ring entry. As we're mixing cache-enabled and cache-inhibited ++ * accesses, this requires a heavy-weight sync. */ ++ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && ++ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { ++ hwsync(); ++ bm_rcr_set_ithresh(&p->p, 1); ++ } ++#endif ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && ++ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { ++ if (flags & BMAN_RELEASE_FLAG_WAIT_INT) ++ /* NB: return success even if signal occurs before ++ * condition is true. pvb_commit guarantees success */ ++ wait_event_interruptible(affine_queue, ++ (p->rcri_owned != pool)); ++ else ++ wait_event(affine_queue, (p->rcri_owned != pool)); ++ } ++#endif ++ return 0; ++} ++ ++int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, ++ u32 flags) ++{ ++ int ret; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (!num || (num > 8)) ++ return -EINVAL; ++ if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE) ++ return -EINVAL; ++#endif ++ /* Without stockpile, this API is a pass-through to the h/w operation */ ++ if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) ++ return __bman_release(pool, bufs, num, flags); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (!atomic_dec_and_test(&pool->in_use)) { ++ pr_crit("Parallel attempts to enter bman_released() detected."); ++ panic("only one instance of bman_released/acquired allowed"); ++ } ++#endif ++ /* Two movements of buffers are possible, and can occur in either order. ++ * A: moving buffers from the caller to the stockpile. ++ * B: moving buffers from the stockpile to hardware. ++ * Order 1: if there is already enough space in the stockpile for A ++ * then we want to do A first, and only do B if we trigger the ++ * stockpile-high threshold. ++ * Order 2: if there is not enough space in the stockpile for A, then ++ * we want to do B first, then do A if B had succeeded. However in this ++ * case B is dependent on how many buffers the user needs to release, ++ * not the stockpile-high threshold. ++ * Due to the different handling of B between the two cases, putting A ++ * and B in a while() loop would require quite obscure logic, so handle ++ * the different sequences explicitly. */ ++ if ((pool->sp_fill + num) <= BMAN_STOCKPILE_SZ) { ++ /* Order 1: do A */ ++ copy_words(pool->sp + pool->sp_fill, bufs, ++ sizeof(struct bm_buffer) * num); ++ pool->sp_fill += num; ++ /* do B relative to STOCKPILE_HIGH */ ++ while (pool->sp_fill >= BMAN_STOCKPILE_HIGH) { ++ ret = __bman_release(pool, ++ pool->sp + (pool->sp_fill - 8), 8, ++ flags); ++ if (ret >= 0) ++ pool->sp_fill -= 8; ++ } ++ } else { ++ /* Order 2: do B relative to 'num' */ ++ do { ++ ret = __bman_release(pool, ++ pool->sp + (pool->sp_fill - 8), 8, ++ flags); ++ if (ret < 0) ++ /* failure */ ++ goto release_done; ++ pool->sp_fill -= 8; ++ } while ((pool->sp_fill + num) > BMAN_STOCKPILE_SZ); ++ /* do A */ ++ copy_words(pool->sp + pool->sp_fill, bufs, ++ sizeof(struct bm_buffer) * num); ++ pool->sp_fill += num; ++ } ++ /* success */ ++ ret = 0; ++release_done: ++#ifdef CONFIG_FSL_DPA_CHECKING ++ atomic_inc(&pool->in_use); ++#endif ++ return ret; ++} ++EXPORT_SYMBOL(bman_release); ++ ++static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, ++ u8 num) ++{ ++ struct bman_portal *p = get_affine_portal(); ++ struct bm_mc_command *mcc; ++ struct bm_mc_result *mcr; ++ __maybe_unused unsigned long irqflags; ++ int ret, i; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ mcc = bm_mc_start(&p->p); ++ mcc->acquire.bpid = pool->params.bpid; ++ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | ++ (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT)); ++ while (!(mcr = bm_mc_result(&p->p))) ++ cpu_relax(); ++ ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT; ++ if (bufs) { ++ for (i = 0; i < num; i++) ++ bufs[i].opaque = ++ be64_to_cpu(mcr->acquire.bufs[i].opaque); ++ } ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (ret != num) ++ ret = -ENOMEM; ++ return ret; ++} ++ ++int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, ++ u32 flags) ++{ ++ int ret; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (!num || (num > 8)) ++ return -EINVAL; ++ if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE) ++ return -EINVAL; ++#endif ++ /* Without stockpile, this API is a pass-through to the h/w operation */ ++ if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) ++ return __bman_acquire(pool, bufs, num); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (!atomic_dec_and_test(&pool->in_use)) { ++ pr_crit("Parallel attempts to enter bman_acquire() detected."); ++ panic("only one instance of bman_released/acquired allowed"); ++ } ++#endif ++ /* Two movements of buffers are possible, and can occur in either order. ++ * A: moving buffers from stockpile to the caller. ++ * B: moving buffers from hardware to the stockpile. ++ * Order 1: if there are already enough buffers in the stockpile for A ++ * then we want to do A first, and only do B if we trigger the ++ * stockpile-low threshold. ++ * Order 2: if there are not enough buffers in the stockpile for A, ++ * then we want to do B first, then do A if B had succeeded. However in ++ * this case B is dependent on how many buffers the user needs, not the ++ * stockpile-low threshold. ++ * Due to the different handling of B between the two cases, putting A ++ * and B in a while() loop would require quite obscure logic, so handle ++ * the different sequences explicitly. */ ++ if (num <= pool->sp_fill) { ++ /* Order 1: do A */ ++ copy_words(bufs, pool->sp + (pool->sp_fill - num), ++ sizeof(struct bm_buffer) * num); ++ pool->sp_fill -= num; ++ /* do B relative to STOCKPILE_LOW */ ++ while (pool->sp_fill <= BMAN_STOCKPILE_LOW) { ++ ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8); ++ if (ret < 0) ++ ret = __bman_acquire(pool, ++ pool->sp + pool->sp_fill, 1); ++ if (ret < 0) ++ break; ++ pool->sp_fill += ret; ++ } ++ } else { ++ /* Order 2: do B relative to 'num' */ ++ do { ++ ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8); ++ if (ret < 0) ++ ret = __bman_acquire(pool, ++ pool->sp + pool->sp_fill, 1); ++ if (ret < 0) ++ /* failure */ ++ goto acquire_done; ++ pool->sp_fill += ret; ++ } while (pool->sp_fill < num); ++ /* do A */ ++ copy_words(bufs, pool->sp + (pool->sp_fill - num), ++ sizeof(struct bm_buffer) * num); ++ pool->sp_fill -= num; ++ } ++ /* success */ ++ ret = num; ++acquire_done: ++#ifdef CONFIG_FSL_DPA_CHECKING ++ atomic_inc(&pool->in_use); ++#endif ++ return ret; ++} ++EXPORT_SYMBOL(bman_acquire); ++ ++int bman_flush_stockpile(struct bman_pool *pool, u32 flags) ++{ ++ u8 num; ++ int ret; ++ ++ while (pool->sp_fill) { ++ num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill); ++ ret = __bman_release(pool, pool->sp + (pool->sp_fill - num), ++ num, flags); ++ if (ret) ++ return ret; ++ pool->sp_fill -= num; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(bman_flush_stockpile); ++ ++int bman_query_pools(struct bm_pool_state *state) ++{ ++ struct bman_portal *p = get_affine_portal(); ++ struct bm_mc_result *mcr; ++ __maybe_unused unsigned long irqflags; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ bm_mc_start(&p->p); ++ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); ++ while (!(mcr = bm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY); ++ *state = mcr->query; ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return 0; ++} ++EXPORT_SYMBOL(bman_query_pools); ++ ++#ifdef CONFIG_FSL_BMAN_CONFIG ++u32 bman_query_free_buffers(struct bman_pool *pool) ++{ ++ return bm_pool_free_buffers(pool->params.bpid); ++} ++EXPORT_SYMBOL(bman_query_free_buffers); ++ ++int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds) ++{ ++ u32 bpid; ++ ++ bpid = bman_get_params(pool)->bpid; ++ ++ return bm_pool_set(bpid, thresholds); ++} ++EXPORT_SYMBOL(bman_update_pool_thresholds); ++#endif ++ ++int bman_shutdown_pool(u32 bpid) ++{ ++ struct bman_portal *p = get_affine_portal(); ++ __maybe_unused unsigned long irqflags; ++ int ret; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ret = bm_shutdown_pool(&p->p, bpid); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(bman_shutdown_pool); ++ ++const struct bm_portal_config *bman_get_bm_portal_config( ++ struct bman_portal *portal) ++{ ++ return portal->sharing_redirect ? NULL : portal->config; ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_low.h +@@ -0,0 +1,559 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "bman_private.h" ++ ++/***************************/ ++/* Portal register assists */ ++/***************************/ ++ ++#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) ++ ++/* Cache-inhibited register offsets */ ++#define BM_REG_RCR_PI_CINH 0x0000 ++#define BM_REG_RCR_CI_CINH 0x0004 ++#define BM_REG_RCR_ITR 0x0008 ++#define BM_REG_CFG 0x0100 ++#define BM_REG_SCN(n) (0x0200 + ((n) << 2)) ++#define BM_REG_ISR 0x0e00 ++#define BM_REG_IIR 0x0e0c ++ ++/* Cache-enabled register offsets */ ++#define BM_CL_CR 0x0000 ++#define BM_CL_RR0 0x0100 ++#define BM_CL_RR1 0x0140 ++#define BM_CL_RCR 0x1000 ++#define BM_CL_RCR_PI_CENA 0x3000 ++#define BM_CL_RCR_CI_CENA 0x3100 ++ ++#endif ++ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ ++/* Cache-inhibited register offsets */ ++#define BM_REG_RCR_PI_CINH 0x3000 ++#define BM_REG_RCR_CI_CINH 0x3100 ++#define BM_REG_RCR_ITR 0x3200 ++#define BM_REG_CFG 0x3300 ++#define BM_REG_SCN(n) (0x3400 + ((n) << 6)) ++#define BM_REG_ISR 0x3e00 ++#define BM_REG_IIR 0x3ec0 ++ ++/* Cache-enabled register offsets */ ++#define BM_CL_CR 0x0000 ++#define BM_CL_RR0 0x0100 ++#define BM_CL_RR1 0x0140 ++#define BM_CL_RCR 0x1000 ++#define BM_CL_RCR_PI_CENA 0x3000 ++#define BM_CL_RCR_CI_CENA 0x3100 ++ ++#endif ++ ++/* BTW, the drivers (and h/w programming model) already obtain the required ++ * synchronisation for portal accesses via lwsync(), hwsync(), and ++ * data-dependencies. Use of barrier()s or other order-preserving primitives ++ * simply degrade performance. Hence the use of the __raw_*() interfaces, which ++ * simply ensure that the compiler treats the portal registers as volatile (ie. ++ * non-coherent). */ ++ ++/* Cache-inhibited register access. */ ++#define __bm_in(bm, o) be32_to_cpu(__raw_readl((bm)->addr_ci + (o))) ++#define __bm_out(bm, o, val) __raw_writel(cpu_to_be32(val), \ ++ (bm)->addr_ci + (o)); ++#define bm_in(reg) __bm_in(&portal->addr, BM_REG_##reg) ++#define bm_out(reg, val) __bm_out(&portal->addr, BM_REG_##reg, val) ++ ++/* Cache-enabled (index) register access */ ++#define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->addr_ce + (o)) ++#define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->addr_ce + (o)) ++#define __bm_cl_in(bm, o) be32_to_cpu(__raw_readl((bm)->addr_ce + (o))) ++#define __bm_cl_out(bm, o, val) \ ++ do { \ ++ u32 *__tmpclout = (bm)->addr_ce + (o); \ ++ __raw_writel(cpu_to_be32(val), __tmpclout); \ ++ dcbf(__tmpclout); \ ++ } while (0) ++#define __bm_cl_invalidate(bm, o) dcbi((bm)->addr_ce + (o)) ++#define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA) ++#define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA) ++#define bm_cl_in(reg) __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA) ++#define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val) ++#define bm_cl_invalidate(reg)\ ++ __bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA) ++ ++/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf ++ * analysis, look at using the "extra" bit in the ring index registers to avoid ++ * cyclic issues. */ ++static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last) ++{ ++ /* 'first' is included, 'last' is excluded */ ++ if (first <= last) ++ return last - first; ++ return ringsize + last - first; ++} ++ ++/* Portal modes. ++ * Enum types; ++ * pmode == production mode ++ * cmode == consumption mode, ++ * Enum values use 3 letter codes. First letter matches the portal mode, ++ * remaining two letters indicate; ++ * ci == cache-inhibited portal register ++ * ce == cache-enabled portal register ++ * vb == in-band valid-bit (cache-enabled) ++ */ ++enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */ ++ bm_rcr_pci = 0, /* PI index, cache-inhibited */ ++ bm_rcr_pce = 1, /* PI index, cache-enabled */ ++ bm_rcr_pvb = 2 /* valid-bit */ ++}; ++enum bm_rcr_cmode { /* s/w-only */ ++ bm_rcr_cci, /* CI index, cache-inhibited */ ++ bm_rcr_cce /* CI index, cache-enabled */ ++}; ++ ++ ++/* ------------------------- */ ++/* --- Portal structures --- */ ++ ++#define BM_RCR_SIZE 8 ++ ++struct bm_rcr { ++ struct bm_rcr_entry *ring, *cursor; ++ u8 ci, available, ithresh, vbit; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ u32 busy; ++ enum bm_rcr_pmode pmode; ++ enum bm_rcr_cmode cmode; ++#endif ++}; ++ ++struct bm_mc { ++ struct bm_mc_command *cr; ++ struct bm_mc_result *rr; ++ u8 rridx, vbit; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ enum { ++ /* Can only be _mc_start()ed */ ++ mc_idle, ++ /* Can only be _mc_commit()ed or _mc_abort()ed */ ++ mc_user, ++ /* Can only be _mc_retry()ed */ ++ mc_hw ++ } state; ++#endif ++}; ++ ++struct bm_addr { ++ void __iomem *addr_ce; /* cache-enabled */ ++ void __iomem *addr_ci; /* cache-inhibited */ ++}; ++ ++struct bm_portal { ++ struct bm_addr addr; ++ struct bm_rcr rcr; ++ struct bm_mc mc; ++ struct bm_portal_config config; ++} ____cacheline_aligned; ++ ++ ++/* --------------- */ ++/* --- RCR API --- */ ++ ++/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ ++#define RCR_CARRYCLEAR(p) \ ++ (void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6))) ++ ++/* Bit-wise logic to convert a ring pointer to a ring index */ ++static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e) ++{ ++ return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1); ++} ++ ++/* Increment the 'cursor' ring pointer, taking 'vbit' into account */ ++static inline void RCR_INC(struct bm_rcr *rcr) ++{ ++ /* NB: this is odd-looking, but experiments show that it generates ++ * fast code with essentially no branching overheads. We increment to ++ * the next RCR pointer and handle overflow and 'vbit'. */ ++ struct bm_rcr_entry *partial = rcr->cursor + 1; ++ rcr->cursor = RCR_CARRYCLEAR(partial); ++ if (partial != rcr->cursor) ++ rcr->vbit ^= BM_RCR_VERB_VBIT; ++} ++ ++static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode, ++ __maybe_unused enum bm_rcr_cmode cmode) ++{ ++ /* This use of 'register', as well as all other occurrences, is because ++ * it has been observed to generate much faster code with gcc than is ++ * otherwise the case. */ ++ register struct bm_rcr *rcr = &portal->rcr; ++ u32 cfg; ++ u8 pi; ++ ++ rcr->ring = portal->addr.addr_ce + BM_CL_RCR; ++ rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); ++ ++ pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1); ++ rcr->cursor = rcr->ring + pi; ++ rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0; ++ rcr->available = BM_RCR_SIZE - 1 ++ - bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi); ++ rcr->ithresh = bm_in(RCR_ITR); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ rcr->busy = 0; ++ rcr->pmode = pmode; ++ rcr->cmode = cmode; ++#endif ++ cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */ ++ bm_out(CFG, cfg); ++ return 0; ++} ++ ++static inline void bm_rcr_finish(struct bm_portal *portal) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1); ++ u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); ++ DPA_ASSERT(!rcr->busy); ++ if (pi != RCR_PTR2IDX(rcr->cursor)) ++ pr_crit("losing uncommited RCR entries\n"); ++ if (ci != rcr->ci) ++ pr_crit("missing existing RCR completions\n"); ++ if (rcr->ci != RCR_PTR2IDX(rcr->cursor)) ++ pr_crit("RCR destroyed unquiesced\n"); ++} ++ ++static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ DPA_ASSERT(!rcr->busy); ++ if (!rcr->available) ++ return NULL; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ rcr->busy = 1; ++#endif ++ dcbz_64(rcr->cursor); ++ return rcr->cursor; ++} ++ ++static inline void bm_rcr_abort(struct bm_portal *portal) ++{ ++ __maybe_unused register struct bm_rcr *rcr = &portal->rcr; ++ DPA_ASSERT(rcr->busy); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ rcr->busy = 0; ++#endif ++} ++ ++static inline struct bm_rcr_entry *bm_rcr_pend_and_next( ++ struct bm_portal *portal, u8 myverb) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ DPA_ASSERT(rcr->busy); ++ DPA_ASSERT(rcr->pmode != bm_rcr_pvb); ++ if (rcr->available == 1) ++ return NULL; ++ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; ++ dcbf_64(rcr->cursor); ++ RCR_INC(rcr); ++ rcr->available--; ++ dcbz_64(rcr->cursor); ++ return rcr->cursor; ++} ++ ++static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ DPA_ASSERT(rcr->busy); ++ DPA_ASSERT(rcr->pmode == bm_rcr_pci); ++ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; ++ RCR_INC(rcr); ++ rcr->available--; ++ hwsync(); ++ bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor)); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ rcr->busy = 0; ++#endif ++} ++ ++static inline void bm_rcr_pce_prefetch(struct bm_portal *portal) ++{ ++ __maybe_unused register struct bm_rcr *rcr = &portal->rcr; ++ DPA_ASSERT(rcr->pmode == bm_rcr_pce); ++ bm_cl_invalidate(RCR_PI); ++ bm_cl_touch_rw(RCR_PI); ++} ++ ++static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ DPA_ASSERT(rcr->busy); ++ DPA_ASSERT(rcr->pmode == bm_rcr_pce); ++ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; ++ RCR_INC(rcr); ++ rcr->available--; ++ lwsync(); ++ bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor)); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ rcr->busy = 0; ++#endif ++} ++ ++static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ struct bm_rcr_entry *rcursor; ++ DPA_ASSERT(rcr->busy); ++ DPA_ASSERT(rcr->pmode == bm_rcr_pvb); ++ lwsync(); ++ rcursor = rcr->cursor; ++ rcursor->__dont_write_directly__verb = myverb | rcr->vbit; ++ dcbf_64(rcursor); ++ RCR_INC(rcr); ++ rcr->available--; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ rcr->busy = 0; ++#endif ++} ++ ++static inline u8 bm_rcr_cci_update(struct bm_portal *portal) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ u8 diff, old_ci = rcr->ci; ++ DPA_ASSERT(rcr->cmode == bm_rcr_cci); ++ rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); ++ diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); ++ rcr->available += diff; ++ return diff; ++} ++ ++static inline void bm_rcr_cce_prefetch(struct bm_portal *portal) ++{ ++ __maybe_unused register struct bm_rcr *rcr = &portal->rcr; ++ DPA_ASSERT(rcr->cmode == bm_rcr_cce); ++ bm_cl_touch_ro(RCR_CI); ++} ++ ++static inline u8 bm_rcr_cce_update(struct bm_portal *portal) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ u8 diff, old_ci = rcr->ci; ++ DPA_ASSERT(rcr->cmode == bm_rcr_cce); ++ rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1); ++ bm_cl_invalidate(RCR_CI); ++ diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); ++ rcr->available += diff; ++ return diff; ++} ++ ++static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ return rcr->ithresh; ++} ++ ++static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ rcr->ithresh = ithresh; ++ bm_out(RCR_ITR, ithresh); ++} ++ ++static inline u8 bm_rcr_get_avail(struct bm_portal *portal) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ return rcr->available; ++} ++ ++static inline u8 bm_rcr_get_fill(struct bm_portal *portal) ++{ ++ register struct bm_rcr *rcr = &portal->rcr; ++ return BM_RCR_SIZE - 1 - rcr->available; ++} ++ ++ ++/* ------------------------------ */ ++/* --- Management command API --- */ ++ ++static inline int bm_mc_init(struct bm_portal *portal) ++{ ++ register struct bm_mc *mc = &portal->mc; ++ mc->cr = portal->addr.addr_ce + BM_CL_CR; ++ mc->rr = portal->addr.addr_ce + BM_CL_RR0; ++ mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) & ++ BM_MCC_VERB_VBIT) ? 0 : 1; ++ mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = mc_idle; ++#endif ++ return 0; ++} ++ ++static inline void bm_mc_finish(struct bm_portal *portal) ++{ ++ __maybe_unused register struct bm_mc *mc = &portal->mc; ++ DPA_ASSERT(mc->state == mc_idle); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (mc->state != mc_idle) ++ pr_crit("Losing incomplete MC command\n"); ++#endif ++} ++ ++static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal) ++{ ++ register struct bm_mc *mc = &portal->mc; ++ DPA_ASSERT(mc->state == mc_idle); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = mc_user; ++#endif ++ dcbz_64(mc->cr); ++ return mc->cr; ++} ++ ++static inline void bm_mc_abort(struct bm_portal *portal) ++{ ++ __maybe_unused register struct bm_mc *mc = &portal->mc; ++ DPA_ASSERT(mc->state == mc_user); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = mc_idle; ++#endif ++} ++ ++static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb) ++{ ++ register struct bm_mc *mc = &portal->mc; ++ struct bm_mc_result *rr = mc->rr + mc->rridx; ++ DPA_ASSERT(mc->state == mc_user); ++ lwsync(); ++ mc->cr->__dont_write_directly__verb = myverb | mc->vbit; ++ dcbf(mc->cr); ++ dcbit_ro(rr); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = mc_hw; ++#endif ++} ++ ++static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal) ++{ ++ register struct bm_mc *mc = &portal->mc; ++ struct bm_mc_result *rr = mc->rr + mc->rridx; ++ DPA_ASSERT(mc->state == mc_hw); ++ /* The inactive response register's verb byte always returns zero until ++ * its command is submitted and completed. This includes the valid-bit, ++ * in case you were wondering... */ ++ if (!__raw_readb(&rr->verb)) { ++ dcbit_ro(rr); ++ return NULL; ++ } ++ mc->rridx ^= 1; ++ mc->vbit ^= BM_MCC_VERB_VBIT; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = mc_idle; ++#endif ++ return rr; ++} ++ ++ ++/* ------------------------------------- */ ++/* --- Portal interrupt register API --- */ ++ ++static inline int bm_isr_init(__always_unused struct bm_portal *portal) ++{ ++ return 0; ++} ++ ++static inline void bm_isr_finish(__always_unused struct bm_portal *portal) ++{ ++} ++ ++#define SCN_REG(bpid) BM_REG_SCN((bpid) / 32) ++#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31)) ++static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid, ++ int enable) ++{ ++ u32 val; ++ DPA_ASSERT(bpid < bman_pool_max); ++ /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */ ++ val = __bm_in(&portal->addr, SCN_REG(bpid)); ++ if (enable) ++ val |= SCN_BIT(bpid); ++ else ++ val &= ~SCN_BIT(bpid); ++ __bm_out(&portal->addr, SCN_REG(bpid), val); ++} ++ ++static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n) ++{ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ return __bm_in(&portal->addr, BM_REG_ISR + (n << 6)); ++#else ++ return __bm_in(&portal->addr, BM_REG_ISR + (n << 2)); ++#endif ++} ++ ++static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, ++ u32 val) ++{ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ __bm_out(&portal->addr, BM_REG_ISR + (n << 6), val); ++#else ++ __bm_out(&portal->addr, BM_REG_ISR + (n << 2), val); ++#endif ++} ++ ++/* Buffer Pool Cleanup */ ++static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid) ++{ ++ struct bm_mc_command *bm_cmd; ++ struct bm_mc_result *bm_res; ++ ++ int aq_count = 0; ++ bool stop = false; ++ while (!stop) { ++ /* Acquire buffers until empty */ ++ bm_cmd = bm_mc_start(p); ++ bm_cmd->acquire.bpid = bpid; ++ bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE | 1); ++ while (!(bm_res = bm_mc_result(p))) ++ cpu_relax(); ++ if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) { ++ /* Pool is empty */ ++ /* TBD : Should we do a few extra iterations in ++ case some other some blocks keep buffers 'on deck', ++ which may also be problematic */ ++ stop = true; ++ } else ++ ++aq_count; ++ } ++ return 0; ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_private.h +@@ -0,0 +1,166 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "dpa_sys.h" ++#include <linux/fsl_bman.h> ++ ++/* Revision info (for errata and feature handling) */ ++#define BMAN_REV10 0x0100 ++#define BMAN_REV20 0x0200 ++#define BMAN_REV21 0x0201 ++#define QBMAN_ANY_PORTAL_IDX 0xffffffff ++extern u16 bman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */ ++ ++/* ++ * Global variables of the max portal/pool number this bman version supported ++ */ ++extern u16 bman_pool_max; ++ ++/* used by CCSR and portal interrupt code */ ++enum bm_isr_reg { ++ bm_isr_status = 0, ++ bm_isr_enable = 1, ++ bm_isr_disable = 2, ++ bm_isr_inhibit = 3 ++}; ++ ++struct bm_portal_config { ++ /* Corenet portal addresses; ++ * [0]==cache-enabled, [1]==cache-inhibited. */ ++ __iomem void *addr_virt[2]; ++ struct resource addr_phys[2]; ++ /* Allow these to be joined in lists */ ++ struct list_head list; ++ /* User-visible portal configuration settings */ ++ struct bman_portal_config public_cfg; ++ /* power management saved data */ ++ u32 saved_isdr; ++}; ++ ++#ifdef CONFIG_FSL_BMAN_CONFIG ++/* Hooks from bman_driver.c to bman_config.c */ ++int bman_init_ccsr(struct device_node *node); ++#endif ++ ++/* Hooks from bman_driver.c in to bman_high.c */ ++struct bman_portal *bman_create_portal( ++ struct bman_portal *portal, ++ const struct bm_portal_config *config); ++struct bman_portal *bman_create_affine_portal( ++ const struct bm_portal_config *config); ++struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect, ++ int cpu); ++void bman_destroy_portal(struct bman_portal *bm); ++ ++const struct bm_portal_config *bman_destroy_affine_portal(void); ++ ++/* Hooks from fsl_usdpaa.c to bman_driver.c */ ++struct bm_portal_config *bm_get_unused_portal(void); ++struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx); ++void bm_put_unused_portal(struct bm_portal_config *pcfg); ++void bm_set_liodns(struct bm_portal_config *pcfg); ++ ++/* Pool logic in the portal driver, during initialisation, needs to know if ++ * there's access to CCSR or not (if not, it'll cripple the pool allocator). */ ++#ifdef CONFIG_FSL_BMAN_CONFIG ++int bman_have_ccsr(void); ++#else ++#define bman_have_ccsr() 0 ++#endif ++ ++/* Stockpile build constants. The _LOW value: when bman_acquire() is called and ++ * the stockpile fill-level is <= _LOW, an acquire is attempted from h/w but it ++ * might fail (if the buffer pool is depleted). So this value provides some ++ * "stagger" in that the bman_acquire() function will only fail if lots of bufs ++ * are requested at once or if h/w has been tested a couple of times without ++ * luck. The _HIGH value: when bman_release() is called and the stockpile ++ * fill-level is >= _HIGH, a release is attempted to h/w but it might fail (if ++ * the release ring is full). So this value provides some "stagger" so that ++ * ring-access is retried a couple of times prior to the API returning a ++ * failure. The following *must* be true; ++ * BMAN_STOCKPILE_HIGH-BMAN_STOCKPILE_LOW > 8 ++ * (to avoid thrashing) ++ * BMAN_STOCKPILE_SZ >= 16 ++ * (as the release logic expects to either send 8 buffers to hw prior to ++ * adding the given buffers to the stockpile or add the buffers to the ++ * stockpile before sending 8 to hw, as the API must be an all-or-nothing ++ * success/fail.) ++ */ ++#define BMAN_STOCKPILE_SZ 16u /* number of bufs in per-pool cache */ ++#define BMAN_STOCKPILE_LOW 2u /* when fill is <= this, acquire from hw */ ++#define BMAN_STOCKPILE_HIGH 14u /* when fill is >= this, release to hw */ ++ ++/*************************************************/ ++/* BMan s/w corenet portal, low-level i/face */ ++/*************************************************/ ++ ++/* Used by all portal interrupt registers except 'inhibit' ++ * This mask contains all the "irqsource" bits visible to API users ++ */ ++#define BM_PIRQ_VISIBLE (BM_PIRQ_RCRI | BM_PIRQ_BSCN) ++ ++/* These are bm_<reg>_<verb>(). So for example, bm_disable_write() means "write ++ * the disable register" rather than "disable the ability to write". */ ++#define bm_isr_status_read(bm) __bm_isr_read(bm, bm_isr_status) ++#define bm_isr_status_clear(bm, m) __bm_isr_write(bm, bm_isr_status, m) ++#define bm_isr_enable_read(bm) __bm_isr_read(bm, bm_isr_enable) ++#define bm_isr_enable_write(bm, v) __bm_isr_write(bm, bm_isr_enable, v) ++#define bm_isr_disable_read(bm) __bm_isr_read(bm, bm_isr_disable) ++#define bm_isr_disable_write(bm, v) __bm_isr_write(bm, bm_isr_disable, v) ++#define bm_isr_inhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 1) ++#define bm_isr_uninhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 0) ++ ++#ifdef CONFIG_FSL_BMAN_CONFIG ++/* Set depletion thresholds associated with a buffer pool. Requires that the ++ * operating system have access to Bman CCSR (ie. compiled in support and ++ * run-time access courtesy of the device-tree). */ ++int bm_pool_set(u32 bpid, const u32 *thresholds); ++#define BM_POOL_THRESH_SW_ENTER 0 ++#define BM_POOL_THRESH_SW_EXIT 1 ++#define BM_POOL_THRESH_HW_ENTER 2 ++#define BM_POOL_THRESH_HW_EXIT 3 ++ ++/* Read the free buffer count for a given buffer */ ++u32 bm_pool_free_buffers(u32 bpid); ++ ++__init int bman_init(void); ++__init int bman_resource_init(void); ++ ++const struct bm_portal_config *bman_get_bm_portal_config( ++ struct bman_portal *portal); ++ ++/* power management */ ++#ifdef CONFIG_SUSPEND ++void suspend_unused_bportal(void); ++void resume_unused_bportal(void); ++#endif ++ ++#endif /* CONFIG_FSL_BMAN_CONFIG */ +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_test.c +@@ -0,0 +1,56 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "bman_test.h" ++ ++MODULE_AUTHOR("Geoff Thorpe"); ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_DESCRIPTION("Bman testing"); ++ ++static int test_init(void) ++{ ++#ifdef CONFIG_FSL_BMAN_TEST_HIGH ++ int loop = 1; ++ while (loop--) ++ bman_test_high(); ++#endif ++#ifdef CONFIG_FSL_BMAN_TEST_THRESH ++ bman_test_thresh(); ++#endif ++ return 0; ++} ++ ++static void test_exit(void) ++{ ++} ++ ++module_init(test_init); ++module_exit(test_exit); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_test.h +@@ -0,0 +1,44 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include <linux/kernel.h> ++#include <linux/errno.h> ++#include <linux/io.h> ++#include <linux/slab.h> ++#include <linux/module.h> ++#include <linux/interrupt.h> ++#include <linux/delay.h> ++#include <linux/kthread.h> ++ ++#include <linux/fsl_bman.h> ++ ++void bman_test_high(void); ++void bman_test_thresh(void); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_test_high.c +@@ -0,0 +1,183 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "bman_test.h" ++#include "bman_private.h" ++ ++/*************/ ++/* constants */ ++/*************/ ++ ++#define PORTAL_OPAQUE ((void *)0xf00dbeef) ++#define POOL_OPAQUE ((void *)0xdeadabba) ++#define NUM_BUFS 93 ++#define LOOPS 3 ++#define BMAN_TOKEN_MASK 0x00FFFFFFFFFFLLU ++ ++/***************/ ++/* global vars */ ++/***************/ ++ ++static struct bman_pool *pool; ++static int depleted; ++static struct bm_buffer bufs_in[NUM_BUFS] ____cacheline_aligned; ++static struct bm_buffer bufs_out[NUM_BUFS] ____cacheline_aligned; ++static int bufs_received; ++ ++/* Predeclare the callback so we can instantiate pool parameters */ ++static void depletion_cb(struct bman_portal *, struct bman_pool *, void *, int); ++ ++/**********************/ ++/* internal functions */ ++/**********************/ ++ ++static void bufs_init(void) ++{ ++ int i; ++ for (i = 0; i < NUM_BUFS; i++) ++ bm_buffer_set64(&bufs_in[i], 0xfedc01234567LLU * i); ++ bufs_received = 0; ++} ++ ++static inline int bufs_cmp(const struct bm_buffer *a, const struct bm_buffer *b) ++{ ++ if ((bman_ip_rev == BMAN_REV20) || (bman_ip_rev == BMAN_REV21)) { ++ ++ /* On SoCs with Bman revison 2.0, Bman only respects the 40 ++ * LS-bits of buffer addresses, masking off the upper 8-bits on ++ * release commands. The API provides for 48-bit addresses ++ * because some SoCs support all 48-bits. When generating ++ * garbage addresses for testing, we either need to zero the ++ * upper 8-bits when releasing to Bman (otherwise we'll be ++ * disappointed when the buffers we acquire back from Bman ++ * don't match), or we need to mask the upper 8-bits off when ++ * comparing. We do the latter. ++ */ ++ if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK) ++ < (bm_buffer_get64(b) & BMAN_TOKEN_MASK)) ++ return -1; ++ if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK) ++ > (bm_buffer_get64(b) & BMAN_TOKEN_MASK)) ++ return 1; ++ } else { ++ if (bm_buffer_get64(a) < bm_buffer_get64(b)) ++ return -1; ++ if (bm_buffer_get64(a) > bm_buffer_get64(b)) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++static void bufs_confirm(void) ++{ ++ int i, j; ++ for (i = 0; i < NUM_BUFS; i++) { ++ int matches = 0; ++ for (j = 0; j < NUM_BUFS; j++) ++ if (!bufs_cmp(&bufs_in[i], &bufs_out[j])) ++ matches++; ++ BUG_ON(matches != 1); ++ } ++} ++ ++/********/ ++/* test */ ++/********/ ++ ++static void depletion_cb(struct bman_portal *__portal, struct bman_pool *__pool, ++ void *pool_ctx, int __depleted) ++{ ++ BUG_ON(__pool != pool); ++ BUG_ON(pool_ctx != POOL_OPAQUE); ++ depleted = __depleted; ++} ++ ++void bman_test_high(void) ++{ ++ struct bman_pool_params pparams = { ++ .flags = BMAN_POOL_FLAG_DEPLETION | BMAN_POOL_FLAG_DYNAMIC_BPID, ++ .cb = depletion_cb, ++ .cb_ctx = POOL_OPAQUE, ++ }; ++ int i, loops = LOOPS; ++ struct bm_buffer tmp_buf; ++ ++ bufs_init(); ++ ++ pr_info("BMAN: --- starting high-level test ---\n"); ++ ++ pool = bman_new_pool(&pparams); ++ BUG_ON(!pool); ++ ++ /*******************/ ++ /* Release buffers */ ++ /*******************/ ++do_loop: ++ i = 0; ++ while (i < NUM_BUFS) { ++ u32 flags = BMAN_RELEASE_FLAG_WAIT; ++ int num = 8; ++ if ((i + num) > NUM_BUFS) ++ num = NUM_BUFS - i; ++ if ((i + num) == NUM_BUFS) ++ flags |= BMAN_RELEASE_FLAG_WAIT_SYNC; ++ if (bman_release(pool, bufs_in + i, num, flags)) ++ panic("bman_release() failed\n"); ++ i += num; ++ } ++ ++ /*******************/ ++ /* Acquire buffers */ ++ /*******************/ ++ while (i > 0) { ++ int tmp, num = 8; ++ if (num > i) ++ num = i; ++ tmp = bman_acquire(pool, bufs_out + i - num, num, 0); ++ BUG_ON(tmp != num); ++ i -= num; ++ } ++ ++ i = bman_acquire(pool, &tmp_buf, 1, 0); ++ BUG_ON(i > 0); ++ ++ bufs_confirm(); ++ ++ if (--loops) ++ goto do_loop; ++ ++ /************/ ++ /* Clean up */ ++ /************/ ++ bman_free_pool(pool); ++ pr_info("BMAN: --- finished high-level test ---\n"); ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/bman_test_thresh.c +@@ -0,0 +1,196 @@ ++/* Copyright 2010-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "bman_test.h" ++ ++/* Test constants */ ++#define TEST_NUMBUFS 129728 ++#define TEST_EXIT 129536 ++#define TEST_ENTRY 129024 ++ ++struct affine_test_data { ++ struct task_struct *t; ++ int cpu; ++ int expect_affinity; ++ int drain; ++ int num_enter; ++ int num_exit; ++ struct list_head node; ++ struct completion wakethread; ++ struct completion wakeparent; ++}; ++ ++static void cb_depletion(struct bman_portal *portal, ++ struct bman_pool *pool, ++ void *opaque, ++ int depleted) ++{ ++ struct affine_test_data *data = opaque; ++ int c = smp_processor_id(); ++ pr_info("cb_depletion: bpid=%d, depleted=%d, cpu=%d, original=%d\n", ++ bman_get_params(pool)->bpid, !!depleted, c, data->cpu); ++ /* We should be executing on the CPU of the thread that owns the pool if ++ * and that CPU has an affine portal (ie. it isn't slaved). */ ++ BUG_ON((c != data->cpu) && data->expect_affinity); ++ BUG_ON((c == data->cpu) && !data->expect_affinity); ++ if (depleted) ++ data->num_enter++; ++ else ++ data->num_exit++; ++} ++ ++/* Params used to set up a pool, this also dynamically allocates a BPID */ ++static const struct bman_pool_params params_nocb = { ++ .flags = BMAN_POOL_FLAG_DYNAMIC_BPID | BMAN_POOL_FLAG_THRESH, ++ .thresholds = { TEST_ENTRY, TEST_EXIT, 0, 0 } ++}; ++ ++/* Params used to set up each cpu's pool with callbacks enabled */ ++static struct bman_pool_params params_cb = { ++ .bpid = 0, /* will be replaced to match pool_nocb */ ++ .flags = BMAN_POOL_FLAG_DEPLETION, ++ .cb = cb_depletion ++}; ++ ++static struct bman_pool *pool_nocb; ++static LIST_HEAD(threads); ++ ++static int affine_test(void *__data) ++{ ++ struct bman_pool *pool; ++ struct affine_test_data *data = __data; ++ struct bman_pool_params my_params = params_cb; ++ ++ pr_info("thread %d: starting\n", data->cpu); ++ /* create the pool */ ++ my_params.cb_ctx = data; ++ pool = bman_new_pool(&my_params); ++ BUG_ON(!pool); ++ complete(&data->wakeparent); ++ wait_for_completion(&data->wakethread); ++ init_completion(&data->wakethread); ++ ++ /* if we're the drainer, we get signalled for that */ ++ if (data->drain) { ++ struct bm_buffer buf; ++ int ret; ++ pr_info("thread %d: draining...\n", data->cpu); ++ do { ++ ret = bman_acquire(pool, &buf, 1, 0); ++ } while (ret > 0); ++ pr_info("thread %d: draining done.\n", data->cpu); ++ complete(&data->wakeparent); ++ wait_for_completion(&data->wakethread); ++ init_completion(&data->wakethread); ++ } ++ ++ /* cleanup */ ++ bman_free_pool(pool); ++ while (!kthread_should_stop()) ++ cpu_relax(); ++ pr_info("thread %d: exiting\n", data->cpu); ++ return 0; ++} ++ ++static struct affine_test_data *start_affine_test(int cpu, int drain) ++{ ++ struct affine_test_data *data = kmalloc(sizeof(*data), GFP_KERNEL); ++ ++ if (!data) ++ return NULL; ++ data->cpu = cpu; ++ data->expect_affinity = cpumask_test_cpu(cpu, bman_affine_cpus()); ++ data->drain = drain; ++ data->num_enter = 0; ++ data->num_exit = 0; ++ init_completion(&data->wakethread); ++ init_completion(&data->wakeparent); ++ list_add_tail(&data->node, &threads); ++ data->t = kthread_create(affine_test, data, "threshtest%d", cpu); ++ BUG_ON(IS_ERR(data->t)); ++ kthread_bind(data->t, cpu); ++ wake_up_process(data->t); ++ return data; ++} ++ ++void bman_test_thresh(void) ++{ ++ int loop = TEST_NUMBUFS; ++ int ret, num_cpus = 0; ++ struct affine_test_data *data, *drainer = NULL; ++ ++ pr_info("bman_test_thresh: start\n"); ++ ++ /* allocate a BPID and seed it */ ++ pool_nocb = bman_new_pool(¶ms_nocb); ++ BUG_ON(!pool_nocb); ++ while (loop--) { ++ struct bm_buffer buf; ++ bm_buffer_set64(&buf, 0x0badbeef + loop); ++ ret = bman_release(pool_nocb, &buf, 1, ++ BMAN_RELEASE_FLAG_WAIT); ++ BUG_ON(ret); ++ } ++ while (!bman_rcr_is_empty()) ++ cpu_relax(); ++ pr_info("bman_test_thresh: buffers are in\n"); ++ ++ /* create threads and wait for them to create pools */ ++ params_cb.bpid = bman_get_params(pool_nocb)->bpid; ++ for_each_cpu(loop, cpu_online_mask) { ++ data = start_affine_test(loop, drainer ? 0 : 1); ++ BUG_ON(!data); ++ if (!drainer) ++ drainer = data; ++ num_cpus++; ++ wait_for_completion(&data->wakeparent); ++ } ++ ++ /* signal the drainer to start draining */ ++ complete(&drainer->wakethread); ++ wait_for_completion(&drainer->wakeparent); ++ init_completion(&drainer->wakeparent); ++ ++ /* tear down */ ++ list_for_each_entry_safe(data, drainer, &threads, node) { ++ complete(&data->wakethread); ++ ret = kthread_stop(data->t); ++ BUG_ON(ret); ++ list_del(&data->node); ++ /* check that we get the expected callbacks (and no others) */ ++ BUG_ON(data->num_enter != 1); ++ BUG_ON(data->num_exit != 0); ++ kfree(data); ++ } ++ bman_free_pool(pool_nocb); ++ ++ pr_info("bman_test_thresh: done\n"); ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/dpa_alloc.c +@@ -0,0 +1,706 @@ ++/* Copyright 2009-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "dpa_sys.h" ++#include <linux/fsl_qman.h> ++#include <linux/fsl_bman.h> ++ ++/* Qman and Bman APIs are front-ends to the common code; */ ++ ++static DECLARE_DPA_ALLOC(bpalloc); /* BPID allocator */ ++static DECLARE_DPA_ALLOC(fqalloc); /* FQID allocator */ ++static DECLARE_DPA_ALLOC(qpalloc); /* pool-channel allocator */ ++static DECLARE_DPA_ALLOC(cgralloc); /* CGR ID allocator */ ++static DECLARE_DPA_ALLOC(ceetm0_challoc); /* CEETM Channel ID allocator */ ++static DECLARE_DPA_ALLOC(ceetm0_lfqidalloc); /* CEETM LFQID allocator */ ++static DECLARE_DPA_ALLOC(ceetm1_challoc); /* CEETM Channel ID allocator */ ++static DECLARE_DPA_ALLOC(ceetm1_lfqidalloc); /* CEETM LFQID allocator */ ++ ++/* This is a sort-of-conditional dpa_alloc_free() routine. Eg. when releasing ++ * FQIDs (probably from user-space), it can filter out those that aren't in the ++ * OOS state (better to leak a h/w resource than to crash). This function ++ * returns the number of invalid IDs that were not released. */ ++static u32 release_id_range(struct dpa_alloc *alloc, u32 id, u32 count, ++ int (*is_valid)(u32 id)) ++{ ++ int valid_mode = 0; ++ u32 loop = id, total_invalid = 0; ++ while (loop < (id + count)) { ++ int isvalid = is_valid ? is_valid(loop) : 1; ++ if (!valid_mode) { ++ /* We're looking for a valid ID to terminate an invalid ++ * range */ ++ if (isvalid) { ++ /* We finished a range of invalid IDs, a valid ++ * range is now underway */ ++ valid_mode = 1; ++ count -= (loop - id); ++ id = loop; ++ } else ++ total_invalid++; ++ } else { ++ /* We're looking for an invalid ID to terminate a ++ * valid range */ ++ if (!isvalid) { ++ /* Release the range of valid IDs, an unvalid ++ * range is now underway */ ++ if (loop > id) ++ dpa_alloc_free(alloc, id, loop - id); ++ valid_mode = 0; ++ } ++ } ++ loop++; ++ } ++ /* Release any unterminated range of valid IDs */ ++ if (valid_mode && count) ++ dpa_alloc_free(alloc, id, count); ++ return total_invalid; ++} ++ ++/* BPID allocator front-end */ ++ ++int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial) ++{ ++ return dpa_alloc_new(&bpalloc, result, count, align, partial); ++} ++EXPORT_SYMBOL(bman_alloc_bpid_range); ++ ++static int bp_cleanup(u32 bpid) ++{ ++ return bman_shutdown_pool(bpid) == 0; ++} ++void bman_release_bpid_range(u32 bpid, u32 count) ++{ ++ u32 total_invalid = release_id_range(&bpalloc, bpid, count, bp_cleanup); ++ if (total_invalid) ++ pr_err("BPID range [%d..%d] (%d) had %d leaks\n", ++ bpid, bpid + count - 1, count, total_invalid); ++} ++EXPORT_SYMBOL(bman_release_bpid_range); ++ ++void bman_seed_bpid_range(u32 bpid, u32 count) ++{ ++ dpa_alloc_seed(&bpalloc, bpid, count); ++} ++EXPORT_SYMBOL(bman_seed_bpid_range); ++ ++int bman_reserve_bpid_range(u32 bpid, u32 count) ++{ ++ return dpa_alloc_reserve(&bpalloc, bpid, count); ++} ++EXPORT_SYMBOL(bman_reserve_bpid_range); ++ ++ ++/* FQID allocator front-end */ ++ ++int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial) ++{ ++ return dpa_alloc_new(&fqalloc, result, count, align, partial); ++} ++EXPORT_SYMBOL(qman_alloc_fqid_range); ++ ++static int fq_cleanup(u32 fqid) ++{ ++ return qman_shutdown_fq(fqid) == 0; ++} ++void qman_release_fqid_range(u32 fqid, u32 count) ++{ ++ u32 total_invalid = release_id_range(&fqalloc, fqid, count, fq_cleanup); ++ if (total_invalid) ++ pr_err("FQID range [%d..%d] (%d) had %d leaks\n", ++ fqid, fqid + count - 1, count, total_invalid); ++} ++EXPORT_SYMBOL(qman_release_fqid_range); ++ ++int qman_reserve_fqid_range(u32 fqid, u32 count) ++{ ++ return dpa_alloc_reserve(&fqalloc, fqid, count); ++} ++EXPORT_SYMBOL(qman_reserve_fqid_range); ++ ++void qman_seed_fqid_range(u32 fqid, u32 count) ++{ ++ dpa_alloc_seed(&fqalloc, fqid, count); ++} ++EXPORT_SYMBOL(qman_seed_fqid_range); ++ ++/* Pool-channel allocator front-end */ ++ ++int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial) ++{ ++ return dpa_alloc_new(&qpalloc, result, count, align, partial); ++} ++EXPORT_SYMBOL(qman_alloc_pool_range); ++ ++static int qpool_cleanup(u32 qp) ++{ ++ /* We query all FQDs starting from ++ * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs ++ * whose destination channel is the pool-channel being released. ++ * When a non-OOS FQD is found we attempt to clean it up */ ++ struct qman_fq fq = { ++ .fqid = 1 ++ }; ++ int err; ++ do { ++ struct qm_mcr_queryfq_np np; ++ err = qman_query_fq_np(&fq, &np); ++ if (err) ++ /* FQID range exceeded, found no problems */ ++ return 1; ++ if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) { ++ struct qm_fqd fqd; ++ err = qman_query_fq(&fq, &fqd); ++ BUG_ON(err); ++ if (fqd.dest.channel == qp) { ++ /* The channel is the FQ's target, clean it */ ++ if (qman_shutdown_fq(fq.fqid) != 0) ++ /* Couldn't shut down the FQ ++ so the pool must be leaked */ ++ return 0; ++ } ++ } ++ /* Move to the next FQID */ ++ fq.fqid++; ++ } while (1); ++} ++void qman_release_pool_range(u32 qp, u32 count) ++{ ++ u32 total_invalid = release_id_range(&qpalloc, qp, ++ count, qpool_cleanup); ++ if (total_invalid) { ++ /* Pool channels are almost always used individually */ ++ if (count == 1) ++ pr_err("Pool channel 0x%x had %d leaks\n", ++ qp, total_invalid); ++ else ++ pr_err("Pool channels [%d..%d] (%d) had %d leaks\n", ++ qp, qp + count - 1, count, total_invalid); ++ } ++} ++EXPORT_SYMBOL(qman_release_pool_range); ++ ++ ++void qman_seed_pool_range(u32 poolid, u32 count) ++{ ++ dpa_alloc_seed(&qpalloc, poolid, count); ++ ++} ++EXPORT_SYMBOL(qman_seed_pool_range); ++ ++int qman_reserve_pool_range(u32 poolid, u32 count) ++{ ++ return dpa_alloc_reserve(&qpalloc, poolid, count); ++} ++EXPORT_SYMBOL(qman_reserve_pool_range); ++ ++ ++/* CGR ID allocator front-end */ ++ ++int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial) ++{ ++ return dpa_alloc_new(&cgralloc, result, count, align, partial); ++} ++EXPORT_SYMBOL(qman_alloc_cgrid_range); ++ ++static int cqr_cleanup(u32 cgrid) ++{ ++ /* We query all FQDs starting from ++ * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs ++ * whose CGR is the CGR being released. ++ */ ++ struct qman_fq fq = { ++ .fqid = 1 ++ }; ++ int err; ++ do { ++ struct qm_mcr_queryfq_np np; ++ err = qman_query_fq_np(&fq, &np); ++ if (err) ++ /* FQID range exceeded, found no problems */ ++ return 1; ++ if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) { ++ struct qm_fqd fqd; ++ err = qman_query_fq(&fq, &fqd); ++ BUG_ON(err); ++ if ((fqd.fq_ctrl & QM_FQCTRL_CGE) && ++ (fqd.cgid == cgrid)) { ++ pr_err("CRGID 0x%x is being used by FQID 0x%x," ++ " CGR will be leaked\n", ++ cgrid, fq.fqid); ++ return 1; ++ } ++ } ++ /* Move to the next FQID */ ++ fq.fqid++; ++ } while (1); ++} ++ ++void qman_release_cgrid_range(u32 cgrid, u32 count) ++{ ++ u32 total_invalid = release_id_range(&cgralloc, cgrid, ++ count, cqr_cleanup); ++ if (total_invalid) ++ pr_err("CGRID range [%d..%d] (%d) had %d leaks\n", ++ cgrid, cgrid + count - 1, count, total_invalid); ++} ++EXPORT_SYMBOL(qman_release_cgrid_range); ++ ++void qman_seed_cgrid_range(u32 cgrid, u32 count) ++{ ++ dpa_alloc_seed(&cgralloc, cgrid, count); ++ ++} ++EXPORT_SYMBOL(qman_seed_cgrid_range); ++ ++/* CEETM CHANNEL ID allocator front-end */ ++int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align, ++ int partial) ++{ ++ return dpa_alloc_new(&ceetm0_challoc, result, count, align, partial); ++} ++EXPORT_SYMBOL(qman_alloc_ceetm0_channel_range); ++ ++int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align, ++ int partial) ++{ ++ return dpa_alloc_new(&ceetm1_challoc, result, count, align, partial); ++} ++EXPORT_SYMBOL(qman_alloc_ceetm1_channel_range); ++ ++void qman_release_ceetm0_channel_range(u32 channelid, u32 count) ++{ ++ u32 total_invalid; ++ ++ total_invalid = release_id_range(&ceetm0_challoc, channelid, count, ++ NULL); ++ if (total_invalid) ++ pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n", ++ channelid, channelid + count - 1, count, total_invalid); ++} ++EXPORT_SYMBOL(qman_release_ceetm0_channel_range); ++ ++void qman_seed_ceetm0_channel_range(u32 channelid, u32 count) ++{ ++ dpa_alloc_seed(&ceetm0_challoc, channelid, count); ++ ++} ++EXPORT_SYMBOL(qman_seed_ceetm0_channel_range); ++ ++void qman_release_ceetm1_channel_range(u32 channelid, u32 count) ++{ ++ u32 total_invalid; ++ total_invalid = release_id_range(&ceetm1_challoc, channelid, count, ++ NULL); ++ if (total_invalid) ++ pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n", ++ channelid, channelid + count - 1, count, total_invalid); ++} ++EXPORT_SYMBOL(qman_release_ceetm1_channel_range); ++ ++void qman_seed_ceetm1_channel_range(u32 channelid, u32 count) ++{ ++ dpa_alloc_seed(&ceetm1_challoc, channelid, count); ++ ++} ++EXPORT_SYMBOL(qman_seed_ceetm1_channel_range); ++ ++/* CEETM LFQID allocator front-end */ ++int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align, ++ int partial) ++{ ++ return dpa_alloc_new(&ceetm0_lfqidalloc, result, count, align, partial); ++} ++EXPORT_SYMBOL(qman_alloc_ceetm0_lfqid_range); ++ ++int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align, ++ int partial) ++{ ++ return dpa_alloc_new(&ceetm1_lfqidalloc, result, count, align, partial); ++} ++EXPORT_SYMBOL(qman_alloc_ceetm1_lfqid_range); ++ ++void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count) ++{ ++ u32 total_invalid; ++ ++ total_invalid = release_id_range(&ceetm0_lfqidalloc, lfqid, count, ++ NULL); ++ if (total_invalid) ++ pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n", ++ lfqid, lfqid + count - 1, count, total_invalid); ++} ++EXPORT_SYMBOL(qman_release_ceetm0_lfqid_range); ++ ++void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count) ++{ ++ dpa_alloc_seed(&ceetm0_lfqidalloc, lfqid, count); ++ ++} ++EXPORT_SYMBOL(qman_seed_ceetm0_lfqid_range); ++ ++void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count) ++{ ++ u32 total_invalid; ++ ++ total_invalid = release_id_range(&ceetm1_lfqidalloc, lfqid, count, ++ NULL); ++ if (total_invalid) ++ pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n", ++ lfqid, lfqid + count - 1, count, total_invalid); ++} ++EXPORT_SYMBOL(qman_release_ceetm1_lfqid_range); ++ ++void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count) ++{ ++ dpa_alloc_seed(&ceetm1_lfqidalloc, lfqid, count); ++ ++} ++EXPORT_SYMBOL(qman_seed_ceetm1_lfqid_range); ++ ++ ++/* Everything else is the common backend to all the allocators */ ++ ++/* The allocator is a (possibly-empty) list of these; */ ++struct alloc_node { ++ struct list_head list; ++ u32 base; ++ u32 num; ++ /* refcount and is_alloced are only set ++ when the node is in the used list */ ++ unsigned int refcount; ++ int is_alloced; ++}; ++ ++/* #define DPA_ALLOC_DEBUG */ ++ ++#ifdef DPA_ALLOC_DEBUG ++#define DPRINT pr_info ++static void DUMP(struct dpa_alloc *alloc) ++{ ++ int off = 0; ++ char buf[256]; ++ struct alloc_node *p; ++ pr_info("Free Nodes\n"); ++ list_for_each_entry(p, &alloc->free, list) { ++ if (off < 255) ++ off += snprintf(buf + off, 255-off, "{%d,%d}", ++ p->base, p->base + p->num - 1); ++ } ++ pr_info("%s\n", buf); ++ ++ off = 0; ++ pr_info("Used Nodes\n"); ++ list_for_each_entry(p, &alloc->used, list) { ++ if (off < 255) ++ off += snprintf(buf + off, 255-off, "{%d,%d}", ++ p->base, p->base + p->num - 1); ++ } ++ pr_info("%s\n", buf); ++ ++ ++ ++} ++#else ++#define DPRINT(x...) ++#define DUMP(a) ++#endif ++ ++int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align, ++ int partial) ++{ ++ struct alloc_node *i = NULL, *next_best = NULL, *used_node = NULL; ++ u32 base, next_best_base = 0, num = 0, next_best_num = 0; ++ struct alloc_node *margin_left, *margin_right; ++ ++ *result = (u32)-1; ++ DPRINT("alloc_range(%d,%d,%d)\n", count, align, partial); ++ DUMP(alloc); ++ /* If 'align' is 0, it should behave as though it was 1 */ ++ if (!align) ++ align = 1; ++ margin_left = kmalloc(sizeof(*margin_left), GFP_KERNEL); ++ if (!margin_left) ++ goto err; ++ margin_right = kmalloc(sizeof(*margin_right), GFP_KERNEL); ++ if (!margin_right) { ++ kfree(margin_left); ++ goto err; ++ } ++ spin_lock_irq(&alloc->lock); ++ list_for_each_entry(i, &alloc->free, list) { ++ base = (i->base + align - 1) / align; ++ base *= align; ++ if ((base - i->base) >= i->num) ++ /* alignment is impossible, regardless of count */ ++ continue; ++ num = i->num - (base - i->base); ++ if (num >= count) { ++ /* this one will do nicely */ ++ num = count; ++ goto done; ++ } ++ if (num > next_best_num) { ++ next_best = i; ++ next_best_base = base; ++ next_best_num = num; ++ } ++ } ++ if (partial && next_best) { ++ i = next_best; ++ base = next_best_base; ++ num = next_best_num; ++ } else ++ i = NULL; ++done: ++ if (i) { ++ if (base != i->base) { ++ margin_left->base = i->base; ++ margin_left->num = base - i->base; ++ list_add_tail(&margin_left->list, &i->list); ++ } else ++ kfree(margin_left); ++ if ((base + num) < (i->base + i->num)) { ++ margin_right->base = base + num; ++ margin_right->num = (i->base + i->num) - ++ (base + num); ++ list_add(&margin_right->list, &i->list); ++ } else ++ kfree(margin_right); ++ list_del(&i->list); ++ kfree(i); ++ *result = base; ++ } else { ++ spin_unlock_irq(&alloc->lock); ++ kfree(margin_left); ++ kfree(margin_right); ++ } ++ ++err: ++ DPRINT("returning %d\n", i ? num : -ENOMEM); ++ DUMP(alloc); ++ if (!i) ++ return -ENOMEM; ++ ++ /* Add the allocation to the used list with a refcount of 1 */ ++ used_node = kmalloc(sizeof(*used_node), GFP_KERNEL); ++ if (!used_node) { ++ spin_unlock_irq(&alloc->lock); ++ return -ENOMEM; ++ } ++ used_node->base = *result; ++ used_node->num = num; ++ used_node->refcount = 1; ++ used_node->is_alloced = 1; ++ list_add_tail(&used_node->list, &alloc->used); ++ spin_unlock_irq(&alloc->lock); ++ return (int)num; ++} ++ ++/* Allocate the list node using GFP_ATOMIC, because we *really* want to avoid ++ * forcing error-handling on to users in the deallocation path. */ ++static void _dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count) ++{ ++ struct alloc_node *i, *node = kmalloc(sizeof(*node), GFP_ATOMIC); ++ BUG_ON(!node); ++ DPRINT("release_range(%d,%d)\n", base_id, count); ++ DUMP(alloc); ++ BUG_ON(!count); ++ spin_lock_irq(&alloc->lock); ++ ++ ++ node->base = base_id; ++ node->num = count; ++ list_for_each_entry(i, &alloc->free, list) { ++ if (i->base >= node->base) { ++ /* BUG_ON(any overlapping) */ ++ BUG_ON(i->base < (node->base + node->num)); ++ list_add_tail(&node->list, &i->list); ++ goto done; ++ } ++ } ++ list_add_tail(&node->list, &alloc->free); ++done: ++ /* Merge to the left */ ++ i = list_entry(node->list.prev, struct alloc_node, list); ++ if (node->list.prev != &alloc->free) { ++ BUG_ON((i->base + i->num) > node->base); ++ if ((i->base + i->num) == node->base) { ++ node->base = i->base; ++ node->num += i->num; ++ list_del(&i->list); ++ kfree(i); ++ } ++ } ++ /* Merge to the right */ ++ i = list_entry(node->list.next, struct alloc_node, list); ++ if (node->list.next != &alloc->free) { ++ BUG_ON((node->base + node->num) > i->base); ++ if ((node->base + node->num) == i->base) { ++ node->num += i->num; ++ list_del(&i->list); ++ kfree(i); ++ } ++ } ++ spin_unlock_irq(&alloc->lock); ++ DUMP(alloc); ++} ++ ++ ++void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count) ++{ ++ struct alloc_node *i = NULL; ++ spin_lock_irq(&alloc->lock); ++ ++ /* First find the node in the used list and decrement its ref count */ ++ list_for_each_entry(i, &alloc->used, list) { ++ if (i->base == base_id && i->num == count) { ++ --i->refcount; ++ if (i->refcount == 0) { ++ list_del(&i->list); ++ spin_unlock_irq(&alloc->lock); ++ if (i->is_alloced) ++ _dpa_alloc_free(alloc, base_id, count); ++ kfree(i); ++ return; ++ } ++ spin_unlock_irq(&alloc->lock); ++ return; ++ } ++ } ++ /* Couldn't find the allocation */ ++ pr_err("Attempt to free ID 0x%x COUNT %d that wasn't alloc'd or reserved\n", ++ base_id, count); ++ spin_unlock_irq(&alloc->lock); ++} ++ ++void dpa_alloc_seed(struct dpa_alloc *alloc, u32 base_id, u32 count) ++{ ++ /* Same as free but no previous allocation checking is needed */ ++ _dpa_alloc_free(alloc, base_id, count); ++} ++ ++ ++int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base, u32 num) ++{ ++ struct alloc_node *i = NULL, *used_node; ++ ++ DPRINT("alloc_reserve(%d,%d)\n", base, num); ++ DUMP(alloc); ++ ++ spin_lock_irq(&alloc->lock); ++ ++ /* Check for the node in the used list. ++ If found, increase it's refcount */ ++ list_for_each_entry(i, &alloc->used, list) { ++ if ((i->base == base) && (i->num == num)) { ++ ++i->refcount; ++ spin_unlock_irq(&alloc->lock); ++ return 0; ++ } ++ if ((base >= i->base) && (base < (i->base + i->num))) { ++ /* This is an attempt to reserve a region that was ++ already reserved or alloced with a different ++ base or num */ ++ pr_err("Cannot reserve %d - %d, it overlaps with" ++ " existing reservation from %d - %d\n", ++ base, base + num - 1, i->base, ++ i->base + i->num - 1); ++ spin_unlock_irq(&alloc->lock); ++ return -1; ++ } ++ } ++ /* Check to make sure this ID isn't in the free list */ ++ list_for_each_entry(i, &alloc->free, list) { ++ if ((base >= i->base) && (base < (i->base + i->num))) { ++ /* yep, the reservation is within this node */ ++ pr_err("Cannot reserve %d - %d, it overlaps with" ++ " free range %d - %d and must be alloced\n", ++ base, base + num - 1, ++ i->base, i->base + i->num - 1); ++ spin_unlock_irq(&alloc->lock); ++ return -1; ++ } ++ } ++ /* Add the allocation to the used list with a refcount of 1 */ ++ used_node = kmalloc(sizeof(*used_node), GFP_KERNEL); ++ if (!used_node) { ++ spin_unlock_irq(&alloc->lock); ++ return -ENOMEM; ++ ++ } ++ used_node->base = base; ++ used_node->num = num; ++ used_node->refcount = 1; ++ used_node->is_alloced = 0; ++ list_add_tail(&used_node->list, &alloc->used); ++ spin_unlock_irq(&alloc->lock); ++ return 0; ++} ++ ++ ++int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count) ++{ ++ struct alloc_node *i = NULL; ++ DPRINT("alloc_pop()\n"); ++ DUMP(alloc); ++ spin_lock_irq(&alloc->lock); ++ if (!list_empty(&alloc->free)) { ++ i = list_entry(alloc->free.next, struct alloc_node, list); ++ list_del(&i->list); ++ } ++ spin_unlock_irq(&alloc->lock); ++ DPRINT("returning %d\n", i ? 0 : -ENOMEM); ++ DUMP(alloc); ++ if (!i) ++ return -ENOMEM; ++ *result = i->base; ++ *count = i->num; ++ kfree(i); ++ return 0; ++} ++ ++int dpa_alloc_check(struct dpa_alloc *list_head, u32 item) ++{ ++ struct alloc_node *i = NULL; ++ int res = 0; ++ DPRINT("alloc_check()\n"); ++ spin_lock_irq(&list_head->lock); ++ ++ list_for_each_entry(i, &list_head->free, list) { ++ if ((item >= i->base) && (item < (i->base + i->num))) { ++ res = 1; ++ break; ++ } ++ } ++ spin_unlock_irq(&list_head->lock); ++ return res; ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/dpa_sys.h +@@ -0,0 +1,259 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef DPA_SYS_H ++#define DPA_SYS_H ++ ++#include <linux/kernel.h> ++#include <linux/errno.h> ++#include <linux/io.h> ++#include <linux/dma-mapping.h> ++#include <linux/bootmem.h> ++#include <linux/slab.h> ++#include <linux/module.h> ++#include <linux/init.h> ++#include <linux/interrupt.h> ++#include <linux/delay.h> ++#include <linux/of_platform.h> ++#include <linux/of_address.h> ++#include <linux/of_irq.h> ++#include <linux/kthread.h> ++#include <linux/memblock.h> ++#include <linux/completion.h> ++#include <linux/log2.h> ++#include <linux/types.h> ++#include <linux/ioctl.h> ++#include <linux/miscdevice.h> ++#include <linux/uaccess.h> ++#include <linux/debugfs.h> ++#include <linux/seq_file.h> ++#include <linux/device.h> ++#include <linux/uio_driver.h> ++#include <linux/smp.h> ++#include <linux/fsl_hypervisor.h> ++#include <linux/vmalloc.h> ++#include <linux/ctype.h> ++#include <linux/math64.h> ++#include <linux/bitops.h> ++ ++#include <linux/fsl_usdpaa.h> ++ ++/* When copying aligned words or shorts, try to avoid memcpy() */ ++#define CONFIG_TRY_BETTER_MEMCPY ++ ++/* For 2-element tables related to cache-inhibited and cache-enabled mappings */ ++#define DPA_PORTAL_CE 0 ++#define DPA_PORTAL_CI 1 ++ ++/***********************/ ++/* Misc inline assists */ ++/***********************/ ++ ++#if defined CONFIG_PPC32 ++#include "dpa_sys_ppc32.h" ++#elif defined CONFIG_PPC64 ++#include "dpa_sys_ppc64.h" ++#elif defined CONFIG_ARM ++#include "dpa_sys_arm.h" ++#elif defined CONFIG_ARM64 ++#include "dpa_sys_arm64.h" ++#endif ++ ++ ++#ifdef CONFIG_FSL_DPA_CHECKING ++#define DPA_ASSERT(x) \ ++ do { \ ++ if (!(x)) { \ ++ pr_crit("ASSERT: (%s:%d) %s\n", __FILE__, __LINE__, \ ++ __stringify_1(x)); \ ++ dump_stack(); \ ++ panic("assertion failure"); \ ++ } \ ++ } while (0) ++#else ++#define DPA_ASSERT(x) ++#endif ++ ++/* memcpy() stuff - when you know alignments in advance */ ++#ifdef CONFIG_TRY_BETTER_MEMCPY ++static inline void copy_words(void *dest, const void *src, size_t sz) ++{ ++ u32 *__dest = dest; ++ const u32 *__src = src; ++ size_t __sz = sz >> 2; ++ BUG_ON((unsigned long)dest & 0x3); ++ BUG_ON((unsigned long)src & 0x3); ++ BUG_ON(sz & 0x3); ++ while (__sz--) ++ *(__dest++) = *(__src++); ++} ++static inline void copy_shorts(void *dest, const void *src, size_t sz) ++{ ++ u16 *__dest = dest; ++ const u16 *__src = src; ++ size_t __sz = sz >> 1; ++ BUG_ON((unsigned long)dest & 0x1); ++ BUG_ON((unsigned long)src & 0x1); ++ BUG_ON(sz & 0x1); ++ while (__sz--) ++ *(__dest++) = *(__src++); ++} ++static inline void copy_bytes(void *dest, const void *src, size_t sz) ++{ ++ u8 *__dest = dest; ++ const u8 *__src = src; ++ while (sz--) ++ *(__dest++) = *(__src++); ++} ++#else ++#define copy_words memcpy ++#define copy_shorts memcpy ++#define copy_bytes memcpy ++#endif ++ ++/************/ ++/* RB-trees */ ++/************/ ++ ++/* We encapsulate RB-trees so that its easier to use non-linux forms in ++ * non-linux systems. This also encapsulates the extra plumbing that linux code ++ * usually provides when using RB-trees. This encapsulation assumes that the ++ * data type held by the tree is u32. */ ++ ++struct dpa_rbtree { ++ struct rb_root root; ++}; ++#define DPA_RBTREE { .root = RB_ROOT } ++ ++static inline void dpa_rbtree_init(struct dpa_rbtree *tree) ++{ ++ tree->root = RB_ROOT; ++} ++ ++#define IMPLEMENT_DPA_RBTREE(name, type, node_field, val_field) \ ++static inline int name##_push(struct dpa_rbtree *tree, type *obj) \ ++{ \ ++ struct rb_node *parent = NULL, **p = &tree->root.rb_node; \ ++ while (*p) { \ ++ u32 item; \ ++ parent = *p; \ ++ item = rb_entry(parent, type, node_field)->val_field; \ ++ if (obj->val_field < item) \ ++ p = &parent->rb_left; \ ++ else if (obj->val_field > item) \ ++ p = &parent->rb_right; \ ++ else \ ++ return -EBUSY; \ ++ } \ ++ rb_link_node(&obj->node_field, parent, p); \ ++ rb_insert_color(&obj->node_field, &tree->root); \ ++ return 0; \ ++} \ ++static inline void name##_del(struct dpa_rbtree *tree, type *obj) \ ++{ \ ++ rb_erase(&obj->node_field, &tree->root); \ ++} \ ++static inline type *name##_find(struct dpa_rbtree *tree, u32 val) \ ++{ \ ++ type *ret; \ ++ struct rb_node *p = tree->root.rb_node; \ ++ while (p) { \ ++ ret = rb_entry(p, type, node_field); \ ++ if (val < ret->val_field) \ ++ p = p->rb_left; \ ++ else if (val > ret->val_field) \ ++ p = p->rb_right; \ ++ else \ ++ return ret; \ ++ } \ ++ return NULL; \ ++} ++ ++/************/ ++/* Bootargs */ ++/************/ ++ ++/* Qman has "qportals=" and Bman has "bportals=", they use the same syntax ++ * though; a comma-separated list of items, each item being a cpu index and/or a ++ * range of cpu indices, and each item optionally be prefixed by "s" to indicate ++ * that the portal associated with that cpu should be shared. See bman_driver.c ++ * for more specifics. */ ++static int __parse_portals_cpu(const char **s, unsigned int *cpu) ++{ ++ *cpu = 0; ++ if (!isdigit(**s)) ++ return -EINVAL; ++ while (isdigit(**s)) ++ *cpu = *cpu * 10 + (*((*s)++) - '0'); ++ return 0; ++} ++static inline int parse_portals_bootarg(char *str, struct cpumask *want_shared, ++ struct cpumask *want_unshared, ++ const char *argname) ++{ ++ const char *s = str; ++ unsigned int shared, cpu1, cpu2, loop; ++ ++keep_going: ++ if (*s == 's') { ++ shared = 1; ++ s++; ++ } else ++ shared = 0; ++ if (__parse_portals_cpu(&s, &cpu1)) ++ goto err; ++ if (*s == '-') { ++ s++; ++ if (__parse_portals_cpu(&s, &cpu2)) ++ goto err; ++ if (cpu2 < cpu1) ++ goto err; ++ } else ++ cpu2 = cpu1; ++ for (loop = cpu1; loop <= cpu2; loop++) ++ cpumask_set_cpu(loop, shared ? want_shared : want_unshared); ++ if (*s == ',') { ++ s++; ++ goto keep_going; ++ } else if ((*s == '\0') || isspace(*s)) ++ return 0; ++err: ++ pr_crit("Malformed %s argument: %s, offset: %lu\n", argname, str, ++ (unsigned long)s - (unsigned long)str); ++ return -EINVAL; ++} ++ ++/* Hooks from fsl_usdpaa_irq.c to fsl_usdpaa.c */ ++int usdpaa_get_portal_config(struct file *filp, void *cinh, ++ enum usdpaa_portal_type ptype, unsigned int *irq, ++ void **iir_reg); ++ ++#endif /* DPA_SYS_H */ +--- /dev/null ++++ b/drivers/staging/fsl_qbman/dpa_sys_arm.h +@@ -0,0 +1,95 @@ ++/* Copyright 2016 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef DPA_SYS_ARM_H ++#define DPA_SYS_ARM_H ++ ++#include <asm/cacheflush.h> ++#include <asm/barrier.h> ++ ++/* Implementation of ARM specific routines */ ++ ++/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler ++ * barriers and that dcb*() won't fall victim to compiler or execution ++ * reordering with respect to other code/instructions that manipulate the same ++ * cacheline. */ ++#define hwsync() { asm volatile("dmb st" : : : "memory"); } ++#define lwsync() { asm volatile("dmb st" : : : "memory"); } ++#define dcbf(p) { asm volatile("mcr p15, 0, %0, c7, c10, 1" : : "r" (p) : "memory"); } ++#define dcbt_ro(p) { asm volatile("pld [%0, #64];": : "r" (p)); } ++#define dcbt_rw(p) { asm volatile("pldw [%0, #64];": : "r" (p)); } ++#define dcbi(p) { asm volatile("mcr p15, 0, %0, c7, c6, 1" : : "r" (p) : "memory"); } ++ ++#define dcbz_64(p) { memset(p, 0, sizeof(*p)); } ++ ++#define dcbf_64(p) \ ++ do { \ ++ dcbf((u32)p); \ ++ } while (0) ++/* Commonly used combo */ ++#define dcbit_ro(p) \ ++ do { \ ++ dcbi((u32)p); \ ++ dcbt_ro((u32)p); \ ++ } while (0) ++ ++static inline u64 mfatb(void) ++{ ++ return get_cycles(); ++} ++ ++static inline u32 in_be32(volatile void *addr) ++{ ++ return be32_to_cpu(*((volatile u32 *) addr)); ++} ++ ++static inline void out_be32(void *addr, u32 val) ++{ ++ *((u32 *) addr) = cpu_to_be32(val); ++} ++ ++ ++static inline void set_bits(unsigned long mask, volatile unsigned long *p) ++{ ++ *p |= mask; ++} ++static inline void clear_bits(unsigned long mask, volatile unsigned long *p) ++{ ++ *p &= ~mask; ++} ++ ++static inline void flush_dcache_range(unsigned long start, unsigned long stop) ++{ ++ __cpuc_flush_dcache_area((void *) start, stop - start); ++} ++ ++#define hard_smp_processor_id() raw_smp_processor_id() ++#endif +--- /dev/null ++++ b/drivers/staging/fsl_qbman/dpa_sys_arm64.h +@@ -0,0 +1,102 @@ ++/* Copyright 2014 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef DPA_SYS_ARM64_H ++#define DPA_SYS_ARM64_H ++ ++#include <asm/cacheflush.h> ++#include <asm/barrier.h> ++ ++/* Implementation of ARM 64 bit specific routines */ ++ ++/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler ++ * barriers and that dcb*() won't fall victim to compiler or execution ++ * reordering with respect to other code/instructions that manipulate the same ++ * cacheline. */ ++#define hwsync() { asm volatile("dmb st" : : : "memory"); } ++#define lwsync() { asm volatile("dmb st" : : : "memory"); } ++#define dcbf(p) { asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); } ++#define dcbt_ro(p) { asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); } ++#define dcbt_rw(p) { asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); } ++#define dcbi(p) { asm volatile("dc ivac, %0" : : "r"(p) : "memory"); } ++#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); } ++ ++#define dcbz_64(p) \ ++ do { \ ++ dcbz(p); \ ++ } while (0) ++ ++#define dcbf_64(p) \ ++ do { \ ++ dcbf(p); \ ++ } while (0) ++/* Commonly used combo */ ++#define dcbit_ro(p) \ ++ do { \ ++ dcbi(p); \ ++ dcbt_ro(p); \ ++ } while (0) ++ ++static inline u64 mfatb(void) ++{ ++ return get_cycles(); ++} ++ ++static inline u32 in_be32(volatile void *addr) ++{ ++ return be32_to_cpu(*((volatile u32 *) addr)); ++} ++ ++static inline void out_be32(void *addr, u32 val) ++{ ++ *((u32 *) addr) = cpu_to_be32(val); ++} ++ ++ ++static inline void set_bits(unsigned long mask, volatile unsigned long *p) ++{ ++ *p |= mask; ++} ++static inline void clear_bits(unsigned long mask, volatile unsigned long *p) ++{ ++ *p &= ~mask; ++} ++ ++static inline void flush_dcache_range(unsigned long start, unsigned long stop) ++{ ++ __flush_dcache_area((void *) start, stop - start); ++} ++ ++#define hard_smp_processor_id() raw_smp_processor_id() ++ ++ ++ ++#endif +--- /dev/null ++++ b/drivers/staging/fsl_qbman/dpa_sys_ppc32.h +@@ -0,0 +1,70 @@ ++/* Copyright 2014 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef DPA_SYS_PPC32_H ++#define DPA_SYS_PPC32_H ++ ++/* Implementation of PowerPC 32 bit specific routines */ ++ ++/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler ++ * barriers and that dcb*() won't fall victim to compiler or execution ++ * reordering with respect to other code/instructions that manipulate the same ++ * cacheline. */ ++#define hwsync() __asm__ __volatile__ ("sync" : : : "memory") ++#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory") ++#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory") ++#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p)) ++#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p)) ++#define dcbi(p) dcbf(p) ++ ++#define dcbzl(p) __asm__ __volatile__ ("dcbzl 0,%0" : : "r" (p)) ++#define dcbz_64(p) dcbzl(p) ++#define dcbf_64(p) dcbf(p) ++ ++/* Commonly used combo */ ++#define dcbit_ro(p) \ ++ do { \ ++ dcbi(p); \ ++ dcbt_ro(p); \ ++ } while (0) ++ ++static inline u64 mfatb(void) ++{ ++ u32 hi, lo, chk; ++ do { ++ hi = mfspr(SPRN_ATBU); ++ lo = mfspr(SPRN_ATBL); ++ chk = mfspr(SPRN_ATBU); ++ } while (unlikely(hi != chk)); ++ return ((u64)hi << 32) | (u64)lo; ++} ++ ++#endif +--- /dev/null ++++ b/drivers/staging/fsl_qbman/dpa_sys_ppc64.h +@@ -0,0 +1,79 @@ ++/* Copyright 2014 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef DPA_SYS_PPC64_H ++#define DPA_SYS_PPC64_H ++ ++/* Implementation of PowerPC 64 bit specific routines */ ++ ++/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler ++ * barriers and that dcb*() won't fall victim to compiler or execution ++ * reordering with respect to other code/instructions that manipulate the same ++ * cacheline. */ ++#define hwsync() __asm__ __volatile__ ("sync" : : : "memory") ++#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory") ++#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory") ++#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p)) ++#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p)) ++#define dcbi(p) dcbf(p) ++ ++#define dcbz(p) __asm__ __volatile__ ("dcbz 0,%0" : : "r" (p)) ++#define dcbz_64(p) \ ++ do { \ ++ dcbz((void*)p + 32); \ ++ dcbz(p); \ ++ } while (0) ++#define dcbf_64(p) \ ++ do { \ ++ dcbf((void*)p + 32); \ ++ dcbf(p); \ ++ } while (0) ++/* Commonly used combo */ ++#define dcbit_ro(p) \ ++ do { \ ++ dcbi(p); \ ++ dcbi((void*)p + 32); \ ++ dcbt_ro(p); \ ++ dcbt_ro((void*)p + 32); \ ++ } while (0) ++ ++static inline u64 mfatb(void) ++{ ++ u32 hi, lo, chk; ++ do { ++ hi = mfspr(SPRN_ATBU); ++ lo = mfspr(SPRN_ATBL); ++ chk = mfspr(SPRN_ATBU); ++ } while (unlikely(hi != chk)); ++ return ((u64)hi << 32) | (u64)lo; ++} ++ ++#endif +--- /dev/null ++++ b/drivers/staging/fsl_qbman/fsl_usdpaa.c +@@ -0,0 +1,1982 @@ ++/* Copyright (C) 2008-2012 Freescale Semiconductor, Inc. ++ * Authors: Andy Fleming <afleming@freescale.com> ++ * Timur Tabi <timur@freescale.com> ++ * Geoff Thorpe <Geoff.Thorpe@freescale.com> ++ * ++ * This file is licensed under the terms of the GNU General Public License ++ * version 2. This program is licensed "as is" without any warranty of any ++ * kind, whether express or implied. ++ */ ++ ++ ++#include <linux/miscdevice.h> ++#include <linux/fs.h> ++#include <linux/cdev.h> ++#include <linux/mm.h> ++#include <linux/of.h> ++#include <linux/memblock.h> ++#include <linux/slab.h> ++#include <linux/mman.h> ++#include <linux/of_reserved_mem.h> ++ ++#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64)) ++#include <mm/mmu_decl.h> ++#endif ++ ++#include "dpa_sys.h" ++#include <linux/fsl_usdpaa.h> ++#include "bman_low.h" ++#include "qman_low.h" ++ ++/* Physical address range of the memory reservation, exported for mm/mem.c */ ++static u64 phys_start; ++static u64 phys_size; ++static u64 arg_phys_size; ++ ++/* PFN versions of the above */ ++static unsigned long pfn_start; ++static unsigned long pfn_size; ++ ++/* Memory reservations are manipulated under this spinlock (which is why 'refs' ++ * isn't atomic_t). */ ++static DEFINE_SPINLOCK(mem_lock); ++ ++/* The range of TLB1 indices */ ++static unsigned int first_tlb; ++static unsigned int num_tlb = 1; ++static unsigned int current_tlb; /* loops around for fault handling */ ++ ++/* Memory reservation is represented as a list of 'mem_fragment's, some of which ++ * may be mapped. Unmapped fragments are always merged where possible. */ ++static LIST_HEAD(mem_list); ++ ++struct mem_mapping; ++ ++/* Memory fragments are in 'mem_list'. */ ++struct mem_fragment { ++ u64 base; ++ u64 len; ++ unsigned long pfn_base; /* PFN version of 'base' */ ++ unsigned long pfn_len; /* PFN version of 'len' */ ++ unsigned int refs; /* zero if unmapped */ ++ u64 root_len; /* Size of the orignal fragment */ ++ unsigned long root_pfn; /* PFN of the orignal fragment */ ++ struct list_head list; ++ /* if mapped, flags+name captured at creation time */ ++ u32 flags; ++ char name[USDPAA_DMA_NAME_MAX]; ++ u64 map_len; ++ /* support multi-process locks per-memory-fragment. */ ++ int has_locking; ++ wait_queue_head_t wq; ++ struct mem_mapping *owner; ++}; ++ ++/* Mappings of memory fragments in 'struct ctx'. These are created from ++ * ioctl(USDPAA_IOCTL_DMA_MAP), though the actual mapping then happens via a ++ * mmap(). */ ++struct mem_mapping { ++ struct mem_fragment *root_frag; ++ u32 frag_count; ++ u64 total_size; ++ struct list_head list; ++ int refs; ++ void *virt_addr; ++}; ++ ++struct portal_mapping { ++ struct usdpaa_ioctl_portal_map user; ++ union { ++ struct qm_portal_config *qportal; ++ struct bm_portal_config *bportal; ++ }; ++ /* Declare space for the portals in case the process ++ exits unexpectedly and needs to be cleaned by the kernel */ ++ union { ++ struct qm_portal qman_portal_low; ++ struct bm_portal bman_portal_low; ++ }; ++ struct list_head list; ++ struct resource *phys; ++ struct iommu_domain *iommu_domain; ++}; ++ ++/* Track the DPAA resources the process is using */ ++struct active_resource { ++ struct list_head list; ++ u32 id; ++ u32 num; ++ unsigned int refcount; ++}; ++ ++/* Per-FD state (which should also be per-process but we don't enforce that) */ ++struct ctx { ++ /* Lock to protect the context */ ++ spinlock_t lock; ++ /* Allocated resources get put here for accounting */ ++ struct list_head resources[usdpaa_id_max]; ++ /* list of DMA maps */ ++ struct list_head maps; ++ /* list of portal maps */ ++ struct list_head portals; ++}; ++ ++/* Different resource classes */ ++static const struct alloc_backend { ++ enum usdpaa_id_type id_type; ++ int (*alloc)(u32 *, u32, u32, int); ++ void (*release)(u32 base, unsigned int count); ++ int (*reserve)(u32 base, unsigned int count); ++ const char *acronym; ++} alloc_backends[] = { ++ { ++ .id_type = usdpaa_id_fqid, ++ .alloc = qman_alloc_fqid_range, ++ .release = qman_release_fqid_range, ++ .reserve = qman_reserve_fqid_range, ++ .acronym = "FQID" ++ }, ++ { ++ .id_type = usdpaa_id_bpid, ++ .alloc = bman_alloc_bpid_range, ++ .release = bman_release_bpid_range, ++ .reserve = bman_reserve_bpid_range, ++ .acronym = "BPID" ++ }, ++ { ++ .id_type = usdpaa_id_qpool, ++ .alloc = qman_alloc_pool_range, ++ .release = qman_release_pool_range, ++ .reserve = qman_reserve_pool_range, ++ .acronym = "QPOOL" ++ }, ++ { ++ .id_type = usdpaa_id_cgrid, ++ .alloc = qman_alloc_cgrid_range, ++ .release = qman_release_cgrid_range, ++ .acronym = "CGRID" ++ }, ++ { ++ .id_type = usdpaa_id_ceetm0_lfqid, ++ .alloc = qman_alloc_ceetm0_lfqid_range, ++ .release = qman_release_ceetm0_lfqid_range, ++ .acronym = "CEETM0_LFQID" ++ }, ++ { ++ .id_type = usdpaa_id_ceetm0_channelid, ++ .alloc = qman_alloc_ceetm0_channel_range, ++ .release = qman_release_ceetm0_channel_range, ++ .acronym = "CEETM0_LFQID" ++ }, ++ { ++ .id_type = usdpaa_id_ceetm1_lfqid, ++ .alloc = qman_alloc_ceetm1_lfqid_range, ++ .release = qman_release_ceetm1_lfqid_range, ++ .acronym = "CEETM1_LFQID" ++ }, ++ { ++ .id_type = usdpaa_id_ceetm1_channelid, ++ .alloc = qman_alloc_ceetm1_channel_range, ++ .release = qman_release_ceetm1_channel_range, ++ .acronym = "CEETM1_LFQID" ++ }, ++ { ++ /* This terminates the array */ ++ .id_type = usdpaa_id_max ++ } ++}; ++ ++/* Determines the largest acceptable page size for a given size ++ The sizes are determined by what the TLB1 acceptable page sizes are */ ++static u32 largest_page_size(u32 size) ++{ ++ int shift = 30; /* Start at 1G size */ ++ if (size < 4096) ++ return 0; ++ do { ++ if (size >= (1<<shift)) ++ return 1<<shift; ++ shift -= 2; ++ } while (shift >= 12); /* Up to 4k */ ++ return 0; ++} ++ ++/* Determine if value is power of 4 */ ++static inline bool is_power_of_4(u64 x) ++{ ++ if (x == 0 || ((x & (x - 1)) != 0)) ++ return false; ++ return !!(x & 0x5555555555555555ull); ++} ++ ++/* Helper for ioctl_dma_map() when we have a larger fragment than we need. This ++ * splits the fragment into 4 and returns the upper-most. (The caller can loop ++ * until it has a suitable fragment size.) */ ++static struct mem_fragment *split_frag(struct mem_fragment *frag) ++{ ++ struct mem_fragment *x[3]; ++ ++ x[0] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC); ++ x[1] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC); ++ x[2] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC); ++ if (!x[0] || !x[1] || !x[2]) { ++ kfree(x[0]); ++ kfree(x[1]); ++ kfree(x[2]); ++ return NULL; ++ } ++ BUG_ON(frag->refs); ++ frag->len >>= 2; ++ frag->pfn_len >>= 2; ++ x[0]->base = frag->base + frag->len; ++ x[1]->base = x[0]->base + frag->len; ++ x[2]->base = x[1]->base + frag->len; ++ x[0]->len = x[1]->len = x[2]->len = frag->len; ++ x[0]->pfn_base = frag->pfn_base + frag->pfn_len; ++ x[1]->pfn_base = x[0]->pfn_base + frag->pfn_len; ++ x[2]->pfn_base = x[1]->pfn_base + frag->pfn_len; ++ x[0]->pfn_len = x[1]->pfn_len = x[2]->pfn_len = frag->pfn_len; ++ x[0]->refs = x[1]->refs = x[2]->refs = 0; ++ x[0]->root_len = x[1]->root_len = x[2]->root_len = frag->root_len; ++ x[0]->root_pfn = x[1]->root_pfn = x[2]->root_pfn = frag->root_pfn; ++ x[0]->name[0] = x[1]->name[0] = x[2]->name[0] = 0; ++ list_add_tail(&x[0]->list, &frag->list); ++ list_add_tail(&x[1]->list, &x[0]->list); ++ list_add_tail(&x[2]->list, &x[1]->list); ++ return x[2]; ++} ++ ++static __maybe_unused void dump_frags(void) ++{ ++ struct mem_fragment *frag; ++ int i = 0; ++ list_for_each_entry(frag, &mem_list, list) { ++ pr_info("FRAG %d: base 0x%llx pfn_base 0x%lx len 0x%llx root_len 0x%llx root_pfn 0x%lx refs %d name %s\n", ++ i, frag->base, frag->pfn_base, ++ frag->len, frag->root_len, frag->root_pfn, ++ frag->refs, frag->name); ++ ++i; ++ } ++} ++ ++/* Walk the list of fragments and adjoin neighbouring segments if possible */ ++static void compress_frags(void) ++{ ++ /* Walk the fragment list and combine fragments */ ++ struct mem_fragment *frag, *nxtfrag; ++ u64 len = 0; ++ ++ int i, numfrags; ++ ++ ++ frag = list_entry(mem_list.next, struct mem_fragment, list); ++ ++ while (&frag->list != &mem_list) { ++ /* Must combine consecutive fragemenst with ++ same root_pfn such that they are power of 4 */ ++ if (frag->refs != 0) { ++ frag = list_entry(frag->list.next, ++ struct mem_fragment, list); ++ continue; /* Not this window */ ++ } ++ len = frag->len; ++ numfrags = 0; ++ nxtfrag = list_entry(frag->list.next, ++ struct mem_fragment, list); ++ while (true) { ++ if (&nxtfrag->list == &mem_list) { ++ numfrags = 0; ++ break; /* End of list */ ++ } ++ if (nxtfrag->refs) { ++ numfrags = 0; ++ break; /* In use still */ ++ } ++ if (nxtfrag->root_pfn != frag->root_pfn) { ++ numfrags = 0; ++ break; /* Crosses root fragment boundary */ ++ } ++ len += nxtfrag->len; ++ numfrags++; ++ if (is_power_of_4(len)) { ++ /* These fragments can be combined */ ++ break; ++ } ++ nxtfrag = list_entry(nxtfrag->list.next, ++ struct mem_fragment, list); ++ } ++ if (numfrags == 0) { ++ frag = list_entry(frag->list.next, ++ struct mem_fragment, list); ++ continue; /* try the next window */ ++ } ++ for (i = 0; i < numfrags; i++) { ++ struct mem_fragment *todel = ++ list_entry(nxtfrag->list.prev, ++ struct mem_fragment, list); ++ nxtfrag->len += todel->len; ++ nxtfrag->pfn_len += todel->pfn_len; ++ list_del(&todel->list); ++ } ++ /* Re evaluate the list, things may merge now */ ++ frag = list_entry(mem_list.next, struct mem_fragment, list); ++ } ++} ++ ++/* Hook from arch/powerpc/mm/mem.c */ ++int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size) ++{ ++ struct mem_fragment *frag; ++ int idx = -1; ++ if ((pfn < pfn_start) || (pfn >= (pfn_start + pfn_size))) ++ return -1; ++ /* It's in-range, we need to find the fragment */ ++ spin_lock(&mem_lock); ++ list_for_each_entry(frag, &mem_list, list) { ++ if ((pfn >= frag->pfn_base) && (pfn < (frag->pfn_base + ++ frag->pfn_len))) { ++ *phys_addr = frag->base; ++ *size = frag->len; ++ idx = current_tlb++; ++ if (current_tlb >= (first_tlb + num_tlb)) ++ current_tlb = first_tlb; ++ break; ++ } ++ } ++ spin_unlock(&mem_lock); ++ return idx; ++} ++ ++static int usdpaa_open(struct inode *inode, struct file *filp) ++{ ++ const struct alloc_backend *backend = &alloc_backends[0]; ++ struct ctx *ctx = kmalloc(sizeof(struct ctx), GFP_KERNEL); ++ if (!ctx) ++ return -ENOMEM; ++ filp->private_data = ctx; ++ ++ while (backend->id_type != usdpaa_id_max) { ++ INIT_LIST_HEAD(&ctx->resources[backend->id_type]); ++ backend++; ++ } ++ ++ INIT_LIST_HEAD(&ctx->maps); ++ INIT_LIST_HEAD(&ctx->portals); ++ spin_lock_init(&ctx->lock); ++ ++ //filp->f_mapping->backing_dev_info = &directly_mappable_cdev_bdi; ++ ++ return 0; ++} ++ ++#define DQRR_MAXFILL 15 ++ ++/* Reset a QMan portal to its default state */ ++static int init_qm_portal(struct qm_portal_config *config, ++ struct qm_portal *portal) ++{ ++ const struct qm_dqrr_entry *dqrr = NULL; ++ int i; ++ ++ portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; ++ portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; ++ ++ /* Make sure interrupts are inhibited */ ++ qm_out(IIR, 1); ++ ++ /* Initialize the DQRR. This will stop any dequeue ++ commands that are in progress */ ++ if (qm_dqrr_init(portal, config, qm_dqrr_dpush, qm_dqrr_pvb, ++ qm_dqrr_cdc, DQRR_MAXFILL)) { ++ pr_err("qm_dqrr_init() failed when trying to" ++ " recover portal, portal will be leaked\n"); ++ return 1; ++ } ++ ++ /* Discard any entries on the DQRR */ ++ /* If we consume the ring twice something is wrong */ ++ for (i = 0; i < DQRR_MAXFILL * 2; i++) { ++ qm_dqrr_pvb_update(portal); ++ dqrr = qm_dqrr_current(portal); ++ if (!dqrr) ++ break; ++ qm_dqrr_cdc_consume_1ptr(portal, dqrr, 0); ++ qm_dqrr_pvb_update(portal); ++ qm_dqrr_next(portal); ++ } ++ /* Initialize the EQCR */ ++ if (qm_eqcr_init(portal, qm_eqcr_pvb, ++ qm_eqcr_get_ci_stashing(portal), 1)) { ++ pr_err("Qman EQCR initialisation failed\n"); ++ return 1; ++ } ++ /* initialize the MR */ ++ if (qm_mr_init(portal, qm_mr_pvb, qm_mr_cci)) { ++ pr_err("Qman MR initialisation failed\n"); ++ return 1; ++ } ++ qm_mr_pvb_update(portal); ++ while (qm_mr_current(portal)) { ++ qm_mr_next(portal); ++ qm_mr_cci_consume_to_current(portal); ++ qm_mr_pvb_update(portal); ++ } ++ ++ if (qm_mc_init(portal)) { ++ pr_err("Qman MC initialisation failed\n"); ++ return 1; ++ } ++ return 0; ++} ++ ++static int init_bm_portal(struct bm_portal_config *config, ++ struct bm_portal *portal) ++{ ++ portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; ++ portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; ++ ++ if (bm_rcr_init(portal, bm_rcr_pvb, bm_rcr_cce)) { ++ pr_err("Bman RCR initialisation failed\n"); ++ return 1; ++ } ++ if (bm_mc_init(portal)) { ++ pr_err("Bman MC initialisation failed\n"); ++ return 1; ++ } ++ return 0; ++} ++ ++/* Function that will scan all FQ's in the system. For each FQ that is not ++ OOS it will call the check_channel helper to determine if the FQ should ++ be torn down. If the check_channel helper returns true the FQ will be ++ transitioned to the OOS state */ ++static int qm_check_and_destroy_fqs(struct qm_portal *portal, void *ctx, ++ bool (*check_channel)(void*, u32)) ++{ ++ u32 fq_id = 0; ++ while (1) { ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ u8 state; ++ u32 channel; ++ ++ /* Determine the channel for the FQID */ ++ mcc = qm_mc_start(portal); ++ mcc->queryfq.fqid = fq_id; ++ qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ); ++ while (!(mcr = qm_mc_result(portal))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ++ == QM_MCR_VERB_QUERYFQ); ++ if (mcr->result != QM_MCR_RESULT_OK) ++ break; /* End of valid FQIDs */ ++ ++ channel = mcr->queryfq.fqd.dest.channel; ++ /* Determine the state of the FQID */ ++ mcc = qm_mc_start(portal); ++ mcc->queryfq_np.fqid = fq_id; ++ qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ_NP); ++ while (!(mcr = qm_mc_result(portal))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ++ == QM_MCR_VERB_QUERYFQ_NP); ++ state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; ++ if (state == QM_MCR_NP_STATE_OOS) ++ /* Already OOS, no need to do anymore checks */ ++ goto next; ++ ++ if (check_channel(ctx, channel)) ++ qm_shutdown_fq(&portal, 1, fq_id); ++ next: ++ ++fq_id; ++ } ++ return 0; ++} ++ ++static bool check_channel_device(void *_ctx, u32 channel) ++{ ++ struct ctx *ctx = _ctx; ++ struct portal_mapping *portal, *tmpportal; ++ struct active_resource *res; ++ ++ /* See if the FQ is destined for one of the portals we're cleaning up */ ++ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) { ++ if (portal->user.type == usdpaa_portal_qman) { ++ if (portal->qportal->public_cfg.channel == channel) { ++ /* This FQs destination is a portal ++ we're cleaning, send a retire */ ++ return true; ++ } ++ } ++ } ++ ++ /* Check the pool channels that will be released as well */ ++ list_for_each_entry(res, &ctx->resources[usdpaa_id_qpool], list) { ++ if ((res->id >= channel) && ++ ((res->id + res->num - 1) <= channel)) ++ return true; ++ } ++ return false; ++} ++ ++static bool check_portal_channel(void *ctx, u32 channel) ++{ ++ u32 portal_channel = *(u32 *)ctx; ++ if (portal_channel == channel) { ++ /* This FQs destination is a portal ++ we're cleaning, send a retire */ ++ return true; ++ } ++ return false; ++} ++ ++ ++ ++ ++static int usdpaa_release(struct inode *inode, struct file *filp) ++{ ++ struct ctx *ctx = filp->private_data; ++ struct mem_mapping *map, *tmpmap; ++ struct portal_mapping *portal, *tmpportal; ++ const struct alloc_backend *backend = &alloc_backends[0]; ++ struct active_resource *res; ++ struct qm_portal *qm_cleanup_portal = NULL; ++ struct bm_portal *bm_cleanup_portal = NULL; ++ struct qm_portal_config *qm_alloced_portal = NULL; ++ struct bm_portal_config *bm_alloced_portal = NULL; ++ ++ struct qm_portal *portal_array[qman_portal_max]; ++ int portal_count = 0; ++ ++ /* Ensure the release operation cannot be migrated to another ++ CPU as CPU specific variables may be needed during cleanup */ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_disable(); ++#endif ++ /* The following logic is used to recover resources that were not ++ correctly released by the process that is closing the FD. ++ Step 1: syncronize the HW with the qm_portal/bm_portal structures ++ in the kernel ++ */ ++ ++ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) { ++ /* Try to recover any portals that weren't shut down */ ++ if (portal->user.type == usdpaa_portal_qman) { ++ portal_array[portal_count] = &portal->qman_portal_low; ++ ++portal_count; ++ init_qm_portal(portal->qportal, ++ &portal->qman_portal_low); ++ if (!qm_cleanup_portal) { ++ qm_cleanup_portal = &portal->qman_portal_low; ++ } else { ++ /* Clean FQs on the dedicated channel */ ++ u32 chan = portal->qportal->public_cfg.channel; ++ qm_check_and_destroy_fqs( ++ &portal->qman_portal_low, &chan, ++ check_portal_channel); ++ } ++ } else { ++ /* BMAN */ ++ init_bm_portal(portal->bportal, ++ &portal->bman_portal_low); ++ if (!bm_cleanup_portal) ++ bm_cleanup_portal = &portal->bman_portal_low; ++ } ++ } ++ /* If no portal was found, allocate one for cleanup */ ++ if (!qm_cleanup_portal) { ++ qm_alloced_portal = qm_get_unused_portal(); ++ if (!qm_alloced_portal) { ++ pr_crit("No QMan portal avalaible for cleanup\n"); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_enable(); ++#endif ++ return -1; ++ } ++ qm_cleanup_portal = kmalloc(sizeof(struct qm_portal), ++ GFP_KERNEL); ++ if (!qm_cleanup_portal) { ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_enable(); ++#endif ++ return -ENOMEM; ++ } ++ init_qm_portal(qm_alloced_portal, qm_cleanup_portal); ++ portal_array[portal_count] = qm_cleanup_portal; ++ ++portal_count; ++ } ++ if (!bm_cleanup_portal) { ++ bm_alloced_portal = bm_get_unused_portal(); ++ if (!bm_alloced_portal) { ++ pr_crit("No BMan portal avalaible for cleanup\n"); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_enable(); ++#endif ++ return -1; ++ } ++ bm_cleanup_portal = kmalloc(sizeof(struct bm_portal), ++ GFP_KERNEL); ++ if (!bm_cleanup_portal) { ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_enable(); ++#endif ++ return -ENOMEM; ++ } ++ init_bm_portal(bm_alloced_portal, bm_cleanup_portal); ++ } ++ ++ /* OOS the FQs associated with this process */ ++ qm_check_and_destroy_fqs(qm_cleanup_portal, ctx, check_channel_device); ++ ++ while (backend->id_type != usdpaa_id_max) { ++ int leaks = 0; ++ list_for_each_entry(res, &ctx->resources[backend->id_type], ++ list) { ++ if (backend->id_type == usdpaa_id_fqid) { ++ int i = 0; ++ for (; i < res->num; i++) { ++ /* Clean FQs with the cleanup portal */ ++ qm_shutdown_fq(portal_array, ++ portal_count, ++ res->id + i); ++ } ++ } ++ leaks += res->num; ++ backend->release(res->id, res->num); ++ } ++ if (leaks) ++ pr_crit("USDPAA process leaking %d %s%s\n", leaks, ++ backend->acronym, (leaks > 1) ? "s" : ""); ++ backend++; ++ } ++ /* Release any DMA regions */ ++ spin_lock(&mem_lock); ++ list_for_each_entry_safe(map, tmpmap, &ctx->maps, list) { ++ struct mem_fragment *current_frag = map->root_frag; ++ int i; ++ if (map->root_frag->has_locking && ++ (map->root_frag->owner == map)) { ++ map->root_frag->owner = NULL; ++ wake_up(&map->root_frag->wq); ++ } ++ /* Check each fragment and merge if the ref count is 0 */ ++ for (i = 0; i < map->frag_count; i++) { ++ --current_frag->refs; ++ current_frag = list_entry(current_frag->list.prev, ++ struct mem_fragment, list); ++ } ++ ++ compress_frags(); ++ list_del(&map->list); ++ kfree(map); ++ } ++ spin_unlock(&mem_lock); ++ ++ /* Return portals */ ++ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) { ++ if (portal->user.type == usdpaa_portal_qman) { ++ /* Give the portal back to the allocator */ ++ init_qm_portal(portal->qportal, ++ &portal->qman_portal_low); ++ qm_put_unused_portal(portal->qportal); ++ } else { ++ init_bm_portal(portal->bportal, ++ &portal->bman_portal_low); ++ bm_put_unused_portal(portal->bportal); ++ } ++ list_del(&portal->list); ++ kfree(portal); ++ } ++ if (qm_alloced_portal) { ++ qm_put_unused_portal(qm_alloced_portal); ++ kfree(qm_cleanup_portal); ++ } ++ if (bm_alloced_portal) { ++ bm_put_unused_portal(bm_alloced_portal); ++ kfree(bm_cleanup_portal); ++ } ++ ++ kfree(ctx); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_enable(); ++#endif ++ return 0; ++} ++ ++static int check_mmap_dma(struct ctx *ctx, struct vm_area_struct *vma, ++ int *match, unsigned long *pfn) ++{ ++ struct mem_mapping *map; ++ ++ list_for_each_entry(map, &ctx->maps, list) { ++ int i; ++ struct mem_fragment *frag = map->root_frag; ++ ++ for (i = 0; i < map->frag_count; i++) { ++ if (frag->pfn_base == vma->vm_pgoff) { ++ *match = 1; ++ *pfn = frag->pfn_base; ++ return 0; ++ } ++ frag = list_entry(frag->list.next, struct mem_fragment, ++ list); ++ } ++ } ++ *match = 0; ++ return 0; ++} ++ ++static int check_mmap_resource(struct resource *res, struct vm_area_struct *vma, ++ int *match, unsigned long *pfn) ++{ ++ *pfn = res->start >> PAGE_SHIFT; ++ if (*pfn == vma->vm_pgoff) { ++ *match = 1; ++ if ((vma->vm_end - vma->vm_start) != resource_size(res)) ++ return -EINVAL; ++ } else ++ *match = 0; ++ return 0; ++} ++ ++static int check_mmap_portal(struct ctx *ctx, struct vm_area_struct *vma, ++ int *match, unsigned long *pfn) ++{ ++ struct portal_mapping *portal; ++ int ret; ++ ++ list_for_each_entry(portal, &ctx->portals, list) { ++ ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CE], vma, ++ match, pfn); ++ if (*match) { ++ vma->vm_page_prot = ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ pgprot_cached_ns(vma->vm_page_prot); ++#else ++ pgprot_cached_noncoherent(vma->vm_page_prot); ++#endif ++ return ret; ++ } ++ ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CI], vma, ++ match, pfn); ++ if (*match) { ++ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); ++ return ret; ++ } ++ } ++ *match = 0; ++ return 0; ++} ++ ++static int usdpaa_mmap(struct file *filp, struct vm_area_struct *vma) ++{ ++ struct ctx *ctx = filp->private_data; ++ unsigned long pfn = 0; ++ int match, ret; ++ ++ spin_lock(&mem_lock); ++ ret = check_mmap_dma(ctx, vma, &match, &pfn); ++ if (!match) ++ ret = check_mmap_portal(ctx, vma, &match, &pfn); ++ spin_unlock(&mem_lock); ++ if (!match) ++ return -EINVAL; ++ if (!ret) ++ ret = remap_pfn_range(vma, vma->vm_start, pfn, ++ vma->vm_end - vma->vm_start, ++ vma->vm_page_prot); ++ return ret; ++} ++ ++/* Return the nearest rounded-up address >= 'addr' that is 'sz'-aligned. 'sz' ++ * must be a power of 2, but both 'addr' and 'sz' can be expressions. */ ++#define USDPAA_MEM_ROUNDUP(addr, sz) \ ++ ({ \ ++ unsigned long foo_align = (sz) - 1; \ ++ ((addr) + foo_align) & ~foo_align; \ ++ }) ++/* Searching for a size-aligned virtual address range starting from 'addr' */ ++static unsigned long usdpaa_get_unmapped_area(struct file *file, ++ unsigned long addr, ++ unsigned long len, ++ unsigned long pgoff, ++ unsigned long flags) ++{ ++ struct vm_area_struct *vma; ++ ++ if (len % PAGE_SIZE) ++ return -EINVAL; ++ if (!len) ++ return -EINVAL; ++ ++ /* Need to align the address to the largest pagesize of the mapping ++ * because the MMU requires the virtual address to have the same ++ * alignment as the physical address */ ++ addr = USDPAA_MEM_ROUNDUP(addr, largest_page_size(len)); ++ vma = find_vma(current->mm, addr); ++ /* Keep searching until we reach the end of currently-used virtual ++ * address-space or we find a big enough gap. */ ++ while (vma) { ++ if ((addr + len) < vma->vm_start) ++ return addr; ++ ++ addr = USDPAA_MEM_ROUNDUP(vma->vm_end, largest_page_size(len)); ++ vma = vma->vm_next; ++ } ++ if ((TASK_SIZE - len) < addr) ++ return -ENOMEM; ++ return addr; ++} ++ ++static long ioctl_id_alloc(struct ctx *ctx, void __user *arg) ++{ ++ struct usdpaa_ioctl_id_alloc i; ++ const struct alloc_backend *backend; ++ struct active_resource *res; ++ int ret = copy_from_user(&i, arg, sizeof(i)); ++ if (ret) ++ return ret; ++ if ((i.id_type >= usdpaa_id_max) || !i.num) ++ return -EINVAL; ++ backend = &alloc_backends[i.id_type]; ++ /* Allocate the required resource type */ ++ ret = backend->alloc(&i.base, i.num, i.align, i.partial); ++ if (ret < 0) ++ return ret; ++ i.num = ret; ++ /* Copy the result to user-space */ ++ ret = copy_to_user(arg, &i, sizeof(i)); ++ if (ret) { ++ backend->release(i.base, i.num); ++ return ret; ++ } ++ /* Assign the allocated range to the FD accounting */ ++ res = kmalloc(sizeof(*res), GFP_KERNEL); ++ if (!res) { ++ backend->release(i.base, i.num); ++ return -ENOMEM; ++ } ++ spin_lock(&ctx->lock); ++ res->id = i.base; ++ res->num = i.num; ++ res->refcount = 1; ++ list_add(&res->list, &ctx->resources[i.id_type]); ++ spin_unlock(&ctx->lock); ++ return 0; ++} ++ ++static long ioctl_id_release(struct ctx *ctx, void __user *arg) ++{ ++ struct usdpaa_ioctl_id_release i; ++ const struct alloc_backend *backend; ++ struct active_resource *tmp, *pos; ++ ++ int ret = copy_from_user(&i, arg, sizeof(i)); ++ if (ret) ++ return ret; ++ if ((i.id_type >= usdpaa_id_max) || !i.num) ++ return -EINVAL; ++ backend = &alloc_backends[i.id_type]; ++ /* Pull the range out of the FD accounting - the range is valid iff this ++ * succeeds. */ ++ spin_lock(&ctx->lock); ++ list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) { ++ if (pos->id == i.base && pos->num == i.num) { ++ pos->refcount--; ++ if (pos->refcount) { ++ spin_unlock(&ctx->lock); ++ return 0; /* Still being used */ ++ } ++ list_del(&pos->list); ++ kfree(pos); ++ spin_unlock(&ctx->lock); ++ goto found; ++ } ++ } ++ /* Failed to find the resource */ ++ spin_unlock(&ctx->lock); ++ pr_err("Couldn't find resource type %d base 0x%x num %d\n", ++ i.id_type, i.base, i.num); ++ return -EINVAL; ++found: ++ /* Release the resource to the backend */ ++ backend->release(i.base, i.num); ++ return 0; ++} ++ ++static long ioctl_id_reserve(struct ctx *ctx, void __user *arg) ++{ ++ struct usdpaa_ioctl_id_reserve i; ++ const struct alloc_backend *backend; ++ struct active_resource *tmp, *pos; ++ ++ int ret = copy_from_user(&i, arg, sizeof(i)); ++ if (ret) ++ return ret; ++ if ((i.id_type >= usdpaa_id_max) || !i.num) ++ return -EINVAL; ++ backend = &alloc_backends[i.id_type]; ++ if (!backend->reserve) ++ return -EINVAL; ++ /* Pull the range out of the FD accounting - the range is valid iff this ++ * succeeds. */ ++ spin_lock(&ctx->lock); ++ list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) { ++ if (pos->id == i.base && pos->num == i.num) { ++ pos->refcount++; ++ spin_unlock(&ctx->lock); ++ return 0; ++ } ++ } ++ ++ /* Failed to find the resource */ ++ spin_unlock(&ctx->lock); ++ ++ /* Reserve the resource in the backend */ ++ ret = backend->reserve(i.base, i.num); ++ if (ret) ++ return ret; ++ /* Assign the reserved range to the FD accounting */ ++ pos = kmalloc(sizeof(*pos), GFP_KERNEL); ++ if (!pos) { ++ backend->release(i.base, i.num); ++ return -ENOMEM; ++ } ++ spin_lock(&ctx->lock); ++ pos->id = i.base; ++ pos->num = i.num; ++ pos->refcount = 1; ++ list_add(&pos->list, &ctx->resources[i.id_type]); ++ spin_unlock(&ctx->lock); ++ return 0; ++} ++ ++static long ioctl_dma_map(struct file *fp, struct ctx *ctx, ++ struct usdpaa_ioctl_dma_map *i) ++{ ++ struct mem_fragment *frag, *start_frag, *next_frag; ++ struct mem_mapping *map, *tmp; ++ int ret = 0; ++ u32 largest_page, so_far = 0; ++ int frag_count = 0; ++ unsigned long next_addr = PAGE_SIZE, populate; ++ ++ /* error checking to ensure values copied from user space are valid */ ++ if (i->len % PAGE_SIZE) ++ return -EINVAL; ++ ++ map = kmalloc(sizeof(*map), GFP_KERNEL); ++ if (!map) ++ return -ENOMEM; ++ ++ spin_lock(&mem_lock); ++ if (i->flags & USDPAA_DMA_FLAG_SHARE) { ++ list_for_each_entry(frag, &mem_list, list) { ++ if (frag->refs && (frag->flags & ++ USDPAA_DMA_FLAG_SHARE) && ++ !strncmp(i->name, frag->name, ++ USDPAA_DMA_NAME_MAX)) { ++ /* Matching entry */ ++ if ((i->flags & USDPAA_DMA_FLAG_CREATE) && ++ !(i->flags & USDPAA_DMA_FLAG_LAZY)) { ++ ret = -EBUSY; ++ goto out; ++ } ++ ++ /* Check to ensure size matches record */ ++ if (i->len != frag->map_len && i->len) { ++ pr_err("ioctl_dma_map() Size requested does not match %s and is none zero. This usage will be disallowed in future release\n", ++ frag->name); ++ } ++ ++ /* Check if this has already been mapped ++ to this process */ ++ list_for_each_entry(tmp, &ctx->maps, list) ++ if (tmp->root_frag == frag) { ++ /* Already mapped, just need to ++ inc ref count */ ++ tmp->refs++; ++ kfree(map); ++ i->did_create = 0; ++ i->len = tmp->total_size; ++ i->phys_addr = frag->base; ++ i->ptr = tmp->virt_addr; ++ spin_unlock(&mem_lock); ++ return 0; ++ } ++ /* Matching entry - just need to map */ ++ i->has_locking = frag->has_locking; ++ i->did_create = 0; ++ i->len = frag->map_len; ++ start_frag = frag; ++ goto do_map; ++ } ++ } ++ /* No matching entry */ ++ if (!(i->flags & USDPAA_DMA_FLAG_CREATE)) { ++ pr_err("ioctl_dma_map() No matching entry\n"); ++ ret = -ENOMEM; ++ goto out; ++ } ++ } ++ /* New fragment required, size must be provided. */ ++ if (!i->len) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ /* Find one of more contiguous fragments that satisfy the total length ++ trying to minimize the number of fragments ++ compute the largest page size that the allocation could use */ ++ largest_page = largest_page_size(i->len); ++ start_frag = NULL; ++ while (largest_page && ++ largest_page <= largest_page_size(phys_size) && ++ start_frag == NULL) { ++ /* Search the list for a frag of that size */ ++ list_for_each_entry(frag, &mem_list, list) { ++ if (!frag->refs && (frag->len == largest_page)) { ++ /* See if the next x fragments are free ++ and can accomidate the size */ ++ u32 found_size = largest_page; ++ next_frag = list_entry(frag->list.prev, ++ struct mem_fragment, ++ list); ++ /* If the fragement is too small check ++ if the neighbours cab support it */ ++ while (found_size < i->len) { ++ if (&mem_list == &next_frag->list) ++ break; /* End of list */ ++ if (next_frag->refs != 0 || ++ next_frag->len == 0) ++ break; /* not enough space */ ++ found_size += next_frag->len; ++ next_frag = list_entry( ++ next_frag->list.prev, ++ struct mem_fragment, ++ list); ++ } ++ if (found_size >= i->len) { ++ /* Success! there is enough contigous ++ free space */ ++ start_frag = frag; ++ break; ++ } ++ } ++ } /* next frag loop */ ++ /* Couldn't statisfy the request with this ++ largest page size, try a smaller one */ ++ largest_page <<= 2; ++ } ++ if (start_frag == NULL) { ++ /* Couldn't find proper amount of space */ ++ ret = -ENOMEM; ++ goto out; ++ } ++ i->did_create = 1; ++do_map: ++ /* Verify there is sufficient space to do the mapping */ ++ down_write(¤t->mm->mmap_sem); ++ next_addr = usdpaa_get_unmapped_area(fp, next_addr, i->len, 0, 0); ++ up_write(¤t->mm->mmap_sem); ++ ++ if (next_addr & ~PAGE_MASK) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ ++ /* We may need to divide the final fragment to accomidate the mapping */ ++ next_frag = start_frag; ++ while (so_far != i->len) { ++ BUG_ON(next_frag->len == 0); ++ while ((next_frag->len + so_far) > i->len) { ++ /* Split frag until they match */ ++ split_frag(next_frag); ++ } ++ so_far += next_frag->len; ++ next_frag->refs++; ++ ++frag_count; ++ next_frag = list_entry(next_frag->list.prev, ++ struct mem_fragment, list); ++ } ++ if (i->did_create) { ++ size_t name_len = 0; ++ start_frag->flags = i->flags; ++ strncpy(start_frag->name, i->name, USDPAA_DMA_NAME_MAX); ++ name_len = strnlen(start_frag->name, USDPAA_DMA_NAME_MAX); ++ if (name_len >= USDPAA_DMA_NAME_MAX) { ++ ret = -EFAULT; ++ goto out; ++ } ++ start_frag->map_len = i->len; ++ start_frag->has_locking = i->has_locking; ++ init_waitqueue_head(&start_frag->wq); ++ start_frag->owner = NULL; ++ } ++ ++ /* Setup the map entry */ ++ map->root_frag = start_frag; ++ map->total_size = i->len; ++ map->frag_count = frag_count; ++ map->refs = 1; ++ list_add(&map->list, &ctx->maps); ++ i->phys_addr = start_frag->base; ++out: ++ spin_unlock(&mem_lock); ++ ++ if (!ret) { ++ unsigned long longret; ++ down_write(¤t->mm->mmap_sem); ++ longret = do_mmap_pgoff(fp, next_addr, map->total_size, ++ PROT_READ | ++ (i->flags & ++ USDPAA_DMA_FLAG_RDONLY ? 0 ++ : PROT_WRITE), ++ MAP_SHARED, ++ start_frag->pfn_base, ++ &populate); ++ up_write(¤t->mm->mmap_sem); ++ if (longret & ~PAGE_MASK) { ++ ret = (int)longret; ++ } else { ++ i->ptr = (void *)longret; ++ map->virt_addr = i->ptr; ++ } ++ } else ++ kfree(map); ++ return ret; ++} ++ ++static long ioctl_dma_unmap(struct ctx *ctx, void __user *arg) ++{ ++ struct mem_mapping *map; ++ struct vm_area_struct *vma; ++ int ret, i; ++ struct mem_fragment *current_frag; ++ size_t sz; ++ unsigned long base; ++ unsigned long vaddr; ++ ++ down_write(¤t->mm->mmap_sem); ++ vma = find_vma(current->mm, (unsigned long)arg); ++ if (!vma || (vma->vm_start > (unsigned long)arg)) { ++ up_write(¤t->mm->mmap_sem); ++ return -EFAULT; ++ } ++ spin_lock(&mem_lock); ++ list_for_each_entry(map, &ctx->maps, list) { ++ if (map->root_frag->pfn_base == vma->vm_pgoff) { ++ /* Drop the map lock if we hold it */ ++ if (map->root_frag->has_locking && ++ (map->root_frag->owner == map)) { ++ map->root_frag->owner = NULL; ++ wake_up(&map->root_frag->wq); ++ } ++ goto map_match; ++ } ++ } ++ /* Failed to find a matching mapping for this process */ ++ ret = -EFAULT; ++ spin_unlock(&mem_lock); ++ goto out; ++map_match: ++ map->refs--; ++ if (map->refs != 0) { ++ /* Another call the dma_map is referencing this */ ++ ret = 0; ++ spin_unlock(&mem_lock); ++ goto out; ++ } ++ ++ current_frag = map->root_frag; ++ vaddr = (unsigned long) map->virt_addr; ++ for (i = 0; i < map->frag_count; i++) { ++ DPA_ASSERT(current_frag->refs > 0); ++ --current_frag->refs; ++#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64)) ++ /* ++ * Make sure we invalidate the TLB entry for ++ * this fragment, otherwise a remap of a different ++ * page to this vaddr would give acces to an ++ * incorrect piece of memory ++ */ ++ cleartlbcam(vaddr, mfspr(SPRN_PID)); ++#endif ++ vaddr += current_frag->len; ++ current_frag = list_entry(current_frag->list.prev, ++ struct mem_fragment, list); ++ } ++ map->root_frag->name[0] = 0; ++ list_del(&map->list); ++ compress_frags(); ++ spin_unlock(&mem_lock); ++ ++ base = vma->vm_start; ++ sz = vma->vm_end - vma->vm_start; ++ do_munmap(current->mm, base, sz); ++ ret = 0; ++ out: ++ up_write(¤t->mm->mmap_sem); ++ return ret; ++} ++ ++static long ioctl_dma_stats(struct ctx *ctx, void __user *arg) ++{ ++ struct mem_fragment *frag; ++ struct usdpaa_ioctl_dma_used result; ++ ++ result.free_bytes = 0; ++ result.total_bytes = phys_size; ++ ++ list_for_each_entry(frag, &mem_list, list) { ++ if (frag->refs == 0) ++ result.free_bytes += frag->len; ++ } ++ ++ return copy_to_user(arg, &result, sizeof(result)); } ++ ++static int test_lock(struct mem_mapping *map) ++{ ++ int ret = 0; ++ spin_lock(&mem_lock); ++ if (!map->root_frag->owner) { ++ map->root_frag->owner = map; ++ ret = 1; ++ } ++ spin_unlock(&mem_lock); ++ return ret; ++} ++ ++static long ioctl_dma_lock(struct ctx *ctx, void __user *arg) ++{ ++ struct mem_mapping *map; ++ struct vm_area_struct *vma; ++ ++ down_read(¤t->mm->mmap_sem); ++ vma = find_vma(current->mm, (unsigned long)arg); ++ if (!vma || (vma->vm_start > (unsigned long)arg)) { ++ up_read(¤t->mm->mmap_sem); ++ return -EFAULT; ++ } ++ spin_lock(&mem_lock); ++ list_for_each_entry(map, &ctx->maps, list) { ++ if (map->root_frag->pfn_base == vma->vm_pgoff) ++ goto map_match; ++ } ++ map = NULL; ++map_match: ++ spin_unlock(&mem_lock); ++ up_read(¤t->mm->mmap_sem); ++ ++ if (!map) ++ return -EFAULT; ++ if (!map->root_frag->has_locking) ++ return -ENODEV; ++ return wait_event_interruptible(map->root_frag->wq, test_lock(map)); ++} ++ ++static long ioctl_dma_unlock(struct ctx *ctx, void __user *arg) ++{ ++ struct mem_mapping *map; ++ struct vm_area_struct *vma; ++ int ret; ++ ++ down_read(¤t->mm->mmap_sem); ++ vma = find_vma(current->mm, (unsigned long)arg); ++ if (!vma || (vma->vm_start > (unsigned long)arg)) ++ ret = -EFAULT; ++ else { ++ spin_lock(&mem_lock); ++ list_for_each_entry(map, &ctx->maps, list) { ++ if (map->root_frag->pfn_base == vma->vm_pgoff) { ++ if (!map->root_frag->has_locking) ++ ret = -ENODEV; ++ else if (map->root_frag->owner == map) { ++ map->root_frag->owner = NULL; ++ wake_up(&map->root_frag->wq); ++ ret = 0; ++ } else ++ ret = -EBUSY; ++ goto map_match; ++ } ++ } ++ ret = -EINVAL; ++map_match: ++ spin_unlock(&mem_lock); ++ } ++ up_read(¤t->mm->mmap_sem); ++ return ret; ++} ++ ++static int portal_mmap(struct file *fp, struct resource *res, void **ptr) ++{ ++ unsigned long longret = 0, populate; ++ resource_size_t len; ++ ++ down_write(¤t->mm->mmap_sem); ++ len = resource_size(res); ++ if (len != (unsigned long)len) ++ return -EINVAL; ++ longret = do_mmap_pgoff(fp, PAGE_SIZE, (unsigned long)len, ++ PROT_READ | PROT_WRITE, MAP_SHARED, ++ res->start >> PAGE_SHIFT, &populate); ++ up_write(¤t->mm->mmap_sem); ++ ++ if (longret & ~PAGE_MASK) ++ return (int)longret; ++ ++ *ptr = (void *) longret; ++ return 0; ++} ++ ++static void portal_munmap(struct resource *res, void *ptr) ++{ ++ down_write(¤t->mm->mmap_sem); ++ do_munmap(current->mm, (unsigned long)ptr, resource_size(res)); ++ up_write(¤t->mm->mmap_sem); ++} ++ ++static long ioctl_portal_map(struct file *fp, struct ctx *ctx, ++ struct usdpaa_ioctl_portal_map *arg) ++{ ++ struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); ++ int ret; ++ ++ if (!mapping) ++ return -ENOMEM; ++ ++ mapping->user = *arg; ++ mapping->iommu_domain = NULL; ++ ++ if (mapping->user.type == usdpaa_portal_qman) { ++ mapping->qportal = ++ qm_get_unused_portal_idx(mapping->user.index); ++ if (!mapping->qportal) { ++ ret = -ENODEV; ++ goto err_get_portal; ++ } ++ mapping->phys = &mapping->qportal->addr_phys[0]; ++ mapping->user.channel = mapping->qportal->public_cfg.channel; ++ mapping->user.pools = mapping->qportal->public_cfg.pools; ++ mapping->user.index = mapping->qportal->public_cfg.index; ++ } else if (mapping->user.type == usdpaa_portal_bman) { ++ mapping->bportal = ++ bm_get_unused_portal_idx(mapping->user.index); ++ if (!mapping->bportal) { ++ ret = -ENODEV; ++ goto err_get_portal; ++ } ++ mapping->phys = &mapping->bportal->addr_phys[0]; ++ mapping->user.index = mapping->bportal->public_cfg.index; ++ } else { ++ ret = -EINVAL; ++ goto err_copy_from_user; ++ } ++ /* Need to put pcfg in ctx's list before the mmaps because the mmap ++ * handlers look it up. */ ++ spin_lock(&mem_lock); ++ list_add(&mapping->list, &ctx->portals); ++ spin_unlock(&mem_lock); ++ ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CE], ++ &mapping->user.addr.cena); ++ if (ret) ++ goto err_mmap_cena; ++ ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CI], ++ &mapping->user.addr.cinh); ++ if (ret) ++ goto err_mmap_cinh; ++ *arg = mapping->user; ++ return ret; ++ ++err_mmap_cinh: ++ portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena); ++err_mmap_cena: ++ if ((mapping->user.type == usdpaa_portal_qman) && mapping->qportal) ++ qm_put_unused_portal(mapping->qportal); ++ else if ((mapping->user.type == usdpaa_portal_bman) && mapping->bportal) ++ bm_put_unused_portal(mapping->bportal); ++ spin_lock(&mem_lock); ++ list_del(&mapping->list); ++ spin_unlock(&mem_lock); ++err_get_portal: ++err_copy_from_user: ++ kfree(mapping); ++ return ret; ++} ++ ++static long ioctl_portal_unmap(struct ctx *ctx, struct usdpaa_portal_map *i) ++{ ++ struct portal_mapping *mapping; ++ struct vm_area_struct *vma; ++ unsigned long pfn; ++ u32 channel; ++ ++ /* Get the PFN corresponding to one of the virt addresses */ ++ down_read(¤t->mm->mmap_sem); ++ vma = find_vma(current->mm, (unsigned long)i->cinh); ++ if (!vma || (vma->vm_start > (unsigned long)i->cinh)) { ++ up_read(¤t->mm->mmap_sem); ++ return -EFAULT; ++ } ++ pfn = vma->vm_pgoff; ++ up_read(¤t->mm->mmap_sem); ++ ++ /* Find the corresponding portal */ ++ spin_lock(&mem_lock); ++ list_for_each_entry(mapping, &ctx->portals, list) { ++ if (pfn == (mapping->phys[DPA_PORTAL_CI].start >> PAGE_SHIFT)) ++ goto found; ++ } ++ mapping = NULL; ++found: ++ if (mapping) ++ list_del(&mapping->list); ++ spin_unlock(&mem_lock); ++ if (!mapping) ++ return -ENODEV; ++ portal_munmap(&mapping->phys[DPA_PORTAL_CI], mapping->user.addr.cinh); ++ portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena); ++ if (mapping->user.type == usdpaa_portal_qman) { ++ init_qm_portal(mapping->qportal, ++ &mapping->qman_portal_low); ++ ++ /* Tear down any FQs this portal is referencing */ ++ channel = mapping->qportal->public_cfg.channel; ++ qm_check_and_destroy_fqs(&mapping->qman_portal_low, ++ &channel, ++ check_portal_channel); ++ qm_put_unused_portal(mapping->qportal); ++ } else if (mapping->user.type == usdpaa_portal_bman) { ++ init_bm_portal(mapping->bportal, ++ &mapping->bman_portal_low); ++ bm_put_unused_portal(mapping->bportal); ++ } ++ kfree(mapping); ++ return 0; ++} ++ ++static void portal_config_pamu(struct qm_portal_config *pcfg, uint8_t sdest, ++ uint32_t cpu, uint32_t cache, uint32_t window) ++{ ++#ifdef CONFIG_FSL_PAMU ++ int ret; ++ int window_count = 1; ++ struct iommu_domain_geometry geom_attr; ++ struct pamu_stash_attribute stash_attr; ++ ++ pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type); ++ if (!pcfg->iommu_domain) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed", ++ __func__); ++ goto _no_iommu; ++ } ++ geom_attr.aperture_start = 0; ++ geom_attr.aperture_end = ++ ((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1; ++ geom_attr.force_aperture = true; ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY, ++ &geom_attr); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS, ++ &window_count); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ stash_attr.cpu = cpu; ++ stash_attr.cache = cache; ++ /* set stash information for the window */ ++ stash_attr.window = 0; ++ ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, ++ DOMAIN_ATTR_FSL_PAMU_STASH, ++ &stash_attr); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36, ++ IOMMU_READ | IOMMU_WRITE); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, ++ DOMAIN_ATTR_FSL_PAMU_ENABLE, ++ &window_count); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_detach_device; ++ } ++_no_iommu: ++#endif ++ ++#ifdef CONFIG_FSL_QMAN_CONFIG ++ if (qman_set_sdest(pcfg->public_cfg.channel, sdest)) ++#endif ++ pr_warn("Failed to set QMan portal's stash request queue\n"); ++ ++ return; ++ ++#ifdef CONFIG_FSL_PAMU ++_iommu_detach_device: ++ iommu_detach_device(pcfg->iommu_domain, NULL); ++_iommu_domain_free: ++ iommu_domain_free(pcfg->iommu_domain); ++#endif ++} ++ ++static long ioctl_allocate_raw_portal(struct file *fp, struct ctx *ctx, ++ struct usdpaa_ioctl_raw_portal *arg) ++{ ++ struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); ++ int ret; ++ ++ if (!mapping) ++ return -ENOMEM; ++ ++ mapping->user.type = arg->type; ++ mapping->iommu_domain = NULL; ++ if (arg->type == usdpaa_portal_qman) { ++ mapping->qportal = qm_get_unused_portal_idx(arg->index); ++ if (!mapping->qportal) { ++ ret = -ENODEV; ++ goto err; ++ } ++ mapping->phys = &mapping->qportal->addr_phys[0]; ++ arg->index = mapping->qportal->public_cfg.index; ++ arg->cinh = mapping->qportal->addr_phys[DPA_PORTAL_CI].start; ++ arg->cena = mapping->qportal->addr_phys[DPA_PORTAL_CE].start; ++ if (arg->enable_stash) { ++ /* Setup the PAMU with the supplied parameters */ ++ portal_config_pamu(mapping->qportal, arg->sdest, ++ arg->cpu, arg->cache, arg->window); ++ } ++ } else if (mapping->user.type == usdpaa_portal_bman) { ++ mapping->bportal = ++ bm_get_unused_portal_idx(arg->index); ++ if (!mapping->bportal) { ++ ret = -ENODEV; ++ goto err; ++ } ++ mapping->phys = &mapping->bportal->addr_phys[0]; ++ arg->index = mapping->bportal->public_cfg.index; ++ arg->cinh = mapping->bportal->addr_phys[DPA_PORTAL_CI].start; ++ arg->cena = mapping->bportal->addr_phys[DPA_PORTAL_CE].start; ++ } else { ++ ret = -EINVAL; ++ goto err; ++ } ++ /* Need to put pcfg in ctx's list before the mmaps because the mmap ++ * handlers look it up. */ ++ spin_lock(&mem_lock); ++ list_add(&mapping->list, &ctx->portals); ++ spin_unlock(&mem_lock); ++ return 0; ++err: ++ kfree(mapping); ++ return ret; ++} ++ ++static long ioctl_free_raw_portal(struct file *fp, struct ctx *ctx, ++ struct usdpaa_ioctl_raw_portal *arg) ++{ ++ struct portal_mapping *mapping; ++ u32 channel; ++ ++ /* Find the corresponding portal */ ++ spin_lock(&mem_lock); ++ list_for_each_entry(mapping, &ctx->portals, list) { ++ if (mapping->phys[DPA_PORTAL_CI].start == arg->cinh) ++ goto found; ++ } ++ mapping = NULL; ++found: ++ if (mapping) ++ list_del(&mapping->list); ++ spin_unlock(&mem_lock); ++ if (!mapping) ++ return -ENODEV; ++ if (mapping->user.type == usdpaa_portal_qman) { ++ init_qm_portal(mapping->qportal, ++ &mapping->qman_portal_low); ++ ++ /* Tear down any FQs this portal is referencing */ ++ channel = mapping->qportal->public_cfg.channel; ++ qm_check_and_destroy_fqs(&mapping->qman_portal_low, ++ &channel, ++ check_portal_channel); ++ qm_put_unused_portal(mapping->qportal); ++ } else if (mapping->user.type == usdpaa_portal_bman) { ++ init_bm_portal(mapping->bportal, ++ &mapping->bman_portal_low); ++ bm_put_unused_portal(mapping->bportal); ++ } ++ kfree(mapping); ++ return 0; ++} ++ ++static long usdpaa_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) ++{ ++ struct ctx *ctx = fp->private_data; ++ void __user *a = (void __user *)arg; ++ switch (cmd) { ++ case USDPAA_IOCTL_ID_ALLOC: ++ return ioctl_id_alloc(ctx, a); ++ case USDPAA_IOCTL_ID_RELEASE: ++ return ioctl_id_release(ctx, a); ++ case USDPAA_IOCTL_ID_RESERVE: ++ return ioctl_id_reserve(ctx, a); ++ case USDPAA_IOCTL_DMA_MAP: ++ { ++ struct usdpaa_ioctl_dma_map input; ++ int ret; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ ret = ioctl_dma_map(fp, ctx, &input); ++ if (copy_to_user(a, &input, sizeof(input))) ++ return -EFAULT; ++ return ret; ++ } ++ case USDPAA_IOCTL_DMA_UNMAP: ++ return ioctl_dma_unmap(ctx, a); ++ case USDPAA_IOCTL_DMA_LOCK: ++ return ioctl_dma_lock(ctx, a); ++ case USDPAA_IOCTL_DMA_UNLOCK: ++ return ioctl_dma_unlock(ctx, a); ++ case USDPAA_IOCTL_PORTAL_MAP: ++ { ++ struct usdpaa_ioctl_portal_map input; ++ int ret; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ ret = ioctl_portal_map(fp, ctx, &input); ++ if (copy_to_user(a, &input, sizeof(input))) ++ return -EFAULT; ++ return ret; ++ } ++ case USDPAA_IOCTL_PORTAL_UNMAP: ++ { ++ struct usdpaa_portal_map input; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ return ioctl_portal_unmap(ctx, &input); ++ } ++ case USDPAA_IOCTL_DMA_USED: ++ return ioctl_dma_stats(ctx, a); ++ case USDPAA_IOCTL_ALLOC_RAW_PORTAL: ++ { ++ struct usdpaa_ioctl_raw_portal input; ++ int ret; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ ret = ioctl_allocate_raw_portal(fp, ctx, &input); ++ if (copy_to_user(a, &input, sizeof(input))) ++ return -EFAULT; ++ return ret; ++ } ++ case USDPAA_IOCTL_FREE_RAW_PORTAL: ++ { ++ struct usdpaa_ioctl_raw_portal input; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ return ioctl_free_raw_portal(fp, ctx, &input); ++ } ++ } ++ return -EINVAL; ++} ++ ++static long usdpaa_ioctl_compat(struct file *fp, unsigned int cmd, ++ unsigned long arg) ++{ ++#ifdef CONFIG_COMPAT ++ struct ctx *ctx = fp->private_data; ++ void __user *a = (void __user *)arg; ++#endif ++ switch (cmd) { ++#ifdef CONFIG_COMPAT ++ case USDPAA_IOCTL_DMA_MAP_COMPAT: ++ { ++ int ret; ++ struct usdpaa_ioctl_dma_map_compat input; ++ struct usdpaa_ioctl_dma_map converted; ++ ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ ++ converted.ptr = compat_ptr(input.ptr); ++ converted.phys_addr = input.phys_addr; ++ converted.len = input.len; ++ converted.flags = input.flags; ++ strncpy(converted.name, input.name, USDPAA_DMA_NAME_MAX); ++ converted.has_locking = input.has_locking; ++ converted.did_create = input.did_create; ++ ++ ret = ioctl_dma_map(fp, ctx, &converted); ++ input.ptr = ptr_to_compat(converted.ptr); ++ input.phys_addr = converted.phys_addr; ++ input.len = converted.len; ++ input.flags = converted.flags; ++ strncpy(input.name, converted.name, USDPAA_DMA_NAME_MAX); ++ input.has_locking = converted.has_locking; ++ input.did_create = converted.did_create; ++ if (copy_to_user(a, &input, sizeof(input))) ++ return -EFAULT; ++ return ret; ++ } ++ case USDPAA_IOCTL_PORTAL_MAP_COMPAT: ++ { ++ int ret; ++ struct compat_usdpaa_ioctl_portal_map input; ++ struct usdpaa_ioctl_portal_map converted; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ converted.type = input.type; ++ converted.index = input.index; ++ ret = ioctl_portal_map(fp, ctx, &converted); ++ input.addr.cinh = ptr_to_compat(converted.addr.cinh); ++ input.addr.cena = ptr_to_compat(converted.addr.cena); ++ input.channel = converted.channel; ++ input.pools = converted.pools; ++ input.index = converted.index; ++ if (copy_to_user(a, &input, sizeof(input))) ++ return -EFAULT; ++ return ret; ++ } ++ case USDPAA_IOCTL_PORTAL_UNMAP_COMPAT: ++ { ++ struct usdpaa_portal_map_compat input; ++ struct usdpaa_portal_map converted; ++ ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ converted.cinh = compat_ptr(input.cinh); ++ converted.cena = compat_ptr(input.cena); ++ return ioctl_portal_unmap(ctx, &converted); ++ } ++ case USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT: ++ { ++ int ret; ++ struct usdpaa_ioctl_raw_portal converted; ++ struct compat_ioctl_raw_portal input; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ converted.type = input.type; ++ converted.index = input.index; ++ converted.enable_stash = input.enable_stash; ++ converted.cpu = input.cpu; ++ converted.cache = input.cache; ++ converted.window = input.window; ++ converted.sdest = input.sdest; ++ ret = ioctl_allocate_raw_portal(fp, ctx, &converted); ++ ++ input.cinh = converted.cinh; ++ input.cena = converted.cena; ++ input.index = converted.index; ++ ++ if (copy_to_user(a, &input, sizeof(input))) ++ return -EFAULT; ++ return ret; ++ } ++ case USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT: ++ { ++ struct usdpaa_ioctl_raw_portal converted; ++ struct compat_ioctl_raw_portal input; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ converted.type = input.type; ++ converted.index = input.index; ++ converted.cinh = input.cinh; ++ converted.cena = input.cena; ++ return ioctl_free_raw_portal(fp, ctx, &converted); ++ } ++#endif ++ default: ++ return usdpaa_ioctl(fp, cmd, arg); ++ } ++ return -EINVAL; ++} ++ ++int usdpaa_get_portal_config(struct file *filp, void *cinh, ++ enum usdpaa_portal_type ptype, unsigned int *irq, ++ void **iir_reg) ++{ ++ /* Walk the list of portals for filp and return the config ++ for the portal that matches the hint */ ++ struct ctx *context; ++ struct portal_mapping *portal; ++ ++ /* First sanitize the filp */ ++ if (filp->f_op->open != usdpaa_open) ++ return -ENODEV; ++ context = filp->private_data; ++ spin_lock(&context->lock); ++ list_for_each_entry(portal, &context->portals, list) { ++ if (portal->user.type == ptype && ++ portal->user.addr.cinh == cinh) { ++ if (ptype == usdpaa_portal_qman) { ++ *irq = portal->qportal->public_cfg.irq; ++ *iir_reg = portal->qportal->addr_virt[1] + ++ QM_REG_IIR; ++ } else { ++ *irq = portal->bportal->public_cfg.irq; ++ *iir_reg = portal->bportal->addr_virt[1] + ++ BM_REG_IIR; ++ } ++ spin_unlock(&context->lock); ++ return 0; ++ } ++ } ++ spin_unlock(&context->lock); ++ return -EINVAL; ++} ++ ++static const struct file_operations usdpaa_fops = { ++ .open = usdpaa_open, ++ .release = usdpaa_release, ++ .mmap = usdpaa_mmap, ++ .get_unmapped_area = usdpaa_get_unmapped_area, ++ .unlocked_ioctl = usdpaa_ioctl, ++ .compat_ioctl = usdpaa_ioctl_compat ++}; ++ ++static struct miscdevice usdpaa_miscdev = { ++ .name = "fsl-usdpaa", ++ .fops = &usdpaa_fops, ++ .minor = MISC_DYNAMIC_MINOR, ++}; ++ ++/* Early-boot memory allocation. The boot-arg "usdpaa_mem=<x>" is used to ++ * indicate how much memory (if any) to allocate during early boot. If the ++ * format "usdpaa_mem=<x>,<y>" is used, then <y> will be interpreted as the ++ * number of TLB1 entries to reserve (default is 1). If there are more mappings ++ * than there are TLB1 entries, fault-handling will occur. */ ++ ++static __init int usdpaa_mem(char *arg) ++{ ++ pr_warn("uspdaa_mem argument is depracated\n"); ++ arg_phys_size = memparse(arg, &arg); ++ num_tlb = 1; ++ if (*arg == ',') { ++ unsigned long ul; ++ int err = kstrtoul(arg + 1, 0, &ul); ++ if (err < 0) { ++ num_tlb = 1; ++ pr_warn("ERROR, usdpaa_mem arg is invalid\n"); ++ } else ++ num_tlb = (unsigned int)ul; ++ } ++ return 0; ++} ++early_param("usdpaa_mem", usdpaa_mem); ++ ++static int usdpaa_mem_init(struct reserved_mem *rmem) ++{ ++ phys_start = rmem->base; ++ phys_size = rmem->size; ++ ++ WARN_ON(!(phys_start && phys_size)); ++ ++ return 0; ++} ++RESERVEDMEM_OF_DECLARE(usdpaa_mem_init, "fsl,usdpaa-mem", usdpaa_mem_init); ++ ++__init int fsl_usdpaa_init_early(void) ++{ ++ if (!phys_size || !phys_start) { ++ pr_info("No USDPAA memory, no 'fsl,usdpaa-mem' in device-tree\n"); ++ return 0; ++ } ++ if (phys_size % PAGE_SIZE) { ++ pr_err("'fsl,usdpaa-mem' size must be a multiple of page size\n"); ++ phys_size = 0; ++ return 0; ++ } ++ if (arg_phys_size && phys_size != arg_phys_size) { ++ pr_err("'usdpaa_mem argument size (0x%llx) does not match device tree size (0x%llx)\n", ++ arg_phys_size, phys_size); ++ phys_size = 0; ++ return 0; ++ } ++ pfn_start = phys_start >> PAGE_SHIFT; ++ pfn_size = phys_size >> PAGE_SHIFT; ++#ifdef CONFIG_PPC ++ first_tlb = current_tlb = tlbcam_index; ++ tlbcam_index += num_tlb; ++#endif ++ pr_info("USDPAA region at %llx:%llx(%lx:%lx), %d TLB1 entries)\n", ++ phys_start, phys_size, pfn_start, pfn_size, num_tlb); ++ return 0; ++} ++subsys_initcall(fsl_usdpaa_init_early); ++ ++ ++static int __init usdpaa_init(void) ++{ ++ struct mem_fragment *frag; ++ int ret; ++ u64 tmp_size = phys_size; ++ u64 tmp_start = phys_start; ++ u64 tmp_pfn_size = pfn_size; ++ u64 tmp_pfn_start = pfn_start; ++ ++ pr_info("Freescale USDPAA process driver\n"); ++ if (!phys_start) { ++ pr_warn("fsl-usdpaa: no region found\n"); ++ return 0; ++ } ++ ++ while (tmp_size != 0) { ++ u32 frag_size = largest_page_size(tmp_size); ++ frag = kmalloc(sizeof(*frag), GFP_KERNEL); ++ if (!frag) { ++ pr_err("Failed to setup USDPAA memory accounting\n"); ++ return -ENOMEM; ++ } ++ frag->base = tmp_start; ++ frag->len = frag->root_len = frag_size; ++ frag->root_pfn = tmp_pfn_start; ++ frag->pfn_base = tmp_pfn_start; ++ frag->pfn_len = frag_size / PAGE_SIZE; ++ frag->refs = 0; ++ init_waitqueue_head(&frag->wq); ++ frag->owner = NULL; ++ list_add(&frag->list, &mem_list); ++ ++ /* Adjust for this frag */ ++ tmp_start += frag_size; ++ tmp_size -= frag_size; ++ tmp_pfn_start += frag_size / PAGE_SIZE; ++ tmp_pfn_size -= frag_size / PAGE_SIZE; ++ } ++ ret = misc_register(&usdpaa_miscdev); ++ if (ret) ++ pr_err("fsl-usdpaa: failed to register misc device\n"); ++ return ret; ++} ++ ++static void __exit usdpaa_exit(void) ++{ ++ misc_deregister(&usdpaa_miscdev); ++} ++ ++module_init(usdpaa_init); ++module_exit(usdpaa_exit); ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("Freescale Semiconductor"); ++MODULE_DESCRIPTION("Freescale USDPAA process driver"); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/fsl_usdpaa_irq.c +@@ -0,0 +1,289 @@ ++/* Copyright (c) 2013 Freescale Semiconductor, Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++/* define a device that allows USPDAA processes to open a file ++ descriptor and specify which IRQ it wants to montior using an ioctl() ++ When an IRQ is received, the device becomes readable so that a process ++ can use read() or select() type calls to monitor for IRQs */ ++ ++#include <linux/miscdevice.h> ++#include <linux/fs.h> ++#include <linux/cdev.h> ++#include <linux/slab.h> ++#include <linux/interrupt.h> ++#include <linux/poll.h> ++#include <linux/uaccess.h> ++#include <linux/fsl_usdpaa.h> ++#include <linux/module.h> ++#include <linux/fdtable.h> ++#include <linux/file.h> ++ ++#include "qman_low.h" ++#include "bman_low.h" ++ ++struct usdpaa_irq_ctx { ++ int irq_set; /* Set to true once the irq is set via ioctl */ ++ unsigned int irq_num; ++ u32 last_irq_count; /* Last value returned from read */ ++ u32 irq_count; /* Number of irqs since last read */ ++ wait_queue_head_t wait_queue; /* Waiting processes */ ++ spinlock_t lock; ++ void *inhibit_addr; /* inhibit register address */ ++ struct file *usdpaa_filp; ++ char irq_name[128]; ++}; ++ ++static int usdpaa_irq_open(struct inode *inode, struct file *filp) ++{ ++ struct usdpaa_irq_ctx *ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); ++ if (!ctx) ++ return -ENOMEM; ++ ctx->irq_set = 0; ++ ctx->irq_count = 0; ++ ctx->last_irq_count = 0; ++ init_waitqueue_head(&ctx->wait_queue); ++ spin_lock_init(&ctx->lock); ++ filp->private_data = ctx; ++ return 0; ++} ++ ++static int usdpaa_irq_release(struct inode *inode, struct file *filp) ++{ ++ struct usdpaa_irq_ctx *ctx = filp->private_data; ++ if (ctx->irq_set) { ++ /* Inhibit the IRQ */ ++ out_be32(ctx->inhibit_addr, 0x1); ++ irq_set_affinity_hint(ctx->irq_num, NULL); ++ free_irq(ctx->irq_num, ctx); ++ ctx->irq_set = 0; ++ fput(ctx->usdpaa_filp); ++ } ++ kfree(filp->private_data); ++ return 0; ++} ++ ++static irqreturn_t usdpaa_irq_handler(int irq, void *_ctx) ++{ ++ unsigned long flags; ++ struct usdpaa_irq_ctx *ctx = _ctx; ++ spin_lock_irqsave(&ctx->lock, flags); ++ ++ctx->irq_count; ++ spin_unlock_irqrestore(&ctx->lock, flags); ++ wake_up_all(&ctx->wait_queue); ++ /* Set the inhibit register. This will be reenabled ++ once the USDPAA code handles the IRQ */ ++ out_be32(ctx->inhibit_addr, 0x1); ++ pr_info("Inhibit at %p count %d", ctx->inhibit_addr, ctx->irq_count); ++ return IRQ_HANDLED; ++} ++ ++static int map_irq(struct file *fp, struct usdpaa_ioctl_irq_map *irq_map) ++{ ++ struct usdpaa_irq_ctx *ctx = fp->private_data; ++ int ret; ++ ++ if (ctx->irq_set) { ++ pr_debug("Setting USDPAA IRQ when it was already set!\n"); ++ return -EBUSY; ++ } ++ ++ ctx->usdpaa_filp = fget(irq_map->fd); ++ if (!ctx->usdpaa_filp) { ++ pr_debug("USDPAA fget(%d) returned NULL\n", irq_map->fd); ++ return -EINVAL; ++ } ++ ++ ret = usdpaa_get_portal_config(ctx->usdpaa_filp, irq_map->portal_cinh, ++ irq_map->type, &ctx->irq_num, ++ &ctx->inhibit_addr); ++ if (ret) { ++ pr_debug("USDPAA IRQ couldn't identify portal\n"); ++ fput(ctx->usdpaa_filp); ++ return ret; ++ } ++ ++ ctx->irq_set = 1; ++ ++ snprintf(ctx->irq_name, sizeof(ctx->irq_name), ++ "usdpaa_irq %d", ctx->irq_num); ++ ++ ret = request_irq(ctx->irq_num, usdpaa_irq_handler, 0, ++ ctx->irq_name, ctx); ++ if (ret) { ++ pr_err("USDPAA request_irq(%d) failed, ret= %d\n", ++ ctx->irq_num, ret); ++ ctx->irq_set = 0; ++ fput(ctx->usdpaa_filp); ++ return ret; ++ } ++ ret = irq_set_affinity(ctx->irq_num, tsk_cpus_allowed(current)); ++ if (ret) ++ pr_err("USDPAA irq_set_affinity() failed, ret= %d\n", ret); ++ ++ ret = irq_set_affinity_hint(ctx->irq_num, tsk_cpus_allowed(current)); ++ if (ret) ++ pr_err("USDPAA irq_set_affinity_hint() failed, ret= %d\n", ret); ++ ++ return 0; ++} ++ ++static long usdpaa_irq_ioctl(struct file *fp, unsigned int cmd, ++ unsigned long arg) ++{ ++ int ret; ++ struct usdpaa_ioctl_irq_map irq_map; ++ ++ if (cmd != USDPAA_IOCTL_PORTAL_IRQ_MAP) { ++ pr_debug("USDPAA IRQ unknown command 0x%x\n", cmd); ++ return -EINVAL; ++ } ++ ++ ret = copy_from_user(&irq_map, (void __user *)arg, ++ sizeof(irq_map)); ++ if (ret) ++ return ret; ++ return map_irq(fp, &irq_map); ++} ++ ++static ssize_t usdpaa_irq_read(struct file *filp, char __user *buff, ++ size_t count, loff_t *offp) ++{ ++ struct usdpaa_irq_ctx *ctx = filp->private_data; ++ int ret; ++ ++ if (!ctx->irq_set) { ++ pr_debug("Reading USDPAA IRQ before it was set\n"); ++ return -EINVAL; ++ } ++ ++ if (count < sizeof(ctx->irq_count)) { ++ pr_debug("USDPAA IRQ Read too small\n"); ++ return -EINVAL; ++ } ++ if (ctx->irq_count == ctx->last_irq_count) { ++ if (filp->f_flags & O_NONBLOCK) ++ return -EAGAIN; ++ ++ ret = wait_event_interruptible(ctx->wait_queue, ++ ctx->irq_count != ctx->last_irq_count); ++ if (ret == -ERESTARTSYS) ++ return ret; ++ } ++ ++ ctx->last_irq_count = ctx->irq_count; ++ ++ if (copy_to_user(buff, &ctx->last_irq_count, ++ sizeof(ctx->last_irq_count))) ++ return -EFAULT; ++ return sizeof(ctx->irq_count); ++} ++ ++static unsigned int usdpaa_irq_poll(struct file *filp, poll_table *wait) ++{ ++ struct usdpaa_irq_ctx *ctx = filp->private_data; ++ unsigned int ret = 0; ++ unsigned long flags; ++ ++ if (!ctx->irq_set) ++ return POLLHUP; ++ ++ poll_wait(filp, &ctx->wait_queue, wait); ++ ++ spin_lock_irqsave(&ctx->lock, flags); ++ if (ctx->irq_count != ctx->last_irq_count) ++ ret |= POLLIN | POLLRDNORM; ++ spin_unlock_irqrestore(&ctx->lock, flags); ++ return ret; ++} ++ ++static long usdpaa_irq_ioctl_compat(struct file *fp, unsigned int cmd, ++ unsigned long arg) ++{ ++#ifdef CONFIG_COMPAT ++ void __user *a = (void __user *)arg; ++#endif ++ switch (cmd) { ++#ifdef CONFIG_COMPAT ++ case USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT: ++ { ++ struct compat_ioctl_irq_map input; ++ struct usdpaa_ioctl_irq_map converted; ++ if (copy_from_user(&input, a, sizeof(input))) ++ return -EFAULT; ++ converted.type = input.type; ++ converted.fd = input.fd; ++ converted.portal_cinh = compat_ptr(input.portal_cinh); ++ return map_irq(fp, &converted); ++ } ++#endif ++ default: ++ return usdpaa_irq_ioctl(fp, cmd, arg); ++ } ++} ++ ++static const struct file_operations usdpaa_irq_fops = { ++ .open = usdpaa_irq_open, ++ .release = usdpaa_irq_release, ++ .unlocked_ioctl = usdpaa_irq_ioctl, ++ .compat_ioctl = usdpaa_irq_ioctl_compat, ++ .read = usdpaa_irq_read, ++ .poll = usdpaa_irq_poll ++}; ++ ++static struct miscdevice usdpaa_miscdev = { ++ .name = "fsl-usdpaa-irq", ++ .fops = &usdpaa_irq_fops, ++ .minor = MISC_DYNAMIC_MINOR, ++}; ++ ++static int __init usdpaa_irq_init(void) ++{ ++ int ret; ++ ++ pr_info("Freescale USDPAA process IRQ driver\n"); ++ ret = misc_register(&usdpaa_miscdev); ++ if (ret) ++ pr_err("fsl-usdpaa-irq: failed to register misc device\n"); ++ return ret; ++} ++ ++static void __exit usdpaa_irq_exit(void) ++{ ++ misc_deregister(&usdpaa_miscdev); ++} ++ ++module_init(usdpaa_irq_init); ++module_exit(usdpaa_irq_exit); ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("Freescale Semiconductor"); ++MODULE_DESCRIPTION("Freescale USDPAA process IRQ driver"); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qbman_driver.c +@@ -0,0 +1,88 @@ ++/* Copyright 2013 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include <linux/time.h> ++#include "qman_private.h" ++#include "bman_private.h" ++__init void qman_init_early(void); ++__init void bman_init_early(void); ++ ++static __init int qbman_init(void) ++{ ++ struct device_node *dn; ++ u32 is_portal_available; ++ ++ bman_init(); ++ qman_init(); ++ ++ is_portal_available = 0; ++ for_each_compatible_node(dn, NULL, "fsl,qman-portal") { ++ if (!of_device_is_available(dn)) ++ continue; ++ else ++ is_portal_available = 1; ++ } ++ ++ if (!qman_have_ccsr() && is_portal_available) { ++ struct qman_fq fq = { ++ .fqid = 1 ++ }; ++ struct qm_mcr_queryfq_np np; ++ int err, retry = CONFIG_FSL_QMAN_INIT_TIMEOUT; ++ struct timespec nowts, diffts, startts = current_kernel_time(); ++ /* Loop while querying given fqid succeeds or time out */ ++ while (1) { ++ err = qman_query_fq_np(&fq, &np); ++ if (!err) { ++ /* success, control-plane has configured QMan */ ++ break; ++ } else if (err != -ERANGE) { ++ pr_err("QMan: I/O error, continuing anyway\n"); ++ break; ++ } ++ nowts = current_kernel_time(); ++ diffts = timespec_sub(nowts, startts); ++ if (diffts.tv_sec > 0) { ++ if (!retry--) { ++ pr_err("QMan: time out, control-plane" ++ " dead?\n"); ++ break; ++ } ++ pr_warn("QMan: polling for the control-plane" ++ " (%d)\n", retry); ++ } ++ } ++ } ++ bman_resource_init(); ++ qman_resource_init(); ++ return 0; ++} ++subsys_initcall(qbman_init); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_config.c +@@ -0,0 +1,1199 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include <asm/cacheflush.h> ++#include "qman_private.h" ++#include <linux/highmem.h> ++#include <linux/of_reserved_mem.h> ++ ++/* Last updated for v00.800 of the BG */ ++ ++/* Register offsets */ ++#define REG_QCSP_LIO_CFG(n) (0x0000 + ((n) * 0x10)) ++#define REG_QCSP_IO_CFG(n) (0x0004 + ((n) * 0x10)) ++#define REG_QCSP_DD_CFG(n) (0x000c + ((n) * 0x10)) ++#define REG_DD_CFG 0x0200 ++#define REG_DCP_CFG(n) (0x0300 + ((n) * 0x10)) ++#define REG_DCP_DD_CFG(n) (0x0304 + ((n) * 0x10)) ++#define REG_DCP_DLM_AVG(n) (0x030c + ((n) * 0x10)) ++#define REG_PFDR_FPC 0x0400 ++#define REG_PFDR_FP_HEAD 0x0404 ++#define REG_PFDR_FP_TAIL 0x0408 ++#define REG_PFDR_FP_LWIT 0x0410 ++#define REG_PFDR_CFG 0x0414 ++#define REG_SFDR_CFG 0x0500 ++#define REG_SFDR_IN_USE 0x0504 ++#define REG_WQ_CS_CFG(n) (0x0600 + ((n) * 0x04)) ++#define REG_WQ_DEF_ENC_WQID 0x0630 ++#define REG_WQ_SC_DD_CFG(n) (0x640 + ((n) * 0x04)) ++#define REG_WQ_PC_DD_CFG(n) (0x680 + ((n) * 0x04)) ++#define REG_WQ_DC0_DD_CFG(n) (0x6c0 + ((n) * 0x04)) ++#define REG_WQ_DC1_DD_CFG(n) (0x700 + ((n) * 0x04)) ++#define REG_WQ_DCn_DD_CFG(n) (0x6c0 + ((n) * 0x40)) /* n=2,3 */ ++#define REG_CM_CFG 0x0800 ++#define REG_ECSR 0x0a00 ++#define REG_ECIR 0x0a04 ++#define REG_EADR 0x0a08 ++#define REG_ECIR2 0x0a0c ++#define REG_EDATA(n) (0x0a10 + ((n) * 0x04)) ++#define REG_SBEC(n) (0x0a80 + ((n) * 0x04)) ++#define REG_MCR 0x0b00 ++#define REG_MCP(n) (0x0b04 + ((n) * 0x04)) ++#define REG_MISC_CFG 0x0be0 ++#define REG_HID_CFG 0x0bf0 ++#define REG_IDLE_STAT 0x0bf4 ++#define REG_IP_REV_1 0x0bf8 ++#define REG_IP_REV_2 0x0bfc ++#define REG_FQD_BARE 0x0c00 ++#define REG_PFDR_BARE 0x0c20 ++#define REG_offset_BAR 0x0004 /* relative to REG_[FQD|PFDR]_BARE */ ++#define REG_offset_AR 0x0010 /* relative to REG_[FQD|PFDR]_BARE */ ++#define REG_QCSP_BARE 0x0c80 ++#define REG_QCSP_BAR 0x0c84 ++#define REG_CI_SCHED_CFG 0x0d00 ++#define REG_SRCIDR 0x0d04 ++#define REG_LIODNR 0x0d08 ++#define REG_CI_RLM_AVG 0x0d14 ++#define REG_ERR_ISR 0x0e00 /* + "enum qm_isr_reg" */ ++#define REG_REV3_QCSP_LIO_CFG(n) (0x1000 + ((n) * 0x10)) ++#define REG_REV3_QCSP_IO_CFG(n) (0x1004 + ((n) * 0x10)) ++#define REG_REV3_QCSP_DD_CFG(n) (0x100c + ((n) * 0x10)) ++#define REG_CEETM_CFG_IDX 0x900 ++#define REG_CEETM_CFG_PRES 0x904 ++#define REG_CEETM_XSFDR_IN_USE 0x908 ++ ++/* Assists for QMAN_MCR */ ++#define MCR_INIT_PFDR 0x01000000 ++#define MCR_get_rslt(v) (u8)((v) >> 24) ++#define MCR_rslt_idle(r) (!rslt || (rslt >= 0xf0)) ++#define MCR_rslt_ok(r) (rslt == 0xf0) ++#define MCR_rslt_eaccess(r) (rslt == 0xf8) ++#define MCR_rslt_inval(r) (rslt == 0xff) ++ ++struct qman; ++ ++/* Follows WQ_CS_CFG0-5 */ ++enum qm_wq_class { ++ qm_wq_portal = 0, ++ qm_wq_pool = 1, ++ qm_wq_fman0 = 2, ++ qm_wq_fman1 = 3, ++ qm_wq_caam = 4, ++ qm_wq_pme = 5, ++ qm_wq_first = qm_wq_portal, ++ qm_wq_last = qm_wq_pme ++}; ++ ++/* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */ ++enum qm_memory { ++ qm_memory_fqd, ++ qm_memory_pfdr ++}; ++ ++/* Used by all error interrupt registers except 'inhibit' */ ++#define QM_EIRQ_CIDE 0x20000000 /* Corenet Initiator Data Error */ ++#define QM_EIRQ_CTDE 0x10000000 /* Corenet Target Data Error */ ++#define QM_EIRQ_CITT 0x08000000 /* Corenet Invalid Target Transaction */ ++#define QM_EIRQ_PLWI 0x04000000 /* PFDR Low Watermark */ ++#define QM_EIRQ_MBEI 0x02000000 /* Multi-bit ECC Error */ ++#define QM_EIRQ_SBEI 0x01000000 /* Single-bit ECC Error */ ++#define QM_EIRQ_PEBI 0x00800000 /* PFDR Enqueues Blocked Interrupt */ ++#define QM_EIRQ_IFSI 0x00020000 /* Invalid FQ Flow Control State */ ++#define QM_EIRQ_ICVI 0x00010000 /* Invalid Command Verb */ ++#define QM_EIRQ_IDDI 0x00000800 /* Invalid Dequeue (Direct-connect) */ ++#define QM_EIRQ_IDFI 0x00000400 /* Invalid Dequeue FQ */ ++#define QM_EIRQ_IDSI 0x00000200 /* Invalid Dequeue Source */ ++#define QM_EIRQ_IDQI 0x00000100 /* Invalid Dequeue Queue */ ++#define QM_EIRQ_IECE 0x00000010 /* Invalid Enqueue Configuration */ ++#define QM_EIRQ_IEOI 0x00000008 /* Invalid Enqueue Overflow */ ++#define QM_EIRQ_IESI 0x00000004 /* Invalid Enqueue State */ ++#define QM_EIRQ_IECI 0x00000002 /* Invalid Enqueue Channel */ ++#define QM_EIRQ_IEQI 0x00000001 /* Invalid Enqueue Queue */ ++ ++/* QMAN_ECIR valid error bit */ ++#define PORTAL_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \ ++ QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \ ++ QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI) ++#define FQID_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \ ++ QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \ ++ QM_EIRQ_IFSI) ++ ++union qman_ecir { ++ u32 ecir_raw; ++ struct { ++ u32 __reserved:2; ++ u32 portal_type:1; ++ u32 portal_num:5; ++ u32 fqid:24; ++ } __packed info; ++}; ++ ++union qman_ecir2 { ++ u32 ecir2_raw; ++ struct { ++ u32 portal_type:1; ++ u32 __reserved:21; ++ u32 portal_num:10; ++ } __packed info; ++}; ++ ++union qman_eadr { ++ u32 eadr_raw; ++ struct { ++ u32 __reserved1:4; ++ u32 memid:4; ++ u32 __reserved2:12; ++ u32 eadr:12; ++ } __packed info; ++ struct { ++ u32 __reserved1:3; ++ u32 memid:5; ++ u32 __reserved:8; ++ u32 eadr:16; ++ } __packed info_rev3; ++}; ++ ++struct qman_hwerr_txt { ++ u32 mask; ++ const char *txt; ++}; ++ ++#define QMAN_HWE_TXT(a, b) { .mask = QM_EIRQ_##a, .txt = b } ++ ++static const struct qman_hwerr_txt qman_hwerr_txts[] = { ++ QMAN_HWE_TXT(CIDE, "Corenet Initiator Data Error"), ++ QMAN_HWE_TXT(CTDE, "Corenet Target Data Error"), ++ QMAN_HWE_TXT(CITT, "Corenet Invalid Target Transaction"), ++ QMAN_HWE_TXT(PLWI, "PFDR Low Watermark"), ++ QMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"), ++ QMAN_HWE_TXT(SBEI, "Single-bit ECC Error"), ++ QMAN_HWE_TXT(PEBI, "PFDR Enqueues Blocked Interrupt"), ++ QMAN_HWE_TXT(ICVI, "Invalid Command Verb"), ++ QMAN_HWE_TXT(IFSI, "Invalid Flow Control State"), ++ QMAN_HWE_TXT(IDDI, "Invalid Dequeue (Direct-connect)"), ++ QMAN_HWE_TXT(IDFI, "Invalid Dequeue FQ"), ++ QMAN_HWE_TXT(IDSI, "Invalid Dequeue Source"), ++ QMAN_HWE_TXT(IDQI, "Invalid Dequeue Queue"), ++ QMAN_HWE_TXT(IECE, "Invalid Enqueue Configuration"), ++ QMAN_HWE_TXT(IEOI, "Invalid Enqueue Overflow"), ++ QMAN_HWE_TXT(IESI, "Invalid Enqueue State"), ++ QMAN_HWE_TXT(IECI, "Invalid Enqueue Channel"), ++ QMAN_HWE_TXT(IEQI, "Invalid Enqueue Queue") ++}; ++#define QMAN_HWE_COUNT (sizeof(qman_hwerr_txts)/sizeof(struct qman_hwerr_txt)) ++ ++struct qman_error_info_mdata { ++ u16 addr_mask; ++ u16 bits; ++ const char *txt; ++}; ++ ++#define QMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c} ++static const struct qman_error_info_mdata error_mdata[] = { ++ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 0"), ++ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 1"), ++ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 2"), ++ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 3"), ++ QMAN_ERR_MDATA(0x0FFF, 512, "FQD cache memory"), ++ QMAN_ERR_MDATA(0x07FF, 128, "SFDR memory"), ++ QMAN_ERR_MDATA(0x01FF, 72, "WQ context memory"), ++ QMAN_ERR_MDATA(0x00FF, 240, "CGR memory"), ++ QMAN_ERR_MDATA(0x00FF, 302, "Internal Order Restoration List memory"), ++ QMAN_ERR_MDATA(0x01FF, 256, "SW portal ring memory"), ++ QMAN_ERR_MDATA(0x07FF, 181, "CEETM class queue descriptor memory"), ++ QMAN_ERR_MDATA(0x0FFF, 140, "CEETM extended SFDR memory"), ++ QMAN_ERR_MDATA(0x0FFF, 25, "CEETM logical FQ mapping memory"), ++ QMAN_ERR_MDATA(0x0FFF, 96, "CEETM dequeue context memory"), ++ QMAN_ERR_MDATA(0x07FF, 396, "CEETM ccgr memory"), ++ QMAN_ERR_MDATA(0x00FF, 146, "CEETM CQ channel shaping memory"), ++ QMAN_ERR_MDATA(0x007F, 256, "CEETM CQ channel scheduling memory"), ++ QMAN_ERR_MDATA(0x01FF, 88, "CEETM dequeue statistics memory"), ++}; ++#define QMAN_ERR_MDATA_COUNT \ ++ (sizeof(error_mdata)/sizeof(struct qman_error_info_mdata)) ++ ++/* Add this in Kconfig */ ++#define QMAN_ERRS_TO_UNENABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI) ++ ++/** ++ * qm_err_isr_<reg>_<verb> - Manipulate global interrupt registers ++ * @v: for accessors that write values, this is the 32-bit value ++ * ++ * Manipulates QMAN_ERR_ISR, QMAN_ERR_IER, QMAN_ERR_ISDR, QMAN_ERR_IIR. All ++ * manipulations except qm_err_isr_[un]inhibit() use 32-bit masks composed of ++ * the QM_EIRQ_*** definitions. Note that "qm_err_isr_enable_write" means ++ * "write the enable register" rather than "enable the write register"! ++ */ ++#define qm_err_isr_status_read(qm) \ ++ __qm_err_isr_read(qm, qm_isr_status) ++#define qm_err_isr_status_clear(qm, m) \ ++ __qm_err_isr_write(qm, qm_isr_status, m) ++#define qm_err_isr_enable_read(qm) \ ++ __qm_err_isr_read(qm, qm_isr_enable) ++#define qm_err_isr_enable_write(qm, v) \ ++ __qm_err_isr_write(qm, qm_isr_enable, v) ++#define qm_err_isr_disable_read(qm) \ ++ __qm_err_isr_read(qm, qm_isr_disable) ++#define qm_err_isr_disable_write(qm, v) \ ++ __qm_err_isr_write(qm, qm_isr_disable, v) ++#define qm_err_isr_inhibit(qm) \ ++ __qm_err_isr_write(qm, qm_isr_inhibit, 1) ++#define qm_err_isr_uninhibit(qm) \ ++ __qm_err_isr_write(qm, qm_isr_inhibit, 0) ++ ++/* ++ * TODO: unimplemented registers ++ * ++ * Keeping a list here of Qman registers I have not yet covered; ++ * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR, ++ * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG, ++ * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12 ++ */ ++ ++/* Encapsulate "struct qman *" as a cast of the register space address. */ ++ ++static struct qman *qm_create(void *regs) ++{ ++ return (struct qman *)regs; ++} ++ ++static inline u32 __qm_in(struct qman *qm, u32 offset) ++{ ++ return in_be32((void *)qm + offset); ++} ++static inline void __qm_out(struct qman *qm, u32 offset, u32 val) ++{ ++ out_be32((void *)qm + offset, val); ++} ++#define qm_in(reg) __qm_in(qm, REG_##reg) ++#define qm_out(reg, val) __qm_out(qm, REG_##reg, val) ++ ++static u32 __qm_err_isr_read(struct qman *qm, enum qm_isr_reg n) ++{ ++ return __qm_in(qm, REG_ERR_ISR + (n << 2)); ++} ++ ++static void __qm_err_isr_write(struct qman *qm, enum qm_isr_reg n, u32 val) ++{ ++ __qm_out(qm, REG_ERR_ISR + (n << 2), val); ++} ++ ++static void qm_set_dc(struct qman *qm, enum qm_dc_portal portal, ++ int ed, u8 sernd) ++{ ++ DPA_ASSERT(!ed || (portal == qm_dc_portal_fman0) || ++ (portal == qm_dc_portal_fman1)); ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) ++ qm_out(DCP_CFG(portal), (ed ? 0x1000 : 0) | (sernd & 0x3ff)); ++ else ++ qm_out(DCP_CFG(portal), (ed ? 0x100 : 0) | (sernd & 0x1f)); ++} ++ ++static void qm_set_wq_scheduling(struct qman *qm, enum qm_wq_class wq_class, ++ u8 cs_elev, u8 csw2, u8 csw3, u8 csw4, u8 csw5, ++ u8 csw6, u8 csw7) ++{ ++ qm_out(WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) | ++ ((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) | ++ ((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) | ++ ((csw6 & 0x7) << 4) | (csw7 & 0x7)); ++} ++ ++static void qm_set_hid(struct qman *qm) ++{ ++ qm_out(HID_CFG, 0); ++} ++ ++static void qm_set_corenet_initiator(struct qman *qm) ++{ ++ qm_out(CI_SCHED_CFG, ++ 0x80000000 | /* write srcciv enable */ ++ (CONFIG_FSL_QMAN_CI_SCHED_CFG_SRCCIV << 24) | ++ (CONFIG_FSL_QMAN_CI_SCHED_CFG_SRQ_W << 8) | ++ (CONFIG_FSL_QMAN_CI_SCHED_CFG_RW_W << 4) | ++ CONFIG_FSL_QMAN_CI_SCHED_CFG_BMAN_W); ++} ++ ++static void qm_get_version(struct qman *qm, u16 *id, u8 *major, u8 *minor, ++ u8 *cfg) ++{ ++ u32 v = qm_in(IP_REV_1); ++ u32 v2 = qm_in(IP_REV_2); ++ *id = (v >> 16); ++ *major = (v >> 8) & 0xff; ++ *minor = v & 0xff; ++ *cfg = v2 & 0xff; ++} ++ ++static void qm_set_memory(struct qman *qm, enum qm_memory memory, u64 ba, ++ int enable, int prio, int stash, u32 size) ++{ ++ u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE; ++ u32 exp = ilog2(size); ++ /* choke if size isn't within range */ ++ DPA_ASSERT((size >= 4096) && (size <= 1073741824) && ++ is_power_of_2(size)); ++ /* choke if 'ba' has lower-alignment than 'size' */ ++ DPA_ASSERT(!(ba & (size - 1))); ++ __qm_out(qm, offset, upper_32_bits(ba)); ++ __qm_out(qm, offset + REG_offset_BAR, lower_32_bits(ba)); ++ __qm_out(qm, offset + REG_offset_AR, ++ (enable ? 0x80000000 : 0) | ++ (prio ? 0x40000000 : 0) | ++ (stash ? 0x20000000 : 0) | ++ (exp - 1)); ++} ++ ++static void qm_set_pfdr_threshold(struct qman *qm, u32 th, u8 k) ++{ ++ qm_out(PFDR_FP_LWIT, th & 0xffffff); ++ qm_out(PFDR_CFG, k); ++} ++ ++static void qm_set_sfdr_threshold(struct qman *qm, u16 th) ++{ ++ qm_out(SFDR_CFG, th & 0x3ff); ++} ++ ++static int qm_init_pfdr(struct qman *qm, u32 pfdr_start, u32 num) ++{ ++ u8 rslt = MCR_get_rslt(qm_in(MCR)); ++ ++ DPA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num); ++ /* Make sure the command interface is 'idle' */ ++ if (!MCR_rslt_idle(rslt)) ++ panic("QMAN_MCR isn't idle"); ++ ++ /* Write the MCR command params then the verb */ ++ qm_out(MCP(0), pfdr_start); ++ /* TODO: remove this - it's a workaround for a model bug that is ++ * corrected in more recent versions. We use the workaround until ++ * everyone has upgraded. */ ++ qm_out(MCP(1), (pfdr_start + num - 16)); ++ lwsync(); ++ qm_out(MCR, MCR_INIT_PFDR); ++ /* Poll for the result */ ++ do { ++ rslt = MCR_get_rslt(qm_in(MCR)); ++ } while (!MCR_rslt_idle(rslt)); ++ if (MCR_rslt_ok(rslt)) ++ return 0; ++ if (MCR_rslt_eaccess(rslt)) ++ return -EACCES; ++ if (MCR_rslt_inval(rslt)) ++ return -EINVAL; ++ pr_crit("Unexpected result from MCR_INIT_PFDR: %02x\n", rslt); ++ return -ENOSYS; ++} ++ ++/*****************/ ++/* Config driver */ ++/*****************/ ++ ++#define DEFAULT_FQD_SZ (PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ) ++#define DEFAULT_PFDR_SZ (PAGE_SIZE << CONFIG_FSL_QMAN_PFDR_SZ) ++ ++/* We support only one of these */ ++static struct qman *qm; ++static struct device_node *qm_node; ++ ++/* And this state belongs to 'qm'. It is set during fsl_qman_init(), but used ++ * during qman_init_ccsr(). */ ++static dma_addr_t fqd_a, pfdr_a; ++static size_t fqd_sz = DEFAULT_FQD_SZ, pfdr_sz = DEFAULT_PFDR_SZ; ++ ++static int qman_fqd(struct reserved_mem *rmem) ++{ ++ fqd_a = rmem->base; ++ fqd_sz = rmem->size; ++ ++ WARN_ON(!(fqd_a && fqd_sz)); ++ ++ return 0; ++} ++RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd); ++ ++static int qman_pfdr(struct reserved_mem *rmem) ++{ ++ pfdr_a = rmem->base; ++ pfdr_sz = rmem->size; ++ ++ WARN_ON(!(pfdr_a && pfdr_sz)); ++ ++ return 0; ++} ++RESERVEDMEM_OF_DECLARE(qman_fbpr, "fsl,qman-pfdr", qman_pfdr); ++ ++size_t get_qman_fqd_size() ++{ ++ return fqd_sz; ++} ++ ++/* Parse the <name> property to extract the memory location and size and ++ * memblock_reserve() it. If it isn't supplied, memblock_alloc() the default ++ * size. Also flush this memory range from data cache so that QMAN originated ++ * transactions for this memory region could be marked non-coherent. ++ */ ++static __init int parse_mem_property(struct device_node *node, const char *name, ++ dma_addr_t *addr, size_t *sz, int zero) ++{ ++ int ret; ++ ++ /* If using a "zero-pma", don't try to zero it, even if you asked */ ++ if (zero && of_find_property(node, "zero-pma", &ret)) { ++ pr_info(" it's a 'zero-pma', not zeroing from s/w\n"); ++ zero = 0; ++ } ++ ++ if (zero) { ++ /* map as cacheable, non-guarded */ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ void __iomem *tmpp = ioremap_cache(*addr, *sz); ++#else ++ void __iomem *tmpp = ioremap(*addr, *sz); ++#endif ++ ++ if (!tmpp) ++ return -ENOMEM; ++ memset_io(tmpp, 0, *sz); ++ flush_dcache_range((unsigned long)tmpp, ++ (unsigned long)tmpp + *sz); ++ iounmap(tmpp); ++ } ++ ++ return 0; ++} ++ ++/* TODO: ++ * - there is obviously no handling of errors, ++ * - the calls to qm_set_memory() hard-code the priority and CPC-stashing for ++ * both memory resources to zero. ++ */ ++static int __init fsl_qman_init(struct device_node *node) ++{ ++ struct resource res; ++ resource_size_t len; ++ u32 __iomem *regs; ++ const char *s; ++ int ret, standby = 0; ++ u16 id; ++ u8 major, minor, cfg; ++ ret = of_address_to_resource(node, 0, &res); ++ if (ret) { ++ pr_err("Can't get %s property '%s'\n", node->full_name, "reg"); ++ return ret; ++ } ++ s = of_get_property(node, "fsl,hv-claimable", &ret); ++ if (s && !strcmp(s, "standby")) ++ standby = 1; ++ if (!standby) { ++ ret = parse_mem_property(node, "fsl,qman-fqd", ++ &fqd_a, &fqd_sz, 1); ++ pr_info("qman-fqd addr 0x%llx size 0x%zx\n", ++ (unsigned long long)fqd_a, fqd_sz); ++ BUG_ON(ret); ++ ret = parse_mem_property(node, "fsl,qman-pfdr", ++ &pfdr_a, &pfdr_sz, 0); ++ pr_info("qman-pfdr addr 0x%llx size 0x%zx\n", ++ (unsigned long long)pfdr_a, pfdr_sz); ++ BUG_ON(ret); ++ } ++ /* Global configuration */ ++ len = resource_size(&res); ++ if (len != (unsigned long)len) ++ return -EINVAL; ++ regs = ioremap(res.start, (unsigned long)len); ++ qm = qm_create(regs); ++ qm_node = node; ++ qm_get_version(qm, &id, &major, &minor, &cfg); ++ pr_info("Qman ver:%04x,%02x,%02x,%02x\n", id, major, minor, cfg); ++ if (!qman_ip_rev) { ++ if ((major == 1) && (minor == 0)) { ++ pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n"); ++ iounmap(regs); ++ return -ENODEV; ++ } else if ((major == 1) && (minor == 1)) ++ qman_ip_rev = QMAN_REV11; ++ else if ((major == 1) && (minor == 2)) ++ qman_ip_rev = QMAN_REV12; ++ else if ((major == 2) && (minor == 0)) ++ qman_ip_rev = QMAN_REV20; ++ else if ((major == 3) && (minor == 0)) ++ qman_ip_rev = QMAN_REV30; ++ else if ((major == 3) && (minor == 1)) ++ qman_ip_rev = QMAN_REV31; ++ else if ((major == 3) && (minor == 2)) ++ qman_ip_rev = QMAN_REV32; ++ else { ++ pr_warn("unknown Qman version, default to rev1.1\n"); ++ qman_ip_rev = QMAN_REV11; ++ } ++ qman_ip_cfg = cfg; ++ } ++ ++ if (standby) { ++ pr_info(" -> in standby mode\n"); ++ return 0; ++ } ++ return 0; ++} ++ ++int qman_have_ccsr(void) ++{ ++ return qm ? 1 : 0; ++} ++ ++__init int qman_init_early(void) ++{ ++ struct device_node *dn; ++ int ret; ++ ++ for_each_compatible_node(dn, NULL, "fsl,qman") { ++ if (qm) ++ pr_err("%s: only one 'fsl,qman' allowed\n", ++ dn->full_name); ++ else { ++ if (!of_device_is_available(dn)) ++ continue; ++ ++ ret = fsl_qman_init(dn); ++ BUG_ON(ret); ++ } ++ } ++ return 0; ++} ++postcore_initcall_sync(qman_init_early); ++ ++static void log_edata_bits(u32 bit_count) ++{ ++ u32 i, j, mask = 0xffffffff; ++ ++ pr_warn("Qman ErrInt, EDATA:\n"); ++ i = bit_count/32; ++ if (bit_count%32) { ++ i++; ++ mask = ~(mask << bit_count%32); ++ } ++ j = 16-i; ++ pr_warn(" 0x%08x\n", qm_in(EDATA(j)) & mask); ++ j++; ++ for (; j < 16; j++) ++ pr_warn(" 0x%08x\n", qm_in(EDATA(j))); ++} ++ ++static void log_additional_error_info(u32 isr_val, u32 ecsr_val) ++{ ++ union qman_ecir ecir_val; ++ union qman_eadr eadr_val; ++ ++ ecir_val.ecir_raw = qm_in(ECIR); ++ /* Is portal info valid */ ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { ++ union qman_ecir2 ecir2_val; ++ ecir2_val.ecir2_raw = qm_in(ECIR2); ++ if (ecsr_val & PORTAL_ECSR_ERR) { ++ pr_warn("Qman ErrInt: %s id %d\n", ++ (ecir2_val.info.portal_type) ? ++ "DCP" : "SWP", ecir2_val.info.portal_num); ++ } ++ if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE)) { ++ pr_warn("Qman ErrInt: ecir.fqid 0x%x\n", ++ ecir_val.info.fqid); ++ } ++ if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) { ++ eadr_val.eadr_raw = qm_in(EADR); ++ pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n", ++ error_mdata[eadr_val.info_rev3.memid].txt, ++ error_mdata[eadr_val.info_rev3.memid].addr_mask ++ & eadr_val.info_rev3.eadr); ++ log_edata_bits( ++ error_mdata[eadr_val.info_rev3.memid].bits); ++ } ++ } else { ++ if (ecsr_val & PORTAL_ECSR_ERR) { ++ pr_warn("Qman ErrInt: %s id %d\n", ++ (ecir_val.info.portal_type) ? ++ "DCP" : "SWP", ecir_val.info.portal_num); ++ } ++ if (ecsr_val & FQID_ECSR_ERR) { ++ pr_warn("Qman ErrInt: ecir.fqid 0x%x\n", ++ ecir_val.info.fqid); ++ } ++ if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) { ++ eadr_val.eadr_raw = qm_in(EADR); ++ pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n", ++ error_mdata[eadr_val.info.memid].txt, ++ error_mdata[eadr_val.info.memid].addr_mask ++ & eadr_val.info.eadr); ++ log_edata_bits(error_mdata[eadr_val.info.memid].bits); ++ } ++ } ++} ++ ++/* Qman interrupt handler */ ++static irqreturn_t qman_isr(int irq, void *ptr) ++{ ++ u32 isr_val, ier_val, ecsr_val, isr_mask, i; ++ ++ ier_val = qm_err_isr_enable_read(qm); ++ isr_val = qm_err_isr_status_read(qm); ++ ecsr_val = qm_in(ECSR); ++ isr_mask = isr_val & ier_val; ++ ++ if (!isr_mask) ++ return IRQ_NONE; ++ for (i = 0; i < QMAN_HWE_COUNT; i++) { ++ if (qman_hwerr_txts[i].mask & isr_mask) { ++ pr_warn("Qman ErrInt: %s\n", qman_hwerr_txts[i].txt); ++ if (qman_hwerr_txts[i].mask & ecsr_val) { ++ log_additional_error_info(isr_mask, ecsr_val); ++ /* Re-arm error capture registers */ ++ qm_out(ECSR, ecsr_val); ++ } ++ if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_UNENABLE) { ++ pr_devel("Qman un-enabling error 0x%x\n", ++ qman_hwerr_txts[i].mask); ++ ier_val &= ~qman_hwerr_txts[i].mask; ++ qm_err_isr_enable_write(qm, ier_val); ++ } ++ } ++ } ++ qm_err_isr_status_clear(qm, isr_val); ++ return IRQ_HANDLED; ++} ++ ++static int __bind_irq(void) ++{ ++ int ret, err_irq; ++ ++ err_irq = of_irq_to_resource(qm_node, 0, NULL); ++ if (err_irq == 0) { ++ pr_info("Can't get %s property '%s'\n", qm_node->full_name, ++ "interrupts"); ++ return -ENODEV; ++ } ++ ret = request_irq(err_irq, qman_isr, IRQF_SHARED, "qman-err", qm_node); ++ if (ret) { ++ pr_err("request_irq() failed %d for '%s'\n", ret, ++ qm_node->full_name); ++ return -ENODEV; ++ } ++ /* Write-to-clear any stale bits, (eg. starvation being asserted prior ++ * to resource allocation during driver init). */ ++ qm_err_isr_status_clear(qm, 0xffffffff); ++ /* Enable Error Interrupts */ ++ qm_err_isr_enable_write(qm, 0xffffffff); ++ return 0; ++} ++ ++int qman_init_ccsr(struct device_node *node) ++{ ++ int ret; ++ if (!qman_have_ccsr()) ++ return 0; ++ if (node != qm_node) ++ return -EINVAL; ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ /* TEMP for LS1043 : should be done in uboot */ ++ qm_out(QCSP_BARE, 0x5); ++ qm_out(QCSP_BAR, 0x0); ++#endif ++ /* FQD memory */ ++ qm_set_memory(qm, qm_memory_fqd, fqd_a, 1, 0, 0, fqd_sz); ++ /* PFDR memory */ ++ qm_set_memory(qm, qm_memory_pfdr, pfdr_a, 1, 0, 0, pfdr_sz); ++ qm_init_pfdr(qm, 8, pfdr_sz / 64 - 8); ++ /* thresholds */ ++ qm_set_pfdr_threshold(qm, 512, 64); ++ qm_set_sfdr_threshold(qm, 128); ++ /* clear stale PEBI bit from interrupt status register */ ++ qm_err_isr_status_clear(qm, QM_EIRQ_PEBI); ++ /* corenet initiator settings */ ++ qm_set_corenet_initiator(qm); ++ /* HID settings */ ++ qm_set_hid(qm); ++ /* Set scheduling weights to defaults */ ++ for (ret = qm_wq_first; ret <= qm_wq_last; ret++) ++ qm_set_wq_scheduling(qm, ret, 0, 0, 0, 0, 0, 0, 0); ++ /* We are not prepared to accept ERNs for hardware enqueues */ ++ qm_set_dc(qm, qm_dc_portal_fman0, 1, 0); ++ qm_set_dc(qm, qm_dc_portal_fman1, 1, 0); ++ /* Initialise Error Interrupt Handler */ ++ ret = __bind_irq(); ++ if (ret) ++ return ret; ++ return 0; ++} ++ ++#define LIO_CFG_LIODN_MASK 0x0fff0000 ++void qman_liodn_fixup(u16 channel) ++{ ++ static int done; ++ static u32 liodn_offset; ++ u32 before, after; ++ int idx = channel - QM_CHANNEL_SWPORTAL0; ++ ++ if (!qman_have_ccsr()) ++ return; ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) ++ before = qm_in(REV3_QCSP_LIO_CFG(idx)); ++ else ++ before = qm_in(QCSP_LIO_CFG(idx)); ++ if (!done) { ++ liodn_offset = before & LIO_CFG_LIODN_MASK; ++ done = 1; ++ return; ++ } ++ after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset; ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) ++ qm_out(REV3_QCSP_LIO_CFG(idx), after); ++ else ++ qm_out(QCSP_LIO_CFG(idx), after); ++} ++ ++#define IO_CFG_SDEST_MASK 0x00ff0000 ++int qman_set_sdest(u16 channel, unsigned int cpu_idx) ++{ ++ int idx = channel - QM_CHANNEL_SWPORTAL0; ++ u32 before, after; ++ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ if ((qman_ip_rev & 0xFF00) == QMAN_REV31) { ++ /* LS1043A - only one L2 cache */ ++ cpu_idx = 0; ++ } ++ ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { ++ before = qm_in(REV3_QCSP_IO_CFG(idx)); ++ /* Each pair of vcpu share the same SRQ(SDEST) */ ++ cpu_idx /= 2; ++ after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16); ++ qm_out(REV3_QCSP_IO_CFG(idx), after); ++ } else { ++ before = qm_in(QCSP_IO_CFG(idx)); ++ after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16); ++ qm_out(QCSP_IO_CFG(idx), after); ++ } ++ return 0; ++} ++ ++#define MISC_CFG_WPM_MASK 0x00000002 ++int qm_set_wpm(int wpm) ++{ ++ u32 before; ++ u32 after; ++ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ ++ before = qm_in(MISC_CFG); ++ after = (before & (~MISC_CFG_WPM_MASK)) | (wpm << 1); ++ qm_out(MISC_CFG, after); ++ return 0; ++} ++ ++int qm_get_wpm(int *wpm) ++{ ++ u32 before; ++ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ ++ before = qm_in(MISC_CFG); ++ *wpm = (before & MISC_CFG_WPM_MASK) >> 1; ++ return 0; ++} ++ ++/* CEETM_CFG_PRES register has PRES field which is calculated by: ++ * PRES = (2^22 / credit update reference period) * QMan clock period ++ * = (2^22 * 10^9)/ CONFIG_QMAN_CEETM_UPDATE_PERIOD) / qman_clk ++ */ ++ ++int qman_ceetm_set_prescaler(enum qm_dc_portal portal) ++{ ++ u64 temp; ++ u16 pres; ++ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ ++ temp = 0x400000 * 100; ++ do_div(temp, CONFIG_QMAN_CEETM_UPDATE_PERIOD); ++ temp *= 10000000; ++ do_div(temp, qman_clk); ++ pres = (u16) temp; ++ qm_out(CEETM_CFG_IDX, portal); ++ qm_out(CEETM_CFG_PRES, pres); ++ return 0; ++} ++ ++int qman_ceetm_get_prescaler(u16 *pres) ++{ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ *pres = (u16)qm_in(CEETM_CFG_PRES); ++ return 0; ++} ++ ++#define DCP_CFG_CEETME_MASK 0xFFFF0000 ++#define QM_SP_ENABLE_CEETM(n) (0x80000000 >> (n)) ++int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal) ++{ ++ u32 dcp_cfg; ++ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ ++ dcp_cfg = qm_in(DCP_CFG(portal)); ++ dcp_cfg |= QM_SP_ENABLE_CEETM(sub_portal); ++ qm_out(DCP_CFG(portal), dcp_cfg); ++ return 0; ++} ++ ++int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal) ++{ ++ u32 dcp_cfg; ++ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ dcp_cfg = qm_in(DCP_CFG(portal)); ++ dcp_cfg &= ~(QM_SP_ENABLE_CEETM(sub_portal)); ++ qm_out(DCP_CFG(portal), dcp_cfg); ++ return 0; ++} ++ ++int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num) ++{ ++ if (!qman_have_ccsr()) ++ return -ENODEV; ++ *num = qm_in(CEETM_XSFDR_IN_USE); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_get_xsfdr); ++ ++#ifdef CONFIG_SYSFS ++ ++#define DRV_NAME "fsl-qman" ++#define DCP_MAX_ID 3 ++#define DCP_MIN_ID 0 ++ ++static ssize_t show_pfdr_fpc(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_FPC)); ++}; ++ ++static ssize_t show_dlm_avg(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ u32 data; ++ int i; ++ ++ if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i)) ++ return -EINVAL; ++ if (i < DCP_MIN_ID || i > DCP_MAX_ID) ++ return -EINVAL; ++ data = qm_in(DCP_DLM_AVG(i)); ++ return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8, ++ (data & 0x000000ff)*390625); ++}; ++ ++static ssize_t set_dlm_avg(struct device *dev, ++ struct device_attribute *dev_attr, const char *buf, size_t count) ++{ ++ unsigned long val; ++ int i; ++ ++ if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i)) ++ return -EINVAL; ++ if (i < DCP_MIN_ID || i > DCP_MAX_ID) ++ return -EINVAL; ++ if (kstrtoul(buf, 0, &val)) { ++ dev_dbg(dev, "invalid input %s\n", buf); ++ return -EINVAL; ++ } ++ qm_out(DCP_DLM_AVG(i), val); ++ return count; ++}; ++ ++static ssize_t show_pfdr_cfg(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_CFG)); ++}; ++ ++static ssize_t set_pfdr_cfg(struct device *dev, ++ struct device_attribute *dev_attr, const char *buf, size_t count) ++{ ++ unsigned long val; ++ ++ if (kstrtoul(buf, 0, &val)) { ++ dev_dbg(dev, "invalid input %s\n", buf); ++ return -EINVAL; ++ } ++ qm_out(PFDR_CFG, val); ++ return count; ++}; ++ ++static ssize_t show_sfdr_in_use(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SFDR_IN_USE)); ++}; ++ ++static ssize_t show_idle_stat(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(IDLE_STAT)); ++}; ++ ++static ssize_t show_ci_rlm_avg(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ u32 data = qm_in(CI_RLM_AVG); ++ return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8, ++ (data & 0x000000ff)*390625); ++}; ++ ++static ssize_t set_ci_rlm_avg(struct device *dev, ++ struct device_attribute *dev_attr, const char *buf, size_t count) ++{ ++ unsigned long val; ++ ++ if (kstrtoul(buf, 0, &val)) { ++ dev_dbg(dev, "invalid input %s\n", buf); ++ return -EINVAL; ++ } ++ qm_out(CI_RLM_AVG, val); ++ return count; ++}; ++ ++static ssize_t show_err_isr(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ return snprintf(buf, PAGE_SIZE, "0x%08x\n", qm_in(ERR_ISR)); ++}; ++ ++#define SBEC_MAX_ID 14 ++#define SBEC_MIN_ID 0 ++ ++static ssize_t show_sbec(struct device *dev, ++ struct device_attribute *dev_attr, char *buf) ++{ ++ int i; ++ ++ if (!sscanf(dev_attr->attr.name, "sbec_%d", &i)) ++ return -EINVAL; ++ if (i < SBEC_MIN_ID || i > SBEC_MAX_ID) ++ return -EINVAL; ++ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SBEC(i))); ++}; ++ ++static DEVICE_ATTR(pfdr_fpc, S_IRUSR, show_pfdr_fpc, NULL); ++static DEVICE_ATTR(pfdr_cfg, S_IRUSR, show_pfdr_cfg, set_pfdr_cfg); ++static DEVICE_ATTR(idle_stat, S_IRUSR, show_idle_stat, NULL); ++static DEVICE_ATTR(ci_rlm_avg, (S_IRUSR|S_IWUSR), ++ show_ci_rlm_avg, set_ci_rlm_avg); ++static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL); ++static DEVICE_ATTR(sfdr_in_use, S_IRUSR, show_sfdr_in_use, NULL); ++ ++static DEVICE_ATTR(dcp0_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); ++static DEVICE_ATTR(dcp1_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); ++static DEVICE_ATTR(dcp2_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); ++static DEVICE_ATTR(dcp3_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); ++ ++static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_2, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_3, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_4, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_5, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_6, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_7, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_8, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_9, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_10, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_11, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_12, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_13, S_IRUSR, show_sbec, NULL); ++static DEVICE_ATTR(sbec_14, S_IRUSR, show_sbec, NULL); ++ ++static struct attribute *qman_dev_attributes[] = { ++ &dev_attr_pfdr_fpc.attr, ++ &dev_attr_pfdr_cfg.attr, ++ &dev_attr_idle_stat.attr, ++ &dev_attr_ci_rlm_avg.attr, ++ &dev_attr_err_isr.attr, ++ &dev_attr_dcp0_dlm_avg.attr, ++ &dev_attr_dcp1_dlm_avg.attr, ++ &dev_attr_dcp2_dlm_avg.attr, ++ &dev_attr_dcp3_dlm_avg.attr, ++ /* sfdr_in_use will be added if necessary */ ++ NULL ++}; ++ ++static struct attribute *qman_dev_ecr_attributes[] = { ++ &dev_attr_sbec_0.attr, ++ &dev_attr_sbec_1.attr, ++ &dev_attr_sbec_2.attr, ++ &dev_attr_sbec_3.attr, ++ &dev_attr_sbec_4.attr, ++ &dev_attr_sbec_5.attr, ++ &dev_attr_sbec_6.attr, ++ &dev_attr_sbec_7.attr, ++ &dev_attr_sbec_8.attr, ++ &dev_attr_sbec_9.attr, ++ &dev_attr_sbec_10.attr, ++ &dev_attr_sbec_11.attr, ++ &dev_attr_sbec_12.attr, ++ &dev_attr_sbec_13.attr, ++ &dev_attr_sbec_14.attr, ++ NULL ++}; ++ ++/* root level */ ++static const struct attribute_group qman_dev_attr_grp = { ++ .name = NULL, ++ .attrs = qman_dev_attributes ++}; ++static const struct attribute_group qman_dev_ecr_grp = { ++ .name = "error_capture", ++ .attrs = qman_dev_ecr_attributes ++}; ++ ++static int of_fsl_qman_remove(struct platform_device *ofdev) ++{ ++ sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp); ++ return 0; ++}; ++ ++static int of_fsl_qman_probe(struct platform_device *ofdev) ++{ ++ int ret; ++ ++ ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_attr_grp); ++ if (ret) ++ goto done; ++ ret = sysfs_add_file_to_group(&ofdev->dev.kobj, ++ &dev_attr_sfdr_in_use.attr, qman_dev_attr_grp.name); ++ if (ret) ++ goto del_group_0; ++ ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_ecr_grp); ++ if (ret) ++ goto del_group_0; ++ ++ goto done; ++ ++del_group_0: ++ sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp); ++done: ++ if (ret) ++ dev_err(&ofdev->dev, ++ "Cannot create dev attributes ret=%d\n", ret); ++ return ret; ++}; ++ ++static struct of_device_id of_fsl_qman_ids[] = { ++ { ++ .compatible = "fsl,qman", ++ }, ++ {} ++}; ++MODULE_DEVICE_TABLE(of, of_fsl_qman_ids); ++ ++#ifdef CONFIG_SUSPEND ++ ++static u32 saved_isdr; ++static int qman_pm_suspend_noirq(struct device *dev) ++{ ++ uint32_t idle_state; ++ ++ suspend_unused_qportal(); ++ /* save isdr, disable all, clear isr */ ++ saved_isdr = qm_err_isr_disable_read(qm); ++ qm_err_isr_disable_write(qm, 0xffffffff); ++ qm_err_isr_status_clear(qm, 0xffffffff); ++ idle_state = qm_in(IDLE_STAT); ++ if (!(idle_state & 0x1)) { ++ pr_err("Qman not idle 0x%x aborting\n", idle_state); ++ qm_err_isr_disable_write(qm, saved_isdr); ++ resume_unused_qportal(); ++ return -EBUSY; ++ } ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Qman suspend code, IDLE_STAT = 0x%x\n", idle_state); ++#endif ++ return 0; ++} ++ ++static int qman_pm_resume_noirq(struct device *dev) ++{ ++ /* restore isdr */ ++ qm_err_isr_disable_write(qm, saved_isdr); ++ resume_unused_qportal(); ++ return 0; ++} ++#else ++#define qman_pm_suspend_noirq NULL ++#define qman_pm_resume_noirq NULL ++#endif ++ ++static const struct dev_pm_ops qman_pm_ops = { ++ .suspend_noirq = qman_pm_suspend_noirq, ++ .resume_noirq = qman_pm_resume_noirq, ++}; ++ ++static struct platform_driver of_fsl_qman_driver = { ++ .driver = { ++ .owner = THIS_MODULE, ++ .name = DRV_NAME, ++ .of_match_table = of_fsl_qman_ids, ++ .pm = &qman_pm_ops, ++ }, ++ .probe = of_fsl_qman_probe, ++ .remove = of_fsl_qman_remove, ++}; ++ ++static int qman_ctrl_init(void) ++{ ++ return platform_driver_register(&of_fsl_qman_driver); ++} ++ ++static void qman_ctrl_exit(void) ++{ ++ platform_driver_unregister(&of_fsl_qman_driver); ++} ++ ++module_init(qman_ctrl_init); ++module_exit(qman_ctrl_exit); ++ ++#endif /* CONFIG_SYSFS */ +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_debugfs.c +@@ -0,0 +1,1594 @@ ++/* Copyright 2010-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++#include "qman_private.h" ++ ++#define MAX_FQID (0x00ffffff) ++#define QM_FQD_BLOCK_SIZE 64 ++#define QM_FQD_AR (0xC10) ++ ++static u32 fqid_max; ++static u64 qman_ccsr_start; ++static u64 qman_ccsr_size; ++ ++static const char * const state_txt[] = { ++ "Out of Service", ++ "Retired", ++ "Tentatively Scheduled", ++ "Truly Scheduled", ++ "Parked", ++ "Active, Active Held or Held Suspended", ++ "Unknown State 6", ++ "Unknown State 7", ++ NULL, ++}; ++ ++static const u8 fqd_states[] = { ++ QM_MCR_NP_STATE_OOS, QM_MCR_NP_STATE_RETIRED, QM_MCR_NP_STATE_TEN_SCHED, ++ QM_MCR_NP_STATE_TRU_SCHED, QM_MCR_NP_STATE_PARKED, ++ QM_MCR_NP_STATE_ACTIVE}; ++ ++struct mask_to_text { ++ u16 mask; ++ const char *txt; ++}; ++ ++struct mask_filter_s { ++ u16 mask; ++ u8 filter; ++}; ++ ++static const struct mask_filter_s mask_filter[] = { ++ {QM_FQCTRL_PREFERINCACHE, 0}, ++ {QM_FQCTRL_PREFERINCACHE, 1}, ++ {QM_FQCTRL_HOLDACTIVE, 0}, ++ {QM_FQCTRL_HOLDACTIVE, 1}, ++ {QM_FQCTRL_AVOIDBLOCK, 0}, ++ {QM_FQCTRL_AVOIDBLOCK, 1}, ++ {QM_FQCTRL_FORCESFDR, 0}, ++ {QM_FQCTRL_FORCESFDR, 1}, ++ {QM_FQCTRL_CPCSTASH, 0}, ++ {QM_FQCTRL_CPCSTASH, 1}, ++ {QM_FQCTRL_CTXASTASHING, 0}, ++ {QM_FQCTRL_CTXASTASHING, 1}, ++ {QM_FQCTRL_ORP, 0}, ++ {QM_FQCTRL_ORP, 1}, ++ {QM_FQCTRL_TDE, 0}, ++ {QM_FQCTRL_TDE, 1}, ++ {QM_FQCTRL_CGE, 0}, ++ {QM_FQCTRL_CGE, 1} ++}; ++ ++static const struct mask_to_text fq_ctrl_text_list[] = { ++ { ++ .mask = QM_FQCTRL_PREFERINCACHE, ++ .txt = "Prefer in cache", ++ }, ++ { ++ .mask = QM_FQCTRL_HOLDACTIVE, ++ .txt = "Hold active in portal", ++ }, ++ { ++ .mask = QM_FQCTRL_AVOIDBLOCK, ++ .txt = "Avoid Blocking", ++ }, ++ { ++ .mask = QM_FQCTRL_FORCESFDR, ++ .txt = "High-priority SFDRs", ++ }, ++ { ++ .mask = QM_FQCTRL_CPCSTASH, ++ .txt = "CPC Stash Enable", ++ }, ++ { ++ .mask = QM_FQCTRL_CTXASTASHING, ++ .txt = "Context-A stashing", ++ }, ++ { ++ .mask = QM_FQCTRL_ORP, ++ .txt = "ORP Enable", ++ }, ++ { ++ .mask = QM_FQCTRL_TDE, ++ .txt = "Tail-Drop Enable", ++ }, ++ { ++ .mask = QM_FQCTRL_CGE, ++ .txt = "Congestion Group Enable", ++ }, ++ { ++ .mask = 0, ++ .txt = NULL, ++ } ++}; ++ ++static const char *get_fqd_ctrl_text(u16 mask) ++{ ++ int i = 0; ++ ++ while (fq_ctrl_text_list[i].txt != NULL) { ++ if (fq_ctrl_text_list[i].mask == mask) ++ return fq_ctrl_text_list[i].txt; ++ i++; ++ } ++ return NULL; ++} ++ ++static const struct mask_to_text stashing_text_list[] = { ++ { ++ .mask = QM_STASHING_EXCL_CTX, ++ .txt = "FQ Ctx Stash" ++ }, ++ { ++ .mask = QM_STASHING_EXCL_DATA, ++ .txt = "Frame Data Stash", ++ }, ++ { ++ .mask = QM_STASHING_EXCL_ANNOTATION, ++ .txt = "Frame Annotation Stash", ++ }, ++ { ++ .mask = 0, ++ .txt = NULL, ++ }, ++}; ++ ++static int user_input_convert(const char __user *user_buf, size_t count, ++ unsigned long *val) ++{ ++ char buf[12]; ++ ++ if (count > sizeof(buf) - 1) ++ return -EINVAL; ++ if (copy_from_user(buf, user_buf, count)) ++ return -EFAULT; ++ buf[count] = '\0'; ++ if (kstrtoul(buf, 0, val)) ++ return -EINVAL; ++ return 0; ++} ++ ++struct line_buffer_fq { ++ u32 buf[8]; ++ u32 buf_cnt; ++ int line_cnt; ++}; ++ ++static void add_to_line_buffer(struct line_buffer_fq *line_buf, u32 fqid, ++ struct seq_file *file) ++{ ++ line_buf->buf[line_buf->buf_cnt] = fqid; ++ line_buf->buf_cnt++; ++ if (line_buf->buf_cnt == 8) { ++ /* Buffer is full, flush it */ ++ if (line_buf->line_cnt != 0) ++ seq_puts(file, ",\n"); ++ seq_printf(file, "0x%06x,0x%06x,0x%06x,0x%06x,0x%06x," ++ "0x%06x,0x%06x,0x%06x", ++ line_buf->buf[0], line_buf->buf[1], line_buf->buf[2], ++ line_buf->buf[3], line_buf->buf[4], line_buf->buf[5], ++ line_buf->buf[6], line_buf->buf[7]); ++ line_buf->buf_cnt = 0; ++ line_buf->line_cnt++; ++ } ++} ++ ++static void flush_line_buffer(struct line_buffer_fq *line_buf, ++ struct seq_file *file) ++{ ++ if (line_buf->buf_cnt) { ++ int y = 0; ++ if (line_buf->line_cnt != 0) ++ seq_puts(file, ",\n"); ++ while (y != line_buf->buf_cnt) { ++ if (y+1 == line_buf->buf_cnt) ++ seq_printf(file, "0x%06x", line_buf->buf[y]); ++ else ++ seq_printf(file, "0x%06x,", line_buf->buf[y]); ++ y++; ++ } ++ line_buf->line_cnt++; ++ } ++ if (line_buf->line_cnt) ++ seq_putc(file, '\n'); ++} ++ ++static struct dentry *dfs_root; /* debugfs root directory */ ++ ++/******************************************************************************* ++ * Query Frame Queue Non Programmable Fields ++ ******************************************************************************/ ++struct query_fq_np_fields_data_s { ++ u32 fqid; ++}; ++static struct query_fq_np_fields_data_s query_fq_np_fields_data = { ++ .fqid = 1, ++}; ++ ++static int query_fq_np_fields_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_mcr_queryfq_np np; ++ struct qman_fq fq; ++ ++ fq.fqid = query_fq_np_fields_data.fqid; ++ ret = qman_query_fq_np(&fq, &np); ++ if (ret) ++ return ret; ++ /* Print state */ ++ seq_printf(file, "Query FQ Non Programmable Fields Result fqid 0x%x\n", ++ fq.fqid); ++ seq_printf(file, " force eligible pending: %s\n", ++ (np.state & QM_MCR_NP_STATE_FE) ? "yes" : "no"); ++ seq_printf(file, " retirement pending: %s\n", ++ (np.state & QM_MCR_NP_STATE_R) ? "yes" : "no"); ++ seq_printf(file, " state: %s\n", ++ state_txt[np.state & QM_MCR_NP_STATE_MASK]); ++ seq_printf(file, " fq_link: 0x%x\n", np.fqd_link); ++ seq_printf(file, " odp_seq: %u\n", np.odp_seq); ++ seq_printf(file, " orp_nesn: %u\n", np.orp_nesn); ++ seq_printf(file, " orp_ea_hseq: %u\n", np.orp_ea_hseq); ++ seq_printf(file, " orp_ea_tseq: %u\n", np.orp_ea_tseq); ++ seq_printf(file, " orp_ea_hptr: 0x%x\n", np.orp_ea_hptr); ++ seq_printf(file, " orp_ea_tptr: 0x%x\n", np.orp_ea_tptr); ++ seq_printf(file, " pfdr_hptr: 0x%x\n", np.pfdr_hptr); ++ seq_printf(file, " pfdr_tptr: 0x%x\n", np.pfdr_tptr); ++ seq_printf(file, " is: ics_surp contains a %s\n", ++ (np.is) ? "deficit" : "surplus"); ++ seq_printf(file, " ics_surp: %u\n", np.ics_surp); ++ seq_printf(file, " byte_cnt: %u\n", np.byte_cnt); ++ seq_printf(file, " frm_cnt: %u\n", np.frm_cnt); ++ seq_printf(file, " ra1_sfdr: 0x%x\n", np.ra1_sfdr); ++ seq_printf(file, " ra2_sfdr: 0x%x\n", np.ra2_sfdr); ++ seq_printf(file, " od1_sfdr: 0x%x\n", np.od1_sfdr); ++ seq_printf(file, " od2_sfdr: 0x%x\n", np.od2_sfdr); ++ seq_printf(file, " od3_sfdr: 0x%x\n", np.od3_sfdr); ++ return 0; ++} ++ ++static int query_fq_np_fields_open(struct inode *inode, ++ struct file *file) ++{ ++ return single_open(file, query_fq_np_fields_show, NULL); ++} ++ ++static ssize_t query_fq_np_fields_write(struct file *f, ++ const char __user *buf, size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ if (val > MAX_FQID) ++ return -EINVAL; ++ query_fq_np_fields_data.fqid = (u32)val; ++ return count; ++} ++ ++static const struct file_operations query_fq_np_fields_fops = { ++ .owner = THIS_MODULE, ++ .open = query_fq_np_fields_open, ++ .read = seq_read, ++ .write = query_fq_np_fields_write, ++ .release = single_release, ++}; ++ ++/******************************************************************************* ++ * Frame Queue Programmable Fields ++ ******************************************************************************/ ++struct query_fq_fields_data_s { ++ u32 fqid; ++}; ++ ++static struct query_fq_fields_data_s query_fq_fields_data = { ++ .fqid = 1, ++}; ++ ++static int query_fq_fields_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_fqd fqd; ++ struct qman_fq fq; ++ int i = 0; ++ ++ memset(&fqd, 0, sizeof(struct qm_fqd)); ++ fq.fqid = query_fq_fields_data.fqid; ++ ret = qman_query_fq(&fq, &fqd); ++ if (ret) ++ return ret; ++ seq_printf(file, "Query FQ Programmable Fields Result fqid 0x%x\n", ++ fq.fqid); ++ seq_printf(file, " orprws: %u\n", fqd.orprws); ++ seq_printf(file, " oa: %u\n", fqd.oa); ++ seq_printf(file, " olws: %u\n", fqd.olws); ++ ++ seq_printf(file, " cgid: %u\n", fqd.cgid); ++ ++ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) == 0) ++ seq_puts(file, " fq_ctrl: None\n"); ++ else { ++ i = 0; ++ seq_puts(file, " fq_ctrl:\n"); ++ while (fq_ctrl_text_list[i].txt != NULL) { ++ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) & ++ fq_ctrl_text_list[i].mask) ++ seq_printf(file, " %s\n", ++ fq_ctrl_text_list[i].txt); ++ i++; ++ } ++ } ++ seq_printf(file, " dest_channel: %u\n", fqd.dest.channel); ++ seq_printf(file, " dest_wq: %u\n", fqd.dest.wq); ++ seq_printf(file, " ics_cred: %u\n", fqd.ics_cred); ++ seq_printf(file, " td_mant: %u\n", fqd.td.mant); ++ seq_printf(file, " td_exp: %u\n", fqd.td.exp); ++ ++ seq_printf(file, " ctx_b: 0x%x\n", fqd.context_b); ++ ++ seq_printf(file, " ctx_a: 0x%llx\n", qm_fqd_stashing_get64(&fqd)); ++ /* Any stashing configured */ ++ if ((fqd.context_a.stashing.exclusive & 0x7) == 0) ++ seq_puts(file, " ctx_a_stash_exclusive: None\n"); ++ else { ++ seq_puts(file, " ctx_a_stash_exclusive:\n"); ++ i = 0; ++ while (stashing_text_list[i].txt != NULL) { ++ if ((fqd.fq_ctrl & 0x7) & stashing_text_list[i].mask) ++ seq_printf(file, " %s\n", ++ stashing_text_list[i].txt); ++ i++; ++ } ++ } ++ seq_printf(file, " ctx_a_stash_annotation_cl: %u\n", ++ fqd.context_a.stashing.annotation_cl); ++ seq_printf(file, " ctx_a_stash_data_cl: %u\n", ++ fqd.context_a.stashing.data_cl); ++ seq_printf(file, " ctx_a_stash_context_cl: %u\n", ++ fqd.context_a.stashing.context_cl); ++ return 0; ++} ++ ++static int query_fq_fields_open(struct inode *inode, ++ struct file *file) ++{ ++ return single_open(file, query_fq_fields_show, NULL); ++} ++ ++static ssize_t query_fq_fields_write(struct file *f, ++ const char __user *buf, size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ if (val > MAX_FQID) ++ return -EINVAL; ++ query_fq_fields_data.fqid = (u32)val; ++ return count; ++} ++ ++static const struct file_operations query_fq_fields_fops = { ++ .owner = THIS_MODULE, ++ .open = query_fq_fields_open, ++ .read = seq_read, ++ .write = query_fq_fields_write, ++ .release = single_release, ++}; ++ ++/******************************************************************************* ++ * Query WQ lengths ++ ******************************************************************************/ ++struct query_wq_lengths_data_s { ++ union { ++ u16 channel_wq; /* ignores wq (3 lsbits) */ ++ struct { ++ u16 id:13; /* qm_channel */ ++ u16 __reserved:3; ++ } __packed channel; ++ }; ++}; ++static struct query_wq_lengths_data_s query_wq_lengths_data; ++static int query_wq_lengths_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_mcr_querywq wq; ++ int i; ++ ++ memset(&wq, 0, sizeof(struct qm_mcr_querywq)); ++ wq.channel.id = query_wq_lengths_data.channel.id; ++ ret = qman_query_wq(0, &wq); ++ if (ret) ++ return ret; ++ seq_printf(file, "Query Result For Channel: 0x%x\n", wq.channel.id); ++ for (i = 0; i < 8; i++) ++ /* mask out upper 4 bits since they are not part of length */ ++ seq_printf(file, " wq%d_len : %u\n", i, wq.wq_len[i] & 0x0fff); ++ return 0; ++} ++ ++static int query_wq_lengths_open(struct inode *inode, ++ struct file *file) ++{ ++ return single_open(file, query_wq_lengths_show, NULL); ++} ++ ++static ssize_t query_wq_lengths_write(struct file *f, ++ const char __user *buf, size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ if (val > 0xfff8) ++ return -EINVAL; ++ query_wq_lengths_data.channel.id = (u16)val; ++ return count; ++} ++ ++static const struct file_operations query_wq_lengths_fops = { ++ .owner = THIS_MODULE, ++ .open = query_wq_lengths_open, ++ .read = seq_read, ++ .write = query_wq_lengths_write, ++ .release = single_release, ++}; ++ ++/******************************************************************************* ++ * Query CGR ++ ******************************************************************************/ ++struct query_cgr_s { ++ u8 cgid; ++}; ++static struct query_cgr_s query_cgr_data; ++ ++static int query_cgr_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_mcr_querycgr cgrd; ++ struct qman_cgr cgr; ++ int i, j; ++ u32 mask; ++ ++ memset(&cgr, 0, sizeof(cgr)); ++ memset(&cgrd, 0, sizeof(cgrd)); ++ cgr.cgrid = query_cgr_data.cgid; ++ ret = qman_query_cgr(&cgr, &cgrd); ++ if (ret) ++ return ret; ++ seq_printf(file, "Query CGR id 0x%x\n", cgr.cgrid); ++ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ cgrd.cgr.wr_parm_g.MA, cgrd.cgr.wr_parm_g.Mn, ++ cgrd.cgr.wr_parm_g.SA, cgrd.cgr.wr_parm_g.Sn, ++ cgrd.cgr.wr_parm_g.Pn); ++ ++ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ cgrd.cgr.wr_parm_y.MA, cgrd.cgr.wr_parm_y.Mn, ++ cgrd.cgr.wr_parm_y.SA, cgrd.cgr.wr_parm_y.Sn, ++ cgrd.cgr.wr_parm_y.Pn); ++ ++ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ cgrd.cgr.wr_parm_r.MA, cgrd.cgr.wr_parm_r.Mn, ++ cgrd.cgr.wr_parm_r.SA, cgrd.cgr.wr_parm_r.Sn, ++ cgrd.cgr.wr_parm_r.Pn); ++ ++ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n", ++ cgrd.cgr.wr_en_g, cgrd.cgr.wr_en_y, cgrd.cgr.wr_en_r); ++ ++ seq_printf(file, " cscn_en: %u\n", cgrd.cgr.cscn_en); ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { ++ seq_puts(file, " cscn_targ_dcp:\n"); ++ mask = 0x80000000; ++ for (i = 0; i < 32; i++) { ++ if (cgrd.cgr.cscn_targ & mask) ++ seq_printf(file, " send CSCN to dcp %u\n", ++ (31 - i)); ++ mask >>= 1; ++ } ++ ++ seq_puts(file, " cscn_targ_swp:\n"); ++ for (i = 0; i < 4; i++) { ++ mask = 0x80000000; ++ for (j = 0; j < 32; j++) { ++ if (cgrd.cscn_targ_swp[i] & mask) ++ seq_printf(file, " send CSCN to swp" ++ " %u\n", (127 - (i * 32) - j)); ++ mask >>= 1; ++ } ++ } ++ } else { ++ seq_printf(file, " cscn_targ: %u\n", cgrd.cgr.cscn_targ); ++ } ++ seq_printf(file, " cstd_en: %u\n", cgrd.cgr.cstd_en); ++ seq_printf(file, " cs: %u\n", cgrd.cgr.cs); ++ ++ seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n", ++ cgrd.cgr.cs_thres.TA, cgrd.cgr.cs_thres.Tn); ++ ++ seq_printf(file, " mode: %s\n", ++ (cgrd.cgr.mode & QMAN_CGR_MODE_FRAME) ? ++ "frame count" : "byte count"); ++ seq_printf(file, " i_bcnt: %llu\n", qm_mcr_querycgr_i_get64(&cgrd)); ++ seq_printf(file, " a_bcnt: %llu\n", qm_mcr_querycgr_a_get64(&cgrd)); ++ ++ return 0; ++} ++ ++static int query_cgr_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, query_cgr_show, NULL); ++} ++ ++static ssize_t query_cgr_write(struct file *f, const char __user *buf, ++ size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ if (val > 0xff) ++ return -EINVAL; ++ query_cgr_data.cgid = (u8)val; ++ return count; ++} ++ ++static const struct file_operations query_cgr_fops = { ++ .owner = THIS_MODULE, ++ .open = query_cgr_open, ++ .read = seq_read, ++ .write = query_cgr_write, ++ .release = single_release, ++}; ++ ++/******************************************************************************* ++ * Test Write CGR ++ ******************************************************************************/ ++struct test_write_cgr_s { ++ u64 i_bcnt; ++ u8 cgid; ++}; ++static struct test_write_cgr_s test_write_cgr_data; ++ ++static int testwrite_cgr_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_mcr_cgrtestwrite result; ++ struct qman_cgr cgr; ++ u64 i_bcnt; ++ ++ memset(&cgr, 0, sizeof(struct qman_cgr)); ++ memset(&result, 0, sizeof(struct qm_mcr_cgrtestwrite)); ++ cgr.cgrid = test_write_cgr_data.cgid; ++ i_bcnt = test_write_cgr_data.i_bcnt; ++ ret = qman_testwrite_cgr(&cgr, i_bcnt, &result); ++ if (ret) ++ return ret; ++ seq_printf(file, "CGR Test Write CGR id 0x%x\n", cgr.cgrid); ++ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ result.cgr.wr_parm_g.MA, result.cgr.wr_parm_g.Mn, ++ result.cgr.wr_parm_g.SA, result.cgr.wr_parm_g.Sn, ++ result.cgr.wr_parm_g.Pn); ++ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ result.cgr.wr_parm_y.MA, result.cgr.wr_parm_y.Mn, ++ result.cgr.wr_parm_y.SA, result.cgr.wr_parm_y.Sn, ++ result.cgr.wr_parm_y.Pn); ++ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ result.cgr.wr_parm_r.MA, result.cgr.wr_parm_r.Mn, ++ result.cgr.wr_parm_r.SA, result.cgr.wr_parm_r.Sn, ++ result.cgr.wr_parm_r.Pn); ++ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n", ++ result.cgr.wr_en_g, result.cgr.wr_en_y, result.cgr.wr_en_r); ++ seq_printf(file, " cscn_en: %u\n", result.cgr.cscn_en); ++ seq_printf(file, " cscn_targ: %u\n", result.cgr.cscn_targ); ++ seq_printf(file, " cstd_en: %u\n", result.cgr.cstd_en); ++ seq_printf(file, " cs: %u\n", result.cgr.cs); ++ seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n", ++ result.cgr.cs_thres.TA, result.cgr.cs_thres.Tn); ++ ++ /* Add Mode for Si 2 */ ++ seq_printf(file, " mode: %s\n", ++ (result.cgr.mode & QMAN_CGR_MODE_FRAME) ? ++ "frame count" : "byte count"); ++ ++ seq_printf(file, " i_bcnt: %llu\n", ++ qm_mcr_cgrtestwrite_i_get64(&result)); ++ seq_printf(file, " a_bcnt: %llu\n", ++ qm_mcr_cgrtestwrite_a_get64(&result)); ++ seq_printf(file, " wr_prob_g: %u\n", result.wr_prob_g); ++ seq_printf(file, " wr_prob_y: %u\n", result.wr_prob_y); ++ seq_printf(file, " wr_prob_r: %u\n", result.wr_prob_r); ++ return 0; ++} ++ ++static int testwrite_cgr_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, testwrite_cgr_show, NULL); ++} ++ ++static const struct file_operations testwrite_cgr_fops = { ++ .owner = THIS_MODULE, ++ .open = testwrite_cgr_open, ++ .read = seq_read, ++ .release = single_release, ++}; ++ ++ ++static int testwrite_cgr_ibcnt_show(struct seq_file *file, void *offset) ++{ ++ seq_printf(file, "i_bcnt: %llu\n", test_write_cgr_data.i_bcnt); ++ return 0; ++} ++static int testwrite_cgr_ibcnt_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, testwrite_cgr_ibcnt_show, NULL); ++} ++ ++static ssize_t testwrite_cgr_ibcnt_write(struct file *f, const char __user *buf, ++ size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ test_write_cgr_data.i_bcnt = val; ++ return count; ++} ++ ++static const struct file_operations teswrite_cgr_ibcnt_fops = { ++ .owner = THIS_MODULE, ++ .open = testwrite_cgr_ibcnt_open, ++ .read = seq_read, ++ .write = testwrite_cgr_ibcnt_write, ++ .release = single_release, ++}; ++ ++static int testwrite_cgr_cgrid_show(struct seq_file *file, void *offset) ++{ ++ seq_printf(file, "cgrid: %u\n", (u32)test_write_cgr_data.cgid); ++ return 0; ++} ++static int testwrite_cgr_cgrid_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, testwrite_cgr_cgrid_show, NULL); ++} ++ ++static ssize_t testwrite_cgr_cgrid_write(struct file *f, const char __user *buf, ++ size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ if (val > 0xff) ++ return -EINVAL; ++ test_write_cgr_data.cgid = (u8)val; ++ return count; ++} ++ ++static const struct file_operations teswrite_cgr_cgrid_fops = { ++ .owner = THIS_MODULE, ++ .open = testwrite_cgr_cgrid_open, ++ .read = seq_read, ++ .write = testwrite_cgr_cgrid_write, ++ .release = single_release, ++}; ++ ++/******************************************************************************* ++ * Query Congestion State ++ ******************************************************************************/ ++static int query_congestion_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_mcr_querycongestion cs; ++ int i, j, in_cong = 0; ++ u32 mask; ++ ++ memset(&cs, 0, sizeof(struct qm_mcr_querycongestion)); ++ ret = qman_query_congestion(&cs); ++ if (ret) ++ return ret; ++ seq_puts(file, "Query Congestion Result\n"); ++ for (i = 0; i < 8; i++) { ++ mask = 0x80000000; ++ for (j = 0; j < 32; j++) { ++ if (cs.state.__state[i] & mask) { ++ in_cong = 1; ++ seq_printf(file, " cg %u: %s\n", (i*32)+j, ++ "in congestion"); ++ } ++ mask >>= 1; ++ } ++ } ++ if (!in_cong) ++ seq_puts(file, " All congestion groups not congested.\n"); ++ return 0; ++} ++ ++static int query_congestion_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, query_congestion_show, NULL); ++} ++ ++static const struct file_operations query_congestion_fops = { ++ .owner = THIS_MODULE, ++ .open = query_congestion_open, ++ .read = seq_read, ++ .release = single_release, ++}; ++ ++/******************************************************************************* ++ * Query CCGR ++ ******************************************************************************/ ++struct query_ccgr_s { ++ u32 ccgid; ++}; ++static struct query_ccgr_s query_ccgr_data; ++ ++static int query_ccgr_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_mcr_ceetm_ccgr_query ccgr_query; ++ struct qm_mcc_ceetm_ccgr_query query_opts; ++ int i, j; ++ u32 mask; ++ ++ memset(&ccgr_query, 0, sizeof(struct qm_mcr_ceetm_ccgr_query)); ++ memset(&query_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_query)); ++ ++ if ((qman_ip_rev & 0xFF00) < QMAN_REV30) ++ return -EINVAL; ++ ++ seq_printf(file, "Query CCGID %x\n", query_ccgr_data.ccgid); ++ query_opts.dcpid = ((query_ccgr_data.ccgid & 0xFF000000) >> 24); ++ query_opts.ccgrid = query_ccgr_data.ccgid & 0x000001FF; ++ ret = qman_ceetm_query_ccgr(&query_opts, &ccgr_query); ++ if (ret) ++ return ret; ++ seq_printf(file, "Query CCGR id %x in DCP %d\n", query_opts.ccgrid, ++ query_opts.dcpid); ++ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ ccgr_query.cm_query.wr_parm_g.MA, ++ ccgr_query.cm_query.wr_parm_g.Mn, ++ ccgr_query.cm_query.wr_parm_g.SA, ++ ccgr_query.cm_query.wr_parm_g.Sn, ++ ccgr_query.cm_query.wr_parm_g.Pn); ++ ++ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ ccgr_query.cm_query.wr_parm_y.MA, ++ ccgr_query.cm_query.wr_parm_y.Mn, ++ ccgr_query.cm_query.wr_parm_y.SA, ++ ccgr_query.cm_query.wr_parm_y.Sn, ++ ccgr_query.cm_query.wr_parm_y.Pn); ++ ++ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", ++ ccgr_query.cm_query.wr_parm_r.MA, ++ ccgr_query.cm_query.wr_parm_r.Mn, ++ ccgr_query.cm_query.wr_parm_r.SA, ++ ccgr_query.cm_query.wr_parm_r.Sn, ++ ccgr_query.cm_query.wr_parm_r.Pn); ++ ++ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n", ++ ccgr_query.cm_query.ctl_wr_en_g, ++ ccgr_query.cm_query.ctl_wr_en_y, ++ ccgr_query.cm_query.ctl_wr_en_r); ++ ++ seq_printf(file, " cscn_en: %u\n", ccgr_query.cm_query.ctl_cscn_en); ++ seq_puts(file, " cscn_targ_dcp:\n"); ++ mask = 0x80000000; ++ for (i = 0; i < 32; i++) { ++ if (ccgr_query.cm_query.cscn_targ_dcp & mask) ++ seq_printf(file, " send CSCN to dcp %u\n", (31 - i)); ++ mask >>= 1; ++ } ++ ++ seq_puts(file, " cscn_targ_swp:\n"); ++ for (i = 0; i < 4; i++) { ++ mask = 0x80000000; ++ for (j = 0; j < 32; j++) { ++ if (ccgr_query.cm_query.cscn_targ_swp[i] & mask) ++ seq_printf(file, " send CSCN to swp" ++ "%u\n", (127 - (i * 32) - j)); ++ mask >>= 1; ++ } ++ } ++ ++ seq_printf(file, " td_en: %u\n", ccgr_query.cm_query.ctl_td_en); ++ ++ seq_printf(file, " cs_thresh_in_TA: %u, cs_thresh_in_Tn: %u\n", ++ ccgr_query.cm_query.cs_thres.TA, ++ ccgr_query.cm_query.cs_thres.Tn); ++ ++ seq_printf(file, " cs_thresh_out_TA: %u, cs_thresh_out_Tn: %u\n", ++ ccgr_query.cm_query.cs_thres_x.TA, ++ ccgr_query.cm_query.cs_thres_x.Tn); ++ ++ seq_printf(file, " td_thresh_TA: %u, td_thresh_Tn: %u\n", ++ ccgr_query.cm_query.td_thres.TA, ++ ccgr_query.cm_query.td_thres.Tn); ++ ++ seq_printf(file, " mode: %s\n", ++ (ccgr_query.cm_query.ctl_mode & ++ QMAN_CGR_MODE_FRAME) ? ++ "frame count" : "byte count"); ++ seq_printf(file, " i_cnt: %llu\n", (u64)ccgr_query.cm_query.i_cnt); ++ seq_printf(file, " a_cnt: %llu\n", (u64)ccgr_query.cm_query.a_cnt); ++ ++ return 0; ++} ++ ++static int query_ccgr_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, query_ccgr_show, NULL); ++} ++ ++static ssize_t query_ccgr_write(struct file *f, const char __user *buf, ++ size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ query_ccgr_data.ccgid = val; ++ return count; ++} ++ ++static const struct file_operations query_ccgr_fops = { ++ .owner = THIS_MODULE, ++ .open = query_ccgr_open, ++ .read = seq_read, ++ .write = query_ccgr_write, ++ .release = single_release, ++}; ++/******************************************************************************* ++ * QMan register ++ ******************************************************************************/ ++struct qman_register_s { ++ u32 val; ++}; ++static struct qman_register_s qman_register_data; ++ ++static void init_ccsrmempeek(void) ++{ ++ struct device_node *dn; ++ const u32 *regaddr_p; ++ ++ dn = of_find_compatible_node(NULL, NULL, "fsl,qman"); ++ if (!dn) { ++ pr_info("No fsl,qman node\n"); ++ return; ++ } ++ regaddr_p = of_get_address(dn, 0, &qman_ccsr_size, NULL); ++ if (!regaddr_p) { ++ of_node_put(dn); ++ return; ++ } ++ qman_ccsr_start = of_translate_address(dn, regaddr_p); ++ of_node_put(dn); ++} ++/* This function provides access to QMan ccsr memory map */ ++static int qman_ccsrmempeek(u32 *val, u32 offset) ++{ ++ void __iomem *addr; ++ u64 phys_addr; ++ ++ if (!qman_ccsr_start) ++ return -EINVAL; ++ ++ if (offset > (qman_ccsr_size - sizeof(u32))) ++ return -EINVAL; ++ ++ phys_addr = qman_ccsr_start + offset; ++ addr = ioremap(phys_addr, sizeof(u32)); ++ if (!addr) { ++ pr_err("ccsrmempeek, ioremap failed\n"); ++ return -EINVAL; ++ } ++ *val = in_be32(addr); ++ iounmap(addr); ++ return 0; ++} ++ ++static int qman_ccsrmempeek_show(struct seq_file *file, void *offset) ++{ ++ u32 b; ++ ++ qman_ccsrmempeek(&b, qman_register_data.val); ++ seq_printf(file, "QMan register offset = 0x%x\n", ++ qman_register_data.val); ++ seq_printf(file, "value = 0x%08x\n", b); ++ ++ return 0; ++} ++ ++static int qman_ccsrmempeek_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, qman_ccsrmempeek_show, NULL); ++} ++ ++static ssize_t qman_ccsrmempeek_write(struct file *f, const char __user *buf, ++ size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ /* multiple of 4 */ ++ if (val > (qman_ccsr_size - sizeof(u32))) { ++ pr_info("Input 0x%lx > 0x%llx\n", ++ val, (qman_ccsr_size - sizeof(u32))); ++ return -EINVAL; ++ } ++ if (val & 0x3) { ++ pr_info("Input 0x%lx not multiple of 4\n", val); ++ return -EINVAL; ++ } ++ qman_register_data.val = val; ++ return count; ++} ++ ++static const struct file_operations qman_ccsrmempeek_fops = { ++ .owner = THIS_MODULE, ++ .open = qman_ccsrmempeek_open, ++ .read = seq_read, ++ .write = qman_ccsrmempeek_write, ++}; ++ ++/******************************************************************************* ++ * QMan state ++ ******************************************************************************/ ++static int qman_fqd_state_show(struct seq_file *file, void *offset) ++{ ++ struct qm_mcr_queryfq_np np; ++ struct qman_fq fq; ++ struct line_buffer_fq line_buf; ++ int ret, i; ++ u8 *state = file->private; ++ u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)]; ++ ++ memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt)); ++ memset(&line_buf, 0, sizeof(line_buf)); ++ ++ seq_printf(file, "List of fq ids in state: %s\n", state_txt[*state]); ++ ++ for (i = 1; i < fqid_max; i++) { ++ fq.fqid = i; ++ ret = qman_query_fq_np(&fq, &np); ++ if (ret) ++ return ret; ++ if (*state == (np.state & QM_MCR_NP_STATE_MASK)) ++ add_to_line_buffer(&line_buf, fq.fqid, file); ++ /* Keep a summary count of all states */ ++ if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states)) ++ qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++; ++ } ++ flush_line_buffer(&line_buf, file); ++ ++ for (i = 0; i < ARRAY_SIZE(fqd_states); i++) { ++ seq_printf(file, "%s count = %u\n", state_txt[i], ++ qm_fq_state_cnt[i]); ++ } ++ return 0; ++} ++ ++static int qman_fqd_state_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, qman_fqd_state_show, inode->i_private); ++} ++ ++static const struct file_operations qman_fqd_state_fops = { ++ .owner = THIS_MODULE, ++ .open = qman_fqd_state_open, ++ .read = seq_read, ++}; ++ ++static int qman_fqd_ctrl_show(struct seq_file *file, void *offset) ++{ ++ struct qm_fqd fqd; ++ struct qman_fq fq; ++ u32 fq_en_cnt = 0, fq_di_cnt = 0; ++ int ret, i; ++ struct mask_filter_s *data = file->private; ++ const char *ctrl_txt = get_fqd_ctrl_text(data->mask); ++ struct line_buffer_fq line_buf; ++ ++ memset(&line_buf, 0, sizeof(line_buf)); ++ seq_printf(file, "List of fq ids with: %s :%s\n", ++ ctrl_txt, (data->filter) ? "enabled" : "disabled"); ++ for (i = 1; i < fqid_max; i++) { ++ fq.fqid = i; ++ memset(&fqd, 0, sizeof(struct qm_fqd)); ++ ret = qman_query_fq(&fq, &fqd); ++ if (ret) ++ return ret; ++ if (data->filter) { ++ if (fqd.fq_ctrl & data->mask) ++ add_to_line_buffer(&line_buf, fq.fqid, file); ++ } else { ++ if (!(fqd.fq_ctrl & data->mask)) ++ add_to_line_buffer(&line_buf, fq.fqid, file); ++ } ++ if (fqd.fq_ctrl & data->mask) ++ fq_en_cnt++; ++ else ++ fq_di_cnt++; ++ } ++ flush_line_buffer(&line_buf, file); ++ ++ seq_printf(file, "Total FQD with: %s : enabled = %u\n", ++ ctrl_txt, fq_en_cnt); ++ seq_printf(file, "Total FQD with: %s : disabled = %u\n", ++ ctrl_txt, fq_di_cnt); ++ return 0; ++} ++ ++/******************************************************************************* ++ * QMan ctrl CGE, TDE, ORP, CTX, CPC, SFDR, BLOCK, HOLD, CACHE ++ ******************************************************************************/ ++static int qman_fqd_ctrl_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, qman_fqd_ctrl_show, inode->i_private); ++} ++ ++static const struct file_operations qman_fqd_ctrl_fops = { ++ .owner = THIS_MODULE, ++ .open = qman_fqd_ctrl_open, ++ .read = seq_read, ++}; ++ ++/******************************************************************************* ++ * QMan ctrl summary ++ ******************************************************************************/ ++/******************************************************************************* ++ * QMan summary state ++ ******************************************************************************/ ++static int qman_fqd_non_prog_summary_show(struct seq_file *file, void *offset) ++{ ++ struct qm_mcr_queryfq_np np; ++ struct qman_fq fq; ++ int ret, i; ++ u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)]; ++ ++ memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt)); ++ ++ for (i = 1; i < fqid_max; i++) { ++ fq.fqid = i; ++ ret = qman_query_fq_np(&fq, &np); ++ if (ret) ++ return ret; ++ /* Keep a summary count of all states */ ++ if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states)) ++ qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++; ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(fqd_states); i++) { ++ seq_printf(file, "%s count = %u\n", state_txt[i], ++ qm_fq_state_cnt[i]); ++ } ++ return 0; ++} ++ ++static int qman_fqd_prog_summary_show(struct seq_file *file, void *offset) ++{ ++ struct qm_fqd fqd; ++ struct qman_fq fq; ++ int ret, i , j; ++ u32 qm_prog_cnt[ARRAY_SIZE(mask_filter)/2]; ++ ++ memset(qm_prog_cnt, 0, sizeof(qm_prog_cnt)); ++ ++ for (i = 1; i < fqid_max; i++) { ++ memset(&fqd, 0, sizeof(struct qm_fqd)); ++ fq.fqid = i; ++ ret = qman_query_fq(&fq, &fqd); ++ if (ret) ++ return ret; ++ /* Keep a summary count of all states */ ++ for (j = 0; j < ARRAY_SIZE(mask_filter); j += 2) ++ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) & ++ mask_filter[j].mask) ++ qm_prog_cnt[j/2]++; ++ } ++ for (i = 0; i < ARRAY_SIZE(mask_filter) / 2; i++) { ++ seq_printf(file, "%s count = %u\n", ++ get_fqd_ctrl_text(mask_filter[i*2].mask), ++ qm_prog_cnt[i]); ++ } ++ return 0; ++} ++ ++static int qman_fqd_summary_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ ++ /* Display summary of non programmable fields */ ++ ret = qman_fqd_non_prog_summary_show(file, offset); ++ if (ret) ++ return ret; ++ seq_puts(file, "-----------------------------------------\n"); ++ /* Display programmable fields */ ++ ret = qman_fqd_prog_summary_show(file, offset); ++ if (ret) ++ return ret; ++ return 0; ++} ++ ++static int qman_fqd_summary_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, qman_fqd_summary_show, NULL); ++} ++ ++static const struct file_operations qman_fqd_summary_fops = { ++ .owner = THIS_MODULE, ++ .open = qman_fqd_summary_open, ++ .read = seq_read, ++}; ++ ++/******************************************************************************* ++ * QMan destination work queue ++ ******************************************************************************/ ++struct qman_dest_wq_s { ++ u16 wq_id; ++}; ++static struct qman_dest_wq_s qman_dest_wq_data = { ++ .wq_id = 0, ++}; ++ ++static int qman_fqd_dest_wq_show(struct seq_file *file, void *offset) ++{ ++ struct qm_fqd fqd; ++ struct qman_fq fq; ++ int ret, i; ++ u16 *wq, wq_id = qman_dest_wq_data.wq_id; ++ struct line_buffer_fq line_buf; ++ ++ memset(&line_buf, 0, sizeof(line_buf)); ++ /* use vmalloc : need to allocate large memory region and don't ++ * require the memory to be physically contiguous. */ ++ wq = vzalloc(sizeof(u16) * (0xFFFF+1)); ++ if (!wq) ++ return -ENOMEM; ++ ++ seq_printf(file, "List of fq ids with destination work queue id" ++ " = 0x%x\n", wq_id); ++ ++ for (i = 1; i < fqid_max; i++) { ++ fq.fqid = i; ++ memset(&fqd, 0, sizeof(struct qm_fqd)); ++ ret = qman_query_fq(&fq, &fqd); ++ if (ret) { ++ vfree(wq); ++ return ret; ++ } ++ if (wq_id == fqd.dest_wq) ++ add_to_line_buffer(&line_buf, fq.fqid, file); ++ wq[fqd.dest_wq]++; ++ } ++ flush_line_buffer(&line_buf, file); ++ ++ seq_puts(file, "Summary of all FQD destination work queue values\n"); ++ for (i = 0; i < 0xFFFF; i++) { ++ if (wq[i]) ++ seq_printf(file, "Channel: 0x%x WQ: 0x%x WQ_ID: 0x%x, " ++ "count = %u\n", i >> 3, i & 0x3, i, wq[i]); ++ } ++ vfree(wq); ++ return 0; ++} ++ ++static ssize_t qman_fqd_dest_wq_write(struct file *f, const char __user *buf, ++ size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ if (val > 0xFFFF) ++ return -EINVAL; ++ qman_dest_wq_data.wq_id = val; ++ return count; ++} ++ ++static int qman_fqd_dest_wq_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, qman_fqd_dest_wq_show, NULL); ++} ++ ++static const struct file_operations qman_fqd_dest_wq_fops = { ++ .owner = THIS_MODULE, ++ .open = qman_fqd_dest_wq_open, ++ .read = seq_read, ++ .write = qman_fqd_dest_wq_write, ++}; ++ ++/******************************************************************************* ++ * QMan Intra-Class Scheduling Credit ++ ******************************************************************************/ ++static int qman_fqd_cred_show(struct seq_file *file, void *offset) ++{ ++ struct qm_fqd fqd; ++ struct qman_fq fq; ++ int ret, i; ++ u32 fq_cnt = 0; ++ struct line_buffer_fq line_buf; ++ ++ memset(&line_buf, 0, sizeof(line_buf)); ++ seq_puts(file, "List of fq ids with Intra-Class Scheduling Credit > 0" ++ "\n"); ++ ++ for (i = 1; i < fqid_max; i++) { ++ fq.fqid = i; ++ memset(&fqd, 0, sizeof(struct qm_fqd)); ++ ret = qman_query_fq(&fq, &fqd); ++ if (ret) ++ return ret; ++ if (fqd.ics_cred > 0) { ++ add_to_line_buffer(&line_buf, fq.fqid, file); ++ fq_cnt++; ++ } ++ } ++ flush_line_buffer(&line_buf, file); ++ ++ seq_printf(file, "Total FQD with ics_cred > 0 = %d\n", fq_cnt); ++ return 0; ++} ++ ++static int qman_fqd_cred_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, qman_fqd_cred_show, NULL); ++} ++ ++static const struct file_operations qman_fqd_cred_fops = { ++ .owner = THIS_MODULE, ++ .open = qman_fqd_cred_open, ++ .read = seq_read, ++}; ++ ++/******************************************************************************* ++ * Class Queue Fields ++ ******************************************************************************/ ++struct query_cq_fields_data_s { ++ u32 cqid; ++}; ++ ++static struct query_cq_fields_data_s query_cq_fields_data = { ++ .cqid = 1, ++}; ++ ++static int query_cq_fields_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ struct qm_mcr_ceetm_cq_query query_result; ++ unsigned int cqid; ++ unsigned int portal; ++ ++ if ((qman_ip_rev & 0xFF00) < QMAN_REV30) ++ return -EINVAL; ++ ++ cqid = query_cq_fields_data.cqid & 0x00FFFFFF; ++ portal = query_cq_fields_data.cqid >> 24; ++ if (portal > qm_dc_portal_fman1) ++ return -EINVAL; ++ ++ ret = qman_ceetm_query_cq(cqid, portal, &query_result); ++ if (ret) ++ return ret; ++ seq_printf(file, "Query CQ Fields Result cqid 0x%x on DCP %d\n", ++ cqid, portal); ++ seq_printf(file, " ccgid: %u\n", query_result.ccgid); ++ seq_printf(file, " state: %u\n", query_result.state); ++ seq_printf(file, " pfdr_hptr: %u\n", query_result.pfdr_hptr); ++ seq_printf(file, " pfdr_tptr: %u\n", query_result.pfdr_tptr); ++ seq_printf(file, " od1_xsfdr: %u\n", query_result.od1_xsfdr); ++ seq_printf(file, " od2_xsfdr: %u\n", query_result.od2_xsfdr); ++ seq_printf(file, " od3_xsfdr: %u\n", query_result.od3_xsfdr); ++ seq_printf(file, " od4_xsfdr: %u\n", query_result.od4_xsfdr); ++ seq_printf(file, " od5_xsfdr: %u\n", query_result.od5_xsfdr); ++ seq_printf(file, " od6_xsfdr: %u\n", query_result.od6_xsfdr); ++ seq_printf(file, " ra1_xsfdr: %u\n", query_result.ra1_xsfdr); ++ seq_printf(file, " ra2_xsfdr: %u\n", query_result.ra2_xsfdr); ++ seq_printf(file, " frame_count: %u\n", query_result.frm_cnt); ++ ++ return 0; ++} ++ ++static int query_cq_fields_open(struct inode *inode, ++ struct file *file) ++{ ++ return single_open(file, query_cq_fields_show, NULL); ++} ++ ++static ssize_t query_cq_fields_write(struct file *f, ++ const char __user *buf, size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ query_cq_fields_data.cqid = (u32)val; ++ return count; ++} ++ ++static const struct file_operations query_cq_fields_fops = { ++ .owner = THIS_MODULE, ++ .open = query_cq_fields_open, ++ .read = seq_read, ++ .write = query_cq_fields_write, ++ .release = single_release, ++}; ++ ++/******************************************************************************* ++ * READ CEETM_XSFDR_IN_USE ++ ******************************************************************************/ ++struct query_ceetm_xsfdr_data_s { ++ enum qm_dc_portal dcp_portal; ++}; ++ ++static struct query_ceetm_xsfdr_data_s query_ceetm_xsfdr_data; ++ ++static int query_ceetm_xsfdr_show(struct seq_file *file, void *offset) ++{ ++ int ret; ++ unsigned int xsfdr_in_use; ++ enum qm_dc_portal portal; ++ ++ ++ if (qman_ip_rev < QMAN_REV31) ++ return -EINVAL; ++ ++ portal = query_ceetm_xsfdr_data.dcp_portal; ++ ret = qman_ceetm_get_xsfdr(portal, &xsfdr_in_use); ++ if (ret) { ++ seq_printf(file, "Read CEETM_XSFDR_IN_USE on DCP %d failed\n", ++ portal); ++ return ret; ++ } ++ ++ seq_printf(file, "DCP%d: CEETM_XSFDR_IN_USE number is %u\n", portal, ++ (xsfdr_in_use & 0x1FFF)); ++ return 0; ++} ++ ++static int query_ceetm_xsfdr_open(struct inode *inode, ++ struct file *file) ++{ ++ return single_open(file, query_ceetm_xsfdr_show, NULL); ++} ++ ++static ssize_t query_ceetm_xsfdr_write(struct file *f, ++ const char __user *buf, size_t count, loff_t *off) ++{ ++ int ret; ++ unsigned long val; ++ ++ ret = user_input_convert(buf, count, &val); ++ if (ret) ++ return ret; ++ if (val > qm_dc_portal_fman1) ++ return -EINVAL; ++ query_ceetm_xsfdr_data.dcp_portal = (u32)val; ++ return count; ++} ++ ++static const struct file_operations query_ceetm_xsfdr_fops = { ++ .owner = THIS_MODULE, ++ .open = query_ceetm_xsfdr_open, ++ .read = seq_read, ++ .write = query_ceetm_xsfdr_write, ++ .release = single_release, ++}; ++ ++/* helper macros used in qman_debugfs_module_init */ ++#define QMAN_DBGFS_ENTRY(name, mode, parent, data, fops) \ ++ do { \ ++ d = debugfs_create_file(name, \ ++ mode, parent, \ ++ data, \ ++ fops); \ ++ if (d == NULL) { \ ++ ret = -ENOMEM; \ ++ goto _return; \ ++ } \ ++ } while (0) ++ ++/* dfs_root as parent */ ++#define QMAN_DBGFS_ENTRY_ROOT(name, mode, data, fops) \ ++ QMAN_DBGFS_ENTRY(name, mode, dfs_root, data, fops) ++ ++/* fqd_root as parent */ ++#define QMAN_DBGFS_ENTRY_FQDROOT(name, mode, data, fops) \ ++ QMAN_DBGFS_ENTRY(name, mode, fqd_root, data, fops) ++ ++/* fqd state */ ++#define QMAN_DBGFS_ENTRY_FQDSTATE(name, index) \ ++ QMAN_DBGFS_ENTRY_FQDROOT(name, S_IRUGO, \ ++ (void *)&mask_filter[index], &qman_fqd_ctrl_fops) ++ ++static int __init qman_debugfs_module_init(void) ++{ ++ int ret = 0; ++ struct dentry *d, *fqd_root; ++ u32 reg; ++ ++ fqid_max = 0; ++ init_ccsrmempeek(); ++ if (qman_ccsr_start) { ++ if (!qman_ccsrmempeek(®, QM_FQD_AR)) { ++ /* extract the size of the FQD window */ ++ reg = reg & 0x3f; ++ /* calculate valid frame queue descriptor range */ ++ fqid_max = (1 << (reg + 1)) / QM_FQD_BLOCK_SIZE; ++ } ++ } ++ dfs_root = debugfs_create_dir("qman", NULL); ++ fqd_root = debugfs_create_dir("fqd", dfs_root); ++ if (dfs_root == NULL || fqd_root == NULL) { ++ ret = -ENOMEM; ++ pr_err("Cannot create qman/fqd debugfs dir\n"); ++ goto _return; ++ } ++ if (fqid_max) { ++ QMAN_DBGFS_ENTRY_ROOT("ccsrmempeek", S_IRUGO | S_IWUGO, ++ NULL, &qman_ccsrmempeek_fops); ++ } ++ QMAN_DBGFS_ENTRY_ROOT("query_fq_np_fields", S_IRUGO | S_IWUGO, ++ &query_fq_np_fields_data, &query_fq_np_fields_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("query_fq_fields", S_IRUGO | S_IWUGO, ++ &query_fq_fields_data, &query_fq_fields_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("query_wq_lengths", S_IRUGO | S_IWUGO, ++ &query_wq_lengths_data, &query_wq_lengths_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("query_cgr", S_IRUGO | S_IWUGO, ++ &query_cgr_data, &query_cgr_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("query_congestion", S_IRUGO, ++ NULL, &query_congestion_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr", S_IRUGO, ++ NULL, &testwrite_cgr_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_cgrid", S_IRUGO | S_IWUGO, ++ NULL, &teswrite_cgr_cgrid_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_ibcnt", S_IRUGO | S_IWUGO, ++ NULL, &teswrite_cgr_ibcnt_fops); ++ ++ QMAN_DBGFS_ENTRY_ROOT("query_ceetm_ccgr", S_IRUGO | S_IWUGO, ++ &query_ccgr_data, &query_ccgr_fops); ++ /* Create files with fqd_root as parent */ ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("stateoos", S_IRUGO, ++ (void *)&fqd_states[QM_MCR_NP_STATE_OOS], &qman_fqd_state_fops); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("state_retired", S_IRUGO, ++ (void *)&fqd_states[QM_MCR_NP_STATE_RETIRED], ++ &qman_fqd_state_fops); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("state_tentatively_sched", S_IRUGO, ++ (void *)&fqd_states[QM_MCR_NP_STATE_TEN_SCHED], ++ &qman_fqd_state_fops); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("state_truly_sched", S_IRUGO, ++ (void *)&fqd_states[QM_MCR_NP_STATE_TRU_SCHED], ++ &qman_fqd_state_fops); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("state_parked", S_IRUGO, ++ (void *)&fqd_states[QM_MCR_NP_STATE_PARKED], ++ &qman_fqd_state_fops); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("state_active", S_IRUGO, ++ (void *)&fqd_states[QM_MCR_NP_STATE_ACTIVE], ++ &qman_fqd_state_fops); ++ QMAN_DBGFS_ENTRY_ROOT("query_cq_fields", S_IRUGO | S_IWUGO, ++ &query_cq_fields_data, &query_cq_fields_fops); ++ QMAN_DBGFS_ENTRY_ROOT("query_ceetm_xsfdr_in_use", S_IRUGO | S_IWUGO, ++ &query_ceetm_xsfdr_data, &query_ceetm_xsfdr_fops); ++ ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("cge_enable", 17); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("cge_disable", 16); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("tde_enable", 15); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("tde_disable", 14); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("orp_enable", 13); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("orp_disable", 12); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_enable", 11); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_disable", 10); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("cpc_enable", 9); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("cpc_disable", 8); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_enable", 7); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_disable", 6); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_enable", 5); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_disable", 4); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_enable", 3); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_disable", 2); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_enable", 1); ++ ++ QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_disable", 0); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("summary", S_IRUGO, ++ NULL, &qman_fqd_summary_fops); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("wq", S_IRUGO | S_IWUGO, ++ NULL, &qman_fqd_dest_wq_fops); ++ ++ QMAN_DBGFS_ENTRY_FQDROOT("cred", S_IRUGO, ++ NULL, &qman_fqd_cred_fops); ++ ++ return 0; ++ ++_return: ++ debugfs_remove_recursive(dfs_root); ++ return ret; ++} ++ ++static void __exit qman_debugfs_module_exit(void) ++{ ++ debugfs_remove_recursive(dfs_root); ++} ++ ++module_init(qman_debugfs_module_init); ++module_exit(qman_debugfs_module_exit); ++MODULE_LICENSE("Dual BSD/GPL"); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_driver.c +@@ -0,0 +1,980 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "qman_private.h" ++ ++#include <asm/smp.h> /* hard_smp_processor_id() if !CONFIG_SMP */ ++#ifdef CONFIG_HOTPLUG_CPU ++#include <linux/cpu.h> ++#endif ++ ++/* Global variable containing revision id (even on non-control plane systems ++ * where CCSR isn't available) */ ++u16 qman_ip_rev; ++EXPORT_SYMBOL(qman_ip_rev); ++u8 qman_ip_cfg; ++EXPORT_SYMBOL(qman_ip_cfg); ++u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1; ++EXPORT_SYMBOL(qm_channel_pool1); ++u16 qm_channel_caam = QMAN_CHANNEL_CAAM; ++EXPORT_SYMBOL(qm_channel_caam); ++u16 qm_channel_pme = QMAN_CHANNEL_PME; ++EXPORT_SYMBOL(qm_channel_pme); ++u16 qm_channel_dce = QMAN_CHANNEL_DCE; ++EXPORT_SYMBOL(qm_channel_dce); ++u16 qman_portal_max; ++EXPORT_SYMBOL(qman_portal_max); ++ ++u32 qman_clk; ++struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX]; ++/* the qman ceetm instances on the given SoC */ ++u8 num_ceetms; ++ ++/* For these variables, and the portal-initialisation logic, the ++ * comments in bman_driver.c apply here so won't be repeated. */ ++static struct qman_portal *shared_portals[NR_CPUS]; ++static int num_shared_portals; ++static int shared_portals_idx; ++static LIST_HEAD(unused_pcfgs); ++static DEFINE_SPINLOCK(unused_pcfgs_lock); ++ ++/* A SDQCR mask comprising all the available/visible pool channels */ ++static u32 pools_sdqcr; ++ ++#define STR_ERR_NOPROP "No '%s' property in node %s\n" ++#define STR_ERR_CELL "'%s' is not a %d-cell range in node %s\n" ++#define STR_FQID_RANGE "fsl,fqid-range" ++#define STR_POOL_CHAN_RANGE "fsl,pool-channel-range" ++#define STR_CGRID_RANGE "fsl,cgrid-range" ++ ++/* A "fsl,fqid-range" node; release the given range to the allocator */ ++static __init int fsl_fqid_range_init(struct device_node *node) ++{ ++ int ret; ++ const u32 *range = of_get_property(node, STR_FQID_RANGE, &ret); ++ if (!range) { ++ pr_err(STR_ERR_NOPROP, STR_FQID_RANGE, node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err(STR_ERR_CELL, STR_FQID_RANGE, 2, node->full_name); ++ return -EINVAL; ++ } ++ qman_seed_fqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ pr_info("Qman: FQID allocator includes range %d:%d\n", ++ be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ return 0; ++} ++ ++/* A "fsl,pool-channel-range" node; add to the SDQCR mask only */ ++static __init int fsl_pool_channel_range_sdqcr(struct device_node *node) ++{ ++ int ret; ++ const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret); ++ if (!chanid) { ++ pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name); ++ return -EINVAL; ++ } ++ for (ret = 0; ret < be32_to_cpu(chanid[1]); ret++) ++ pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(be32_to_cpu(chanid[0]) + ret); ++ return 0; ++} ++ ++/* A "fsl,pool-channel-range" node; release the given range to the allocator */ ++static __init int fsl_pool_channel_range_init(struct device_node *node) ++{ ++ int ret; ++ const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret); ++ if (!chanid) { ++ pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name); ++ return -EINVAL; ++ } ++ qman_seed_pool_range(be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1])); ++ pr_info("Qman: pool channel allocator includes range %d:%d\n", ++ be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1])); ++ return 0; ++} ++ ++/* A "fsl,cgrid-range" node; release the given range to the allocator */ ++static __init int fsl_cgrid_range_init(struct device_node *node) ++{ ++ struct qman_cgr cgr; ++ int ret, errors = 0; ++ const u32 *range = of_get_property(node, STR_CGRID_RANGE, &ret); ++ if (!range) { ++ pr_err(STR_ERR_NOPROP, STR_CGRID_RANGE, node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err(STR_ERR_CELL, STR_CGRID_RANGE, 2, node->full_name); ++ return -EINVAL; ++ } ++ qman_seed_cgrid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ pr_info("Qman: CGRID allocator includes range %d:%d\n", ++ be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ for (cgr.cgrid = 0; cgr.cgrid < __CGR_NUM; cgr.cgrid++) { ++ ret = qman_modify_cgr(&cgr, QMAN_CGR_FLAG_USE_INIT, NULL); ++ if (ret) ++ errors++; ++ } ++ if (errors) ++ pr_err("Warning: %d error%s while initialising CGRs %d:%d\n", ++ errors, (errors > 1) ? "s" : "", range[0], range[1]); ++ return 0; ++} ++ ++static __init int fsl_ceetm_init(struct device_node *node) ++{ ++ enum qm_dc_portal dcp_portal; ++ struct qm_ceetm_sp *sp; ++ struct qm_ceetm_lni *lni; ++ int ret, i; ++ const u32 *range; ++ ++ /* Find LFQID range */ ++ range = of_get_property(node, "fsl,ceetm-lfqid-range", &ret); ++ if (!range) { ++ pr_err("No fsl,ceetm-lfqid-range in node %s\n", ++ node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err("fsl,ceetm-lfqid-range is not a 2-cell range in node" ++ " %s\n", node->full_name); ++ return -EINVAL; ++ } ++ ++ dcp_portal = (be32_to_cpu(range[0]) & 0x0F0000) >> 16; ++ if (dcp_portal > qm_dc_portal_fman1) { ++ pr_err("The DCP portal %d doesn't support CEETM\n", dcp_portal); ++ return -EINVAL; ++ } ++ ++ if (dcp_portal == qm_dc_portal_fman0) ++ qman_seed_ceetm0_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ if (dcp_portal == qm_dc_portal_fman1) ++ qman_seed_ceetm1_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ pr_debug("Qman: The lfqid allocator of CEETM %d includes range" ++ " 0x%x:0x%x\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ ++ qman_ceetms[dcp_portal].idx = dcp_portal; ++ INIT_LIST_HEAD(&qman_ceetms[dcp_portal].sub_portals); ++ INIT_LIST_HEAD(&qman_ceetms[dcp_portal].lnis); ++ ++ /* Find Sub-portal range */ ++ range = of_get_property(node, "fsl,ceetm-sp-range", &ret); ++ if (!range) { ++ pr_err("No fsl,ceetm-sp-range in node %s\n", node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err("fsl,ceetm-sp-range is not a 2-cell range in node %s\n", ++ node->full_name); ++ return -EINVAL; ++ } ++ ++ for (i = 0; i < be32_to_cpu(range[1]); i++) { ++ sp = kzalloc(sizeof(*sp), GFP_KERNEL); ++ if (!sp) { ++ pr_err("Can't alloc memory for sub-portal %d\n", ++ range[0] + i); ++ return -ENOMEM; ++ } ++ sp->idx = be32_to_cpu(range[0]) + i; ++ sp->dcp_idx = dcp_portal; ++ sp->is_claimed = 0; ++ list_add_tail(&sp->node, &qman_ceetms[dcp_portal].sub_portals); ++ sp++; ++ } ++ pr_debug("Qman: Reserve sub-portal %d:%d for CEETM %d\n", ++ be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal); ++ qman_ceetms[dcp_portal].sp_range[0] = be32_to_cpu(range[0]); ++ qman_ceetms[dcp_portal].sp_range[1] = be32_to_cpu(range[1]); ++ ++ /* Find LNI range */ ++ range = of_get_property(node, "fsl,ceetm-lni-range", &ret); ++ if (!range) { ++ pr_err("No fsl,ceetm-lni-range in node %s\n", node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err("fsl,ceetm-lni-range is not a 2-cell range in node %s\n", ++ node->full_name); ++ return -EINVAL; ++ } ++ ++ for (i = 0; i < be32_to_cpu(range[1]); i++) { ++ lni = kzalloc(sizeof(*lni), GFP_KERNEL); ++ if (!lni) { ++ pr_err("Can't alloc memory for LNI %d\n", ++ range[0] + i); ++ return -ENOMEM; ++ } ++ lni->idx = be32_to_cpu(range[0]) + i; ++ lni->dcp_idx = dcp_portal; ++ lni->is_claimed = 0; ++ INIT_LIST_HEAD(&lni->channels); ++ list_add_tail(&lni->node, &qman_ceetms[dcp_portal].lnis); ++ lni++; ++ } ++ pr_debug("Qman: Reserve LNI %d:%d for CEETM %d\n", ++ be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal); ++ qman_ceetms[dcp_portal].lni_range[0] = be32_to_cpu(range[0]); ++ qman_ceetms[dcp_portal].lni_range[1] = be32_to_cpu(range[1]); ++ ++ /* Find CEETM channel range */ ++ range = of_get_property(node, "fsl,ceetm-channel-range", &ret); ++ if (!range) { ++ pr_err("No fsl,ceetm-channel-range in node %s\n", ++ node->full_name); ++ return -EINVAL; ++ } ++ if (ret != 8) { ++ pr_err("fsl,ceetm-channel-range is not a 2-cell range in node" ++ "%s\n", node->full_name); ++ return -EINVAL; ++ } ++ ++ if (dcp_portal == qm_dc_portal_fman0) ++ qman_seed_ceetm0_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ if (dcp_portal == qm_dc_portal_fman1) ++ qman_seed_ceetm1_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ pr_debug("Qman: The channel allocator of CEETM %d includes" ++ " range %d:%d\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1])); ++ ++ /* Set CEETM PRES register */ ++ ret = qman_ceetm_set_prescaler(dcp_portal); ++ if (ret) ++ return ret; ++ return 0; ++} ++ ++static void qman_get_ip_revision(struct device_node *dn) ++{ ++ u16 ip_rev = 0; ++ u8 ip_cfg = QMAN_REV_CFG_0; ++ for_each_compatible_node(dn, NULL, "fsl,qman-portal") { ++ if (!of_device_is_available(dn)) ++ continue; ++ if (of_device_is_compatible(dn, "fsl,qman-portal-1.0") || ++ of_device_is_compatible(dn, "fsl,qman-portal-1.0.0")) { ++ pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n"); ++ BUG_ON(1); ++ } else if (of_device_is_compatible(dn, "fsl,qman-portal-1.1") || ++ of_device_is_compatible(dn, "fsl,qman-portal-1.1.0")) { ++ ip_rev = QMAN_REV11; ++ qman_portal_max = 10; ++ } else if (of_device_is_compatible(dn, "fsl,qman-portal-1.2") || ++ of_device_is_compatible(dn, "fsl,qman-portal-1.2.0")) { ++ ip_rev = QMAN_REV12; ++ qman_portal_max = 10; ++ } else if (of_device_is_compatible(dn, "fsl,qman-portal-2.0") || ++ of_device_is_compatible(dn, "fsl,qman-portal-2.0.0")) { ++ ip_rev = QMAN_REV20; ++ qman_portal_max = 3; ++ } else if (of_device_is_compatible(dn, ++ "fsl,qman-portal-3.0.0")) { ++ ip_rev = QMAN_REV30; ++ qman_portal_max = 50; ++ } else if (of_device_is_compatible(dn, ++ "fsl,qman-portal-3.0.1")) { ++ ip_rev = QMAN_REV30; ++ qman_portal_max = 25; ++ ip_cfg = QMAN_REV_CFG_1; ++ } else if (of_device_is_compatible(dn, ++ "fsl,qman-portal-3.1.0")) { ++ ip_rev = QMAN_REV31; ++ qman_portal_max = 50; ++ } else if (of_device_is_compatible(dn, ++ "fsl,qman-portal-3.1.1")) { ++ ip_rev = QMAN_REV31; ++ qman_portal_max = 25; ++ ip_cfg = QMAN_REV_CFG_1; ++ } else if (of_device_is_compatible(dn, ++ "fsl,qman-portal-3.1.2")) { ++ ip_rev = QMAN_REV31; ++ qman_portal_max = 18; ++ ip_cfg = QMAN_REV_CFG_2; ++ } else if (of_device_is_compatible(dn, ++ "fsl,qman-portal-3.1.3")) { ++ ip_rev = QMAN_REV31; ++ qman_portal_max = 10; ++ ip_cfg = QMAN_REV_CFG_3; ++ } else if (of_device_is_compatible(dn, ++ "fsl,qman-portal-3.2.0")) { ++ ip_rev = QMAN_REV32; ++ qman_portal_max = 10; ++ ip_cfg = QMAN_REV_CFG_3; // TODO: Verify for ls1043 ++ } else { ++ pr_warn("unknown QMan version in portal node," ++ "default to rev1.1\n"); ++ ip_rev = QMAN_REV11; ++ qman_portal_max = 10; ++ } ++ ++ if (!qman_ip_rev) { ++ if (ip_rev) { ++ qman_ip_rev = ip_rev; ++ qman_ip_cfg = ip_cfg; ++ } else { ++ pr_warn("unknown Qman version," ++ " default to rev1.1\n"); ++ qman_ip_rev = QMAN_REV11; ++ qman_ip_cfg = QMAN_REV_CFG_0; ++ } ++ } else if (ip_rev && (qman_ip_rev != ip_rev)) ++ pr_warn("Revision=0x%04x, but portal '%s' has" ++ " 0x%04x\n", ++ qman_ip_rev, dn->full_name, ip_rev); ++ if (qman_ip_rev == ip_rev) ++ break; ++ } ++} ++ ++/* Parse a portal node, perform generic mapping duties and return the config. It ++ * is not known at this stage for what purpose (or even if) the portal will be ++ * used. */ ++static struct qm_portal_config * __init parse_pcfg(struct device_node *node) ++{ ++ struct qm_portal_config *pcfg; ++ const u32 *index_p, *channel_p; ++ u32 index, channel; ++ int irq, ret; ++ resource_size_t len; ++ ++ pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL); ++ if (!pcfg) { ++ pr_err("can't allocate portal config"); ++ return NULL; ++ } ++ ++ /* ++ * This is a *horrible hack*, but the IOMMU/PAMU driver needs a ++ * 'struct device' in order to get the PAMU stashing setup and the QMan ++ * portal [driver] won't function at all without ring stashing ++ * ++ * Making the QMan portal driver nice and proper is part of the ++ * upstreaming effort ++ */ ++ pcfg->dev.bus = &platform_bus_type; ++ pcfg->dev.of_node = node; ++#ifdef CONFIG_FSL_PAMU ++ pcfg->dev.archdata.iommu_domain = NULL; ++#endif ++ ++ ret = of_address_to_resource(node, DPA_PORTAL_CE, ++ &pcfg->addr_phys[DPA_PORTAL_CE]); ++ if (ret) { ++ pr_err("Can't get %s property '%s'\n", node->full_name, ++ "reg::CE"); ++ goto err; ++ } ++ ret = of_address_to_resource(node, DPA_PORTAL_CI, ++ &pcfg->addr_phys[DPA_PORTAL_CI]); ++ if (ret) { ++ pr_err("Can't get %s property '%s'\n", node->full_name, ++ "reg::CI"); ++ goto err; ++ } ++ index_p = of_get_property(node, "cell-index", &ret); ++ if (!index_p || (ret != 4)) { ++ pr_err("Can't get %s property '%s'\n", node->full_name, ++ "cell-index"); ++ goto err; ++ } ++ index = be32_to_cpu(*index_p); ++ if (index >= qman_portal_max) { ++ pr_err("QMan portal index %d is beyond max (%d)\n", ++ index, qman_portal_max); ++ goto err; ++ } ++ ++ channel_p = of_get_property(node, "fsl,qman-channel-id", &ret); ++ if (!channel_p || (ret != 4)) { ++ pr_err("Can't get %s property '%s'\n", node->full_name, ++ "fsl,qman-channel-id"); ++ goto err; ++ } ++ channel = be32_to_cpu(*channel_p); ++ if (channel != (index + QM_CHANNEL_SWPORTAL0)) ++ pr_err("Warning: node %s has mismatched %s and %s\n", ++ node->full_name, "cell-index", "fsl,qman-channel-id"); ++ pcfg->public_cfg.channel = channel; ++ pcfg->public_cfg.cpu = -1; ++ irq = irq_of_parse_and_map(node, 0); ++ if (irq == 0) { ++ pr_err("Can't get %s property '%s'\n", node->full_name, ++ "interrupts"); ++ goto err; ++ } ++ pcfg->public_cfg.irq = irq; ++ pcfg->public_cfg.index = index; ++#ifdef CONFIG_FSL_QMAN_CONFIG ++ /* We need the same LIODN offset for all portals */ ++ qman_liodn_fixup(pcfg->public_cfg.channel); ++#endif ++ ++ len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]); ++ if (len != (unsigned long)len) ++ goto err; ++ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns( ++ pcfg->addr_phys[DPA_PORTAL_CE].start, ++ resource_size(&pcfg->addr_phys[DPA_PORTAL_CE])); ++ ++ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap( ++ pcfg->addr_phys[DPA_PORTAL_CI].start, ++ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI])); ++#else ++ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot( ++ pcfg->addr_phys[DPA_PORTAL_CE].start, ++ (unsigned long)len, ++ 0); ++ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot( ++ pcfg->addr_phys[DPA_PORTAL_CI].start, ++ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]), ++ _PAGE_GUARDED | _PAGE_NO_CACHE); ++#endif ++ return pcfg; ++err: ++ kfree(pcfg); ++ return NULL; ++} ++ ++static struct qm_portal_config *get_pcfg(struct list_head *list) ++{ ++ struct qm_portal_config *pcfg; ++ if (list_empty(list)) ++ return NULL; ++ pcfg = list_entry(list->prev, struct qm_portal_config, list); ++ list_del(&pcfg->list); ++ return pcfg; ++} ++ ++static struct qm_portal_config *get_pcfg_idx(struct list_head *list, u32 idx) ++{ ++ struct qm_portal_config *pcfg; ++ if (list_empty(list)) ++ return NULL; ++ list_for_each_entry(pcfg, list, list) { ++ if (pcfg->public_cfg.index == idx) { ++ list_del(&pcfg->list); ++ return pcfg; ++ } ++ } ++ return NULL; ++} ++ ++static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) ++{ ++#ifdef CONFIG_FSL_PAMU ++ int ret; ++ int window_count = 1; ++ struct iommu_domain_geometry geom_attr; ++ struct pamu_stash_attribute stash_attr; ++ ++ pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type); ++ if (!pcfg->iommu_domain) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed", ++ __func__); ++ goto _no_iommu; ++ } ++ geom_attr.aperture_start = 0; ++ geom_attr.aperture_end = ++ ((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1; ++ geom_attr.force_aperture = true; ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY, ++ &geom_attr); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS, ++ &window_count); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ stash_attr.cpu = cpu; ++ stash_attr.cache = PAMU_ATTR_CACHE_L1; ++ /* set stash information for the window */ ++ stash_attr.window = 0; ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, ++ DOMAIN_ATTR_FSL_PAMU_STASH, ++ &stash_attr); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36, ++ IOMMU_READ | IOMMU_WRITE); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d", ++ __func__, ret); ++ goto _iommu_domain_free; ++ } ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, ++ DOMAIN_ATTR_FSL_PAMU_ENABLE, ++ &window_count); ++ if (ret < 0) { ++ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", ++ __func__, ret); ++ goto _iommu_detach_device; ++ } ++ ++_no_iommu: ++#endif ++#ifdef CONFIG_FSL_QMAN_CONFIG ++ if (qman_set_sdest(pcfg->public_cfg.channel, cpu)) ++#endif ++ pr_warn("Failed to set QMan portal's stash request queue\n"); ++ ++ return; ++ ++#ifdef CONFIG_FSL_PAMU ++_iommu_detach_device: ++ iommu_detach_device(pcfg->iommu_domain, NULL); ++_iommu_domain_free: ++ iommu_domain_free(pcfg->iommu_domain); ++#endif ++} ++ ++struct qm_portal_config *qm_get_unused_portal_idx(u32 idx) ++{ ++ struct qm_portal_config *ret; ++ spin_lock(&unused_pcfgs_lock); ++ if (idx == QBMAN_ANY_PORTAL_IDX) ++ ret = get_pcfg(&unused_pcfgs); ++ else ++ ret = get_pcfg_idx(&unused_pcfgs, idx); ++ spin_unlock(&unused_pcfgs_lock); ++ /* Bind stashing LIODNs to the CPU we are currently executing on, and ++ * set the portal to use the stashing request queue corresonding to the ++ * cpu as well. The user-space driver assumption is that the pthread has ++ * to already be affine to one cpu only before opening a portal. If that ++ * check is circumvented, the only risk is a performance degradation - ++ * stashing will go to whatever cpu they happened to be running on when ++ * opening the device file, and if that isn't the cpu they subsequently ++ * bind to and do their polling on, tough. */ ++ if (ret) ++ portal_set_cpu(ret, hard_smp_processor_id()); ++ return ret; ++} ++ ++struct qm_portal_config *qm_get_unused_portal(void) ++{ ++ return qm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX); ++} ++ ++void qm_put_unused_portal(struct qm_portal_config *pcfg) ++{ ++ spin_lock(&unused_pcfgs_lock); ++ list_add(&pcfg->list, &unused_pcfgs); ++ spin_unlock(&unused_pcfgs_lock); ++} ++ ++static struct qman_portal *init_pcfg(struct qm_portal_config *pcfg) ++{ ++ struct qman_portal *p; ++ ++ pcfg->iommu_domain = NULL; ++ portal_set_cpu(pcfg, pcfg->public_cfg.cpu); ++ p = qman_create_affine_portal(pcfg, NULL); ++ if (p) { ++ u32 irq_sources = 0; ++ /* Determine what should be interrupt-vs-poll driven */ ++#ifdef CONFIG_FSL_DPA_PIRQ_SLOW ++ irq_sources |= QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI | ++ QM_PIRQ_CSCI | QM_PIRQ_CCSCI; ++#endif ++#ifdef CONFIG_FSL_DPA_PIRQ_FAST ++ irq_sources |= QM_PIRQ_DQRI; ++#endif ++ qman_p_irqsource_add(p, irq_sources); ++ pr_info("Qman portal %sinitialised, cpu %d\n", ++ pcfg->public_cfg.is_shared ? "(shared) " : "", ++ pcfg->public_cfg.cpu); ++ } else ++ pr_crit("Qman portal failure on cpu %d\n", ++ pcfg->public_cfg.cpu); ++ return p; ++} ++ ++static void init_slave(int cpu) ++{ ++ struct qman_portal *p; ++ struct cpumask oldmask = *tsk_cpus_allowed(current); ++ set_cpus_allowed_ptr(current, get_cpu_mask(cpu)); ++ p = qman_create_affine_slave(shared_portals[shared_portals_idx++], cpu); ++ if (!p) ++ pr_err("Qman slave portal failure on cpu %d\n", cpu); ++ else ++ pr_info("Qman portal %sinitialised, cpu %d\n", "(slave) ", cpu); ++ set_cpus_allowed_ptr(current, &oldmask); ++ if (shared_portals_idx >= num_shared_portals) ++ shared_portals_idx = 0; ++} ++ ++static struct cpumask want_unshared __initdata; ++static struct cpumask want_shared __initdata; ++ ++static int __init parse_qportals(char *str) ++{ ++ return parse_portals_bootarg(str, &want_shared, &want_unshared, ++ "qportals"); ++} ++__setup("qportals=", parse_qportals); ++ ++static void qman_portal_update_sdest(const struct qm_portal_config *pcfg, ++ unsigned int cpu) ++{ ++#ifdef CONFIG_FSL_PAMU ++ struct pamu_stash_attribute stash_attr; ++ int ret; ++ ++ if (pcfg->iommu_domain) { ++ stash_attr.cpu = cpu; ++ stash_attr.cache = PAMU_ATTR_CACHE_L1; ++ /* set stash information for the window */ ++ stash_attr.window = 0; ++ ret = iommu_domain_set_attr(pcfg->iommu_domain, ++ DOMAIN_ATTR_FSL_PAMU_STASH, &stash_attr); ++ if (ret < 0) { ++ pr_err("Failed to update pamu stash setting\n"); ++ return; ++ } ++ } ++#endif ++#ifdef CONFIG_FSL_QMAN_CONFIG ++ if (qman_set_sdest(pcfg->public_cfg.channel, cpu)) ++ pr_warn("Failed to update portal's stash request queue\n"); ++#endif ++} ++ ++static void qman_offline_cpu(unsigned int cpu) ++{ ++ struct qman_portal *p; ++ const struct qm_portal_config *pcfg; ++ p = (struct qman_portal *)affine_portals[cpu]; ++ if (p) { ++ pcfg = qman_get_qm_portal_config(p); ++ if (pcfg) { ++ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0)); ++ qman_portal_update_sdest(pcfg, 0); ++ } ++ } ++} ++ ++#ifdef CONFIG_HOTPLUG_CPU ++static void qman_online_cpu(unsigned int cpu) ++{ ++ struct qman_portal *p; ++ const struct qm_portal_config *pcfg; ++ p = (struct qman_portal *)affine_portals[cpu]; ++ if (p) { ++ pcfg = qman_get_qm_portal_config(p); ++ if (pcfg) { ++ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu)); ++ qman_portal_update_sdest(pcfg, cpu); ++ } ++ } ++} ++ ++static int qman_hotplug_cpu_callback(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ unsigned int cpu = (unsigned long)hcpu; ++ ++ switch (action) { ++ case CPU_ONLINE: ++ case CPU_ONLINE_FROZEN: ++ qman_online_cpu(cpu); ++ break; ++ case CPU_DOWN_PREPARE: ++ case CPU_DOWN_PREPARE_FROZEN: ++ qman_offline_cpu(cpu); ++ default: ++ break; ++ } ++ return NOTIFY_OK; ++} ++ ++static struct notifier_block qman_hotplug_cpu_notifier = { ++ .notifier_call = qman_hotplug_cpu_callback, ++}; ++#endif /* CONFIG_HOTPLUG_CPU */ ++ ++__init int qman_init(void) ++{ ++ struct cpumask slave_cpus; ++ struct cpumask unshared_cpus = *cpu_none_mask; ++ struct cpumask shared_cpus = *cpu_none_mask; ++ LIST_HEAD(unshared_pcfgs); ++ LIST_HEAD(shared_pcfgs); ++ struct device_node *dn; ++ struct qm_portal_config *pcfg; ++ struct qman_portal *p; ++ int cpu, ret; ++ const u32 *clk; ++ struct cpumask offline_cpus; ++ ++ /* Initialise the Qman (CCSR) device */ ++ for_each_compatible_node(dn, NULL, "fsl,qman") { ++ if (!qman_init_ccsr(dn)) ++ pr_info("Qman err interrupt handler present\n"); ++ else ++ pr_err("Qman CCSR setup failed\n"); ++ ++ clk = of_get_property(dn, "clock-frequency", NULL); ++ if (!clk) ++ pr_warn("Can't find Qman clock frequency\n"); ++ else ++ qman_clk = be32_to_cpu(*clk); ++ } ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ /* Setup lookup table for FQ demux */ ++ ret = qman_setup_fq_lookup_table(get_qman_fqd_size()/64); ++ if (ret) ++ return ret; ++#endif ++ ++ /* Get qman ip revision */ ++ qman_get_ip_revision(dn); ++ if ((qman_ip_rev & 0xff00) >= QMAN_REV30) { ++ qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3; ++ qm_channel_caam = QMAN_CHANNEL_CAAM_REV3; ++ qm_channel_pme = QMAN_CHANNEL_PME_REV3; ++ } ++ ++ if ((qman_ip_rev == QMAN_REV31) && (qman_ip_cfg == QMAN_REV_CFG_2)) ++ qm_channel_dce = QMAN_CHANNEL_DCE_QMANREV312; ++ ++ /* ++ * Parse the ceetm node to get how many ceetm instances are supported ++ * on the current silicon. num_ceetms must be confirmed before portals ++ * are intiailized. ++ */ ++ num_ceetms = 0; ++ for_each_compatible_node(dn, NULL, "fsl,qman-ceetm") ++ num_ceetms++; ++ ++ /* Parse pool channels into the SDQCR mask. (Must happen before portals ++ * are initialised.) */ ++ for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") { ++ ret = fsl_pool_channel_range_sdqcr(dn); ++ if (ret) ++ return ret; ++ } ++ ++ memset(affine_portals, 0, sizeof(void *) * num_possible_cpus()); ++ /* Initialise portals. See bman_driver.c for comments */ ++ for_each_compatible_node(dn, NULL, "fsl,qman-portal") { ++ if (!of_device_is_available(dn)) ++ continue; ++ pcfg = parse_pcfg(dn); ++ if (pcfg) { ++ pcfg->public_cfg.pools = pools_sdqcr; ++ list_add_tail(&pcfg->list, &unused_pcfgs); ++ } ++ } ++ for_each_possible_cpu(cpu) { ++ if (cpumask_test_cpu(cpu, &want_shared)) { ++ pcfg = get_pcfg(&unused_pcfgs); ++ if (!pcfg) ++ break; ++ pcfg->public_cfg.cpu = cpu; ++ list_add_tail(&pcfg->list, &shared_pcfgs); ++ cpumask_set_cpu(cpu, &shared_cpus); ++ } ++ if (cpumask_test_cpu(cpu, &want_unshared)) { ++ if (cpumask_test_cpu(cpu, &shared_cpus)) ++ continue; ++ pcfg = get_pcfg(&unused_pcfgs); ++ if (!pcfg) ++ break; ++ pcfg->public_cfg.cpu = cpu; ++ list_add_tail(&pcfg->list, &unshared_pcfgs); ++ cpumask_set_cpu(cpu, &unshared_cpus); ++ } ++ } ++ if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) { ++ for_each_online_cpu(cpu) { ++ pcfg = get_pcfg(&unused_pcfgs); ++ if (!pcfg) ++ break; ++ pcfg->public_cfg.cpu = cpu; ++ list_add_tail(&pcfg->list, &unshared_pcfgs); ++ cpumask_set_cpu(cpu, &unshared_cpus); ++ } ++ } ++ cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus); ++ cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus); ++ if (cpumask_empty(&slave_cpus)) { ++ if (!list_empty(&shared_pcfgs)) { ++ cpumask_or(&unshared_cpus, &unshared_cpus, ++ &shared_cpus); ++ cpumask_clear(&shared_cpus); ++ list_splice_tail(&shared_pcfgs, &unshared_pcfgs); ++ INIT_LIST_HEAD(&shared_pcfgs); ++ } ++ } else { ++ if (list_empty(&shared_pcfgs)) { ++ pcfg = get_pcfg(&unshared_pcfgs); ++ if (!pcfg) { ++ pr_crit("No QMan portals available!\n"); ++ return 0; ++ } ++ cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus); ++ cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus); ++ list_add_tail(&pcfg->list, &shared_pcfgs); ++ } ++ } ++ list_for_each_entry(pcfg, &unshared_pcfgs, list) { ++ pcfg->public_cfg.is_shared = 0; ++ p = init_pcfg(pcfg); ++ if (!p) { ++ pr_crit("Unable to configure portals\n"); ++ return 0; ++ } ++ } ++ list_for_each_entry(pcfg, &shared_pcfgs, list) { ++ pcfg->public_cfg.is_shared = 1; ++ p = init_pcfg(pcfg); ++ if (p) ++ shared_portals[num_shared_portals++] = p; ++ } ++ if (!cpumask_empty(&slave_cpus)) ++ for_each_cpu(cpu, &slave_cpus) ++ init_slave(cpu); ++ pr_info("Qman portals initialised\n"); ++ cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask); ++ for_each_cpu(cpu, &offline_cpus) ++ qman_offline_cpu(cpu); ++#ifdef CONFIG_HOTPLUG_CPU ++ register_hotcpu_notifier(&qman_hotplug_cpu_notifier); ++#endif ++ return 0; ++} ++ ++__init int qman_resource_init(void) ++{ ++ struct device_node *dn; ++ int ret; ++ ++ /* Initialise FQID allocation ranges */ ++ for_each_compatible_node(dn, NULL, "fsl,fqid-range") { ++ ret = fsl_fqid_range_init(dn); ++ if (ret) ++ return ret; ++ } ++ /* Initialise CGRID allocation ranges */ ++ for_each_compatible_node(dn, NULL, "fsl,cgrid-range") { ++ ret = fsl_cgrid_range_init(dn); ++ if (ret) ++ return ret; ++ } ++ /* Parse pool channels into the allocator. (Must happen after portals ++ * are initialised.) */ ++ for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") { ++ ret = fsl_pool_channel_range_init(dn); ++ if (ret) ++ return ret; ++ } ++ ++ /* Parse CEETM */ ++ for_each_compatible_node(dn, NULL, "fsl,qman-ceetm") { ++ ret = fsl_ceetm_init(dn); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++#ifdef CONFIG_SUSPEND ++void suspend_unused_qportal(void) ++{ ++ struct qm_portal_config *pcfg; ++ ++ if (list_empty(&unused_pcfgs)) ++ return; ++ ++ list_for_each_entry(pcfg, &unused_pcfgs, list) { ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Need to save qportal %d\n", pcfg->public_cfg.index); ++#endif ++ /* save isdr, disable all via isdr, clear isr */ ++ pcfg->saved_isdr = ++ __raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); ++ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + ++ 0xe08); ++ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + ++ 0xe00); ++ } ++ return; ++} ++ ++void resume_unused_qportal(void) ++{ ++ struct qm_portal_config *pcfg; ++ ++ if (list_empty(&unused_pcfgs)) ++ return; ++ ++ list_for_each_entry(pcfg, &unused_pcfgs, list) { ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Need to resume qportal %d\n", pcfg->public_cfg.index); ++#endif ++ /* restore isdr */ ++ __raw_writel(pcfg->saved_isdr, ++ pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); ++ } ++ return; ++} ++#endif ++ +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_high.c +@@ -0,0 +1,5568 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "qman_low.h" ++ ++/* Compilation constants */ ++#define DQRR_MAXFILL 15 ++#define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ ++#define IRQNAME "QMan portal %d" ++#define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ ++ ++/* Divide 'n' by 'd', rounding down if 'r' is negative, rounding up if it's ++ * positive, and rounding to the closest value if it's zero. NB, this macro ++ * implicitly upgrades parameters to unsigned 64-bit, so feed it with types ++ * that are compatible with this. NB, these arguments should not be expressions ++ * unless it is safe for them to be evaluated multiple times. Eg. do not pass ++ * in "some_value++" as a parameter to the macro! */ ++#define ROUNDING(n, d, r) \ ++ (((r) < 0) ? div64_u64((n), (d)) : \ ++ (((r) > 0) ? div64_u64(((n) + (d) - 1), (d)) : \ ++ div64_u64(((n) + ((d) / 2)), (d)))) ++ ++/* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about ++ * inter-processor locking only. Note, FQLOCK() is always called either under a ++ * local_irq_save() or from interrupt context - hence there's no need for irq ++ * protection (and indeed, attempting to nest irq-protection doesn't work, as ++ * the "irq en/disable" machinery isn't recursive...). */ ++#define FQLOCK(fq) \ ++ do { \ ++ struct qman_fq *__fq478 = (fq); \ ++ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ ++ spin_lock(&__fq478->fqlock); \ ++ } while (0) ++#define FQUNLOCK(fq) \ ++ do { \ ++ struct qman_fq *__fq478 = (fq); \ ++ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ ++ spin_unlock(&__fq478->fqlock); \ ++ } while (0) ++ ++static inline void fq_set(struct qman_fq *fq, u32 mask) ++{ ++ set_bits(mask, &fq->flags); ++} ++static inline void fq_clear(struct qman_fq *fq, u32 mask) ++{ ++ clear_bits(mask, &fq->flags); ++} ++static inline int fq_isset(struct qman_fq *fq, u32 mask) ++{ ++ return fq->flags & mask; ++} ++static inline int fq_isclear(struct qman_fq *fq, u32 mask) ++{ ++ return !(fq->flags & mask); ++} ++ ++struct qman_portal { ++ struct qm_portal p; ++ unsigned long bits; /* PORTAL_BITS_*** - dynamic, strictly internal */ ++ unsigned long irq_sources; ++ u32 use_eqcr_ci_stashing; ++ u32 slowpoll; /* only used when interrupts are off */ ++ struct qman_fq *vdqcr_owned; /* only 1 volatile dequeue at a time */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ struct qman_fq *eqci_owned; /* only 1 enqueue WAIT_SYNC at a time */ ++#endif ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ raw_spinlock_t sharing_lock; /* only used if is_shared */ ++ int is_shared; ++ struct qman_portal *sharing_redirect; ++#endif ++ u32 sdqcr; ++ int dqrr_disable_ref; ++ /* A portal-specific handler for DCP ERNs. If this is NULL, the global ++ * handler is called instead. */ ++ qman_cb_dc_ern cb_dc_ern; ++ /* When the cpu-affine portal is activated, this is non-NULL */ ++ const struct qm_portal_config *config; ++ /* This is needed for providing a non-NULL device to dma_map_***() */ ++ struct platform_device *pdev; ++ struct dpa_rbtree retire_table; ++ char irqname[MAX_IRQNAME]; ++ /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ ++ struct qman_cgrs *cgrs; ++ /* linked-list of CSCN handlers. */ ++ struct list_head cgr_cbs; ++ /* list lock */ ++ spinlock_t cgr_lock; ++ /* 2-element array. ccgrs[0] is mask, ccgrs[1] is snapshot. */ ++ struct qman_ccgrs *ccgrs[QMAN_CEETM_MAX]; ++ /* 256-element array, each is a linked-list of CCSCN handlers. */ ++ struct list_head ccgr_cbs[QMAN_CEETM_MAX]; ++ /* list lock */ ++ spinlock_t ccgr_lock; ++ /* track if memory was allocated by the driver */ ++ u8 alloced; ++ /* power management data */ ++ u32 save_isdr; ++#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ++ /* Keep a shadow copy of the DQRR on LE systems ++ as the SW needs to do byteswaps of read only ++ memory. Must be aligned to the size of the ++ ring to ensure easy index calcualtions based ++ on address */ ++ struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] ++ __attribute__((aligned(512))); ++#endif ++}; ++ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++#define PORTAL_IRQ_LOCK(p, irqflags) \ ++ do { \ ++ if ((p)->is_shared) \ ++ raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \ ++ else \ ++ local_irq_save(irqflags); \ ++ } while (0) ++#define PORTAL_IRQ_UNLOCK(p, irqflags) \ ++ do { \ ++ if ((p)->is_shared) \ ++ raw_spin_unlock_irqrestore(&(p)->sharing_lock, \ ++ irqflags); \ ++ else \ ++ local_irq_restore(irqflags); \ ++ } while (0) ++#else ++#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags) ++#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags) ++#endif ++ ++/* Global handler for DCP ERNs. Used when the portal receiving the message does ++ * not have a portal-specific handler. */ ++static qman_cb_dc_ern cb_dc_ern; ++ ++static cpumask_t affine_mask; ++static DEFINE_SPINLOCK(affine_mask_lock); ++static u16 affine_channels[NR_CPUS]; ++static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal); ++void *affine_portals[NR_CPUS]; ++ ++/* "raw" gets the cpu-local struct whether it's a redirect or not. */ ++static inline struct qman_portal *get_raw_affine_portal(void) ++{ ++ return &get_cpu_var(qman_affine_portal); ++} ++/* For ops that can redirect, this obtains the portal to use */ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++static inline struct qman_portal *get_affine_portal(void) ++{ ++ struct qman_portal *p = get_raw_affine_portal(); ++ if (p->sharing_redirect) ++ return p->sharing_redirect; ++ return p; ++} ++#else ++#define get_affine_portal() get_raw_affine_portal() ++#endif ++/* For every "get", there must be a "put" */ ++static inline void put_affine_portal(void) ++{ ++ put_cpu_var(qman_affine_portal); ++} ++/* Exception: poll functions assume the caller is cpu-affine and in no risk of ++ * re-entrance, which are the two reasons we usually use the get/put_cpu_var() ++ * semantic - ie. to disable pre-emption. Some use-cases expect the execution ++ * context to remain as non-atomic during poll-triggered callbacks as it was ++ * when the poll API was first called (eg. NAPI), so we go out of our way in ++ * this case to not disable pre-emption. */ ++static inline struct qman_portal *get_poll_portal(void) ++{ ++ return &get_cpu_var(qman_affine_portal); ++} ++#define put_poll_portal() ++ ++/* This gives a FQID->FQ lookup to cover the fact that we can't directly demux ++ * retirement notifications (the fact they are sometimes h/w-consumed means that ++ * contextB isn't always a s/w demux - and as we can't know which case it is ++ * when looking at the notification, we have to use the slow lookup for all of ++ * them). NB, it's possible to have multiple FQ objects refer to the same FQID ++ * (though at most one of them should be the consumer), so this table isn't for ++ * all FQs - FQs are added when retirement commands are issued, and removed when ++ * they complete, which also massively reduces the size of this table. */ ++IMPLEMENT_DPA_RBTREE(fqtree, struct qman_fq, node, fqid); ++ ++/* This is what everything can wait on, even if it migrates to a different cpu ++ * to the one whose affine portal it is waiting on. */ ++static DECLARE_WAIT_QUEUE_HEAD(affine_queue); ++ ++static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) ++{ ++ int ret = fqtree_push(&p->retire_table, fq); ++ if (ret) ++ pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); ++ return ret; ++} ++ ++static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) ++{ ++ fqtree_del(&p->retire_table, fq); ++} ++ ++static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) ++{ ++ return fqtree_find(&p->retire_table, fqid); ++} ++ ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++static void **qman_fq_lookup_table; ++static size_t qman_fq_lookup_table_size; ++ ++int qman_setup_fq_lookup_table(size_t num_entries) ++{ ++ num_entries++; ++ /* Allocate 1 more entry since the first entry is not used */ ++ qman_fq_lookup_table = vzalloc((num_entries * sizeof(void *))); ++ if (!qman_fq_lookup_table) { ++ pr_err("QMan: Could not allocate fq lookup table\n"); ++ return -ENOMEM; ++ } ++ qman_fq_lookup_table_size = num_entries; ++ pr_info("QMan: Allocated lookup table at %p, entry count %lu\n", ++ qman_fq_lookup_table, ++ (unsigned long)qman_fq_lookup_table_size); ++ return 0; ++} ++ ++/* global structure that maintains fq object mapping */ ++static DEFINE_SPINLOCK(fq_hash_table_lock); ++ ++static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) ++{ ++ u32 i; ++ ++ spin_lock(&fq_hash_table_lock); ++ /* Can't use index zero because this has special meaning ++ * in context_b field. */ ++ for (i = 1; i < qman_fq_lookup_table_size; i++) { ++ if (qman_fq_lookup_table[i] == NULL) { ++ *entry = i; ++ qman_fq_lookup_table[i] = fq; ++ spin_unlock(&fq_hash_table_lock); ++ return 0; ++ } ++ } ++ spin_unlock(&fq_hash_table_lock); ++ return -ENOMEM; ++} ++ ++static void clear_fq_table_entry(u32 entry) ++{ ++ spin_lock(&fq_hash_table_lock); ++ BUG_ON(entry >= qman_fq_lookup_table_size); ++ qman_fq_lookup_table[entry] = NULL; ++ spin_unlock(&fq_hash_table_lock); ++} ++ ++static inline struct qman_fq *get_fq_table_entry(u32 entry) ++{ ++ BUG_ON(entry >= qman_fq_lookup_table_size); ++ return qman_fq_lookup_table[entry]; ++} ++#endif ++ ++static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) ++{ ++ /* Byteswap the FQD to HW format */ ++ fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); ++ fqd->dest_wq = cpu_to_be16(fqd->dest_wq); ++ fqd->ics_cred = cpu_to_be16(fqd->ics_cred); ++ fqd->context_b = cpu_to_be32(fqd->context_b); ++ fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); ++} ++ ++static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) ++{ ++ /* Byteswap the FQD to CPU format */ ++ fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); ++ fqd->dest_wq = be16_to_cpu(fqd->dest_wq); ++ fqd->ics_cred = be16_to_cpu(fqd->ics_cred); ++ fqd->context_b = be32_to_cpu(fqd->context_b); ++ fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); ++} ++ ++/* Swap a 40 bit address */ ++static inline u64 cpu_to_be40(u64 in) ++{ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ return in; ++#else ++ u64 out = 0; ++ u8 *p = (u8 *) &out; ++ p[0] = in >> 32; ++ p[1] = in >> 24; ++ p[2] = in >> 16; ++ p[3] = in >> 8; ++ p[4] = in >> 0; ++ return out; ++#endif ++} ++static inline u64 be40_to_cpu(u64 in) ++{ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ return in; ++#else ++ u64 out = 0; ++ u8 *pout = (u8 *) &out; ++ u8 *pin = (u8 *) ∈ ++ pout[0] = pin[4]; ++ pout[1] = pin[3]; ++ pout[2] = pin[2]; ++ pout[3] = pin[1]; ++ pout[4] = pin[0]; ++ return out; ++#endif ++} ++ ++/* Swap a 24 bit value */ ++static inline u32 cpu_to_be24(u32 in) ++{ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ return in; ++#else ++ u32 out = 0; ++ u8 *p = (u8 *) &out; ++ p[0] = in >> 16; ++ p[1] = in >> 8; ++ p[2] = in >> 0; ++ return out; ++#endif ++} ++ ++static inline u32 be24_to_cpu(u32 in) ++{ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ return in; ++#else ++ u32 out = 0; ++ u8 *pout = (u8 *) &out; ++ u8 *pin = (u8 *) ∈ ++ pout[0] = pin[2]; ++ pout[1] = pin[1]; ++ pout[2] = pin[0]; ++ return out; ++#endif ++} ++ ++static inline u64 be48_to_cpu(u64 in) ++{ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ return in; ++#else ++ u64 out = 0; ++ u8 *pout = (u8 *) &out; ++ u8 *pin = (u8 *) ∈ ++ ++ pout[0] = pin[5]; ++ pout[1] = pin[4]; ++ pout[2] = pin[3]; ++ pout[3] = pin[2]; ++ pout[4] = pin[1]; ++ pout[5] = pin[0]; ++ return out; ++#endif ++} ++static inline void cpu_to_hw_fd(struct qm_fd *fd) ++{ ++ fd->addr = cpu_to_be40(fd->addr); ++ fd->status = cpu_to_be32(fd->status); ++ fd->opaque = cpu_to_be32(fd->opaque); ++} ++ ++static inline void hw_fd_to_cpu(struct qm_fd *fd) ++{ ++ fd->addr = be40_to_cpu(fd->addr); ++ fd->status = be32_to_cpu(fd->status); ++ fd->opaque = be32_to_cpu(fd->opaque); ++} ++ ++static inline void hw_cq_query_to_cpu(struct qm_mcr_ceetm_cq_query *cq_query) ++{ ++ cq_query->ccgid = be16_to_cpu(cq_query->ccgid); ++ cq_query->state = be16_to_cpu(cq_query->state); ++ cq_query->pfdr_hptr = be24_to_cpu(cq_query->pfdr_hptr); ++ cq_query->pfdr_tptr = be24_to_cpu(cq_query->pfdr_tptr); ++ cq_query->od1_xsfdr = be16_to_cpu(cq_query->od1_xsfdr); ++ cq_query->od2_xsfdr = be16_to_cpu(cq_query->od2_xsfdr); ++ cq_query->od3_xsfdr = be16_to_cpu(cq_query->od3_xsfdr); ++ cq_query->od4_xsfdr = be16_to_cpu(cq_query->od4_xsfdr); ++ cq_query->od5_xsfdr = be16_to_cpu(cq_query->od5_xsfdr); ++ cq_query->od6_xsfdr = be16_to_cpu(cq_query->od6_xsfdr); ++ cq_query->ra1_xsfdr = be16_to_cpu(cq_query->ra1_xsfdr); ++ cq_query->ra2_xsfdr = be16_to_cpu(cq_query->ra2_xsfdr); ++ cq_query->frm_cnt = be24_to_cpu(cq_query->frm_cnt); ++} ++ ++static inline void hw_ccgr_query_to_cpu(struct qm_mcr_ceetm_ccgr_query *ccgr_q) ++{ ++ int i; ++ ++ ccgr_q->cm_query.cscn_targ_dcp = ++ be16_to_cpu(ccgr_q->cm_query.cscn_targ_dcp); ++ ccgr_q->cm_query.i_cnt = be40_to_cpu(ccgr_q->cm_query.i_cnt); ++ ccgr_q->cm_query.a_cnt = be40_to_cpu(ccgr_q->cm_query.a_cnt); ++ for (i = 0; i < ARRAY_SIZE(ccgr_q->cm_query.cscn_targ_swp); i++) ++ ccgr_q->cm_query.cscn_targ_swp[i] = ++ be32_to_cpu(ccgr_q->cm_query.cscn_targ_swp[i]); ++} ++ ++/* In the case that slow- and fast-path handling are both done by qman_poll() ++ * (ie. because there is no interrupt handling), we ought to balance how often ++ * we do the fast-path poll versus the slow-path poll. We'll use two decrementer ++ * sources, so we call the fast poll 'n' times before calling the slow poll ++ * once. The idle decrementer constant is used when the last slow-poll detected ++ * no work to do, and the busy decrementer constant when the last slow-poll had ++ * work to do. */ ++#define SLOW_POLL_IDLE 1000 ++#define SLOW_POLL_BUSY 10 ++static u32 __poll_portal_slow(struct qman_portal *p, u32 is); ++static inline unsigned int __poll_portal_fast(struct qman_portal *p, ++ unsigned int poll_limit); ++ ++/* Portal interrupt handler */ ++static irqreturn_t portal_isr(__always_unused int irq, void *ptr) ++{ ++ struct qman_portal *p = ptr; ++ /* ++ * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because ++ * it could race against a Query Congestion State command also given ++ * as part of the handling of this interrupt source. We mustn't ++ * clear it a second time in this top-level function. ++ */ ++ u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & ++ ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); ++ u32 is = qm_isr_status_read(&p->p) & p->irq_sources; ++ /* DQRR-handling if it's interrupt-driven */ ++ if (is & QM_PIRQ_DQRI) ++ __poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT); ++ /* Handling of anything else that's interrupt-driven */ ++ clear |= __poll_portal_slow(p, is); ++ qm_isr_status_clear(&p->p, clear); ++ return IRQ_HANDLED; ++} ++ ++/* This inner version is used privately by qman_create_affine_portal(), as well ++ * as by the exported qman_stop_dequeues(). */ ++static inline void qman_stop_dequeues_ex(struct qman_portal *p) ++{ ++ unsigned long irqflags __maybe_unused; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ if (!(p->dqrr_disable_ref++)) ++ qm_dqrr_set_maxfill(&p->p, 0); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++} ++ ++static int drain_mr(struct qm_portal *p) ++{ ++ const struct qm_mr_entry *msg; ++loop: ++ msg = qm_mr_current(p); ++ if (!msg) { ++ /* if MR was full and h/w had other FQRNI entries to produce, we ++ * need to allow it time to produce those entries once the ++ * existing entries are consumed. A worst-case situation ++ * (fully-loaded system) means h/w sequencers may have to do 3-4 ++ * other things before servicing the portal's MR pump, each of ++ * which (if slow) may take ~50 qman cycles (which is ~200 ++ * processor cycles). So rounding up and then multiplying this ++ * worst-case estimate by a factor of 10, just to be ++ * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume ++ * one entry at a time, so h/w has an opportunity to produce new ++ * entries well before the ring has been fully consumed, so ++ * we're being *really* paranoid here. */ ++ u64 now, then = mfatb(); ++ do { ++ now = mfatb(); ++ } while ((then + 10000) > now); ++ msg = qm_mr_current(p); ++ if (!msg) ++ return 0; ++ } ++ qm_mr_next(p); ++ qm_mr_cci_consume(p, 1); ++ goto loop; ++} ++ ++#ifdef CONFIG_SUSPEND ++static int _qman_portal_suspend_noirq(struct device *dev) ++{ ++ struct qman_portal *p = (struct qman_portal *)dev->platform_data; ++#ifdef CONFIG_PM_DEBUG ++ struct platform_device *pdev = to_platform_device(dev); ++#endif ++ ++ p->save_isdr = qm_isr_disable_read(&p->p); ++ qm_isr_disable_write(&p->p, 0xffffffff); ++ qm_isr_status_clear(&p->p, 0xffffffff); ++#ifdef CONFIG_PM_DEBUG ++ pr_info("Suspend for %s\n", pdev->name); ++#endif ++ return 0; ++} ++ ++static int _qman_portal_resume_noirq(struct device *dev) ++{ ++ struct qman_portal *p = (struct qman_portal *)dev->platform_data; ++ ++ /* restore isdr */ ++ qm_isr_disable_write(&p->p, p->save_isdr); ++ return 0; ++} ++#else ++#define _qman_portal_suspend_noirq NULL ++#define _qman_portal_resume_noirq NULL ++#endif ++ ++struct dev_pm_domain qman_portal_device_pm_domain = { ++ .ops = { ++ USE_PLATFORM_PM_SLEEP_OPS ++ .suspend_noirq = _qman_portal_suspend_noirq, ++ .resume_noirq = _qman_portal_resume_noirq, ++ } ++}; ++ ++struct qman_portal *qman_create_portal( ++ struct qman_portal *portal, ++ const struct qm_portal_config *config, ++ const struct qman_cgrs *cgrs) ++{ ++ struct qm_portal *__p; ++ char buf[16]; ++ int ret; ++ u32 isdr; ++ ++ if (!portal) { ++ portal = kmalloc(sizeof(*portal), GFP_KERNEL); ++ if (!portal) ++ return portal; ++ portal->alloced = 1; ++ } else ++ portal->alloced = 0; ++ ++ __p = &portal->p; ++ ++#ifdef CONFIG_FSL_PAMU ++ /* PAMU is required for stashing */ ++ portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? ++ 1 : 0); ++#else ++ portal->use_eqcr_ci_stashing = 0; ++#endif ++ ++ /* prep the low-level portal struct with the mapped addresses from the ++ * config, everything that follows depends on it and "config" is more ++ * for (de)reference... */ ++ __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; ++ __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; ++ /* ++ * If CI-stashing is used, the current defaults use a threshold of 3, ++ * and stash with high-than-DQRR priority. ++ */ ++ if (qm_eqcr_init(__p, qm_eqcr_pvb, ++ portal->use_eqcr_ci_stashing ? 3 : 0, 1)) { ++ pr_err("Qman EQCR initialisation failed\n"); ++ goto fail_eqcr; ++ } ++ if (qm_dqrr_init(__p, config, qm_dqrr_dpush, qm_dqrr_pvb, ++ qm_dqrr_cdc, DQRR_MAXFILL)) { ++ pr_err("Qman DQRR initialisation failed\n"); ++ goto fail_dqrr; ++ } ++ if (qm_mr_init(__p, qm_mr_pvb, qm_mr_cci)) { ++ pr_err("Qman MR initialisation failed\n"); ++ goto fail_mr; ++ } ++ if (qm_mc_init(__p)) { ++ pr_err("Qman MC initialisation failed\n"); ++ goto fail_mc; ++ } ++ if (qm_isr_init(__p)) { ++ pr_err("Qman ISR initialisation failed\n"); ++ goto fail_isr; ++ } ++ /* static interrupt-gating controls */ ++ qm_dqrr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_DQRR_ITHRESH); ++ qm_mr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_MR_ITHRESH); ++ qm_isr_set_iperiod(__p, CONFIG_FSL_QMAN_PIRQ_IPERIOD); ++ portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); ++ if (!portal->cgrs) ++ goto fail_cgrs; ++ /* initial snapshot is no-depletion */ ++ qman_cgrs_init(&portal->cgrs[1]); ++ if (cgrs) ++ portal->cgrs[0] = *cgrs; ++ else ++ /* if the given mask is NULL, assume all CGRs can be seen */ ++ qman_cgrs_fill(&portal->cgrs[0]); ++ INIT_LIST_HEAD(&portal->cgr_cbs); ++ spin_lock_init(&portal->cgr_lock); ++ if (num_ceetms) { ++ for (ret = 0; ret < num_ceetms; ret++) { ++ portal->ccgrs[ret] = kmalloc(2 * ++ sizeof(struct qman_ccgrs), GFP_KERNEL); ++ if (!portal->ccgrs[ret]) ++ goto fail_ccgrs; ++ qman_ccgrs_init(&portal->ccgrs[ret][1]); ++ qman_ccgrs_fill(&portal->ccgrs[ret][0]); ++ INIT_LIST_HEAD(&portal->ccgr_cbs[ret]); ++ } ++ } ++ spin_lock_init(&portal->ccgr_lock); ++ portal->bits = 0; ++ portal->slowpoll = 0; ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ portal->eqci_owned = NULL; ++#endif ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ raw_spin_lock_init(&portal->sharing_lock); ++ portal->is_shared = config->public_cfg.is_shared; ++ portal->sharing_redirect = NULL; ++#endif ++ portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | ++ QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | ++ QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; ++ portal->dqrr_disable_ref = 0; ++ portal->cb_dc_ern = NULL; ++ sprintf(buf, "qportal-%d", config->public_cfg.channel); ++ portal->pdev = platform_device_alloc(buf, -1); ++ if (!portal->pdev) { ++ pr_err("qman_portal - platform_device_alloc() failed\n"); ++ goto fail_devalloc; ++ } ++#ifdef CONFIG_ARM ++ portal->pdev->dev.coherent_dma_mask = DMA_BIT_MASK(40); ++ portal->pdev->dev.dma_mask = &portal->pdev->dev.coherent_dma_mask; ++#else ++ if (dma_set_mask(&portal->pdev->dev, DMA_BIT_MASK(40))) { ++ pr_err("qman_portal - dma_set_mask() failed\n"); ++ goto fail_devadd; ++ } ++#endif ++ portal->pdev->dev.pm_domain = &qman_portal_device_pm_domain; ++ portal->pdev->dev.platform_data = portal; ++ ret = platform_device_add(portal->pdev); ++ if (ret) { ++ pr_err("qman_portal - platform_device_add() failed\n"); ++ goto fail_devadd; ++ } ++ dpa_rbtree_init(&portal->retire_table); ++ isdr = 0xffffffff; ++ qm_isr_disable_write(__p, isdr); ++ portal->irq_sources = 0; ++ qm_isr_enable_write(__p, portal->irq_sources); ++ qm_isr_status_clear(__p, 0xffffffff); ++ snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu); ++ if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname, ++ portal)) { ++ pr_err("request_irq() failed\n"); ++ goto fail_irq; ++ } ++ if ((config->public_cfg.cpu != -1) && ++ irq_can_set_affinity(config->public_cfg.irq) && ++ irq_set_affinity(config->public_cfg.irq, ++ cpumask_of(config->public_cfg.cpu))) { ++ pr_err("irq_set_affinity() failed\n"); ++ goto fail_affinity; ++ } ++ ++ /* Need EQCR to be empty before continuing */ ++ isdr ^= QM_PIRQ_EQCI; ++ qm_isr_disable_write(__p, isdr); ++ ret = qm_eqcr_get_fill(__p); ++ if (ret) { ++ pr_err("Qman EQCR unclean\n"); ++ goto fail_eqcr_empty; ++ } ++ isdr ^= (QM_PIRQ_DQRI | QM_PIRQ_MRI); ++ qm_isr_disable_write(__p, isdr); ++ while (qm_dqrr_current(__p) != NULL) ++ qm_dqrr_cdc_consume_n(__p, 0xffff); ++ drain_mr(__p); ++ /* Success */ ++ portal->config = config; ++ qm_isr_disable_write(__p, 0); ++ qm_isr_uninhibit(__p); ++ /* Write a sane SDQCR */ ++ qm_dqrr_sdqcr_set(__p, portal->sdqcr); ++ return portal; ++fail_eqcr_empty: ++fail_affinity: ++ free_irq(config->public_cfg.irq, portal); ++fail_irq: ++ platform_device_del(portal->pdev); ++fail_devadd: ++ platform_device_put(portal->pdev); ++fail_devalloc: ++ if (num_ceetms) ++ for (ret = 0; ret < num_ceetms; ret++) ++ kfree(portal->ccgrs[ret]); ++fail_ccgrs: ++ kfree(portal->cgrs); ++fail_cgrs: ++ qm_isr_finish(__p); ++fail_isr: ++ qm_mc_finish(__p); ++fail_mc: ++ qm_mr_finish(__p); ++fail_mr: ++ qm_dqrr_finish(__p); ++fail_dqrr: ++ qm_eqcr_finish(__p); ++fail_eqcr: ++ if (portal->alloced) ++ kfree(portal); ++ return NULL; ++} ++ ++struct qman_portal *qman_create_affine_portal( ++ const struct qm_portal_config *config, ++ const struct qman_cgrs *cgrs) ++{ ++ struct qman_portal *res; ++ struct qman_portal *portal; ++ ++ portal = &per_cpu(qman_affine_portal, config->public_cfg.cpu); ++ res = qman_create_portal(portal, config, cgrs); ++ if (res) { ++ spin_lock(&affine_mask_lock); ++ cpumask_set_cpu(config->public_cfg.cpu, &affine_mask); ++ affine_channels[config->public_cfg.cpu] = ++ config->public_cfg.channel; ++ affine_portals[config->public_cfg.cpu] = portal; ++ spin_unlock(&affine_mask_lock); ++ } ++ return res; ++} ++ ++/* These checks are BUG_ON()s because the driver is already supposed to avoid ++ * these cases. */ ++struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect, ++ int cpu) ++{ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ struct qman_portal *p; ++ p = &per_cpu(qman_affine_portal, cpu); ++ /* Check that we don't already have our own portal */ ++ BUG_ON(p->config); ++ /* Check that we aren't already slaving to another portal */ ++ BUG_ON(p->is_shared); ++ /* Check that 'redirect' is prepared to have us */ ++ BUG_ON(!redirect->config->public_cfg.is_shared); ++ /* These are the only elements to initialise when redirecting */ ++ p->irq_sources = 0; ++ p->sharing_redirect = redirect; ++ affine_portals[cpu] = p; ++ return p; ++#else ++ BUG(); ++ return NULL; ++#endif ++} ++ ++void qman_destroy_portal(struct qman_portal *qm) ++{ ++ const struct qm_portal_config *pcfg; ++ int i; ++ ++ /* Stop dequeues on the portal */ ++ qm_dqrr_sdqcr_set(&qm->p, 0); ++ ++ /* NB we do this to "quiesce" EQCR. If we add enqueue-completions or ++ * something related to QM_PIRQ_EQCI, this may need fixing. ++ * Also, due to the prefetching model used for CI updates in the enqueue ++ * path, this update will only invalidate the CI cacheline *after* ++ * working on it, so we need to call this twice to ensure a full update ++ * irrespective of where the enqueue processing was at when the teardown ++ * began. */ ++ qm_eqcr_cce_update(&qm->p); ++ qm_eqcr_cce_update(&qm->p); ++ pcfg = qm->config; ++ ++ free_irq(pcfg->public_cfg.irq, qm); ++ ++ kfree(qm->cgrs); ++ if (num_ceetms) ++ for (i = 0; i < num_ceetms; i++) ++ kfree(qm->ccgrs[i]); ++ qm_isr_finish(&qm->p); ++ qm_mc_finish(&qm->p); ++ qm_mr_finish(&qm->p); ++ qm_dqrr_finish(&qm->p); ++ qm_eqcr_finish(&qm->p); ++ ++ platform_device_del(qm->pdev); ++ platform_device_put(qm->pdev); ++ ++ qm->config = NULL; ++ if (qm->alloced) ++ kfree(qm); ++} ++ ++const struct qm_portal_config *qman_destroy_affine_portal(void) ++{ ++ /* We don't want to redirect if we're a slave, use "raw" */ ++ struct qman_portal *qm = get_raw_affine_portal(); ++ const struct qm_portal_config *pcfg; ++ int cpu; ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (qm->sharing_redirect) { ++ qm->sharing_redirect = NULL; ++ put_affine_portal(); ++ return NULL; ++ } ++ qm->is_shared = 0; ++#endif ++ pcfg = qm->config; ++ cpu = pcfg->public_cfg.cpu; ++ ++ qman_destroy_portal(qm); ++ ++ spin_lock(&affine_mask_lock); ++ cpumask_clear_cpu(cpu, &affine_mask); ++ spin_unlock(&affine_mask_lock); ++ put_affine_portal(); ++ return pcfg; ++} ++ ++const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal *p) ++{ ++ return &p->config->public_cfg; ++} ++EXPORT_SYMBOL(qman_p_get_portal_config); ++ ++const struct qman_portal_config *qman_get_portal_config(void) ++{ ++ struct qman_portal *p = get_affine_portal(); ++ const struct qman_portal_config *ret = qman_p_get_portal_config(p); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_get_portal_config); ++ ++/* Inline helper to reduce nesting in __poll_portal_slow() */ ++static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_mr_entry *msg, u8 verb) ++{ ++ FQLOCK(fq); ++ switch (verb) { ++ case QM_MR_VERB_FQRL: ++ DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); ++ fq_clear(fq, QMAN_FQ_STATE_ORL); ++ table_del_fq(p, fq); ++ break; ++ case QM_MR_VERB_FQRN: ++ DPA_ASSERT((fq->state == qman_fq_state_parked) || ++ (fq->state == qman_fq_state_sched)); ++ DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); ++ fq_clear(fq, QMAN_FQ_STATE_CHANGING); ++ if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) ++ fq_set(fq, QMAN_FQ_STATE_NE); ++ if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) ++ fq_set(fq, QMAN_FQ_STATE_ORL); ++ else ++ table_del_fq(p, fq); ++ fq->state = qman_fq_state_retired; ++ break; ++ case QM_MR_VERB_FQPN: ++ DPA_ASSERT(fq->state == qman_fq_state_sched); ++ DPA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); ++ fq->state = qman_fq_state_parked; ++ } ++ FQUNLOCK(fq); ++} ++ ++static u32 __poll_portal_slow(struct qman_portal *p, u32 is) ++{ ++ const struct qm_mr_entry *msg; ++ ++ if (is & QM_PIRQ_CSCI) { ++ struct qman_cgrs rr, c; ++ struct qm_mc_result *mcr; ++ struct qman_cgr *cgr; ++ unsigned long irqflags __maybe_unused; ++ ++ spin_lock_irqsave(&p->cgr_lock, irqflags); ++ /* ++ * The CSCI bit must be cleared _before_ issuing the ++ * Query Congestion State command, to ensure that a long ++ * CGR State Change callback cannot miss an intervening ++ * state change. ++ */ ++ qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); ++ qm_mc_start(&p->p); ++ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ /* mask out the ones I'm not interested in */ ++ qman_cgrs_and(&rr, (const struct qman_cgrs *) ++ &mcr->querycongestion.state, &p->cgrs[0]); ++ /* check previous snapshot for delta, enter/exit congestion */ ++ qman_cgrs_xor(&c, &rr, &p->cgrs[1]); ++ /* update snapshot */ ++ qman_cgrs_cp(&p->cgrs[1], &rr); ++ /* Invoke callback */ ++ list_for_each_entry(cgr, &p->cgr_cbs, node) ++ if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) ++ cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); ++ spin_unlock_irqrestore(&p->cgr_lock, irqflags); ++ } ++ if (is & QM_PIRQ_CCSCI) { ++ struct qman_ccgrs rr, c, congestion_result; ++ struct qm_mc_result *mcr; ++ struct qm_mc_command *mcc; ++ struct qm_ceetm_ccg *ccg; ++ unsigned long irqflags __maybe_unused; ++ int i, j, k; ++ ++ spin_lock_irqsave(&p->ccgr_lock, irqflags); ++ /* ++ * The CCSCI bit must be cleared _before_ issuing the ++ * Query Congestion State command, to ensure that a long ++ * CCGR State Change callback cannot miss an intervening ++ * state change. ++ */ ++ qm_isr_status_clear(&p->p, QM_PIRQ_CCSCI); ++ ++ for (i = 0; i < num_ceetms; i++) { ++ for (j = 0; j < 2; j++) { ++ mcc = qm_mc_start(&p->p); ++ mcc->ccgr_query.ccgrid = cpu_to_be16( ++ CEETM_QUERY_CONGESTION_STATE | j); ++ mcc->ccgr_query.dcpid = i; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ for (k = 0; k < 8; k++) ++ mcr->ccgr_query.congestion_state.state. ++ __state[k] = be32_to_cpu( ++ mcr->ccgr_query. ++ congestion_state.state. ++ __state[k]); ++ congestion_result.q[j] = ++ mcr->ccgr_query.congestion_state.state; ++ } ++ /* mask out the ones I'm not interested in */ ++ qman_ccgrs_and(&rr, &congestion_result, ++ &p->ccgrs[i][0]); ++ /* ++ * check previous snapshot for delta, enter/exit ++ * congestion. ++ */ ++ qman_ccgrs_xor(&c, &rr, &p->ccgrs[i][1]); ++ /* update snapshot */ ++ qman_ccgrs_cp(&p->ccgrs[i][1], &rr); ++ /* Invoke callback */ ++ list_for_each_entry(ccg, &p->ccgr_cbs[i], cb_node) ++ if (ccg->cb && qman_ccgrs_get(&c, ++ (ccg->parent->idx << 4) | ccg->idx)) ++ ccg->cb(ccg, ccg->cb_ctx, ++ qman_ccgrs_get(&rr, ++ (ccg->parent->idx << 4) ++ | ccg->idx)); ++ } ++ spin_unlock_irqrestore(&p->ccgr_lock, irqflags); ++ } ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (is & QM_PIRQ_EQCI) { ++ unsigned long irqflags; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ p->eqci_owned = NULL; ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ wake_up(&affine_queue); ++ } ++#endif ++ ++ if (is & QM_PIRQ_EQRI) { ++ unsigned long irqflags __maybe_unused; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ qm_eqcr_cce_update(&p->p); ++ qm_eqcr_set_ithresh(&p->p, 0); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ wake_up(&affine_queue); ++ } ++ ++ if (is & QM_PIRQ_MRI) { ++ struct qman_fq *fq; ++ u8 verb, num = 0; ++mr_loop: ++ qm_mr_pvb_update(&p->p); ++ msg = qm_mr_current(&p->p); ++ if (!msg) ++ goto mr_done; ++ verb = msg->verb & QM_MR_VERB_TYPE_MASK; ++ /* The message is a software ERN iff the 0x20 bit is set */ ++ if (verb & 0x20) { ++ switch (verb) { ++ case QM_MR_VERB_FQRNI: ++ /* nada, we drop FQRNIs on the floor */ ++ break; ++ case QM_MR_VERB_FQRN: ++ case QM_MR_VERB_FQRL: ++ /* Lookup in the retirement table */ ++ fq = table_find_fq(p, be32_to_cpu(msg->fq.fqid)); ++ BUG_ON(!fq); ++ fq_state_change(p, fq, msg, verb); ++ if (fq->cb.fqs) ++ fq->cb.fqs(p, fq, msg); ++ break; ++ case QM_MR_VERB_FQPN: ++ /* Parked */ ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ fq = get_fq_table_entry( ++ be32_to_cpu(msg->fq.contextB)); ++#else ++ fq = (void *)(uintptr_t) ++ be32_to_cpu(msg->fq.contextB); ++#endif ++ fq_state_change(p, fq, msg, verb); ++ if (fq->cb.fqs) ++ fq->cb.fqs(p, fq, msg); ++ break; ++ case QM_MR_VERB_DC_ERN: ++ /* DCP ERN */ ++ if (p->cb_dc_ern) ++ p->cb_dc_ern(p, msg); ++ else if (cb_dc_ern) ++ cb_dc_ern(p, msg); ++ else { ++ static int warn_once; ++ if (!warn_once) { ++ pr_crit("Leaking DCP ERNs!\n"); ++ warn_once = 1; ++ } ++ } ++ break; ++ default: ++ pr_crit("Invalid MR verb 0x%02x\n", verb); ++ } ++ } else { ++ /* Its a software ERN */ ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ pr_info("ROY\n"); ++ fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); ++#else ++ fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); ++#endif ++ fq->cb.ern(p, fq, msg); ++ } ++ num++; ++ qm_mr_next(&p->p); ++ goto mr_loop; ++mr_done: ++ qm_mr_cci_consume(&p->p, num); ++ } ++ /* ++ * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific ++ * processing. If that interrupt source has meanwhile been re-asserted, ++ * we mustn't clear it here (or in the top-level interrupt handler). ++ */ ++ return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); ++} ++ ++/* remove some slowish-path stuff from the "fast path" and make sure it isn't ++ * inlined. */ ++static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) ++{ ++ p->vdqcr_owned = NULL; ++ FQLOCK(fq); ++ fq_clear(fq, QMAN_FQ_STATE_VDQCR); ++ FQUNLOCK(fq); ++ wake_up(&affine_queue); ++} ++ ++/* Look: no locks, no irq_save()s, no preempt_disable()s! :-) The only states ++ * that would conflict with other things if they ran at the same time on the ++ * same cpu are; ++ * ++ * (i) setting/clearing vdqcr_owned, and ++ * (ii) clearing the NE (Not Empty) flag. ++ * ++ * Both are safe. Because; ++ * ++ * (i) this clearing can only occur after qman_volatile_dequeue() has set the ++ * vdqcr_owned field (which it does before setting VDQCR), and ++ * qman_volatile_dequeue() blocks interrupts and preemption while this is ++ * done so that we can't interfere. ++ * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as ++ * with (i) that API prevents us from interfering until it's safe. ++ * ++ * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far ++ * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett ++ * advantage comes from this function not having to "lock" anything at all. ++ * ++ * Note also that the callbacks are invoked at points which are safe against the ++ * above potential conflicts, but that this function itself is not re-entrant ++ * (this is because the function tracks one end of each FIFO in the portal and ++ * we do *not* want to lock that). So the consequence is that it is safe for ++ * user callbacks to call into any Qman API *except* qman_poll() (as that's the ++ * sole API that could be invoking the callback through this function). ++ */ ++static inline unsigned int __poll_portal_fast(struct qman_portal *p, ++ unsigned int poll_limit) ++{ ++ const struct qm_dqrr_entry *dq; ++ struct qman_fq *fq; ++ enum qman_cb_dqrr_result res; ++ unsigned int limit = 0; ++#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ++ struct qm_dqrr_entry *shadow; ++#endif ++loop: ++ qm_dqrr_pvb_update(&p->p); ++ dq = qm_dqrr_current(&p->p); ++ if (!dq) ++ goto done; ++#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ ++ /* If running on an LE system the fields of the ++ dequeue entry must be swapped. Because the ++ QMan HW will ignore writes the DQRR entry is ++ copied and the index stored within the copy */ ++ shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; ++ *shadow = *dq; ++ dq = shadow; ++ shadow->fqid = be32_to_cpu(shadow->fqid); ++ shadow->contextB = be32_to_cpu(shadow->contextB); ++ shadow->seqnum = be16_to_cpu(shadow->seqnum); ++ hw_fd_to_cpu(&shadow->fd); ++#endif ++ if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { ++ /* VDQCR: don't trust contextB as the FQ may have been ++ * configured for h/w consumption and we're draining it ++ * post-retirement. */ ++ fq = p->vdqcr_owned; ++ /* We only set QMAN_FQ_STATE_NE when retiring, so we only need ++ * to check for clearing it when doing volatile dequeues. It's ++ * one less thing to check in the critical path (SDQCR). */ ++ if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) ++ fq_clear(fq, QMAN_FQ_STATE_NE); ++ /* this is duplicated from the SDQCR code, but we have stuff to ++ * do before *and* after this callback, and we don't want ++ * multiple if()s in the critical path (SDQCR). */ ++ res = fq->cb.dqrr(p, fq, dq); ++ if (res == qman_cb_dqrr_stop) ++ goto done; ++ /* Check for VDQCR completion */ ++ if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) ++ clear_vdqcr(p, fq); ++ } else { ++ /* SDQCR: contextB points to the FQ */ ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ fq = get_fq_table_entry(dq->contextB); ++#else ++ fq = (void *)(uintptr_t)dq->contextB; ++#endif ++ /* Now let the callback do its stuff */ ++ res = fq->cb.dqrr(p, fq, dq); ++ ++ /* The callback can request that we exit without consuming this ++ * entry nor advancing; */ ++ if (res == qman_cb_dqrr_stop) ++ goto done; ++ } ++ /* Interpret 'dq' from a driver perspective. */ ++ /* Parking isn't possible unless HELDACTIVE was set. NB, ++ * FORCEELIGIBLE implies HELDACTIVE, so we only need to ++ * check for HELDACTIVE to cover both. */ ++ DPA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || ++ (res != qman_cb_dqrr_park)); ++ /* Defer just means "skip it, I'll consume it myself later on" */ ++ if (res != qman_cb_dqrr_defer) ++ qm_dqrr_cdc_consume_1ptr(&p->p, dq, (res == qman_cb_dqrr_park)); ++ /* Move forward */ ++ qm_dqrr_next(&p->p); ++ /* Entry processed and consumed, increment our counter. The callback can ++ * request that we exit after consuming the entry, and we also exit if ++ * we reach our processing limit, so loop back only if neither of these ++ * conditions is met. */ ++ if ((++limit < poll_limit) && (res != qman_cb_dqrr_consume_stop)) ++ goto loop; ++done: ++ return limit; ++} ++ ++u32 qman_irqsource_get(void) ++{ ++ /* "irqsource" and "poll" APIs mustn't redirect when sharing, they ++ * should shut the user out if they are not the primary CPU hosting the ++ * portal. That's why we use the "raw" interface. */ ++ struct qman_portal *p = get_raw_affine_portal(); ++ u32 ret = p->irq_sources & QM_PIRQ_VISIBLE; ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_irqsource_get); ++ ++int qman_p_irqsource_add(struct qman_portal *p, u32 bits __maybe_unused) ++{ ++ __maybe_unused unsigned long irqflags; ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (p->sharing_redirect) ++ return -EINVAL; ++ else ++#endif ++ { ++ PORTAL_IRQ_LOCK(p, irqflags); ++ set_bits(bits & QM_PIRQ_VISIBLE, &p->irq_sources); ++ qm_isr_enable_write(&p->p, p->irq_sources); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_p_irqsource_add); ++ ++int qman_irqsource_add(u32 bits __maybe_unused) ++{ ++ struct qman_portal *p = get_raw_affine_portal(); ++ int ret; ++ ret = qman_p_irqsource_add(p, bits); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_irqsource_add); ++ ++int qman_p_irqsource_remove(struct qman_portal *p, u32 bits) ++{ ++ __maybe_unused unsigned long irqflags; ++ u32 ier; ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (p->sharing_redirect) { ++ put_affine_portal(); ++ return -EINVAL; ++ } ++#endif ++ /* Our interrupt handler only processes+clears status register bits that ++ * are in p->irq_sources. As we're trimming that mask, if one of them ++ * were to assert in the status register just before we remove it from ++ * the enable register, there would be an interrupt-storm when we ++ * release the IRQ lock. So we wait for the enable register update to ++ * take effect in h/w (by reading it back) and then clear all other bits ++ * in the status register. Ie. we clear them from ISR once it's certain ++ * IER won't allow them to reassert. */ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ bits &= QM_PIRQ_VISIBLE; ++ clear_bits(bits, &p->irq_sources); ++ qm_isr_enable_write(&p->p, p->irq_sources); ++ ++ ier = qm_isr_enable_read(&p->p); ++ /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a ++ * data-dependency, ie. to protect against re-ordering. */ ++ qm_isr_status_clear(&p->p, ~ier); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ return 0; ++} ++EXPORT_SYMBOL(qman_p_irqsource_remove); ++ ++int qman_irqsource_remove(u32 bits) ++{ ++ struct qman_portal *p = get_raw_affine_portal(); ++ int ret; ++ ret = qman_p_irqsource_remove(p, bits); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_irqsource_remove); ++ ++const cpumask_t *qman_affine_cpus(void) ++{ ++ return &affine_mask; ++} ++EXPORT_SYMBOL(qman_affine_cpus); ++ ++u16 qman_affine_channel(int cpu) ++{ ++ if (cpu < 0) { ++ struct qman_portal *portal = get_raw_affine_portal(); ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ BUG_ON(portal->sharing_redirect); ++#endif ++ cpu = portal->config->public_cfg.cpu; ++ put_affine_portal(); ++ } ++ BUG_ON(!cpumask_test_cpu(cpu, &affine_mask)); ++ return affine_channels[cpu]; ++} ++EXPORT_SYMBOL(qman_affine_channel); ++ ++void *qman_get_affine_portal(int cpu) ++{ ++ return affine_portals[cpu]; ++} ++EXPORT_SYMBOL(qman_get_affine_portal); ++ ++int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit) ++{ ++ int ret; ++ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (unlikely(p->sharing_redirect)) ++ ret = -EINVAL; ++ else ++#endif ++ { ++ BUG_ON(p->irq_sources & QM_PIRQ_DQRI); ++ ret = __poll_portal_fast(p, limit); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(qman_p_poll_dqrr); ++ ++int qman_poll_dqrr(unsigned int limit) ++{ ++ struct qman_portal *p = get_poll_portal(); ++ int ret; ++ ret = qman_p_poll_dqrr(p, limit); ++ put_poll_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_poll_dqrr); ++ ++u32 qman_p_poll_slow(struct qman_portal *p) ++{ ++ u32 ret; ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (unlikely(p->sharing_redirect)) ++ ret = (u32)-1; ++ else ++#endif ++ { ++ u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; ++ ret = __poll_portal_slow(p, is); ++ qm_isr_status_clear(&p->p, ret); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(qman_p_poll_slow); ++ ++u32 qman_poll_slow(void) ++{ ++ struct qman_portal *p = get_poll_portal(); ++ u32 ret; ++ ret = qman_p_poll_slow(p); ++ put_poll_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_poll_slow); ++ ++/* Legacy wrapper */ ++void qman_p_poll(struct qman_portal *p) ++{ ++#ifdef CONFIG_FSL_DPA_PORTAL_SHARE ++ if (unlikely(p->sharing_redirect)) ++ return; ++#endif ++ if ((~p->irq_sources) & QM_PIRQ_SLOW) { ++ if (!(p->slowpoll--)) { ++ u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; ++ u32 active = __poll_portal_slow(p, is); ++ if (active) { ++ qm_isr_status_clear(&p->p, active); ++ p->slowpoll = SLOW_POLL_BUSY; ++ } else ++ p->slowpoll = SLOW_POLL_IDLE; ++ } ++ } ++ if ((~p->irq_sources) & QM_PIRQ_DQRI) ++ __poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT); ++} ++EXPORT_SYMBOL(qman_p_poll); ++ ++void qman_poll(void) ++{ ++ struct qman_portal *p = get_poll_portal(); ++ qman_p_poll(p); ++ put_poll_portal(); ++} ++EXPORT_SYMBOL(qman_poll); ++ ++void qman_p_stop_dequeues(struct qman_portal *p) ++{ ++ qman_stop_dequeues_ex(p); ++} ++EXPORT_SYMBOL(qman_p_stop_dequeues); ++ ++void qman_stop_dequeues(void) ++{ ++ struct qman_portal *p = get_affine_portal(); ++ qman_p_stop_dequeues(p); ++ put_affine_portal(); ++} ++EXPORT_SYMBOL(qman_stop_dequeues); ++ ++void qman_p_start_dequeues(struct qman_portal *p) ++{ ++ unsigned long irqflags __maybe_unused; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ DPA_ASSERT(p->dqrr_disable_ref > 0); ++ if (!(--p->dqrr_disable_ref)) ++ qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++} ++EXPORT_SYMBOL(qman_p_start_dequeues); ++ ++void qman_start_dequeues(void) ++{ ++ struct qman_portal *p = get_affine_portal(); ++ qman_p_start_dequeues(p); ++ put_affine_portal(); ++} ++EXPORT_SYMBOL(qman_start_dequeues); ++ ++void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools) ++{ ++ unsigned long irqflags __maybe_unused; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ pools &= p->config->public_cfg.pools; ++ p->sdqcr |= pools; ++ qm_dqrr_sdqcr_set(&p->p, p->sdqcr); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++} ++EXPORT_SYMBOL(qman_p_static_dequeue_add); ++ ++void qman_static_dequeue_add(u32 pools) ++{ ++ struct qman_portal *p = get_affine_portal(); ++ qman_p_static_dequeue_add(p, pools); ++ put_affine_portal(); ++} ++EXPORT_SYMBOL(qman_static_dequeue_add); ++ ++void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools) ++{ ++ unsigned long irqflags __maybe_unused; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ pools &= p->config->public_cfg.pools; ++ p->sdqcr &= ~pools; ++ qm_dqrr_sdqcr_set(&p->p, p->sdqcr); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++} ++EXPORT_SYMBOL(qman_p_static_dequeue_del); ++ ++void qman_static_dequeue_del(u32 pools) ++{ ++ struct qman_portal *p = get_affine_portal(); ++ qman_p_static_dequeue_del(p, pools); ++ put_affine_portal(); ++} ++EXPORT_SYMBOL(qman_static_dequeue_del); ++ ++u32 qman_p_static_dequeue_get(struct qman_portal *p) ++{ ++ return p->sdqcr; ++} ++EXPORT_SYMBOL(qman_p_static_dequeue_get); ++ ++u32 qman_static_dequeue_get(void) ++{ ++ struct qman_portal *p = get_affine_portal(); ++ u32 ret = qman_p_static_dequeue_get(p); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_static_dequeue_get); ++ ++void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq, ++ int park_request) ++{ ++ qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); ++} ++EXPORT_SYMBOL(qman_p_dca); ++ ++void qman_dca(struct qm_dqrr_entry *dq, int park_request) ++{ ++ struct qman_portal *p = get_affine_portal(); ++ qman_p_dca(p, dq, park_request); ++ put_affine_portal(); ++} ++EXPORT_SYMBOL(qman_dca); ++ ++/*******************/ ++/* Frame queue API */ ++/*******************/ ++ ++static const char *mcr_result_str(u8 result) ++{ ++ switch (result) { ++ case QM_MCR_RESULT_NULL: ++ return "QM_MCR_RESULT_NULL"; ++ case QM_MCR_RESULT_OK: ++ return "QM_MCR_RESULT_OK"; ++ case QM_MCR_RESULT_ERR_FQID: ++ return "QM_MCR_RESULT_ERR_FQID"; ++ case QM_MCR_RESULT_ERR_FQSTATE: ++ return "QM_MCR_RESULT_ERR_FQSTATE"; ++ case QM_MCR_RESULT_ERR_NOTEMPTY: ++ return "QM_MCR_RESULT_ERR_NOTEMPTY"; ++ case QM_MCR_RESULT_PENDING: ++ return "QM_MCR_RESULT_PENDING"; ++ case QM_MCR_RESULT_ERR_BADCOMMAND: ++ return "QM_MCR_RESULT_ERR_BADCOMMAND"; ++ } ++ return "<unknown MCR result>"; ++} ++ ++int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) ++{ ++ struct qm_fqd fqd; ++ struct qm_mcr_queryfq_np np; ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ ++ if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { ++ int ret = qman_alloc_fqid(&fqid); ++ if (ret) ++ return ret; ++ } ++ spin_lock_init(&fq->fqlock); ++ fq->fqid = fqid; ++ fq->flags = flags; ++ fq->state = qman_fq_state_oos; ++ fq->cgr_groupid = 0; ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) ++ return -ENOMEM; ++#endif ++ if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) ++ return 0; ++ /* Everything else is AS_IS support */ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ mcc = qm_mc_start(&p->p); ++ mcc->queryfq.fqid = cpu_to_be32(fqid); ++ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); ++ if (mcr->result != QM_MCR_RESULT_OK) { ++ pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); ++ goto err; ++ } ++ fqd = mcr->queryfq.fqd; ++ hw_fqd_to_cpu(&fqd); ++ mcc = qm_mc_start(&p->p); ++ mcc->queryfq_np.fqid = cpu_to_be32(fqid); ++ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); ++ if (mcr->result != QM_MCR_RESULT_OK) { ++ pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); ++ goto err; ++ } ++ np = mcr->queryfq_np; ++ /* Phew, have queryfq and queryfq_np results, stitch together ++ * the FQ object from those. */ ++ fq->cgr_groupid = fqd.cgid; ++ switch (np.state & QM_MCR_NP_STATE_MASK) { ++ case QM_MCR_NP_STATE_OOS: ++ break; ++ case QM_MCR_NP_STATE_RETIRED: ++ fq->state = qman_fq_state_retired; ++ if (np.frm_cnt) ++ fq_set(fq, QMAN_FQ_STATE_NE); ++ break; ++ case QM_MCR_NP_STATE_TEN_SCHED: ++ case QM_MCR_NP_STATE_TRU_SCHED: ++ case QM_MCR_NP_STATE_ACTIVE: ++ fq->state = qman_fq_state_sched; ++ if (np.state & QM_MCR_NP_STATE_R) ++ fq_set(fq, QMAN_FQ_STATE_CHANGING); ++ break; ++ case QM_MCR_NP_STATE_PARKED: ++ fq->state = qman_fq_state_parked; ++ break; ++ default: ++ DPA_ASSERT(NULL == "invalid FQ state"); ++ } ++ if (fqd.fq_ctrl & QM_FQCTRL_CGE) ++ fq->state |= QMAN_FQ_STATE_CGR_EN; ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return 0; ++err: ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) ++ qman_release_fqid(fqid); ++ return -EIO; ++} ++EXPORT_SYMBOL(qman_create_fq); ++ ++void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) ++{ ++ ++ /* We don't need to lock the FQ as it is a pre-condition that the FQ be ++ * quiesced. Instead, run some checks. */ ++ switch (fq->state) { ++ case qman_fq_state_parked: ++ DPA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); ++ case qman_fq_state_oos: ++ if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) ++ qman_release_fqid(fq->fqid); ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ clear_fq_table_entry(fq->key); ++#endif ++ return; ++ default: ++ break; ++ } ++ DPA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); ++} ++EXPORT_SYMBOL(qman_destroy_fq); ++ ++u32 qman_fq_fqid(struct qman_fq *fq) ++{ ++ return fq->fqid; ++} ++EXPORT_SYMBOL(qman_fq_fqid); ++ ++void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) ++{ ++ if (state) ++ *state = fq->state; ++ if (flags) ++ *flags = fq->flags; ++} ++EXPORT_SYMBOL(qman_fq_state); ++ ++int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? ++ QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; ++ ++ if ((fq->state != qman_fq_state_oos) && ++ (fq->state != qman_fq_state_parked)) ++ return -EINVAL; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) ++ return -EINVAL; ++#endif ++ if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { ++ /* And can't be set at the same time as TDTHRESH */ ++ if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) ++ return -EINVAL; ++ } ++ /* Issue an INITFQ_[PARKED|SCHED] management command */ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ FQLOCK(fq); ++ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || ++ ((fq->state != qman_fq_state_oos) && ++ (fq->state != qman_fq_state_parked)))) { ++ FQUNLOCK(fq); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return -EBUSY; ++ } ++ mcc = qm_mc_start(&p->p); ++ if (opts) ++ mcc->initfq = *opts; ++ mcc->initfq.fqid = cpu_to_be32(fq->fqid); ++ mcc->initfq.count = 0; ++ ++ /* If the FQ does *not* have the TO_DCPORTAL flag, contextB is set as a ++ * demux pointer. Otherwise, the caller-provided value is allowed to ++ * stand, don't overwrite it. */ ++ if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { ++ dma_addr_t phys_fq; ++ mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ mcc->initfq.fqd.context_b = fq->key; ++#else ++ mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; ++#endif ++ /* and the physical address - NB, if the user wasn't trying to ++ * set CONTEXTA, clear the stashing settings. */ ++ if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { ++ mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; ++ memset(&mcc->initfq.fqd.context_a, 0, ++ sizeof(mcc->initfq.fqd.context_a)); ++ } else { ++ phys_fq = dma_map_single(&p->pdev->dev, fq, sizeof(*fq), ++ DMA_TO_DEVICE); ++ qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); ++ } ++ } ++ if (flags & QMAN_INITFQ_FLAG_LOCAL) { ++ mcc->initfq.fqd.dest.channel = p->config->public_cfg.channel; ++ if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { ++ mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; ++ mcc->initfq.fqd.dest.wq = 4; ++ } ++ } ++ mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); ++ cpu_to_hw_fqd(&mcc->initfq.fqd); ++ qm_mc_commit(&p->p, myverb); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ FQUNLOCK(fq); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return -EIO; ++ } ++ if (opts) { ++ if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { ++ if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) ++ fq_set(fq, QMAN_FQ_STATE_CGR_EN); ++ else ++ fq_clear(fq, QMAN_FQ_STATE_CGR_EN); ++ } ++ if (opts->we_mask & QM_INITFQ_WE_CGID) ++ fq->cgr_groupid = opts->fqd.cgid; ++ } ++ fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? ++ qman_fq_state_sched : qman_fq_state_parked; ++ FQUNLOCK(fq); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return 0; ++} ++EXPORT_SYMBOL(qman_init_fq); ++ ++int qman_schedule_fq(struct qman_fq *fq) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ int ret = 0; ++ u8 res; ++ ++ if (fq->state != qman_fq_state_parked) ++ return -EINVAL; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) ++ return -EINVAL; ++#endif ++ /* Issue a ALTERFQ_SCHED management command */ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ FQLOCK(fq); ++ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || ++ (fq->state != qman_fq_state_parked))) { ++ ret = -EBUSY; ++ goto out; ++ } ++ mcc = qm_mc_start(&p->p); ++ mcc->alterfq.fqid = cpu_to_be32(fq->fqid); ++ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ ret = -EIO; ++ goto out; ++ } ++ fq->state = qman_fq_state_sched; ++out: ++ FQUNLOCK(fq); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_schedule_fq); ++ ++int qman_retire_fq(struct qman_fq *fq, u32 *flags) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ int rval; ++ u8 res; ++ ++ if ((fq->state != qman_fq_state_parked) && ++ (fq->state != qman_fq_state_sched)) ++ return -EINVAL; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) ++ return -EINVAL; ++#endif ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ FQLOCK(fq); ++ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || ++ (fq->state == qman_fq_state_retired) || ++ (fq->state == qman_fq_state_oos))) { ++ rval = -EBUSY; ++ goto out; ++ } ++ rval = table_push_fq(p, fq); ++ if (rval) ++ goto out; ++ mcc = qm_mc_start(&p->p); ++ mcc->alterfq.fqid = cpu_to_be32(fq->fqid); ++ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); ++ res = mcr->result; ++ /* "Elegant" would be to treat OK/PENDING the same way; set CHANGING, ++ * and defer the flags until FQRNI or FQRN (respectively) show up. But ++ * "Friendly" is to process OK immediately, and not set CHANGING. We do ++ * friendly, otherwise the caller doesn't necessarily have a fully ++ * "retired" FQ on return even if the retirement was immediate. However ++ * this does mean some code duplication between here and ++ * fq_state_change(). */ ++ if (likely(res == QM_MCR_RESULT_OK)) { ++ rval = 0; ++ /* Process 'fq' right away, we'll ignore FQRNI */ ++ if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) ++ fq_set(fq, QMAN_FQ_STATE_NE); ++ if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) ++ fq_set(fq, QMAN_FQ_STATE_ORL); ++ else ++ table_del_fq(p, fq); ++ if (flags) ++ *flags = fq->flags; ++ fq->state = qman_fq_state_retired; ++ if (fq->cb.fqs) { ++ /* Another issue with supporting "immediate" retirement ++ * is that we're forced to drop FQRNIs, because by the ++ * time they're seen it may already be "too late" (the ++ * fq may have been OOS'd and free()'d already). But if ++ * the upper layer wants a callback whether it's ++ * immediate or not, we have to fake a "MR" entry to ++ * look like an FQRNI... */ ++ struct qm_mr_entry msg; ++ msg.verb = QM_MR_VERB_FQRNI; ++ msg.fq.fqs = mcr->alterfq.fqs; ++ msg.fq.fqid = fq->fqid; ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ msg.fq.contextB = fq->key; ++#else ++ msg.fq.contextB = (u32)(uintptr_t)fq; ++#endif ++ fq->cb.fqs(p, fq, &msg); ++ } ++ } else if (res == QM_MCR_RESULT_PENDING) { ++ rval = 1; ++ fq_set(fq, QMAN_FQ_STATE_CHANGING); ++ } else { ++ rval = -EIO; ++ table_del_fq(p, fq); ++ } ++out: ++ FQUNLOCK(fq); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return rval; ++} ++EXPORT_SYMBOL(qman_retire_fq); ++ ++int qman_oos_fq(struct qman_fq *fq) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ int ret = 0; ++ u8 res; ++ ++ if (fq->state != qman_fq_state_retired) ++ return -EINVAL; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) ++ return -EINVAL; ++#endif ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ FQLOCK(fq); ++ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || ++ (fq->state != qman_fq_state_retired))) { ++ ret = -EBUSY; ++ goto out; ++ } ++ mcc = qm_mc_start(&p->p); ++ mcc->alterfq.fqid = cpu_to_be32(fq->fqid); ++ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ ret = -EIO; ++ goto out; ++ } ++ fq->state = qman_fq_state_oos; ++out: ++ FQUNLOCK(fq); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_oos_fq); ++ ++int qman_fq_flow_control(struct qman_fq *fq, int xon) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ int ret = 0; ++ u8 res; ++ u8 myverb; ++ ++ if ((fq->state == qman_fq_state_oos) || ++ (fq->state == qman_fq_state_retired) || ++ (fq->state == qman_fq_state_parked)) ++ return -EINVAL; ++ ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) ++ return -EINVAL; ++#endif ++ /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ FQLOCK(fq); ++ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || ++ (fq->state == qman_fq_state_parked) || ++ (fq->state == qman_fq_state_oos) || ++ (fq->state == qman_fq_state_retired))) { ++ ret = -EBUSY; ++ goto out; ++ } ++ mcc = qm_mc_start(&p->p); ++ mcc->alterfq.fqid = fq->fqid; ++ mcc->alterfq.count = 0; ++ myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; ++ ++ qm_mc_commit(&p->p, myverb); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); ++ ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ ret = -EIO; ++ goto out; ++ } ++out: ++ FQUNLOCK(fq); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_fq_flow_control); ++ ++int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ mcc = qm_mc_start(&p->p); ++ mcc->queryfq.fqid = cpu_to_be32(fq->fqid); ++ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) ++ memcpy_fromio(fqd, &mcr->queryfq.fqd, sizeof(*fqd)); ++ hw_fqd_to_cpu(fqd); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res != QM_MCR_RESULT_OK) ++ return -EIO; ++ return 0; ++} ++EXPORT_SYMBOL(qman_query_fq); ++ ++int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ mcc = qm_mc_start(&p->p); ++ mcc->queryfq.fqid = cpu_to_be32(fq->fqid); ++ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) { ++ memcpy_fromio(np, &mcr->queryfq_np, sizeof(*np)); ++ np->fqd_link = be24_to_cpu(np->fqd_link); ++ np->odp_seq = be16_to_cpu(np->odp_seq); ++ np->orp_nesn = be16_to_cpu(np->orp_nesn); ++ np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); ++ np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); ++ np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); ++ np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); ++ np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); ++ np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); ++ np->ics_surp = be16_to_cpu(np->ics_surp); ++ np->byte_cnt = be32_to_cpu(np->byte_cnt); ++ np->frm_cnt = be24_to_cpu(np->frm_cnt); ++ np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); ++ np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); ++ np->od1_sfdr = be16_to_cpu(np->od1_sfdr); ++ np->od2_sfdr = be16_to_cpu(np->od2_sfdr); ++ np->od3_sfdr = be16_to_cpu(np->od3_sfdr); ++ ++ ++ } ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res == QM_MCR_RESULT_ERR_FQID) ++ return -ERANGE; ++ else if (res != QM_MCR_RESULT_OK) ++ return -EIO; ++ return 0; ++} ++EXPORT_SYMBOL(qman_query_fq_np); ++ ++int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res, myverb; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : ++ QM_MCR_VERB_QUERYWQ; ++ mcc = qm_mc_start(&p->p); ++ mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); ++ qm_mc_commit(&p->p, myverb); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) { ++ int i, array_len; ++ wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); ++ array_len = ARRAY_SIZE(mcr->querywq.wq_len); ++ for (i = 0; i < array_len; i++) ++ wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); ++ } ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); ++ return -EIO; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_query_wq); ++ ++int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, ++ struct qm_mcr_cgrtestwrite *result) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ mcc = qm_mc_start(&p->p); ++ mcc->cgrtestwrite.cgid = cgr->cgrid; ++ mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); ++ mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; ++ qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) ++ memcpy_fromio(result, &mcr->cgrtestwrite, sizeof(*result)); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); ++ return -EIO; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_testwrite_cgr); ++ ++int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ int i; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ mcc = qm_mc_start(&p->p); ++ mcc->querycgr.cgid = cgr->cgrid; ++ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) ++ memcpy_fromio(cgrd, &mcr->querycgr, sizeof(*cgrd)); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); ++ return -EIO; ++ } ++ cgrd->cgr.wr_parm_g.word = ++ be32_to_cpu(cgrd->cgr.wr_parm_g.word); ++ cgrd->cgr.wr_parm_y.word = ++ be32_to_cpu(cgrd->cgr.wr_parm_y.word); ++ cgrd->cgr.wr_parm_r.word = ++ be32_to_cpu(cgrd->cgr.wr_parm_r.word); ++ cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); ++ cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); ++ for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) ++ be32_to_cpus(&cgrd->cscn_targ_swp[i]); ++ return 0; ++} ++EXPORT_SYMBOL(qman_query_cgr); ++ ++int qman_query_congestion(struct qm_mcr_querycongestion *congestion) ++{ ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ int i; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ qm_mc_start(&p->p); ++ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_MCC_VERB_QUERYCONGESTION); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) ++ memcpy_fromio(congestion, &mcr->querycongestion, ++ sizeof(*congestion)); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); ++ return -EIO; ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(congestion->state.__state); i++) ++ be32_to_cpus(&congestion->state.__state[i]); ++ return 0; ++} ++EXPORT_SYMBOL(qman_query_congestion); ++ ++/* internal function used as a wait_event() expression */ ++static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr) ++{ ++ unsigned long irqflags __maybe_unused; ++ int ret = -EBUSY; ++ PORTAL_IRQ_LOCK(p, irqflags); ++ if (!p->vdqcr_owned) { ++ FQLOCK(fq); ++ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) ++ goto escape; ++ fq_set(fq, QMAN_FQ_STATE_VDQCR); ++ FQUNLOCK(fq); ++ p->vdqcr_owned = fq; ++ ret = 0; ++ } ++escape: ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ if (!ret) ++ qm_dqrr_vdqcr_set(&p->p, vdqcr); ++ return ret; ++} ++ ++static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr) ++{ ++ int ret; ++ *p = get_affine_portal(); ++ ret = set_p_vdqcr(*p, fq, vdqcr); ++ put_affine_portal(); ++ return ret; ++} ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++static int wait_p_vdqcr_start(struct qman_portal *p, struct qman_fq *fq, ++ u32 vdqcr, u32 flags) ++{ ++ int ret = 0; ++ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) ++ ret = wait_event_interruptible(affine_queue, ++ !(ret = set_p_vdqcr(p, fq, vdqcr))); ++ else ++ wait_event(affine_queue, !(ret = set_p_vdqcr(p, fq, vdqcr))); ++ return ret; ++} ++ ++static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq, ++ u32 vdqcr, u32 flags) ++{ ++ int ret = 0; ++ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) ++ ret = wait_event_interruptible(affine_queue, ++ !(ret = set_vdqcr(p, fq, vdqcr))); ++ else ++ wait_event(affine_queue, !(ret = set_vdqcr(p, fq, vdqcr))); ++ return ret; ++} ++#endif ++ ++int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq, ++ u32 flags __maybe_unused, u32 vdqcr) ++{ ++ int ret; ++ ++ if ((fq->state != qman_fq_state_parked) && ++ (fq->state != qman_fq_state_retired)) ++ return -EINVAL; ++ if (vdqcr & QM_VDQCR_FQID_MASK) ++ return -EINVAL; ++ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) ++ return -EBUSY; ++ vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_VOLATILE_FLAG_WAIT) ++ ret = wait_p_vdqcr_start(p, fq, vdqcr, flags); ++ else ++#endif ++ ret = set_p_vdqcr(p, fq, vdqcr); ++ if (ret) ++ return ret; ++ /* VDQCR is set */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_VOLATILE_FLAG_FINISH) { ++ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) ++ /* NB: don't propagate any error - the caller wouldn't ++ * know whether the VDQCR was issued or not. A signal ++ * could arrive after returning anyway, so the caller ++ * can check signal_pending() if that's an issue. */ ++ wait_event_interruptible(affine_queue, ++ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); ++ else ++ wait_event(affine_queue, ++ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_p_volatile_dequeue); ++ ++int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, ++ u32 vdqcr) ++{ ++ struct qman_portal *p; ++ int ret; ++ ++ if ((fq->state != qman_fq_state_parked) && ++ (fq->state != qman_fq_state_retired)) ++ return -EINVAL; ++ if (vdqcr & QM_VDQCR_FQID_MASK) ++ return -EINVAL; ++ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) ++ return -EBUSY; ++ vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_VOLATILE_FLAG_WAIT) ++ ret = wait_vdqcr_start(&p, fq, vdqcr, flags); ++ else ++#endif ++ ret = set_vdqcr(&p, fq, vdqcr); ++ if (ret) ++ return ret; ++ /* VDQCR is set */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_VOLATILE_FLAG_FINISH) { ++ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) ++ /* NB: don't propagate any error - the caller wouldn't ++ * know whether the VDQCR was issued or not. A signal ++ * could arrive after returning anyway, so the caller ++ * can check signal_pending() if that's an issue. */ ++ wait_event_interruptible(affine_queue, ++ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); ++ else ++ wait_event(affine_queue, ++ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_volatile_dequeue); ++ ++static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) ++{ ++ if (avail) ++ qm_eqcr_cce_prefetch(&p->p); ++ else ++ qm_eqcr_cce_update(&p->p); ++} ++ ++int qman_eqcr_is_empty(void) ++{ ++ unsigned long irqflags __maybe_unused; ++ struct qman_portal *p = get_affine_portal(); ++ u8 avail; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ update_eqcr_ci(p, 0); ++ avail = qm_eqcr_get_fill(&p->p); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return avail == 0; ++} ++EXPORT_SYMBOL(qman_eqcr_is_empty); ++ ++void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) ++{ ++ if (affine) { ++ unsigned long irqflags __maybe_unused; ++ struct qman_portal *p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ p->cb_dc_ern = handler; ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ } else ++ cb_dc_ern = handler; ++} ++EXPORT_SYMBOL(qman_set_dc_ern); ++ ++static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, ++ unsigned long *irqflags __maybe_unused, ++ struct qman_fq *fq, ++ const struct qm_fd *fd, ++ u32 flags) ++{ ++ struct qm_eqcr_entry *eq; ++ u8 avail; ++ PORTAL_IRQ_LOCK(p, (*irqflags)); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { ++ if (p->eqci_owned) { ++ PORTAL_IRQ_UNLOCK(p, (*irqflags)); ++ return NULL; ++ } ++ p->eqci_owned = fq; ++ } ++#endif ++ if (p->use_eqcr_ci_stashing) { ++ /* ++ * The stashing case is easy, only update if we need to in ++ * order to try and liberate ring entries. ++ */ ++ eq = qm_eqcr_start_stash(&p->p); ++ } else { ++ /* ++ * The non-stashing case is harder, need to prefetch ahead of ++ * time. ++ */ ++ avail = qm_eqcr_get_avail(&p->p); ++ if (avail < 2) ++ update_eqcr_ci(p, avail); ++ eq = qm_eqcr_start_no_stash(&p->p); ++ } ++ ++ if (unlikely(!eq)) { ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) ++ p->eqci_owned = NULL; ++#endif ++ PORTAL_IRQ_UNLOCK(p, (*irqflags)); ++ return NULL; ++ } ++ if (flags & QMAN_ENQUEUE_FLAG_DCA) ++ eq->dca = QM_EQCR_DCA_ENABLE | ++ ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? ++ QM_EQCR_DCA_PARK : 0) | ++ ((flags >> 8) & QM_EQCR_DCA_IDXMASK); ++ eq->fqid = cpu_to_be32(fq->fqid); ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ eq->tag = cpu_to_be32(fq->key); ++#else ++ eq->tag = cpu_to_be32((u32)(uintptr_t)fq); ++#endif ++ eq->fd = *fd; ++ cpu_to_hw_fd(&eq->fd); ++ return eq; ++} ++ ++static inline struct qm_eqcr_entry *try_eq_start(struct qman_portal **p, ++ unsigned long *irqflags __maybe_unused, ++ struct qman_fq *fq, ++ const struct qm_fd *fd, ++ u32 flags) ++{ ++ struct qm_eqcr_entry *eq; ++ *p = get_affine_portal(); ++ eq = try_p_eq_start(*p, irqflags, fq, fd, flags); ++ if (!eq) ++ put_affine_portal(); ++ return eq; ++} ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++static noinline struct qm_eqcr_entry *__wait_eq_start(struct qman_portal **p, ++ unsigned long *irqflags __maybe_unused, ++ struct qman_fq *fq, ++ const struct qm_fd *fd, ++ u32 flags) ++{ ++ struct qm_eqcr_entry *eq = try_eq_start(p, irqflags, fq, fd, flags); ++ if (!eq) ++ qm_eqcr_set_ithresh(&(*p)->p, EQCR_ITHRESH); ++ return eq; ++} ++static noinline struct qm_eqcr_entry *wait_eq_start(struct qman_portal **p, ++ unsigned long *irqflags __maybe_unused, ++ struct qman_fq *fq, ++ const struct qm_fd *fd, ++ u32 flags) ++{ ++ struct qm_eqcr_entry *eq; ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return NULL if signal occurs before completion. Signal ++ * can occur during return. Caller must check for signal */ ++ wait_event_interruptible(affine_queue, ++ (eq = __wait_eq_start(p, irqflags, fq, fd, flags))); ++ else ++ wait_event(affine_queue, ++ (eq = __wait_eq_start(p, irqflags, fq, fd, flags))); ++ return eq; ++} ++static noinline struct qm_eqcr_entry *__wait_p_eq_start(struct qman_portal *p, ++ unsigned long *irqflags __maybe_unused, ++ struct qman_fq *fq, ++ const struct qm_fd *fd, ++ u32 flags) ++{ ++ struct qm_eqcr_entry *eq = try_p_eq_start(p, irqflags, fq, fd, flags); ++ if (!eq) ++ qm_eqcr_set_ithresh(&p->p, EQCR_ITHRESH); ++ return eq; ++} ++static noinline struct qm_eqcr_entry *wait_p_eq_start(struct qman_portal *p, ++ unsigned long *irqflags __maybe_unused, ++ struct qman_fq *fq, ++ const struct qm_fd *fd, ++ u32 flags) ++{ ++ struct qm_eqcr_entry *eq; ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return NULL if signal occurs before completion. Signal ++ * can occur during return. Caller must check for signal */ ++ wait_event_interruptible(affine_queue, ++ (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags))); ++ else ++ wait_event(affine_queue, ++ (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags))); ++ return eq; ++} ++#endif ++ ++int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_fd *fd, u32 flags) ++{ ++ struct qm_eqcr_entry *eq; ++ unsigned long irqflags __maybe_unused; ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT) ++ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); ++ else ++#endif ++ eq = try_p_eq_start(p, &irqflags, fq, fd, flags); ++ if (!eq) ++ return -EBUSY; ++ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ ++ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | ++ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); ++ /* Factor the below out, it's used from qman_enqueue_orp() too */ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return success even if signal occurs before ++ * condition is true. pvb_commit guarantees success */ ++ wait_event_interruptible(affine_queue, ++ (p->eqci_owned != fq)); ++ else ++ wait_event(affine_queue, (p->eqci_owned != fq)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_p_enqueue); ++ ++int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) ++{ ++ struct qman_portal *p; ++ struct qm_eqcr_entry *eq; ++ unsigned long irqflags __maybe_unused; ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT) ++ eq = wait_eq_start(&p, &irqflags, fq, fd, flags); ++ else ++#endif ++ eq = try_eq_start(&p, &irqflags, fq, fd, flags); ++ if (!eq) ++ return -EBUSY; ++ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ ++ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | ++ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); ++ /* Factor the below out, it's used from qman_enqueue_orp() too */ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return success even if signal occurs before ++ * condition is true. pvb_commit guarantees success */ ++ wait_event_interruptible(affine_queue, ++ (p->eqci_owned != fq)); ++ else ++ wait_event(affine_queue, (p->eqci_owned != fq)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_enqueue); ++ ++int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_fd *fd, u32 flags, ++ struct qman_fq *orp, u16 orp_seqnum) ++{ ++ struct qm_eqcr_entry *eq; ++ unsigned long irqflags __maybe_unused; ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT) ++ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); ++ else ++#endif ++ eq = try_p_eq_start(p, &irqflags, fq, fd, flags); ++ if (!eq) ++ return -EBUSY; ++ /* Process ORP-specifics here */ ++ if (flags & QMAN_ENQUEUE_FLAG_NLIS) ++ orp_seqnum |= QM_EQCR_SEQNUM_NLIS; ++ else { ++ orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; ++ if (flags & QMAN_ENQUEUE_FLAG_NESN) ++ orp_seqnum |= QM_EQCR_SEQNUM_NESN; ++ else ++ /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ ++ orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; ++ } ++ eq->seqnum = cpu_to_be16(orp_seqnum); ++ eq->orp = cpu_to_be32(orp->fqid); ++ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ ++ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | ++ ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? ++ 0 : QM_EQCR_VERB_CMD_ENQUEUE) | ++ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return success even if signal occurs before ++ * condition is true. pvb_commit guarantees success */ ++ wait_event_interruptible(affine_queue, ++ (p->eqci_owned != fq)); ++ else ++ wait_event(affine_queue, (p->eqci_owned != fq)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_p_enqueue_orp); ++ ++int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, ++ struct qman_fq *orp, u16 orp_seqnum) ++{ ++ struct qman_portal *p; ++ struct qm_eqcr_entry *eq; ++ unsigned long irqflags __maybe_unused; ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT) ++ eq = wait_eq_start(&p, &irqflags, fq, fd, flags); ++ else ++#endif ++ eq = try_eq_start(&p, &irqflags, fq, fd, flags); ++ if (!eq) ++ return -EBUSY; ++ /* Process ORP-specifics here */ ++ if (flags & QMAN_ENQUEUE_FLAG_NLIS) ++ orp_seqnum |= QM_EQCR_SEQNUM_NLIS; ++ else { ++ orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; ++ if (flags & QMAN_ENQUEUE_FLAG_NESN) ++ orp_seqnum |= QM_EQCR_SEQNUM_NESN; ++ else ++ /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ ++ orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; ++ } ++ eq->seqnum = cpu_to_be16(orp_seqnum); ++ eq->orp = cpu_to_be32(orp->fqid); ++ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ ++ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | ++ ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? ++ 0 : QM_EQCR_VERB_CMD_ENQUEUE) | ++ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return success even if signal occurs before ++ * condition is true. pvb_commit guarantees success */ ++ wait_event_interruptible(affine_queue, ++ (p->eqci_owned != fq)); ++ else ++ wait_event(affine_queue, (p->eqci_owned != fq)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_enqueue_orp); ++ ++int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_fd *fd, u32 flags, ++ qman_cb_precommit cb, void *cb_arg) ++{ ++ struct qm_eqcr_entry *eq; ++ unsigned long irqflags __maybe_unused; ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT) ++ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); ++ else ++#endif ++ eq = try_p_eq_start(p, &irqflags, fq, fd, flags); ++ if (!eq) ++ return -EBUSY; ++ /* invoke user supplied callback function before writing commit verb */ ++ if (cb(cb_arg)) { ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ return -EINVAL; ++ } ++ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ ++ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | ++ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); ++ /* Factor the below out, it's used from qman_enqueue_orp() too */ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return success even if signal occurs before ++ * condition is true. pvb_commit guarantees success */ ++ wait_event_interruptible(affine_queue, ++ (p->eqci_owned != fq)); ++ else ++ wait_event(affine_queue, (p->eqci_owned != fq)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_p_enqueue_precommit); ++ ++int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd, ++ u32 flags, qman_cb_precommit cb, void *cb_arg) ++{ ++ struct qman_portal *p; ++ struct qm_eqcr_entry *eq; ++ unsigned long irqflags __maybe_unused; ++ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT) ++ eq = wait_eq_start(&p, &irqflags, fq, fd, flags); ++ else ++#endif ++ eq = try_eq_start(&p, &irqflags, fq, fd, flags); ++ if (!eq) ++ return -EBUSY; ++ /* invoke user supplied callback function before writing commit verb */ ++ if (cb(cb_arg)) { ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return -EINVAL; ++ } ++ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ ++ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | ++ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); ++ /* Factor the below out, it's used from qman_enqueue_orp() too */ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && ++ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { ++ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) ++ /* NB: return success even if signal occurs before ++ * condition is true. pvb_commit guarantees success */ ++ wait_event_interruptible(affine_queue, ++ (p->eqci_owned != fq)); ++ else ++ wait_event(affine_queue, (p->eqci_owned != fq)); ++ } ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_enqueue_precommit); ++ ++int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, ++ struct qm_mcc_initcgr *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ u8 verb = QM_MCC_VERB_MODIFYCGR; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ mcc = qm_mc_start(&p->p); ++ if (opts) ++ mcc->initcgr = *opts; ++ mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); ++ mcc->initcgr.cgr.wr_parm_g.word = ++ cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); ++ mcc->initcgr.cgr.wr_parm_y.word = ++ cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); ++ mcc->initcgr.cgr.wr_parm_r.word = ++ cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); ++ mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); ++ mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); ++ ++ mcc->initcgr.cgid = cgr->cgrid; ++ if (flags & QMAN_CGR_FLAG_USE_INIT) ++ verb = QM_MCC_VERB_INITCGR; ++ qm_mc_commit(&p->p, verb); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); ++ res = mcr->result; ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; ++} ++EXPORT_SYMBOL(qman_modify_cgr); ++ ++#define TARG_MASK(n) (0x80000000 >> (n->config->public_cfg.channel - \ ++ QM_CHANNEL_SWPORTAL0)) ++#define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) ++#define PORTAL_IDX(n) (n->config->public_cfg.channel - QM_CHANNEL_SWPORTAL0) ++ ++static u8 qman_cgr_cpus[__CGR_NUM]; ++ ++int qman_create_cgr(struct qman_cgr *cgr, u32 flags, ++ struct qm_mcc_initcgr *opts) ++{ ++ unsigned long irqflags __maybe_unused; ++ struct qm_mcr_querycgr cgr_state; ++ struct qm_mcc_initcgr local_opts; ++ int ret; ++ struct qman_portal *p; ++ ++ /* We have to check that the provided CGRID is within the limits of the ++ * data-structures, for obvious reasons. However we'll let h/w take ++ * care of determining whether it's within the limits of what exists on ++ * the SoC. */ ++ if (cgr->cgrid >= __CGR_NUM) ++ return -EINVAL; ++ ++ preempt_disable(); ++ p = get_affine_portal(); ++ qman_cgr_cpus[cgr->cgrid] = smp_processor_id(); ++ preempt_enable(); ++ ++ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); ++ cgr->chan = p->config->public_cfg.channel; ++ spin_lock_irqsave(&p->cgr_lock, irqflags); ++ ++ /* if no opts specified, just add it to the list */ ++ if (!opts) ++ goto add_list; ++ ++ ret = qman_query_cgr(cgr, &cgr_state); ++ if (ret) ++ goto release_lock; ++ if (opts) ++ local_opts = *opts; ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) ++ local_opts.cgr.cscn_targ_upd_ctrl = ++ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); ++ else ++ /* Overwrite TARG */ ++ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | ++ TARG_MASK(p); ++ local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; ++ ++ /* send init if flags indicate so */ ++ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) ++ ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); ++ else ++ ret = qman_modify_cgr(cgr, 0, &local_opts); ++ if (ret) ++ goto release_lock; ++add_list: ++ list_add(&cgr->node, &p->cgr_cbs); ++ ++ /* Determine if newly added object requires its callback to be called */ ++ ret = qman_query_cgr(cgr, &cgr_state); ++ if (ret) { ++ /* we can't go back, so proceed and return success, but screen ++ * and wail to the log file */ ++ pr_crit("CGR HW state partially modified\n"); ++ ret = 0; ++ goto release_lock; ++ } ++ if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], ++ cgr->cgrid)) ++ cgr->cb(p, cgr, 1); ++release_lock: ++ spin_unlock_irqrestore(&p->cgr_lock, irqflags); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_create_cgr); ++ ++int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, ++ struct qm_mcc_initcgr *opts) ++{ ++ unsigned long irqflags __maybe_unused; ++ struct qm_mcc_initcgr local_opts; ++ struct qm_mcr_querycgr cgr_state; ++ int ret; ++ ++ /* We have to check that the provided CGRID is within the limits of the ++ * data-structures, for obvious reasons. However we'll let h/w take ++ * care of determining whether it's within the limits of what exists on ++ * the SoC. ++ */ ++ if (cgr->cgrid >= __CGR_NUM) ++ return -EINVAL; ++ ++ ret = qman_query_cgr(cgr, &cgr_state); ++ if (ret) ++ return ret; ++ ++ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); ++ if (opts) ++ local_opts = *opts; ++ ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) ++ local_opts.cgr.cscn_targ_upd_ctrl = ++ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | ++ QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; ++ else ++ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | ++ TARG_DCP_MASK(dcp_portal); ++ local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; ++ ++ /* send init if flags indicate so */ ++ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) ++ ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, ++ &local_opts); ++ else ++ ret = qman_modify_cgr(cgr, 0, &local_opts); ++ ++ return ret; ++} ++EXPORT_SYMBOL(qman_create_cgr_to_dcp); ++ ++int qman_delete_cgr(struct qman_cgr *cgr) ++{ ++ unsigned long irqflags __maybe_unused; ++ struct qm_mcr_querycgr cgr_state; ++ struct qm_mcc_initcgr local_opts; ++ int ret = 0; ++ struct qman_cgr *i; ++ struct qman_portal *p = get_affine_portal(); ++ ++ if (cgr->chan != p->config->public_cfg.channel) { ++ pr_crit("Attempting to delete cgr from different portal " ++ "than it was create: create 0x%x, delete 0x%x\n", ++ cgr->chan, p->config->public_cfg.channel); ++ ret = -EINVAL; ++ goto put_portal; ++ } ++ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); ++ spin_lock_irqsave(&p->cgr_lock, irqflags); ++ list_del(&cgr->node); ++ /* ++ * If there are no other CGR objects for this CGRID in the list, update ++ * CSCN_TARG accordingly ++ */ ++ list_for_each_entry(i, &p->cgr_cbs, node) ++ if ((i->cgrid == cgr->cgrid) && i->cb) ++ goto release_lock; ++ ret = qman_query_cgr(cgr, &cgr_state); ++ if (ret) { ++ /* add back to the list */ ++ list_add(&cgr->node, &p->cgr_cbs); ++ goto release_lock; ++ } ++ /* Overwrite TARG */ ++ local_opts.we_mask = QM_CGR_WE_CSCN_TARG; ++ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) ++ local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); ++ else ++ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & ++ ~(TARG_MASK(p)); ++ ret = qman_modify_cgr(cgr, 0, &local_opts); ++ if (ret) ++ /* add back to the list */ ++ list_add(&cgr->node, &p->cgr_cbs); ++release_lock: ++ spin_unlock_irqrestore(&p->cgr_lock, irqflags); ++put_portal: ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_delete_cgr); ++ ++struct cgr_comp { ++ struct qman_cgr *cgr; ++ struct completion completion; ++}; ++ ++static int qman_delete_cgr_thread(void *p) ++{ ++ struct cgr_comp *cgr_comp = (struct cgr_comp *)p; ++ int res; ++ ++ res = qman_delete_cgr((struct qman_cgr *)cgr_comp->cgr); ++ complete(&cgr_comp->completion); ++ ++ return res; ++} ++ ++void qman_delete_cgr_safe(struct qman_cgr *cgr) ++{ ++ struct task_struct *thread; ++ struct cgr_comp cgr_comp; ++ ++ preempt_disable(); ++ if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) { ++ init_completion(&cgr_comp.completion); ++ cgr_comp.cgr = cgr; ++ thread = kthread_create(qman_delete_cgr_thread, &cgr_comp, ++ "cgr_del"); ++ ++ if (likely(!IS_ERR(thread))) { ++ kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]); ++ wake_up_process(thread); ++ wait_for_completion(&cgr_comp.completion); ++ preempt_enable(); ++ return; ++ } ++ } ++ qman_delete_cgr(cgr); ++ preempt_enable(); ++} ++EXPORT_SYMBOL(qman_delete_cgr_safe); ++ ++int qm_get_clock(u64 *clock_hz) ++{ ++ if (!qman_clk) { ++ pr_warn("Qman clock speed is unknown\n"); ++ return -EINVAL; ++ } ++ *clock_hz = (u64)qman_clk; ++ return 0; ++} ++EXPORT_SYMBOL(qm_get_clock); ++ ++int qm_set_clock(u64 clock_hz) ++{ ++ if (qman_clk) ++ return -1; ++ qman_clk = (u32)clock_hz; ++ return 0; ++} ++EXPORT_SYMBOL(qm_set_clock); ++ ++/* CEETM management command */ ++static int qman_ceetm_configure_lfqmt(struct qm_mcc_ceetm_lfqmt_config *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->lfqmt_config = *opts; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_CONFIG); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_LFQMT_CONFIG); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: CONFIGURE LFQMT failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++ ++int qman_ceetm_query_lfqmt(int lfqid, ++ struct qm_mcr_ceetm_lfqmt_query *lfqmt_query) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->lfqmt_query.lfqid = lfqid; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_QUERY); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_LFQMT_QUERY); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) ++ *lfqmt_query = mcr->lfqmt_query; ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: QUERY LFQMT failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_query_lfqmt); ++ ++static int qman_ceetm_configure_cq(struct qm_mcc_ceetm_cq_config *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->cq_config = *opts; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_CONFIG); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ res = mcr->result; ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_CONFIG); ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: CONFIGURE CQ failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++ ++int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid, ++ struct qm_mcr_ceetm_cq_query *cq_query) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->cq_query.cqid = cpu_to_be16(cqid); ++ mcc->cq_query.dcpid = dcpid; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_QUERY); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_QUERY); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) { ++ *cq_query = mcr->cq_query; ++ hw_cq_query_to_cpu(cq_query); ++ } ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: QUERY CQ failed\n"); ++ return -EIO; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_query_cq); ++ ++static int qman_ceetm_configure_dct(struct qm_mcc_ceetm_dct_config *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->dct_config = *opts; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_CONFIG); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_CONFIG); ++ res = mcr->result; ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: CONFIGURE DCT failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++ ++static int qman_ceetm_query_dct(struct qm_mcc_ceetm_dct_query *opts, ++ struct qm_mcr_ceetm_dct_query *dct_query) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p = get_affine_portal(); ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->dct_query = *opts; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_QUERY); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_QUERY); ++ res = mcr->result; ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: QUERY DCT failed\n"); ++ return -EIO; ++ } ++ ++ *dct_query = mcr->dct_query; ++ return 0; ++} ++ ++static int qman_ceetm_configure_class_scheduler( ++ struct qm_mcc_ceetm_class_scheduler_config *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->csch_config = *opts; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG); ++ res = mcr->result; ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: CONFIGURE CLASS SCHEDULER failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++ ++static int qman_ceetm_query_class_scheduler(struct qm_ceetm_channel *channel, ++ struct qm_mcr_ceetm_class_scheduler_query *query) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->csch_query.cqcid = channel->idx; ++ mcc->csch_query.dcpid = channel->dcp_idx; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_QUERY); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_CLASS_SCHEDULER_QUERY); ++ res = mcr->result; ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: QUERY CLASS SCHEDULER failed\n"); ++ return -EIO; ++ } ++ *query = mcr->csch_query; ++ return 0; ++} ++ ++static int qman_ceetm_configure_mapping_shaper_tcfc( ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->mst_config = *opts; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG); ++ res = mcr->result; ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: CONFIGURE CHANNEL MAPPING failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++ ++static int qman_ceetm_query_mapping_shaper_tcfc( ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query *opts, ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query *response) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->mst_query = *opts; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY); ++ res = mcr->result; ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: QUERY CHANNEL MAPPING failed\n"); ++ return -EIO; ++ } ++ ++ *response = mcr->mst_query; ++ return 0; ++} ++ ++static int qman_ceetm_configure_ccgr(struct qm_mcc_ceetm_ccgr_config *opts) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->ccgr_config = *opts; ++ ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_CONFIG); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_CONFIG); ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: CONFIGURE CCGR failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++ ++int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query, ++ struct qm_mcr_ceetm_ccgr_query *response) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->ccgr_query.ccgrid = cpu_to_be16(ccgr_query->ccgrid); ++ mcc->ccgr_query.dcpid = ccgr_query->dcpid; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY); ++ ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_QUERY); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) { ++ *response = mcr->ccgr_query; ++ hw_ccgr_query_to_cpu(response); ++ } ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: QUERY CCGR failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_query_ccgr); ++ ++static int qman_ceetm_cq_peek_pop_xsfdrread(struct qm_ceetm_cq *cq, ++ u8 command_type, u16 xsfdr, ++ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread *cq_ppxr) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ switch (command_type) { ++ case 0: ++ case 1: ++ mcc->cq_ppxr.cqid = (cq->parent->idx << 4) | cq->idx; ++ break; ++ case 2: ++ mcc->cq_ppxr.xsfdr = xsfdr; ++ break; ++ default: ++ break; ++ } ++ mcc->cq_ppxr.ct = command_type; ++ mcc->cq_ppxr.dcpid = cq->parent->dcp_idx; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD); ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD); ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: CQ PEEK/POP/XSFDR READ failed\n"); ++ return -EIO; ++ } ++ *cq_ppxr = mcr->cq_ppxr; ++ return 0; ++} ++ ++static int qman_ceetm_query_statistics(u16 cid, ++ enum qm_dc_portal dcp_idx, ++ u16 command_type, ++ struct qm_mcr_ceetm_statistics_query *query_result) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->stats_query_write.cid = cid; ++ mcc->stats_query_write.dcpid = dcp_idx; ++ mcc->stats_query_write.ct = command_type; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE); ++ ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_STATISTICS_QUERY_WRITE); ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: STATISTICS QUERY failed\n"); ++ return -EIO; ++ } ++ *query_result = mcr->stats_query; ++ return 0; ++} ++ ++int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx, ++ u16 command_type, u64 frame_count, ++ u64 byte_count) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ mcc->stats_query_write.cid = cid; ++ mcc->stats_query_write.dcpid = dcp_idx; ++ mcc->stats_query_write.ct = command_type; ++ mcc->stats_query_write.frm_cnt = frame_count; ++ mcc->stats_query_write.byte_cnt = byte_count; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE); ++ ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_STATISTICS_QUERY_WRITE); ++ ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ ++ res = mcr->result; ++ if (res != QM_MCR_RESULT_OK) { ++ pr_err("CEETM: STATISTICS WRITE failed\n"); ++ return -EIO; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_query_write_statistics); ++ ++int qman_ceetm_bps2tokenrate(u64 bps, struct qm_ceetm_rate *token_rate, ++ int rounding) ++{ ++ u16 pres; ++ u64 temp; ++ u64 qman_freq; ++ int ret; ++ ++ /* Read PRES from CEET_CFG_PRES register */ ++ ret = qman_ceetm_get_prescaler(&pres); ++ if (ret) ++ return -EINVAL; ++ ++ ret = qm_get_clock(&qman_freq); ++ if (ret) ++ return -EINVAL; ++ ++ /* token-rate = bytes-per-second * update-reference-period ++ * ++ * Where token-rate is N/8192 for a integer N, and ++ * update-reference-period is (2^22)/(PRES*QHz), where PRES ++ * is the prescalar value and QHz is the QMan clock frequency. ++ * So: ++ * ++ * token-rate = (byte-per-second*2^22)/PRES*QHZ) ++ * ++ * Converting to bits-per-second gives; ++ * ++ * token-rate = (bps*2^19) / (PRES*QHZ) ++ * N = (bps*2^32) / (PRES*QHz) ++ * ++ * And to avoid 64-bit overflow if 'bps' is larger than 4Gbps ++ * (yet minimise rounding error if 'bps' is small), we reorganise ++ * the formula to use two 16-bit shifts rather than 1 32-bit shift. ++ * N = (((bps*2^16)/PRES)*2^16)/QHz ++ */ ++ temp = ROUNDING((bps << 16), pres, rounding); ++ temp = ROUNDING((temp << 16), qman_freq, rounding); ++ token_rate->whole = temp >> 13; ++ token_rate->fraction = temp & (((u64)1 << 13) - 1); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_bps2tokenrate); ++ ++int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, u64 *bps, ++ int rounding) ++{ ++ u16 pres; ++ u64 temp; ++ u64 qman_freq; ++ int ret; ++ ++ /* Read PRES from CEET_CFG_PRES register */ ++ ret = qman_ceetm_get_prescaler(&pres); ++ if (ret) ++ return -EINVAL; ++ ++ ret = qm_get_clock(&qman_freq); ++ if (ret) ++ return -EINVAL; ++ ++ /* bytes-per-second = token-rate / update-reference-period ++ * ++ * where "token-rate" is N/8192 for an integer N, and ++ * "update-reference-period" is (2^22)/(PRES*QHz), where PRES is ++ * the prescalar value and QHz is the QMan clock frequency. So; ++ * ++ * bytes-per-second = (N/8192) / (4194304/PRES*QHz) ++ * = N*PRES*QHz / (4194304*8192) ++ * = N*PRES*QHz / (2^35) ++ * ++ * Converting to bits-per-second gives; ++ * ++ * bps = N*PRES*QHZ / (2^32) ++ * ++ * Note, the numerator has a maximum width of 72 bits! So to ++ * avoid 64-bit overflow errors, we calculate PRES*QHZ (maximum ++ * width 48 bits) divided by 2^9 (reducing to maximum 39 bits), before ++ * multiplying by N (goes to maximum of 63 bits). ++ * ++ * temp = PRES*QHZ / (2^16) ++ * kbps = temp*N / (2^16) ++ */ ++ temp = ROUNDING(qman_freq * pres, (u64)1 << 16 , rounding); ++ temp *= ((token_rate->whole << 13) + token_rate->fraction); ++ *bps = ROUNDING(temp, (u64)(1) << 16, rounding); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_tokenrate2bps); ++ ++int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, enum qm_dc_portal dcp_idx, ++ unsigned int sp_idx) ++{ ++ struct qm_ceetm_sp *p; ++ ++ DPA_ASSERT((dcp_idx == qm_dc_portal_fman0) || ++ (dcp_idx == qm_dc_portal_fman1)); ++ ++ if ((sp_idx < qman_ceetms[dcp_idx].sp_range[0]) || ++ (sp_idx > (qman_ceetms[dcp_idx].sp_range[0] + ++ qman_ceetms[dcp_idx].sp_range[1]))) { ++ pr_err("Sub-portal index doesn't exist\n"); ++ return -EINVAL; ++ } ++ ++ list_for_each_entry(p, &qman_ceetms[dcp_idx].sub_portals, node) { ++ if ((p->idx == sp_idx) && (p->is_claimed == 0)) { ++ p->is_claimed = 1; ++ *sp = p; ++ return 0; ++ } ++ } ++ pr_err("The sub-portal#%d is not available!\n", sp_idx); ++ return -ENODEV; ++} ++EXPORT_SYMBOL(qman_ceetm_sp_claim); ++ ++int qman_ceetm_sp_release(struct qm_ceetm_sp *sp) ++{ ++ struct qm_ceetm_sp *p; ++ ++ if (sp->lni && sp->lni->is_claimed == 1) { ++ pr_err("The dependency of sub-portal has not been released!\n"); ++ return -EBUSY; ++ } ++ ++ list_for_each_entry(p, &qman_ceetms[sp->dcp_idx].sub_portals, node) { ++ if (p->idx == sp->idx) { ++ p->is_claimed = 0; ++ p->lni = NULL; ++ } ++ } ++ /* Disable CEETM mode of this sub-portal */ ++ qman_sp_disable_ceetm_mode(sp->dcp_idx, sp->idx); ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_sp_release); ++ ++int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, enum qm_dc_portal dcp_idx, ++ unsigned int lni_idx) ++{ ++ struct qm_ceetm_lni *p; ++ ++ if ((lni_idx < qman_ceetms[dcp_idx].lni_range[0]) || ++ (lni_idx > (qman_ceetms[dcp_idx].lni_range[0] + ++ qman_ceetms[dcp_idx].lni_range[1]))) { ++ pr_err("The lni index is out of range\n"); ++ return -EINVAL; ++ } ++ ++ list_for_each_entry(p, &qman_ceetms[dcp_idx].lnis, node) { ++ if ((p->idx == lni_idx) && (p->is_claimed == 0)) { ++ *lni = p; ++ p->is_claimed = 1; ++ return 0; ++ } ++ } ++ ++ pr_err("The LNI#%d is not available!\n", lni_idx); ++ return -EINVAL; ++} ++EXPORT_SYMBOL(qman_ceetm_lni_claim); ++ ++int qman_ceetm_lni_release(struct qm_ceetm_lni *lni) ++{ ++ struct qm_ceetm_lni *p; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ if (!list_empty(&lni->channels)) { ++ pr_err("The LNI dependencies are not released!\n"); ++ return -EBUSY; ++ } ++ ++ list_for_each_entry(p, &qman_ceetms[lni->dcp_idx].lnis, node) { ++ if (p->idx == lni->idx) { ++ p->shaper_enable = 0; ++ p->shaper_couple = 0; ++ p->cr_token_rate.whole = 0; ++ p->cr_token_rate.fraction = 0; ++ p->er_token_rate.whole = 0; ++ p->er_token_rate.fraction = 0; ++ p->cr_token_bucket_limit = 0; ++ p->er_token_bucket_limit = 0; ++ p->is_claimed = 0; ++ } ++ } ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ config_opts.dcpid = lni->dcp_idx; ++ memset(&config_opts.shaper_config, 0, ++ sizeof(config_opts.shaper_config)); ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_release); ++ ++int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, struct qm_ceetm_lni *lni) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx); ++ config_opts.dcpid = sp->dcp_idx; ++ config_opts.sp_mapping.map_lni_id = lni->idx; ++ sp->lni = lni; ++ ++ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) ++ return -EINVAL; ++ ++ /* Enable CEETM mode for this sub-portal */ ++ return qman_sp_enable_ceetm_mode(sp->dcp_idx, sp->idx); ++} ++EXPORT_SYMBOL(qman_ceetm_sp_set_lni); ++ ++int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, unsigned int *lni_idx) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx); ++ query_opts.dcpid = sp->dcp_idx; ++ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { ++ pr_err("Can't get SP <-> LNI mapping\n"); ++ return -EINVAL; ++ } ++ *lni_idx = query_result.sp_mapping_query.map_lni_id; ++ sp->lni->idx = query_result.sp_mapping_query.map_lni_id; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_sp_get_lni); ++ ++int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled, ++ int oal) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ lni->shaper_enable = 1; ++ lni->shaper_couple = coupled; ++ lni->oal = oal; ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ config_opts.dcpid = lni->dcp_idx; ++ config_opts.shaper_config.cpl = coupled; ++ config_opts.shaper_config.oal = oal; ++ config_opts.shaper_config.crtcr = cpu_to_be24((lni->cr_token_rate.whole ++ << 13) | lni->cr_token_rate.fraction); ++ config_opts.shaper_config.ertcr = cpu_to_be24((lni->er_token_rate.whole ++ << 13) | lni->er_token_rate.fraction); ++ config_opts.shaper_config.crtbl = ++ cpu_to_be16(lni->cr_token_bucket_limit); ++ config_opts.shaper_config.ertbl = ++ cpu_to_be16(lni->er_token_bucket_limit); ++ config_opts.shaper_config.mps = 60; ++ ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_enable_shaper); ++ ++int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ if (!lni->shaper_enable) { ++ pr_err("The shaper has been disabled\n"); ++ return -EINVAL; ++ } ++ ++ config_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx; ++ config_opts.dcpid = lni->dcp_idx; ++ config_opts.shaper_config.cpl = (lni->shaper_couple << 7) | lni->oal; ++ config_opts.shaper_config.crtbl = lni->cr_token_bucket_limit; ++ config_opts.shaper_config.ertbl = lni->er_token_bucket_limit; ++ /* Set CR/ER rate with all 1's to configure an infinite rate, thus ++ * disable the shaping. ++ */ ++ config_opts.shaper_config.crtcr = 0xFFFFFF; ++ config_opts.shaper_config.ertcr = 0xFFFFFF; ++ config_opts.shaper_config.mps = 60; ++ lni->shaper_enable = 0; ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_disable_shaper); ++ ++int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni) ++{ ++ return lni->shaper_enable; ++} ++EXPORT_SYMBOL(qman_ceetm_lni_is_shaper_enabled); ++ ++int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ lni->cr_token_rate.whole = token_rate->whole; ++ lni->cr_token_rate.fraction = token_rate->fraction; ++ lni->cr_token_bucket_limit = token_limit; ++ if (!lni->shaper_enable) ++ return 0; ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ query_opts.dcpid = lni->dcp_idx; ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, ++ &query_result); ++ if (ret) { ++ pr_err("Fail to get current LNI shaper setting\n"); ++ return -EINVAL; ++ } ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ config_opts.dcpid = lni->dcp_idx; ++ config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole << 13) ++ | (token_rate->fraction)); ++ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit); ++ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; ++ config_opts.shaper_config.oal = query_result.shaper_query.oal; ++ config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr; ++ config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl; ++ config_opts.shaper_config.mps = query_result.shaper_query.mps; ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate); ++ ++int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni, ++ u64 bps, ++ u16 token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); ++ if (ret) { ++ pr_err("Can not convert bps to token rate\n"); ++ return -EINVAL; ++ } ++ ++ return qman_ceetm_lni_set_commit_rate(lni, &token_rate, token_limit); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate_bps); ++ ++int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ query_opts.dcpid = lni->dcp_idx; ++ ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret) { ++ pr_err("The LNI CR rate or limit is not set\n"); ++ return -EINVAL; ++ } ++ token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13; ++ token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) & ++ 0x1FFF; ++ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate); ++ ++int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni, ++ u64 *bps, u16 *token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_lni_get_commit_rate(lni, &token_rate, token_limit); ++ if (ret) { ++ pr_err("The LNI CR rate or limit is not available\n"); ++ return -EINVAL; ++ } ++ ++ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate_bps); ++ ++int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ lni->er_token_rate.whole = token_rate->whole; ++ lni->er_token_rate.fraction = token_rate->fraction; ++ lni->er_token_bucket_limit = token_limit; ++ if (!lni->shaper_enable) ++ return 0; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ query_opts.dcpid = lni->dcp_idx; ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, ++ &query_result); ++ if (ret) { ++ pr_err("Fail to get current LNI shaper setting\n"); ++ return -EINVAL; ++ } ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ config_opts.dcpid = lni->dcp_idx; ++ config_opts.shaper_config.ertcr = cpu_to_be24( ++ (token_rate->whole << 13) | (token_rate->fraction)); ++ config_opts.shaper_config.ertbl = cpu_to_be16(token_limit); ++ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; ++ config_opts.shaper_config.oal = query_result.shaper_query.oal; ++ config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr; ++ config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl; ++ config_opts.shaper_config.mps = query_result.shaper_query.mps; ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate); ++ ++int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni, ++ u64 bps, ++ u16 token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); ++ if (ret) { ++ pr_err("Can not convert bps to token rate\n"); ++ return -EINVAL; ++ } ++ return qman_ceetm_lni_set_excess_rate(lni, &token_rate, token_limit); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate_bps); ++ ++int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); ++ query_opts.dcpid = lni->dcp_idx; ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret) { ++ pr_err("The LNI ER rate or limit is not set\n"); ++ return -EINVAL; ++ } ++ token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13; ++ token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) & ++ 0x1FFF; ++ *token_limit = be16_to_cpu(query_result.shaper_query.ertbl); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate); ++ ++int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni, ++ u64 *bps, u16 *token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_lni_get_excess_rate(lni, &token_rate, token_limit); ++ if (ret) { ++ pr_err("The LNI ER rate or limit is not available\n"); ++ return -EINVAL; ++ } ++ ++ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate_bps); ++ ++#define QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(n) ((15 - n) * 4) ++#define QMAN_CEETM_LNITCFCC_ENABLE 0x8 ++int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni, ++ unsigned int cq_level, ++ int traffic_class) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ u64 lnitcfcc; ++ ++ if ((cq_level > 15) | (traffic_class > 7)) { ++ pr_err("The CQ or traffic class id is out of range\n"); ++ return -EINVAL; ++ } ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx); ++ query_opts.dcpid = lni->dcp_idx; ++ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { ++ pr_err("Fail to query tcfcc\n"); ++ return -EINVAL; ++ } ++ ++ lnitcfcc = be64_to_cpu(query_result.tcfc_query.lnitcfcc); ++ if (traffic_class == -1) { ++ /* disable tcfc for this CQ */ ++ lnitcfcc &= ~((u64)QMAN_CEETM_LNITCFCC_ENABLE << ++ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level)); ++ } else { ++ lnitcfcc &= ~((u64)0xF << ++ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level)); ++ lnitcfcc |= ((u64)(QMAN_CEETM_LNITCFCC_ENABLE | ++ traffic_class)) << ++ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level); ++ } ++ config_opts.tcfc_config.lnitcfcc = cpu_to_be64(lnitcfcc); ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx); ++ config_opts.dcpid = lni->dcp_idx; ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_lni_set_tcfcc); ++ ++#define QMAN_CEETM_LNITCFCC_TC_MASK 0x7 ++int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni, unsigned int cq_level, ++ int *traffic_class) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ u8 lnitcfcc; ++ ++ if (cq_level > 15) { ++ pr_err("the CQ level is out of range\n"); ++ return -EINVAL; ++ } ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx); ++ query_opts.dcpid = lni->dcp_idx; ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret) ++ return ret; ++ lnitcfcc = (u8)be64_to_cpu((query_result.tcfc_query.lnitcfcc) >> ++ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level)); ++ if (lnitcfcc & QMAN_CEETM_LNITCFCC_ENABLE) ++ *traffic_class = lnitcfcc & QMAN_CEETM_LNITCFCC_TC_MASK; ++ else ++ *traffic_class = -1; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_lni_get_tcfcc); ++ ++int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel, ++ struct qm_ceetm_lni *lni) ++{ ++ struct qm_ceetm_channel *p; ++ u32 channel_idx; ++ int ret = 0; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ if (lni->dcp_idx == qm_dc_portal_fman0) { ++ ret = qman_alloc_ceetm0_channel(&channel_idx); ++ } else if (lni->dcp_idx == qm_dc_portal_fman1) { ++ ret = qman_alloc_ceetm1_channel(&channel_idx); ++ } else { ++ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", ++ lni->dcp_idx); ++ return -EINVAL; ++ } ++ ++ if (ret) { ++ pr_err("The is no channel available for LNI#%d\n", lni->idx); ++ return -ENODEV; ++ } ++ ++ p = kzalloc(sizeof(*p), GFP_KERNEL); ++ if (!p) ++ return -ENOMEM; ++ p->idx = channel_idx; ++ p->dcp_idx = lni->dcp_idx; ++ list_add_tail(&p->node, &lni->channels); ++ INIT_LIST_HEAD(&p->class_queues); ++ INIT_LIST_HEAD(&p->ccgs); ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | ++ channel_idx); ++ config_opts.dcpid = lni->dcp_idx; ++ config_opts.channel_mapping.map_lni_id = lni->idx; ++ config_opts.channel_mapping.map_shaped = 0; ++ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) { ++ pr_err("Can't map channel#%d for LNI#%d\n", ++ channel_idx, lni->idx); ++ return -EINVAL; ++ } ++ *channel = p; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_claim); ++ ++int qman_ceetm_channel_release(struct qm_ceetm_channel *channel) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ if (!list_empty(&channel->class_queues)) { ++ pr_err("CEETM channel#%d has class queue unreleased!\n", ++ channel->idx); ++ return -EBUSY; ++ } ++ if (!list_empty(&channel->ccgs)) { ++ pr_err("CEETM channel#%d has ccg unreleased!\n", ++ channel->idx); ++ return -EBUSY; ++ } ++ ++ /* channel->dcp_idx corresponds to known fman validation */ ++ if ((channel->dcp_idx != qm_dc_portal_fman0) && ++ (channel->dcp_idx != qm_dc_portal_fman1)) { ++ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", ++ channel->dcp_idx); ++ return -EINVAL; ++ } ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ config_opts.dcpid = channel->dcp_idx; ++ memset(&config_opts.shaper_config, 0, ++ sizeof(config_opts.shaper_config)); ++ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) { ++ pr_err("Can't reset channel shapping parameters\n"); ++ return -EINVAL; ++ } ++ ++ if (channel->dcp_idx == qm_dc_portal_fman0) { ++ qman_release_ceetm0_channelid(channel->idx); ++ } else if (channel->dcp_idx == qm_dc_portal_fman1) { ++ qman_release_ceetm1_channelid(channel->idx); ++ } else { ++ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", ++ channel->dcp_idx); ++ return -EINVAL; ++ } ++ list_del(&channel->node); ++ kfree(channel); ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_release); ++ ++int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel, ++ int coupled) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ if (channel->shaper_enable == 1) { ++ pr_err("This channel shaper has been enabled!\n"); ++ return -EINVAL; ++ } ++ ++ channel->shaper_enable = 1; ++ channel->shaper_couple = coupled; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | ++ channel->idx); ++ query_opts.dcpid = (u8)channel->dcp_idx; ++ ++ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { ++ pr_err("Can't query channel mapping\n"); ++ return -EINVAL; ++ } ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | ++ channel->idx); ++ config_opts.dcpid = channel->dcp_idx; ++ config_opts.channel_mapping.map_lni_id = ++ query_result.channel_mapping_query.map_lni_id; ++ config_opts.channel_mapping.map_shaped = 1; ++ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) { ++ pr_err("Can't enable shaper for channel #%d\n", channel->idx); ++ return -EINVAL; ++ } ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ config_opts.shaper_config.cpl = coupled; ++ config_opts.shaper_config.crtcr = cpu_to_be24((channel->cr_token_rate. ++ whole << 13) | ++ channel->cr_token_rate.fraction); ++ config_opts.shaper_config.ertcr = cpu_to_be24((channel->er_token_rate. ++ whole << 13) | ++ channel->er_token_rate.fraction); ++ config_opts.shaper_config.crtbl = ++ cpu_to_be16(channel->cr_token_bucket_limit); ++ config_opts.shaper_config.ertbl = ++ cpu_to_be16(channel->er_token_bucket_limit); ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_enable_shaper); ++ ++int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | ++ channel->idx); ++ query_opts.dcpid = channel->dcp_idx; ++ ++ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { ++ pr_err("Can't query channel mapping\n"); ++ return -EINVAL; ++ } ++ ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | ++ channel->idx); ++ config_opts.dcpid = channel->dcp_idx; ++ config_opts.channel_mapping.map_shaped = 0; ++ config_opts.channel_mapping.map_lni_id = ++ query_result.channel_mapping_query.map_lni_id; ++ ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_disable_shaper); ++ ++int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ ++ query_opts.cid = CEETM_COMMAND_CHANNEL_MAPPING | channel->idx; ++ query_opts.dcpid = channel->dcp_idx; ++ ++ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { ++ pr_err("Can't query channel mapping\n"); ++ return -EINVAL; ++ } ++ ++ return query_result.channel_mapping_query.map_shaped; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_is_shaper_enabled); ++ ++int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ query_opts.dcpid = channel->dcp_idx; ++ ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret) { ++ pr_err("Fail to get the current channel shaper setting\n"); ++ return -EINVAL; ++ } ++ ++ channel->cr_token_rate.whole = token_rate->whole; ++ channel->cr_token_rate.fraction = token_rate->fraction; ++ channel->cr_token_bucket_limit = token_limit; ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ config_opts.dcpid = channel->dcp_idx; ++ config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole ++ << 13) | (token_rate->fraction)); ++ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit); ++ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; ++ config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr; ++ config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl; ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate); ++ ++int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel, ++ u64 bps, u16 token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); ++ if (ret) { ++ pr_err("Can not convert bps to token rate\n"); ++ return -EINVAL; ++ } ++ return qman_ceetm_channel_set_commit_rate(channel, &token_rate, ++ token_limit); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate_bps); ++ ++int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ query_opts.dcpid = channel->dcp_idx; ++ ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret | !query_result.shaper_query.crtcr | ++ !query_result.shaper_query.crtbl) { ++ pr_err("The channel commit rate or limit is not set\n"); ++ return -EINVAL; ++ } ++ token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13; ++ token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) & ++ 0x1FFF; ++ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate); ++ ++int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel, ++ u64 *bps, u16 *token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_channel_get_commit_rate(channel, &token_rate, ++ token_limit); ++ if (ret) { ++ pr_err("The channel CR rate or limit is not available\n"); ++ return -EINVAL; ++ } ++ ++ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate_bps); ++ ++int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ query_opts.dcpid = channel->dcp_idx; ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret) { ++ pr_err("Fail to get the current channel shaper setting\n"); ++ return -EINVAL; ++ } ++ ++ channel->er_token_rate.whole = token_rate->whole; ++ channel->er_token_rate.fraction = token_rate->fraction; ++ channel->er_token_bucket_limit = token_limit; ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ config_opts.dcpid = channel->dcp_idx; ++ config_opts.shaper_config.ertcr = cpu_to_be24( ++ (token_rate->whole << 13) | (token_rate->fraction)); ++ config_opts.shaper_config.ertbl = cpu_to_be16(token_limit); ++ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; ++ config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr; ++ config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl; ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate); ++ ++int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel, ++ u64 bps, u16 token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); ++ if (ret) { ++ pr_err("Can not convert bps to token rate\n"); ++ return -EINVAL; ++ } ++ return qman_ceetm_channel_set_excess_rate(channel, &token_rate, ++ token_limit); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate_bps); ++ ++int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ query_opts.dcpid = channel->dcp_idx; ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret | !query_result.shaper_query.ertcr | ++ !query_result.shaper_query.ertbl) { ++ pr_err("The channel excess rate or limit is not set\n"); ++ return -EINVAL; ++ } ++ token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13; ++ token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) & ++ 0x1FFF; ++ *token_limit = be16_to_cpu(query_result.shaper_query.ertbl); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate); ++ ++int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel, ++ u64 *bps, u16 *token_limit) ++{ ++ struct qm_ceetm_rate token_rate; ++ int ret; ++ ++ ret = qman_ceetm_channel_get_excess_rate(channel, &token_rate, ++ token_limit); ++ if (ret) { ++ pr_err("The channel ER rate or limit is not available\n"); ++ return -EINVAL; ++ } ++ ++ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate_bps); ++ ++int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel, ++ u16 token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; ++ ++ if (channel->shaper_enable) { ++ pr_err("This channel is a shaped one\n"); ++ return -EINVAL; ++ } ++ ++ channel->cr_token_bucket_limit = token_limit; ++ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ config_opts.dcpid = channel->dcp_idx; ++ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit); ++ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_weight); ++ ++int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel, ++ u16 *token_limit) ++{ ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; ++ int ret; ++ ++ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | ++ channel->idx); ++ query_opts.dcpid = channel->dcp_idx; ++ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); ++ if (ret | !query_result.shaper_query.crtbl) { ++ pr_err("This unshaped channel's uFQ wight is unavailable\n"); ++ return -EINVAL; ++ } ++ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_get_weight); ++ ++int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, int group_b, ++ unsigned int prio_a, unsigned int prio_b) ++{ ++ struct qm_mcc_ceetm_class_scheduler_config config_opts; ++ struct qm_mcr_ceetm_class_scheduler_query query_result; ++ int i; ++ ++ if (prio_a > 7) { ++ pr_err("The priority of group A is out of range\n"); ++ return -EINVAL; ++ } ++ if (group_b && (prio_b > 7)) { ++ pr_err("The priority of group B is out of range\n"); ++ return -EINVAL; ++ } ++ ++ if (qman_ceetm_query_class_scheduler(channel, &query_result)) { ++ pr_err("Can't query channel#%d's scheduler!\n", channel->idx); ++ return -EINVAL; ++ } ++ ++ config_opts.cqcid = cpu_to_be16(channel->idx); ++ config_opts.dcpid = channel->dcp_idx; ++ config_opts.gpc_combine_flag = !group_b; ++ config_opts.gpc_prio_a = prio_a; ++ config_opts.gpc_prio_b = prio_b; ++ ++ for (i = 0; i < 8; i++) ++ config_opts.w[i] = query_result.w[i]; ++ config_opts.crem = query_result.crem; ++ config_opts.erem = query_result.erem; ++ ++ return qman_ceetm_configure_class_scheduler(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_group); ++ ++int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, int *group_b, ++ unsigned int *prio_a, unsigned int *prio_b) ++{ ++ struct qm_mcr_ceetm_class_scheduler_query query_result; ++ ++ if (qman_ceetm_query_class_scheduler(channel, &query_result)) { ++ pr_err("Can't query channel#%d's scheduler!\n", channel->idx); ++ return -EINVAL; ++ } ++ *group_b = !query_result.gpc_combine_flag; ++ *prio_a = query_result.gpc_prio_a; ++ *prio_b = query_result.gpc_prio_b; ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_get_group); ++ ++#define GROUP_A_ELIGIBILITY_SET (1 << 8) ++#define GROUP_B_ELIGIBILITY_SET (1 << 9) ++#define CQ_ELIGIBILITY_SET(n) (1 << (7 - n)) ++int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel ++ *channel, int group_b, int cre) ++{ ++ struct qm_mcc_ceetm_class_scheduler_config csch_config; ++ struct qm_mcr_ceetm_class_scheduler_query csch_query; ++ int i; ++ ++ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { ++ pr_err("Cannot get the channel %d scheduler setting.\n", ++ channel->idx); ++ return -EINVAL; ++ } ++ csch_config.cqcid = cpu_to_be16(channel->idx); ++ csch_config.dcpid = channel->dcp_idx; ++ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; ++ csch_config.gpc_prio_a = csch_query.gpc_prio_a; ++ csch_config.gpc_prio_b = csch_query.gpc_prio_b; ++ ++ for (i = 0; i < 8; i++) ++ csch_config.w[i] = csch_query.w[i]; ++ csch_config.erem = csch_query.erem; ++ if (group_b) ++ csch_config.crem = (be16_to_cpu(csch_query.crem) ++ & ~GROUP_B_ELIGIBILITY_SET) ++ | (cre ? GROUP_B_ELIGIBILITY_SET : 0); ++ else ++ csch_config.crem = (be16_to_cpu(csch_query.crem) ++ & ~GROUP_A_ELIGIBILITY_SET) ++ | (cre ? GROUP_A_ELIGIBILITY_SET : 0); ++ ++ csch_config.crem = cpu_to_be16(csch_config.crem); ++ ++ if (qman_ceetm_configure_class_scheduler(&csch_config)) { ++ pr_err("Cannot config channel %d's scheduler with " ++ "group_%c's cr eligibility\n", channel->idx, ++ group_b ? 'b' : 'a'); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_group_cr_eligibility); ++ ++int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel ++ *channel, int group_b, int ere) ++{ ++ struct qm_mcc_ceetm_class_scheduler_config csch_config; ++ struct qm_mcr_ceetm_class_scheduler_query csch_query; ++ int i; ++ ++ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { ++ pr_err("Cannot get the channel %d scheduler setting.\n", ++ channel->idx); ++ return -EINVAL; ++ } ++ csch_config.cqcid = cpu_to_be16(channel->idx); ++ csch_config.dcpid = channel->dcp_idx; ++ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; ++ csch_config.gpc_prio_a = csch_query.gpc_prio_a; ++ csch_config.gpc_prio_b = csch_query.gpc_prio_b; ++ ++ for (i = 0; i < 8; i++) ++ csch_config.w[i] = csch_query.w[i]; ++ csch_config.crem = csch_query.crem; ++ if (group_b) ++ csch_config.erem = (be16_to_cpu(csch_query.erem) ++ & ~GROUP_B_ELIGIBILITY_SET) ++ | (ere ? GROUP_B_ELIGIBILITY_SET : 0); ++ else ++ csch_config.erem = (be16_to_cpu(csch_query.erem) ++ & ~GROUP_A_ELIGIBILITY_SET) ++ | (ere ? GROUP_A_ELIGIBILITY_SET : 0); ++ ++ csch_config.erem = cpu_to_be16(csch_config.erem); ++ ++ if (qman_ceetm_configure_class_scheduler(&csch_config)) { ++ pr_err("Cannot config channel %d's scheduler with " ++ "group_%c's er eligibility\n", channel->idx, ++ group_b ? 'b' : 'a'); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_group_er_eligibility); ++ ++int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel, ++ unsigned int idx, int cre) ++{ ++ struct qm_mcc_ceetm_class_scheduler_config csch_config; ++ struct qm_mcr_ceetm_class_scheduler_query csch_query; ++ int i; ++ ++ if (idx > 7) { ++ pr_err("CQ index is out of range\n"); ++ return -EINVAL; ++ } ++ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { ++ pr_err("Cannot get the channel %d scheduler setting.\n", ++ channel->idx); ++ return -EINVAL; ++ } ++ csch_config.cqcid = cpu_to_be16(channel->idx); ++ csch_config.dcpid = channel->dcp_idx; ++ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; ++ csch_config.gpc_prio_a = csch_query.gpc_prio_a; ++ csch_config.gpc_prio_b = csch_query.gpc_prio_b; ++ for (i = 0; i < 8; i++) ++ csch_config.w[i] = csch_query.w[i]; ++ csch_config.erem = csch_query.erem; ++ csch_config.crem = (be16_to_cpu(csch_query.crem) ++ & ~CQ_ELIGIBILITY_SET(idx)) | ++ (cre ? CQ_ELIGIBILITY_SET(idx) : 0); ++ csch_config.crem = cpu_to_be16(csch_config.crem); ++ if (qman_ceetm_configure_class_scheduler(&csch_config)) { ++ pr_err("Cannot config channel scheduler to set " ++ "cr eligibility mask for CQ#%d\n", idx); ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_cq_cr_eligibility); ++ ++int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel, ++ unsigned int idx, int ere) ++{ ++ struct qm_mcc_ceetm_class_scheduler_config csch_config; ++ struct qm_mcr_ceetm_class_scheduler_query csch_query; ++ int i; ++ ++ if (idx > 7) { ++ pr_err("CQ index is out of range\n"); ++ return -EINVAL; ++ } ++ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { ++ pr_err("Cannot get the channel %d scheduler setting.\n", ++ channel->idx); ++ return -EINVAL; ++ } ++ csch_config.cqcid = cpu_to_be16(channel->idx); ++ csch_config.dcpid = channel->dcp_idx; ++ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; ++ csch_config.gpc_prio_a = csch_query.gpc_prio_a; ++ csch_config.gpc_prio_b = csch_query.gpc_prio_b; ++ for (i = 0; i < 8; i++) ++ csch_config.w[i] = csch_query.w[i]; ++ csch_config.crem = csch_query.crem; ++ csch_config.erem = (be16_to_cpu(csch_query.erem) ++ & ~CQ_ELIGIBILITY_SET(idx)) | ++ (ere ? CQ_ELIGIBILITY_SET(idx) : 0); ++ csch_config.erem = cpu_to_be16(csch_config.erem); ++ if (qman_ceetm_configure_class_scheduler(&csch_config)) { ++ pr_err("Cannot config channel scheduler to set " ++ "er eligibility mask for CQ#%d\n", idx); ++ return -EINVAL; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_channel_set_cq_er_eligibility); ++ ++int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq, ++ struct qm_ceetm_channel *channel, unsigned int idx, ++ struct qm_ceetm_ccg *ccg) ++{ ++ struct qm_ceetm_cq *p; ++ struct qm_mcc_ceetm_cq_config cq_config; ++ ++ if (idx > 7) { ++ pr_err("The independent class queue id is out of range\n"); ++ return -EINVAL; ++ } ++ ++ list_for_each_entry(p, &channel->class_queues, node) { ++ if (p->idx == idx) { ++ pr_err("The CQ#%d has been claimed!\n", idx); ++ return -EINVAL; ++ } ++ } ++ ++ p = kmalloc(sizeof(*p), GFP_KERNEL); ++ if (!p) { ++ pr_err("Can't allocate memory for CQ#%d!\n", idx); ++ return -ENOMEM; ++ } ++ ++ list_add_tail(&p->node, &channel->class_queues); ++ p->idx = idx; ++ p->is_claimed = 1; ++ p->parent = channel; ++ INIT_LIST_HEAD(&p->bound_lfqids); ++ ++ if (ccg) { ++ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx); ++ cq_config.dcpid = channel->dcp_idx; ++ cq_config.ccgid = cpu_to_be16(ccg->idx); ++ if (qman_ceetm_configure_cq(&cq_config)) { ++ pr_err("Can't configure the CQ#%d with CCGRID#%d\n", ++ idx, ccg->idx); ++ list_del(&p->node); ++ kfree(p); ++ return -EINVAL; ++ } ++ } ++ ++ *cq = p; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cq_claim); ++ ++int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq, ++ struct qm_ceetm_channel *channel, unsigned int idx, ++ struct qm_ceetm_ccg *ccg) ++{ ++ struct qm_ceetm_cq *p; ++ struct qm_mcc_ceetm_cq_config cq_config; ++ ++ if ((idx < 8) || (idx > 15)) { ++ pr_err("This grouped class queue id is out of range\n"); ++ return -EINVAL; ++ } ++ ++ list_for_each_entry(p, &channel->class_queues, node) { ++ if (p->idx == idx) { ++ pr_err("The CQ#%d has been claimed!\n", idx); ++ return -EINVAL; ++ } ++ } ++ ++ p = kmalloc(sizeof(*p), GFP_KERNEL); ++ if (!p) { ++ pr_err("Can't allocate memory for CQ#%d!\n", idx); ++ return -ENOMEM; ++ } ++ ++ list_add_tail(&p->node, &channel->class_queues); ++ p->idx = idx; ++ p->is_claimed = 1; ++ p->parent = channel; ++ INIT_LIST_HEAD(&p->bound_lfqids); ++ ++ if (ccg) { ++ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx); ++ cq_config.dcpid = channel->dcp_idx; ++ cq_config.ccgid = cpu_to_be16(ccg->idx); ++ if (qman_ceetm_configure_cq(&cq_config)) { ++ pr_err("Can't configure the CQ#%d with CCGRID#%d\n", ++ idx, ccg->idx); ++ list_del(&p->node); ++ kfree(p); ++ return -EINVAL; ++ } ++ } ++ *cq = p; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cq_claim_A); ++ ++int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq, ++ struct qm_ceetm_channel *channel, unsigned int idx, ++ struct qm_ceetm_ccg *ccg) ++{ ++ struct qm_ceetm_cq *p; ++ struct qm_mcc_ceetm_cq_config cq_config; ++ ++ if ((idx < 12) || (idx > 15)) { ++ pr_err("This grouped class queue id is out of range\n"); ++ return -EINVAL; ++ } ++ ++ list_for_each_entry(p, &channel->class_queues, node) { ++ if (p->idx == idx) { ++ pr_err("The CQ#%d has been claimed!\n", idx); ++ return -EINVAL; ++ } ++ } ++ ++ p = kmalloc(sizeof(*p), GFP_KERNEL); ++ if (!p) { ++ pr_err("Can't allocate memory for CQ#%d!\n", idx); ++ return -ENOMEM; ++ } ++ ++ list_add_tail(&p->node, &channel->class_queues); ++ p->idx = idx; ++ p->is_claimed = 1; ++ p->parent = channel; ++ INIT_LIST_HEAD(&p->bound_lfqids); ++ ++ if (ccg) { ++ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx); ++ cq_config.dcpid = channel->dcp_idx; ++ cq_config.ccgid = cpu_to_be16(ccg->idx); ++ if (qman_ceetm_configure_cq(&cq_config)) { ++ pr_err("Can't configure the CQ#%d with CCGRID#%d\n", ++ idx, ccg->idx); ++ list_del(&p->node); ++ kfree(p); ++ return -EINVAL; ++ } ++ } ++ *cq = p; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cq_claim_B); ++ ++int qman_ceetm_cq_release(struct qm_ceetm_cq *cq) ++{ ++ if (!list_empty(&cq->bound_lfqids)) { ++ pr_err("The CQ#%d has unreleased LFQID\n", cq->idx); ++ return -EBUSY; ++ } ++ list_del(&cq->node); ++ qman_ceetm_drain_cq(cq); ++ kfree(cq); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cq_release); ++ ++int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq, ++ struct qm_ceetm_weight_code *weight_code) ++{ ++ struct qm_mcc_ceetm_class_scheduler_config config_opts; ++ struct qm_mcr_ceetm_class_scheduler_query query_result; ++ int i; ++ ++ if (cq->idx < 8) { ++ pr_err("Can not set weight for ungrouped class queue\n"); ++ return -EINVAL; ++ } ++ ++ if (qman_ceetm_query_class_scheduler(cq->parent, &query_result)) { ++ pr_err("Can't query channel#%d's scheduler!\n", ++ cq->parent->idx); ++ return -EINVAL; ++ } ++ ++ config_opts.cqcid = cpu_to_be16(cq->parent->idx); ++ config_opts.dcpid = cq->parent->dcp_idx; ++ config_opts.crem = query_result.crem; ++ config_opts.erem = query_result.erem; ++ config_opts.gpc_combine_flag = query_result.gpc_combine_flag; ++ config_opts.gpc_prio_a = query_result.gpc_prio_a; ++ config_opts.gpc_prio_b = query_result.gpc_prio_b; ++ ++ for (i = 0; i < 8; i++) ++ config_opts.w[i] = query_result.w[i]; ++ config_opts.w[cq->idx - 8] = ((weight_code->y << 3) | ++ (weight_code->x & 0x7)); ++ return qman_ceetm_configure_class_scheduler(&config_opts); ++} ++EXPORT_SYMBOL(qman_ceetm_set_queue_weight); ++ ++int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq, ++ struct qm_ceetm_weight_code *weight_code) ++{ ++ struct qm_mcr_ceetm_class_scheduler_query query_result; ++ ++ if (cq->idx < 8) { ++ pr_err("Can not get weight for ungrouped class queue\n"); ++ return -EINVAL; ++ } ++ ++ if (qman_ceetm_query_class_scheduler(cq->parent, ++ &query_result)) { ++ pr_err("Can't get the weight code for CQ#%d!\n", cq->idx); ++ return -EINVAL; ++ } ++ weight_code->y = query_result.w[cq->idx - 8] >> 3; ++ weight_code->x = query_result.w[cq->idx - 8] & 0x7; ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_get_queue_weight); ++ ++/* The WBFS code is represent as {x,y}, the effect wieght can be calculated as: ++ * effective weight = 2^x / (1 - (y/64)) ++ * = 2^(x+6) / (64 - y) ++ */ ++static void reduce_fraction(u32 *n, u32 *d) ++{ ++ u32 factor = 2; ++ u32 lesser = (*n < *d) ? *n : *d; ++ /* If factor exceeds the square-root of the lesser of *n and *d, ++ * then there's no point continuing. Proof: if there was a factor ++ * bigger than the square root, that would imply there exists ++ * another factor smaller than the square-root with which it ++ * multiplies to give 'lesser' - but that's a contradiction ++ * because the other factor would have already been found and ++ * divided out. ++ */ ++ while ((factor * factor) <= lesser) { ++ /* If 'factor' is a factor of *n and *d, divide them both ++ * by 'factor' as many times as possible. ++ */ ++ while (!(*n % factor) && !(*d % factor)) { ++ *n /= factor; ++ *d /= factor; ++ lesser /= factor; ++ } ++ if (factor == 2) ++ factor = 3; ++ else ++ factor += 2; ++ } ++} ++ ++int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code, ++ u32 *numerator, ++ u32 *denominator) ++{ ++ *numerator = (u32) 1 << (weight_code->x + 6); ++ *denominator = 64 - weight_code->y; ++ reduce_fraction(numerator, denominator); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_wbfs2ratio); ++ ++/* For a given x, the weight is between 2^x (inclusive) and 2^(x+1) (exclusive). ++ * So find 'x' by range, and then estimate 'y' using: ++ * 64 - y = 2^(x + 6) / weight ++ * = 2^(x + 6) / (n/d) ++ * = d * 2^(x+6) / n ++ * y = 64 - (d * 2^(x+6) / n) ++ */ ++int qman_ceetm_ratio2wbfs(u32 numerator, ++ u32 denominator, ++ struct qm_ceetm_weight_code *weight_code, ++ int rounding) ++{ ++ unsigned int y, x = 0; ++ /* search incrementing 'x' until: ++ * weight < 2^(x+1) ++ * n/d < 2^(x+1) ++ * n < d * 2^(x+1) ++ */ ++ while ((x < 8) && (numerator >= (denominator << (x + 1)))) ++ x++; ++ if (x >= 8) ++ return -ERANGE; ++ /* because of the subtraction, use '-rounding' */ ++ y = 64 - ROUNDING(denominator << (x + 6), numerator, -rounding); ++ if (y >= 32) ++ return -ERANGE; ++ weight_code->x = x; ++ weight_code->y = y; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_ratio2wbfs); ++ ++int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio) ++{ ++ struct qm_ceetm_weight_code weight_code; ++ ++ if (qman_ceetm_ratio2wbfs(ratio, 100, &weight_code, 0)) { ++ pr_err("Cannot get wbfs code for cq %x\n", cq->idx); ++ return -EINVAL; ++ } ++ return qman_ceetm_set_queue_weight(cq, &weight_code); ++} ++EXPORT_SYMBOL(qman_ceetm_set_queue_weight_in_ratio); ++ ++int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio) ++{ ++ struct qm_ceetm_weight_code weight_code; ++ u32 n, d; ++ ++ if (qman_ceetm_get_queue_weight(cq, &weight_code)) { ++ pr_err("Cannot query the weight code for cq%x\n", cq->idx); ++ return -EINVAL; ++ } ++ ++ if (qman_ceetm_wbfs2ratio(&weight_code, &n, &d)) { ++ pr_err("Cannot get the ratio with wbfs code\n"); ++ return -EINVAL; ++ } ++ ++ *ratio = (n * (u32)100) / d; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_get_queue_weight_in_ratio); ++ ++int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags, ++ u64 *frame_count, u64 *byte_count) ++{ ++ struct qm_mcr_ceetm_statistics_query result; ++ u16 cid, command_type; ++ enum qm_dc_portal dcp_idx; ++ int ret; ++ ++ cid = cpu_to_be16((cq->parent->idx << 4) | cq->idx); ++ dcp_idx = cq->parent->dcp_idx; ++ if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER) ++ command_type = CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS; ++ else ++ command_type = CEETM_QUERY_DEQUEUE_STATISTICS; ++ ++ ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result); ++ if (ret) { ++ pr_err("Can't query the statistics of CQ#%d!\n", cq->idx); ++ return -EINVAL; ++ } ++ ++ *frame_count = be40_to_cpu(result.frm_cnt); ++ *byte_count = be48_to_cpu(result.byte_cnt); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cq_get_dequeue_statistics); ++ ++int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq) ++{ ++ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread ppxr; ++ int ret; ++ ++ do { ++ ret = qman_ceetm_cq_peek_pop_xsfdrread(cq, 1, 0, &ppxr); ++ if (ret) { ++ pr_err("Failed to pop frame from CQ\n"); ++ return -EINVAL; ++ } ++ } while (!(ppxr.stat & 0x2)); ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_drain_cq); ++ ++#define CEETM_LFQMT_LFQID_MSB 0xF00000 ++#define CEETM_LFQMT_LFQID_LSB 0x000FFF ++int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq, ++ struct qm_ceetm_cq *cq) ++{ ++ struct qm_ceetm_lfq *p; ++ u32 lfqid; ++ int ret = 0; ++ struct qm_mcc_ceetm_lfqmt_config lfqmt_config; ++ ++ if (cq->parent->dcp_idx == qm_dc_portal_fman0) { ++ ret = qman_alloc_ceetm0_lfqid(&lfqid); ++ } else if (cq->parent->dcp_idx == qm_dc_portal_fman1) { ++ ret = qman_alloc_ceetm1_lfqid(&lfqid); ++ } else { ++ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", ++ cq->parent->dcp_idx); ++ return -EINVAL; ++ } ++ ++ if (ret) { ++ pr_err("There is no lfqid avalaible for CQ#%d!\n", cq->idx); ++ return -ENODEV; ++ } ++ p = kmalloc(sizeof(*p), GFP_KERNEL); ++ if (!p) ++ return -ENOMEM; ++ p->idx = lfqid; ++ p->dctidx = (u16)(lfqid & CEETM_LFQMT_LFQID_LSB); ++ p->parent = cq->parent; ++ list_add_tail(&p->node, &cq->bound_lfqids); ++ ++ lfqmt_config.lfqid = cpu_to_be24(CEETM_LFQMT_LFQID_MSB | ++ (cq->parent->dcp_idx << 16) | ++ (lfqid & CEETM_LFQMT_LFQID_LSB)); ++ lfqmt_config.cqid = cpu_to_be16((cq->parent->idx << 4) | (cq->idx)); ++ lfqmt_config.dctidx = cpu_to_be16(p->dctidx); ++ if (qman_ceetm_configure_lfqmt(&lfqmt_config)) { ++ pr_err("Can't configure LFQMT for LFQID#%d @ CQ#%d\n", ++ lfqid, cq->idx); ++ list_del(&p->node); ++ kfree(p); ++ return -EINVAL; ++ } ++ *lfq = p; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_lfq_claim); ++ ++int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq) ++{ ++ if (lfq->parent->dcp_idx == qm_dc_portal_fman0) { ++ qman_release_ceetm0_lfqid(lfq->idx); ++ } else if (lfq->parent->dcp_idx == qm_dc_portal_fman1) { ++ qman_release_ceetm1_lfqid(lfq->idx); ++ } else { ++ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", ++ lfq->parent->dcp_idx); ++ return -EINVAL; ++ } ++ list_del(&lfq->node); ++ kfree(lfq); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_lfq_release); ++ ++int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, u64 context_a, ++ u32 context_b) ++{ ++ struct qm_mcc_ceetm_dct_config dct_config; ++ lfq->context_a = context_a; ++ lfq->context_b = context_b; ++ dct_config.dctidx = cpu_to_be16(lfq->dctidx); ++ dct_config.dcpid = lfq->parent->dcp_idx; ++ dct_config.context_b = cpu_to_be32(context_b); ++ dct_config.context_a = cpu_to_be64(context_a); ++ return qman_ceetm_configure_dct(&dct_config); ++} ++EXPORT_SYMBOL(qman_ceetm_lfq_set_context); ++ ++int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, u64 *context_a, ++ u32 *context_b) ++{ ++ struct qm_mcc_ceetm_dct_query dct_query; ++ struct qm_mcr_ceetm_dct_query query_result; ++ ++ dct_query.dctidx = cpu_to_be16(lfq->dctidx); ++ dct_query.dcpid = lfq->parent->dcp_idx; ++ if (qman_ceetm_query_dct(&dct_query, &query_result)) { ++ pr_err("Can't query LFQID#%d's context!\n", lfq->idx); ++ return -EINVAL; ++ } ++ *context_a = be64_to_cpu(query_result.context_a); ++ *context_b = be32_to_cpu(query_result.context_b); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_lfq_get_context); ++ ++int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq) ++{ ++ spin_lock_init(&fq->fqlock); ++ fq->fqid = lfq->idx; ++ fq->flags = QMAN_FQ_FLAG_NO_MODIFY; ++ if (lfq->ern) ++ fq->cb.ern = lfq->ern; ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) ++ return -ENOMEM; ++#endif ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_create_fq); ++ ++int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg, ++ struct qm_ceetm_channel *channel, ++ unsigned int idx, ++ void (*cscn)(struct qm_ceetm_ccg *, ++ void *cb_ctx, ++ int congested), ++ void *cb_ctx) ++{ ++ struct qm_ceetm_ccg *p; ++ ++ if (idx > 15) { ++ pr_err("The given ccg index is out of range\n"); ++ return -EINVAL; ++ } ++ ++ list_for_each_entry(p, &channel->ccgs, node) { ++ if (p->idx == idx) { ++ pr_err("The CCG#%d has been claimed\n", idx); ++ return -EINVAL; ++ } ++ } ++ ++ p = kmalloc(sizeof(*p), GFP_KERNEL); ++ if (!p) { ++ pr_err("Can't allocate memory for CCG#%d!\n", idx); ++ return -ENOMEM; ++ } ++ ++ list_add_tail(&p->node, &channel->ccgs); ++ ++ p->idx = idx; ++ p->parent = channel; ++ p->cb = cscn; ++ p->cb_ctx = cb_ctx; ++ INIT_LIST_HEAD(&p->cb_node); ++ ++ *ccg = p; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_ccg_claim); ++ ++int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg) ++{ ++ unsigned long irqflags __maybe_unused; ++ struct qm_mcc_ceetm_ccgr_config config_opts; ++ int ret = 0; ++ struct qman_portal *p = get_affine_portal(); ++ ++ memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config)); ++ spin_lock_irqsave(&p->ccgr_lock, irqflags); ++ if (!list_empty(&ccg->cb_node)) ++ list_del(&ccg->cb_node); ++ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE | ++ (ccg->parent->idx << 4) | ccg->idx); ++ config_opts.dcpid = ccg->parent->dcp_idx; ++ config_opts.we_mask = cpu_to_be16(QM_CCGR_WE_CSCN_TUPD); ++ config_opts.cm_config.cscn_tupd = cpu_to_be16(PORTAL_IDX(p)); ++ ret = qman_ceetm_configure_ccgr(&config_opts); ++ spin_unlock_irqrestore(&p->ccgr_lock, irqflags); ++ put_affine_portal(); ++ ++ list_del(&ccg->node); ++ kfree(ccg); ++ return ret; ++} ++EXPORT_SYMBOL(qman_ceetm_ccg_release); ++ ++int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, u16 we_mask, ++ const struct qm_ceetm_ccg_params *params) ++{ ++ struct qm_mcc_ceetm_ccgr_config config_opts; ++ unsigned long irqflags __maybe_unused; ++ int ret; ++ struct qman_portal *p; ++ ++ if (((ccg->parent->idx << 4) | ccg->idx) >= (2 * __CGR_NUM)) ++ return -EINVAL; ++ ++ p = get_affine_portal(); ++ ++ memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config)); ++ spin_lock_irqsave(&p->ccgr_lock, irqflags); ++ ++ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE | ++ (ccg->parent->idx << 4) | ccg->idx); ++ config_opts.dcpid = ccg->parent->dcp_idx; ++ config_opts.we_mask = we_mask; ++ if (we_mask & QM_CCGR_WE_CSCN_EN) { ++ config_opts.we_mask |= QM_CCGR_WE_CSCN_TUPD; ++ config_opts.cm_config.cscn_tupd = cpu_to_be16( ++ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p)); ++ } ++ config_opts.we_mask = cpu_to_be16(config_opts.we_mask); ++ config_opts.cm_config.ctl_wr_en_g = params->wr_en_g; ++ config_opts.cm_config.ctl_wr_en_y = params->wr_en_y; ++ config_opts.cm_config.ctl_wr_en_r = params->wr_en_r; ++ config_opts.cm_config.ctl_td_en = params->td_en; ++ config_opts.cm_config.ctl_td_mode = params->td_mode; ++ config_opts.cm_config.ctl_cscn_en = params->cscn_en; ++ config_opts.cm_config.ctl_mode = params->mode; ++ config_opts.cm_config.oal = params->oal; ++ config_opts.cm_config.cs_thres.hword = ++ cpu_to_be16(params->cs_thres_in.hword); ++ config_opts.cm_config.cs_thres_x.hword = ++ cpu_to_be16(params->cs_thres_out.hword); ++ config_opts.cm_config.td_thres.hword = ++ cpu_to_be16(params->td_thres.hword); ++ config_opts.cm_config.wr_parm_g.word = ++ cpu_to_be32(params->wr_parm_g.word); ++ config_opts.cm_config.wr_parm_y.word = ++ cpu_to_be32(params->wr_parm_y.word); ++ config_opts.cm_config.wr_parm_r.word = ++ cpu_to_be32(params->wr_parm_r.word); ++ ret = qman_ceetm_configure_ccgr(&config_opts); ++ if (ret) { ++ pr_err("Configure CCGR CM failed!\n"); ++ goto release_lock; ++ } ++ ++ if (we_mask & QM_CCGR_WE_CSCN_EN) ++ if (list_empty(&ccg->cb_node)) ++ list_add(&ccg->cb_node, ++ &p->ccgr_cbs[ccg->parent->dcp_idx]); ++release_lock: ++ spin_unlock_irqrestore(&p->ccgr_lock, irqflags); ++ put_affine_portal(); ++ return ret; ++} ++EXPORT_SYMBOL(qman_ceetm_ccg_set); ++ ++#define CEETM_CCGR_CTL_MASK 0x01 ++int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg, ++ struct qm_ceetm_ccg_params *params) ++{ ++ struct qm_mcc_ceetm_ccgr_query query_opts; ++ struct qm_mcr_ceetm_ccgr_query query_result; ++ ++ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY | ++ (ccg->parent->idx << 4) | ccg->idx); ++ query_opts.dcpid = ccg->parent->dcp_idx; ++ ++ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) { ++ pr_err("Can't query CCGR#%d\n", ccg->idx); ++ return -EINVAL; ++ } ++ ++ params->wr_parm_r.word = ++ be32_to_cpu(query_result.cm_query.wr_parm_r.word); ++ params->wr_parm_y.word = ++ be32_to_cpu(query_result.cm_query.wr_parm_y.word); ++ params->wr_parm_g.word = ++ be32_to_cpu(query_result.cm_query.wr_parm_g.word); ++ params->td_thres.hword = ++ be16_to_cpu(query_result.cm_query.td_thres.hword); ++ params->cs_thres_out.hword = ++ be16_to_cpu(query_result.cm_query.cs_thres_x.hword); ++ params->cs_thres_in.hword = ++ be16_to_cpu(query_result.cm_query.cs_thres.hword); ++ params->oal = query_result.cm_query.oal; ++ params->wr_en_g = query_result.cm_query.ctl_wr_en_g; ++ params->wr_en_y = query_result.cm_query.ctl_wr_en_y; ++ params->wr_en_r = query_result.cm_query.ctl_wr_en_r; ++ params->td_en = query_result.cm_query.ctl_td_en; ++ params->td_mode = query_result.cm_query.ctl_td_mode; ++ params->cscn_en = query_result.cm_query.ctl_cscn_en; ++ params->mode = query_result.cm_query.ctl_mode; ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_ccg_get); ++ ++int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags, ++ u64 *frame_count, u64 *byte_count) ++{ ++ struct qm_mcr_ceetm_statistics_query result; ++ u16 cid, command_type; ++ enum qm_dc_portal dcp_idx; ++ int ret; ++ ++ cid = cpu_to_be16((ccg->parent->idx << 4) | ccg->idx); ++ dcp_idx = ccg->parent->dcp_idx; ++ if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER) ++ command_type = CEETM_QUERY_REJECT_CLEAR_STATISTICS; ++ else ++ command_type = CEETM_QUERY_REJECT_STATISTICS; ++ ++ ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result); ++ if (ret) { ++ pr_err("Can't query the statistics of CCG#%d!\n", ccg->idx); ++ return -EINVAL; ++ } ++ ++ *frame_count = be40_to_cpu(result.frm_cnt); ++ *byte_count = be48_to_cpu(result.byte_cnt); ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_ccg_get_reject_statistics); ++ ++int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg, ++ u16 swp_idx, ++ unsigned int *cscn_enabled) ++{ ++ struct qm_mcc_ceetm_ccgr_query query_opts; ++ struct qm_mcr_ceetm_ccgr_query query_result; ++ int i; ++ ++ DPA_ASSERT(swp_idx < 127); ++ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY | ++ (ccg->parent->idx << 4) | ccg->idx); ++ query_opts.dcpid = ccg->parent->dcp_idx; ++ ++ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) { ++ pr_err("Can't query CCGR#%d\n", ccg->idx); ++ return -EINVAL; ++ } ++ ++ i = swp_idx / 32; ++ i = 3 - i; ++ *cscn_enabled = be32_to_cpu(query_result.cm_query.cscn_targ_swp[i]) >> ++ (31 - swp_idx % 32); ++ ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cscn_swp_get); ++ ++int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg, ++ u16 dcp_idx, ++ u8 vcgid, ++ unsigned int cscn_enabled, ++ u16 we_mask, ++ const struct qm_ceetm_ccg_params *params) ++{ ++ struct qm_mcc_ceetm_ccgr_config config_opts; ++ int ret; ++ ++ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE | ++ (ccg->parent->idx << 4) | ccg->idx); ++ config_opts.dcpid = ccg->parent->dcp_idx; ++ config_opts.we_mask = cpu_to_be16(we_mask | QM_CCGR_WE_CSCN_TUPD | ++ QM_CCGR_WE_CDV); ++ config_opts.cm_config.cdv = vcgid; ++ config_opts.cm_config.cscn_tupd = cpu_to_be16((cscn_enabled << 15) | ++ QM_CGR_TARG_UDP_CTRL_DCP | dcp_idx); ++ config_opts.cm_config.ctl_wr_en_g = params->wr_en_g; ++ config_opts.cm_config.ctl_wr_en_y = params->wr_en_y; ++ config_opts.cm_config.ctl_wr_en_r = params->wr_en_r; ++ config_opts.cm_config.ctl_td_en = params->td_en; ++ config_opts.cm_config.ctl_td_mode = params->td_mode; ++ config_opts.cm_config.ctl_cscn_en = params->cscn_en; ++ config_opts.cm_config.ctl_mode = params->mode; ++ config_opts.cm_config.cs_thres.hword = ++ cpu_to_be16(params->cs_thres_in.hword); ++ config_opts.cm_config.cs_thres_x.hword = ++ cpu_to_be16(params->cs_thres_out.hword); ++ config_opts.cm_config.td_thres.hword = ++ cpu_to_be16(params->td_thres.hword); ++ config_opts.cm_config.wr_parm_g.word = ++ cpu_to_be32(params->wr_parm_g.word); ++ config_opts.cm_config.wr_parm_y.word = ++ cpu_to_be32(params->wr_parm_y.word); ++ config_opts.cm_config.wr_parm_r.word = ++ cpu_to_be32(params->wr_parm_r.word); ++ ++ ret = qman_ceetm_configure_ccgr(&config_opts); ++ if (ret) { ++ pr_err("Configure CSCN_TARG_DCP failed!\n"); ++ return -EINVAL; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cscn_dcp_set); ++ ++int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg, ++ u16 dcp_idx, ++ u8 *vcgid, ++ unsigned int *cscn_enabled) ++{ ++ struct qm_mcc_ceetm_ccgr_query query_opts; ++ struct qm_mcr_ceetm_ccgr_query query_result; ++ ++ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY | ++ (ccg->parent->idx << 4) | ccg->idx); ++ query_opts.dcpid = ccg->parent->dcp_idx; ++ ++ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) { ++ pr_err("Can't query CCGR#%d\n", ccg->idx); ++ return -EINVAL; ++ } ++ ++ *vcgid = query_result.cm_query.cdv; ++ *cscn_enabled = (be16_to_cpu(query_result.cm_query.cscn_targ_dcp >> ++ dcp_idx)) & 0x1; ++ return 0; ++} ++EXPORT_SYMBOL(qman_ceetm_cscn_dcp_get); ++ ++int qman_ceetm_querycongestion(struct __qm_mcr_querycongestion *ccg_state, ++ unsigned int dcp_idx) ++{ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ u8 res; ++ int i, j; ++ ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ ++ mcc = qm_mc_start(&p->p); ++ for (i = 0; i < 2; i++) { ++ mcc->ccgr_query.ccgrid = ++ cpu_to_be16(CEETM_QUERY_CONGESTION_STATE | i); ++ mcc->ccgr_query.dcpid = dcp_idx; ++ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY); ++ ++ while (!(mcr = qm_mc_result(&p->p))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_CEETM_VERB_CCGR_QUERY); ++ res = mcr->result; ++ if (res == QM_MCR_RESULT_OK) { ++ for (j = 0; j < 8; j++) ++ mcr->ccgr_query.congestion_state.state. ++ __state[j] = ++ be32_to_cpu(mcr->ccgr_query. ++ congestion_state.state.__state[j]); ++ ++ *(ccg_state + i) = ++ mcr->ccgr_query.congestion_state.state; ++ } else { ++ pr_err("QUERY CEETM CONGESTION STATE failed\n"); ++ return -EIO; ++ } ++ } ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return 0; ++} ++ ++int qman_set_wpm(int wpm_enable) ++{ ++ return qm_set_wpm(wpm_enable); ++} ++EXPORT_SYMBOL(qman_set_wpm); ++ ++int qman_get_wpm(int *wpm_enable) ++{ ++ return qm_get_wpm(wpm_enable); ++} ++EXPORT_SYMBOL(qman_get_wpm); ++ ++int qman_shutdown_fq(u32 fqid) ++{ ++ struct qman_portal *p; ++ unsigned long irqflags __maybe_unused; ++ int ret; ++ struct qm_portal *low_p; ++ p = get_affine_portal(); ++ PORTAL_IRQ_LOCK(p, irqflags); ++ low_p = &p->p; ++ ret = qm_shutdown_fq(&low_p, 1, fqid); ++ PORTAL_IRQ_UNLOCK(p, irqflags); ++ put_affine_portal(); ++ return ret; ++} ++ ++const struct qm_portal_config *qman_get_qm_portal_config( ++ struct qman_portal *portal) ++{ ++ return portal->sharing_redirect ? NULL : portal->config; ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_low.h +@@ -0,0 +1,1407 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "qman_private.h" ++ ++/***************************/ ++/* Portal register assists */ ++/***************************/ ++ ++/* Cache-inhibited register offsets */ ++#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) ++ ++#define QM_REG_EQCR_PI_CINH 0x0000 ++#define QM_REG_EQCR_CI_CINH 0x0004 ++#define QM_REG_EQCR_ITR 0x0008 ++#define QM_REG_DQRR_PI_CINH 0x0040 ++#define QM_REG_DQRR_CI_CINH 0x0044 ++#define QM_REG_DQRR_ITR 0x0048 ++#define QM_REG_DQRR_DCAP 0x0050 ++#define QM_REG_DQRR_SDQCR 0x0054 ++#define QM_REG_DQRR_VDQCR 0x0058 ++#define QM_REG_DQRR_PDQCR 0x005c ++#define QM_REG_MR_PI_CINH 0x0080 ++#define QM_REG_MR_CI_CINH 0x0084 ++#define QM_REG_MR_ITR 0x0088 ++#define QM_REG_CFG 0x0100 ++#define QM_REG_ISR 0x0e00 ++#define QM_REG_IIR 0x0e0c ++#define QM_REG_ITPR 0x0e14 ++ ++/* Cache-enabled register offsets */ ++#define QM_CL_EQCR 0x0000 ++#define QM_CL_DQRR 0x1000 ++#define QM_CL_MR 0x2000 ++#define QM_CL_EQCR_PI_CENA 0x3000 ++#define QM_CL_EQCR_CI_CENA 0x3100 ++#define QM_CL_DQRR_PI_CENA 0x3200 ++#define QM_CL_DQRR_CI_CENA 0x3300 ++#define QM_CL_MR_PI_CENA 0x3400 ++#define QM_CL_MR_CI_CENA 0x3500 ++#define QM_CL_CR 0x3800 ++#define QM_CL_RR0 0x3900 ++#define QM_CL_RR1 0x3940 ++ ++#endif ++ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ ++#define QM_REG_EQCR_PI_CINH 0x3000 ++#define QM_REG_EQCR_CI_CINH 0x3040 ++#define QM_REG_EQCR_ITR 0x3080 ++#define QM_REG_DQRR_PI_CINH 0x3100 ++#define QM_REG_DQRR_CI_CINH 0x3140 ++#define QM_REG_DQRR_ITR 0x3180 ++#define QM_REG_DQRR_DCAP 0x31C0 ++#define QM_REG_DQRR_SDQCR 0x3200 ++#define QM_REG_DQRR_VDQCR 0x3240 ++#define QM_REG_DQRR_PDQCR 0x3280 ++#define QM_REG_MR_PI_CINH 0x3300 ++#define QM_REG_MR_CI_CINH 0x3340 ++#define QM_REG_MR_ITR 0x3380 ++#define QM_REG_CFG 0x3500 ++#define QM_REG_ISR 0x3600 ++#define QM_REG_IIR 0x36C0 ++#define QM_REG_ITPR 0x3740 ++ ++/* Cache-enabled register offsets */ ++#define QM_CL_EQCR 0x0000 ++#define QM_CL_DQRR 0x1000 ++#define QM_CL_MR 0x2000 ++#define QM_CL_EQCR_PI_CENA 0x3000 ++#define QM_CL_EQCR_CI_CENA 0x3040 ++#define QM_CL_DQRR_PI_CENA 0x3100 ++#define QM_CL_DQRR_CI_CENA 0x3140 ++#define QM_CL_MR_PI_CENA 0x3300 ++#define QM_CL_MR_CI_CENA 0x3340 ++#define QM_CL_CR 0x3800 ++#define QM_CL_RR0 0x3900 ++#define QM_CL_RR1 0x3940 ++ ++#endif ++ ++ ++/* BTW, the drivers (and h/w programming model) already obtain the required ++ * synchronisation for portal accesses via lwsync(), hwsync(), and ++ * data-dependencies. Use of barrier()s or other order-preserving primitives ++ * simply degrade performance. Hence the use of the __raw_*() interfaces, which ++ * simply ensure that the compiler treats the portal registers as volatile (ie. ++ * non-coherent). */ ++ ++/* Cache-inhibited register access. */ ++#define __qm_in(qm, o) be32_to_cpu(__raw_readl((qm)->addr_ci + (o))) ++#define __qm_out(qm, o, val) __raw_writel((cpu_to_be32(val)), \ ++ (qm)->addr_ci + (o)); ++#define qm_in(reg) __qm_in(&portal->addr, QM_REG_##reg) ++#define qm_out(reg, val) __qm_out(&portal->addr, QM_REG_##reg, val) ++ ++/* Cache-enabled (index) register access */ ++#define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->addr_ce + (o)) ++#define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->addr_ce + (o)) ++#define __qm_cl_in(qm, o) be32_to_cpu(__raw_readl((qm)->addr_ce + (o))) ++#define __qm_cl_out(qm, o, val) \ ++ do { \ ++ u32 *__tmpclout = (qm)->addr_ce + (o); \ ++ __raw_writel(cpu_to_be32(val), __tmpclout); \ ++ dcbf(__tmpclout); \ ++ } while (0) ++#define __qm_cl_invalidate(qm, o) dcbi((qm)->addr_ce + (o)) ++#define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA) ++#define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA) ++#define qm_cl_in(reg) __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA) ++#define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val) ++#define qm_cl_invalidate(reg)\ ++ __qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA) ++ ++/* Cache-enabled ring access */ ++#define qm_cl(base, idx) ((void *)base + ((idx) << 6)) ++ ++/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf ++ * analysis, look at using the "extra" bit in the ring index registers to avoid ++ * cyclic issues. */ ++static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last) ++{ ++ /* 'first' is included, 'last' is excluded */ ++ if (first <= last) ++ return last - first; ++ return ringsize + last - first; ++} ++ ++/* Portal modes. ++ * Enum types; ++ * pmode == production mode ++ * cmode == consumption mode, ++ * dmode == h/w dequeue mode. ++ * Enum values use 3 letter codes. First letter matches the portal mode, ++ * remaining two letters indicate; ++ * ci == cache-inhibited portal register ++ * ce == cache-enabled portal register ++ * vb == in-band valid-bit (cache-enabled) ++ * dc == DCA (Discrete Consumption Acknowledgement), DQRR-only ++ * As for "enum qm_dqrr_dmode", it should be self-explanatory. ++ */ ++enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */ ++ qm_eqcr_pci = 0, /* PI index, cache-inhibited */ ++ qm_eqcr_pce = 1, /* PI index, cache-enabled */ ++ qm_eqcr_pvb = 2 /* valid-bit */ ++}; ++enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */ ++ qm_dqrr_dpush = 0, /* SDQCR + VDQCR */ ++ qm_dqrr_dpull = 1 /* PDQCR */ ++}; ++enum qm_dqrr_pmode { /* s/w-only */ ++ qm_dqrr_pci, /* reads DQRR_PI_CINH */ ++ qm_dqrr_pce, /* reads DQRR_PI_CENA */ ++ qm_dqrr_pvb /* reads valid-bit */ ++}; ++enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */ ++ qm_dqrr_cci = 0, /* CI index, cache-inhibited */ ++ qm_dqrr_cce = 1, /* CI index, cache-enabled */ ++ qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgement */ ++}; ++enum qm_mr_pmode { /* s/w-only */ ++ qm_mr_pci, /* reads MR_PI_CINH */ ++ qm_mr_pce, /* reads MR_PI_CENA */ ++ qm_mr_pvb /* reads valid-bit */ ++}; ++enum qm_mr_cmode { /* matches QCSP_CFG::MM */ ++ qm_mr_cci = 0, /* CI index, cache-inhibited */ ++ qm_mr_cce = 1 /* CI index, cache-enabled */ ++}; ++ ++ ++/* ------------------------- */ ++/* --- Portal structures --- */ ++ ++#define QM_EQCR_SIZE 8 ++#define QM_DQRR_SIZE 16 ++#define QM_MR_SIZE 8 ++ ++struct qm_eqcr { ++ struct qm_eqcr_entry *ring, *cursor; ++ u8 ci, available, ithresh, vbit; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ u32 busy; ++ enum qm_eqcr_pmode pmode; ++#endif ++}; ++ ++struct qm_dqrr { ++ const struct qm_dqrr_entry *ring, *cursor; ++ u8 pi, ci, fill, ithresh, vbit; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ enum qm_dqrr_dmode dmode; ++ enum qm_dqrr_pmode pmode; ++ enum qm_dqrr_cmode cmode; ++#endif ++}; ++ ++struct qm_mr { ++ const struct qm_mr_entry *ring, *cursor; ++ u8 pi, ci, fill, ithresh, vbit; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ enum qm_mr_pmode pmode; ++ enum qm_mr_cmode cmode; ++#endif ++}; ++ ++struct qm_mc { ++ struct qm_mc_command *cr; ++ struct qm_mc_result *rr; ++ u8 rridx, vbit; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ enum { ++ /* Can be _mc_start()ed */ ++ qman_mc_idle, ++ /* Can be _mc_commit()ed or _mc_abort()ed */ ++ qman_mc_user, ++ /* Can only be _mc_retry()ed */ ++ qman_mc_hw ++ } state; ++#endif ++}; ++ ++#define QM_PORTAL_ALIGNMENT ____cacheline_aligned ++ ++struct qm_addr { ++ void __iomem *addr_ce; /* cache-enabled */ ++ void __iomem *addr_ci; /* cache-inhibited */ ++}; ++ ++struct qm_portal { ++ /* In the non-CONFIG_FSL_DPA_CHECKING case, the following stuff up to ++ * and including 'mc' fits within a cacheline (yay!). The 'config' part ++ * is setup-only, so isn't a cause for a concern. In other words, don't ++ * rearrange this structure on a whim, there be dragons ... */ ++ struct qm_addr addr; ++ struct qm_eqcr eqcr; ++ struct qm_dqrr dqrr; ++ struct qm_mr mr; ++ struct qm_mc mc; ++} QM_PORTAL_ALIGNMENT; ++ ++ ++/* ---------------- */ ++/* --- EQCR API --- */ ++ ++/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ ++#define EQCR_CARRYCLEAR(p) \ ++ (void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6))) ++ ++/* Bit-wise logic to convert a ring pointer to a ring index */ ++static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e) ++{ ++ return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1); ++} ++ ++/* Increment the 'cursor' ring pointer, taking 'vbit' into account */ ++static inline void EQCR_INC(struct qm_eqcr *eqcr) ++{ ++ /* NB: this is odd-looking, but experiments show that it generates fast ++ * code with essentially no branching overheads. We increment to the ++ * next EQCR pointer and handle overflow and 'vbit'. */ ++ struct qm_eqcr_entry *partial = eqcr->cursor + 1; ++ eqcr->cursor = EQCR_CARRYCLEAR(partial); ++ if (partial != eqcr->cursor) ++ eqcr->vbit ^= QM_EQCR_VERB_VBIT; ++} ++ ++static inline int qm_eqcr_init(struct qm_portal *portal, ++ enum qm_eqcr_pmode pmode, ++ unsigned int eq_stash_thresh, ++ int eq_stash_prio) ++{ ++ /* This use of 'register', as well as all other occurrences, is because ++ * it has been observed to generate much faster code with gcc than is ++ * otherwise the case. */ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ u32 cfg; ++ u8 pi; ++ ++ eqcr->ring = portal->addr.addr_ce + QM_CL_EQCR; ++ eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); ++ qm_cl_invalidate(EQCR_CI); ++ pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); ++ eqcr->cursor = eqcr->ring + pi; ++ eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? ++ QM_EQCR_VERB_VBIT : 0; ++ eqcr->available = QM_EQCR_SIZE - 1 - ++ qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); ++ eqcr->ithresh = qm_in(EQCR_ITR); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ eqcr->busy = 0; ++ eqcr->pmode = pmode; ++#endif ++ cfg = (qm_in(CFG) & 0x00ffffff) | ++ (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ ++ (eq_stash_prio << 26) | /* QCSP_CFG: EP */ ++ ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ ++ qm_out(CFG, cfg); ++ return 0; ++} ++ ++static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal) ++{ ++ return (qm_in(CFG) >> 28) & 0x7; ++} ++ ++static inline void qm_eqcr_finish(struct qm_portal *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ u8 pi, ci; ++ u32 cfg; ++ ++ /* ++ * Disable EQCI stashing because the QMan only ++ * presents the value it previously stashed to ++ * maintain coherency. Setting the stash threshold ++ * to 1 then 0 ensures that QMan has resyncronized ++ * its internal copy so that the portal is clean ++ * when it is reinitialized in the future ++ */ ++ cfg = (qm_in(CFG) & 0x0fffffff) | ++ (1 << 28); /* QCSP_CFG: EST */ ++ qm_out(CFG, cfg); ++ cfg &= 0x0fffffff; /* stash threshold = 0 */ ++ qm_out(CFG, cfg); ++ ++ pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); ++ ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); ++ ++ /* Refresh EQCR CI cache value */ ++ qm_cl_invalidate(EQCR_CI); ++ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); ++ ++ DPA_ASSERT(!eqcr->busy); ++ if (pi != EQCR_PTR2IDX(eqcr->cursor)) ++ pr_crit("losing uncommited EQCR entries\n"); ++ if (ci != eqcr->ci) ++ pr_crit("missing existing EQCR completions\n"); ++ if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) ++ pr_crit("EQCR destroyed unquiesced\n"); ++} ++ ++static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal ++ *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ DPA_ASSERT(!eqcr->busy); ++ if (!eqcr->available) ++ return NULL; ++ ++ ++#ifdef CONFIG_FSL_DPA_CHECKING ++ eqcr->busy = 1; ++#endif ++ dcbz_64(eqcr->cursor); ++ return eqcr->cursor; ++} ++ ++static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal ++ *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ u8 diff, old_ci; ++ ++ DPA_ASSERT(!eqcr->busy); ++ if (!eqcr->available) { ++ old_ci = eqcr->ci; ++ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); ++ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); ++ eqcr->available += diff; ++ if (!diff) ++ return NULL; ++ } ++#ifdef CONFIG_FSL_DPA_CHECKING ++ eqcr->busy = 1; ++#endif ++ dcbz_64(eqcr->cursor); ++ return eqcr->cursor; ++} ++ ++static inline void qm_eqcr_abort(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; ++ DPA_ASSERT(eqcr->busy); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ eqcr->busy = 0; ++#endif ++} ++ ++static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next( ++ struct qm_portal *portal, u8 myverb) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ DPA_ASSERT(eqcr->busy); ++ DPA_ASSERT(eqcr->pmode != qm_eqcr_pvb); ++ if (eqcr->available == 1) ++ return NULL; ++ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; ++ dcbf(eqcr->cursor); ++ EQCR_INC(eqcr); ++ eqcr->available--; ++ dcbz_64(eqcr->cursor); ++ return eqcr->cursor; ++} ++ ++#define EQCR_COMMIT_CHECKS(eqcr) \ ++do { \ ++ DPA_ASSERT(eqcr->busy); \ ++ DPA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \ ++ DPA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \ ++} while (0) ++ ++static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ EQCR_COMMIT_CHECKS(eqcr); ++ DPA_ASSERT(eqcr->pmode == qm_eqcr_pci); ++ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; ++ EQCR_INC(eqcr); ++ eqcr->available--; ++ dcbf(eqcr->cursor); ++ hwsync(); ++ qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor)); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ eqcr->busy = 0; ++#endif ++} ++ ++static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; ++ DPA_ASSERT(eqcr->pmode == qm_eqcr_pce); ++ qm_cl_invalidate(EQCR_PI); ++ qm_cl_touch_rw(EQCR_PI); ++} ++ ++static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ EQCR_COMMIT_CHECKS(eqcr); ++ DPA_ASSERT(eqcr->pmode == qm_eqcr_pce); ++ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; ++ EQCR_INC(eqcr); ++ eqcr->available--; ++ dcbf(eqcr->cursor); ++ lwsync(); ++ qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor)); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ eqcr->busy = 0; ++#endif ++} ++ ++static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ struct qm_eqcr_entry *eqcursor; ++ EQCR_COMMIT_CHECKS(eqcr); ++ DPA_ASSERT(eqcr->pmode == qm_eqcr_pvb); ++ lwsync(); ++ eqcursor = eqcr->cursor; ++ eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit; ++ dcbf(eqcursor); ++ EQCR_INC(eqcr); ++ eqcr->available--; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ eqcr->busy = 0; ++#endif ++} ++ ++static inline u8 qm_eqcr_cci_update(struct qm_portal *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ u8 diff, old_ci = eqcr->ci; ++ eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); ++ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); ++ eqcr->available += diff; ++ return diff; ++} ++ ++static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; ++ qm_cl_touch_ro(EQCR_CI); ++} ++ ++static inline u8 qm_eqcr_cce_update(struct qm_portal *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ u8 diff, old_ci = eqcr->ci; ++ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); ++ qm_cl_invalidate(EQCR_CI); ++ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); ++ eqcr->available += diff; ++ return diff; ++} ++ ++static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ return eqcr->ithresh; ++} ++ ++static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ eqcr->ithresh = ithresh; ++ qm_out(EQCR_ITR, ithresh); ++} ++ ++static inline u8 qm_eqcr_get_avail(struct qm_portal *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ return eqcr->available; ++} ++ ++static inline u8 qm_eqcr_get_fill(struct qm_portal *portal) ++{ ++ register struct qm_eqcr *eqcr = &portal->eqcr; ++ return QM_EQCR_SIZE - 1 - eqcr->available; ++} ++ ++ ++/* ---------------- */ ++/* --- DQRR API --- */ ++ ++/* FIXME: many possible improvements; ++ * - look at changing the API to use pointer rather than index parameters now ++ * that 'cursor' is a pointer, ++ * - consider moving other parameters to pointer if it could help (ci) ++ */ ++ ++#define DQRR_CARRYCLEAR(p) \ ++ (void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6))) ++ ++static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e) ++{ ++ return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1); ++} ++ ++static inline const struct qm_dqrr_entry *DQRR_INC( ++ const struct qm_dqrr_entry *e) ++{ ++ return DQRR_CARRYCLEAR(e + 1); ++} ++ ++static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf) ++{ ++ qm_out(CFG, (qm_in(CFG) & 0xff0fffff) | ++ ((mf & (QM_DQRR_SIZE - 1)) << 20)); ++} ++ ++static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cci); ++ dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1); ++ qm_out(DQRR_CI_CINH, dqrr->ci); ++} ++ ++static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce); ++ dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1); ++ qm_cl_out(DQRR_CI, dqrr->ci); ++} ++ ++static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); ++ qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */ ++ ((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */ ++ dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); ++ dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); ++} ++ ++static inline int qm_dqrr_init(struct qm_portal *portal, ++ const struct qm_portal_config *config, ++ enum qm_dqrr_dmode dmode, ++ __maybe_unused enum qm_dqrr_pmode pmode, ++ enum qm_dqrr_cmode cmode, u8 max_fill) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ u32 cfg; ++ ++ /* Make sure the DQRR will be idle when we enable */ ++ qm_out(DQRR_SDQCR, 0); ++ qm_out(DQRR_VDQCR, 0); ++ qm_out(DQRR_PDQCR, 0); ++ dqrr->ring = portal->addr.addr_ce + QM_CL_DQRR; ++ dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); ++ dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); ++ dqrr->cursor = dqrr->ring + dqrr->ci; ++ dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); ++ dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? ++ QM_DQRR_VERB_VBIT : 0; ++ dqrr->ithresh = qm_in(DQRR_ITR); ++ ++ /* Free up pending DQRR entries if any as per current DCM */ ++ if (dqrr->fill) { ++ enum qm_dqrr_cmode dcm = (qm_in(CFG) >> 16) & 3; ++ ++#ifdef CONFIG_FSL_DPA_CHECKING ++ dqrr->cmode = dcm; ++#endif ++ switch (dcm) { ++ case qm_dqrr_cci: ++ qm_dqrr_cci_consume(portal, dqrr->fill); ++ break; ++ case qm_dqrr_cce: ++ qm_dqrr_cce_consume(portal, dqrr->fill); ++ break; ++ case qm_dqrr_cdc: ++ qm_dqrr_cdc_consume_n(portal, (QM_DQRR_SIZE - 1)); ++ break; ++ default: ++ DPA_ASSERT(0); ++ } ++ } ++ ++#ifdef CONFIG_FSL_DPA_CHECKING ++ dqrr->dmode = dmode; ++ dqrr->pmode = pmode; ++ dqrr->cmode = cmode; ++#endif ++ /* Invalidate every ring entry before beginning */ ++ for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) ++ dcbi(qm_cl(dqrr->ring, cfg)); ++ cfg = (qm_in(CFG) & 0xff000f00) | ++ ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ ++ ((dmode & 1) << 18) | /* DP */ ++ ((cmode & 3) << 16) | /* DCM */ ++ 0xa0 | /* RE+SE */ ++ (0 ? 0x40 : 0) | /* Ignore RP */ ++ (0 ? 0x10 : 0); /* Ignore SP */ ++ qm_out(CFG, cfg); ++ qm_dqrr_set_maxfill(portal, max_fill); ++ return 0; ++} ++ ++static inline void qm_dqrr_finish(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if ((dqrr->cmode != qm_dqrr_cdc) && ++ (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) ++ pr_crit("Ignoring completed DQRR entries\n"); ++#endif ++} ++ ++static inline const struct qm_dqrr_entry *qm_dqrr_current( ++ struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ if (!dqrr->fill) ++ return NULL; ++ return dqrr->cursor; ++} ++ ++static inline u8 qm_dqrr_cursor(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ return DQRR_PTR2IDX(dqrr->cursor); ++} ++ ++static inline u8 qm_dqrr_next(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->fill); ++ dqrr->cursor = DQRR_INC(dqrr->cursor); ++ return --dqrr->fill; ++} ++ ++static inline u8 qm_dqrr_pci_update(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ u8 diff, old_pi = dqrr->pi; ++ DPA_ASSERT(dqrr->pmode == qm_dqrr_pci); ++ dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); ++ diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); ++ dqrr->fill += diff; ++ return diff; ++} ++ ++static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->pmode == qm_dqrr_pce); ++ qm_cl_invalidate(DQRR_PI); ++ qm_cl_touch_ro(DQRR_PI); ++} ++ ++static inline u8 qm_dqrr_pce_update(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ u8 diff, old_pi = dqrr->pi; ++ DPA_ASSERT(dqrr->pmode == qm_dqrr_pce); ++ dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1); ++ diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); ++ dqrr->fill += diff; ++ return diff; ++} ++ ++static inline void qm_dqrr_pvb_update(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi); ++ DPA_ASSERT(dqrr->pmode == qm_dqrr_pvb); ++#ifndef CONFIG_FSL_PAMU ++ /* ++ * If PAMU is not available we need to invalidate the cache. ++ * When PAMU is available the cache is updated by stash ++ */ ++ dcbi(res); ++ dcbt_ro(res); ++#endif ++ ++ /* when accessing 'verb', use __raw_readb() to ensure that compiler ++ * inlining doesn't try to optimise out "excess reads". */ ++ if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) { ++ dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1); ++ if (!dqrr->pi) ++ dqrr->vbit ^= QM_DQRR_VERB_VBIT; ++ dqrr->fill++; ++ } ++} ++ ++ ++static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cci); ++ dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); ++ qm_out(DQRR_CI_CINH, dqrr->ci); ++} ++ ++static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce); ++ qm_cl_invalidate(DQRR_CI); ++ qm_cl_touch_rw(DQRR_CI); ++} ++ ++static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce); ++ dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); ++ qm_cl_out(DQRR_CI, dqrr->ci); ++} ++ ++static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx, ++ int park) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); ++ DPA_ASSERT(idx < QM_DQRR_SIZE); ++ qm_out(DQRR_DCAP, (0 << 8) | /* S */ ++ ((park ? 1 : 0) << 6) | /* PK */ ++ idx); /* DCAP_CI */ ++} ++ ++static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal, ++ const struct qm_dqrr_entry *dq, ++ int park) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ u8 idx = DQRR_PTR2IDX(dq); ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); ++ DPA_ASSERT((dqrr->ring + idx) == dq); ++ DPA_ASSERT(idx < QM_DQRR_SIZE); ++ qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */ ++ ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */ ++ idx); /* DQRR_DCAP::DCAP_CI */ ++} ++ ++static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); ++ return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); ++} ++ ++static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); ++ qm_cl_invalidate(DQRR_CI); ++ qm_cl_touch_ro(DQRR_CI); ++} ++ ++static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); ++ return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1); ++} ++ ++static inline u8 qm_dqrr_get_ci(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc); ++ return dqrr->ci; ++} ++ ++static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx) ++{ ++ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc); ++ qm_out(DQRR_DCAP, (0 << 8) | /* S */ ++ (1 << 6) | /* PK */ ++ (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */ ++} ++ ++static inline void qm_dqrr_park_current(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc); ++ qm_out(DQRR_DCAP, (0 << 8) | /* S */ ++ (1 << 6) | /* PK */ ++ DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */ ++} ++ ++static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr) ++{ ++ qm_out(DQRR_SDQCR, sdqcr); ++} ++ ++static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal) ++{ ++ return qm_in(DQRR_SDQCR); ++} ++ ++static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr) ++{ ++ qm_out(DQRR_VDQCR, vdqcr); ++} ++ ++static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal) ++{ ++ return qm_in(DQRR_VDQCR); ++} ++ ++static inline void qm_dqrr_pdqcr_set(struct qm_portal *portal, u32 pdqcr) ++{ ++ qm_out(DQRR_PDQCR, pdqcr); ++} ++ ++static inline u32 qm_dqrr_pdqcr_get(struct qm_portal *portal) ++{ ++ return qm_in(DQRR_PDQCR); ++} ++ ++static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal) ++{ ++ register struct qm_dqrr *dqrr = &portal->dqrr; ++ return dqrr->ithresh; ++} ++ ++static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh) ++{ ++ qm_out(DQRR_ITR, ithresh); ++} ++ ++static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal) ++{ ++ return (qm_in(CFG) & 0x00f00000) >> 20; ++} ++ ++ ++/* -------------- */ ++/* --- MR API --- */ ++ ++#define MR_CARRYCLEAR(p) \ ++ (void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6))) ++ ++static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e) ++{ ++ return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1); ++} ++ ++static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e) ++{ ++ return MR_CARRYCLEAR(e + 1); ++} ++ ++static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode, ++ enum qm_mr_cmode cmode) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ u32 cfg; ++ ++ mr->ring = portal->addr.addr_ce + QM_CL_MR; ++ mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); ++ mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); ++ mr->cursor = mr->ring + mr->ci; ++ mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); ++ mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; ++ mr->ithresh = qm_in(MR_ITR); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mr->pmode = pmode; ++ mr->cmode = cmode; ++#endif ++ cfg = (qm_in(CFG) & 0xfffff0ff) | ++ ((cmode & 1) << 8); /* QCSP_CFG:MM */ ++ qm_out(CFG, cfg); ++ return 0; ++} ++ ++static inline void qm_mr_finish(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ if (mr->ci != MR_PTR2IDX(mr->cursor)) ++ pr_crit("Ignoring completed MR entries\n"); ++} ++ ++static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ if (!mr->fill) ++ return NULL; ++ return mr->cursor; ++} ++ ++static inline u8 qm_mr_cursor(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ return MR_PTR2IDX(mr->cursor); ++} ++ ++static inline u8 qm_mr_next(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ DPA_ASSERT(mr->fill); ++ mr->cursor = MR_INC(mr->cursor); ++ return --mr->fill; ++} ++ ++static inline u8 qm_mr_pci_update(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ u8 diff, old_pi = mr->pi; ++ DPA_ASSERT(mr->pmode == qm_mr_pci); ++ mr->pi = qm_in(MR_PI_CINH); ++ diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi); ++ mr->fill += diff; ++ return diff; ++} ++ ++static inline void qm_mr_pce_prefetch(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_mr *mr = &portal->mr; ++ DPA_ASSERT(mr->pmode == qm_mr_pce); ++ qm_cl_invalidate(MR_PI); ++ qm_cl_touch_ro(MR_PI); ++} ++ ++static inline u8 qm_mr_pce_update(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ u8 diff, old_pi = mr->pi; ++ DPA_ASSERT(mr->pmode == qm_mr_pce); ++ mr->pi = qm_cl_in(MR_PI) & (QM_MR_SIZE - 1); ++ diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi); ++ mr->fill += diff; ++ return diff; ++} ++ ++static inline void qm_mr_pvb_update(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); ++ DPA_ASSERT(mr->pmode == qm_mr_pvb); ++ /* when accessing 'verb', use __raw_readb() to ensure that compiler ++ * inlining doesn't try to optimise out "excess reads". */ ++ if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) { ++ mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); ++ if (!mr->pi) ++ mr->vbit ^= QM_MR_VERB_VBIT; ++ mr->fill++; ++ res = MR_INC(res); ++ } ++ dcbit_ro(res); ++} ++ ++static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ DPA_ASSERT(mr->cmode == qm_mr_cci); ++ mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1); ++ qm_out(MR_CI_CINH, mr->ci); ++} ++ ++static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ DPA_ASSERT(mr->cmode == qm_mr_cci); ++ mr->ci = MR_PTR2IDX(mr->cursor); ++ qm_out(MR_CI_CINH, mr->ci); ++} ++ ++static inline void qm_mr_cce_prefetch(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_mr *mr = &portal->mr; ++ DPA_ASSERT(mr->cmode == qm_mr_cce); ++ qm_cl_invalidate(MR_CI); ++ qm_cl_touch_rw(MR_CI); ++} ++ ++static inline void qm_mr_cce_consume(struct qm_portal *portal, u8 num) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ DPA_ASSERT(mr->cmode == qm_mr_cce); ++ mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1); ++ qm_cl_out(MR_CI, mr->ci); ++} ++ ++static inline void qm_mr_cce_consume_to_current(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ DPA_ASSERT(mr->cmode == qm_mr_cce); ++ mr->ci = MR_PTR2IDX(mr->cursor); ++ qm_cl_out(MR_CI, mr->ci); ++} ++ ++static inline u8 qm_mr_get_ci(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ return mr->ci; ++} ++ ++static inline u8 qm_mr_get_ithresh(struct qm_portal *portal) ++{ ++ register struct qm_mr *mr = &portal->mr; ++ return mr->ithresh; ++} ++ ++static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh) ++{ ++ qm_out(MR_ITR, ithresh); ++} ++ ++ ++/* ------------------------------ */ ++/* --- Management command API --- */ ++ ++static inline int qm_mc_init(struct qm_portal *portal) ++{ ++ register struct qm_mc *mc = &portal->mc; ++ mc->cr = portal->addr.addr_ce + QM_CL_CR; ++ mc->rr = portal->addr.addr_ce + QM_CL_RR0; ++ mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) & ++ QM_MCC_VERB_VBIT) ? 0 : 1; ++ mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = qman_mc_idle; ++#endif ++ return 0; ++} ++ ++static inline void qm_mc_finish(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_mc *mc = &portal->mc; ++ DPA_ASSERT(mc->state == qman_mc_idle); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ if (mc->state != qman_mc_idle) ++ pr_crit("Losing incomplete MC command\n"); ++#endif ++} ++ ++static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal) ++{ ++ register struct qm_mc *mc = &portal->mc; ++ DPA_ASSERT(mc->state == qman_mc_idle); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = qman_mc_user; ++#endif ++ dcbz_64(mc->cr); ++ return mc->cr; ++} ++ ++static inline void qm_mc_abort(struct qm_portal *portal) ++{ ++ __maybe_unused register struct qm_mc *mc = &portal->mc; ++ DPA_ASSERT(mc->state == qman_mc_user); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = qman_mc_idle; ++#endif ++} ++ ++static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb) ++{ ++ register struct qm_mc *mc = &portal->mc; ++ struct qm_mc_result *rr = mc->rr + mc->rridx; ++ DPA_ASSERT(mc->state == qman_mc_user); ++ lwsync(); ++ mc->cr->__dont_write_directly__verb = myverb | mc->vbit; ++ dcbf(mc->cr); ++ dcbit_ro(rr); ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = qman_mc_hw; ++#endif ++} ++ ++static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal) ++{ ++ register struct qm_mc *mc = &portal->mc; ++ struct qm_mc_result *rr = mc->rr + mc->rridx; ++ DPA_ASSERT(mc->state == qman_mc_hw); ++ /* The inactive response register's verb byte always returns zero until ++ * its command is submitted and completed. This includes the valid-bit, ++ * in case you were wondering... */ ++ if (!__raw_readb(&rr->verb)) { ++ dcbit_ro(rr); ++ return NULL; ++ } ++ mc->rridx ^= 1; ++ mc->vbit ^= QM_MCC_VERB_VBIT; ++#ifdef CONFIG_FSL_DPA_CHECKING ++ mc->state = qman_mc_idle; ++#endif ++ return rr; ++} ++ ++ ++/* ------------------------------------- */ ++/* --- Portal interrupt register API --- */ ++ ++static inline int qm_isr_init(__always_unused struct qm_portal *portal) ++{ ++ return 0; ++} ++ ++static inline void qm_isr_finish(__always_unused struct qm_portal *portal) ++{ ++} ++ ++static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod) ++{ ++ qm_out(ITPR, iperiod); ++} ++ ++static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n) ++{ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ return __qm_in(&portal->addr, QM_REG_ISR + (n << 6)); ++#else ++ return __qm_in(&portal->addr, QM_REG_ISR + (n << 2)); ++#endif ++} ++ ++static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n, ++ u32 val) ++{ ++#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) ++ __qm_out(&portal->addr, QM_REG_ISR + (n << 6), val); ++#else ++ __qm_out(&portal->addr, QM_REG_ISR + (n << 2), val); ++#endif ++} ++ ++/* Cleanup FQs */ ++static inline int qm_shutdown_fq(struct qm_portal **portal, int portal_count, ++ u32 fqid) ++{ ++ ++ struct qm_mc_command *mcc; ++ struct qm_mc_result *mcr; ++ u8 state; ++ int orl_empty, fq_empty, i, drain = 0; ++ u32 result; ++ u32 channel, wq; ++ u16 dest_wq; ++ ++ /* Determine the state of the FQID */ ++ mcc = qm_mc_start(portal[0]); ++ mcc->queryfq_np.fqid = cpu_to_be32(fqid); ++ qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ_NP); ++ while (!(mcr = qm_mc_result(portal[0]))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); ++ state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; ++ if (state == QM_MCR_NP_STATE_OOS) ++ return 0; /* Already OOS, no need to do anymore checks */ ++ ++ /* Query which channel the FQ is using */ ++ mcc = qm_mc_start(portal[0]); ++ mcc->queryfq.fqid = cpu_to_be32(fqid); ++ qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ); ++ while (!(mcr = qm_mc_result(portal[0]))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); ++ ++ /* Need to store these since the MCR gets reused */ ++ dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); ++ wq = dest_wq & 0x7; ++ channel = dest_wq>>3; ++ ++ switch (state) { ++ case QM_MCR_NP_STATE_TEN_SCHED: ++ case QM_MCR_NP_STATE_TRU_SCHED: ++ case QM_MCR_NP_STATE_ACTIVE: ++ case QM_MCR_NP_STATE_PARKED: ++ orl_empty = 0; ++ mcc = qm_mc_start(portal[0]); ++ mcc->alterfq.fqid = cpu_to_be32(fqid); ++ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_RETIRE); ++ while (!(mcr = qm_mc_result(portal[0]))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_MCR_VERB_ALTER_RETIRE); ++ result = mcr->result; /* Make a copy as we reuse MCR below */ ++ ++ if (result == QM_MCR_RESULT_PENDING) { ++ /* Need to wait for the FQRN in the message ring, which ++ will only occur once the FQ has been drained. In ++ order for the FQ to drain the portal needs to be set ++ to dequeue from the channel the FQ is scheduled on */ ++ const struct qm_mr_entry *msg; ++ const struct qm_dqrr_entry *dqrr = NULL; ++ int found_fqrn = 0; ++ u16 dequeue_wq = 0; ++ ++ /* Flag that we need to drain FQ */ ++ drain = 1; ++ ++ if (channel >= qm_channel_pool1 && ++ channel < (qm_channel_pool1 + 15)) { ++ /* Pool channel, enable the bit in the portal */ ++ dequeue_wq = (channel - ++ qm_channel_pool1 + 1)<<4 | wq; ++ } else if (channel < qm_channel_pool1) { ++ /* Dedicated channel */ ++ dequeue_wq = wq; ++ } else { ++ pr_info("Cannot recover FQ 0x%x, it is " ++ "scheduled on channel 0x%x", ++ fqid, channel); ++ return -EBUSY; ++ } ++ /* Set the sdqcr to drain this channel */ ++ if (channel < qm_channel_pool1) ++ for (i = 0; i < portal_count; i++) ++ qm_dqrr_sdqcr_set(portal[i], ++ QM_SDQCR_TYPE_ACTIVE | ++ QM_SDQCR_CHANNELS_DEDICATED); ++ else ++ for (i = 0; i < portal_count; i++) ++ qm_dqrr_sdqcr_set( ++ portal[i], ++ QM_SDQCR_TYPE_ACTIVE | ++ QM_SDQCR_CHANNELS_POOL_CONV ++ (channel)); ++ while (!found_fqrn) { ++ /* Keep draining DQRR while checking the MR*/ ++ for (i = 0; i < portal_count; i++) { ++ qm_dqrr_pvb_update(portal[i]); ++ dqrr = qm_dqrr_current(portal[i]); ++ while (dqrr) { ++ qm_dqrr_cdc_consume_1ptr( ++ portal[i], dqrr, 0); ++ qm_dqrr_pvb_update(portal[i]); ++ qm_dqrr_next(portal[i]); ++ dqrr = qm_dqrr_current( ++ portal[i]); ++ } ++ /* Process message ring too */ ++ qm_mr_pvb_update(portal[i]); ++ msg = qm_mr_current(portal[i]); ++ while (msg) { ++ if ((msg->verb & ++ QM_MR_VERB_TYPE_MASK) ++ == QM_MR_VERB_FQRN) ++ found_fqrn = 1; ++ qm_mr_next(portal[i]); ++ qm_mr_cci_consume_to_current( ++ portal[i]); ++ qm_mr_pvb_update(portal[i]); ++ msg = qm_mr_current(portal[i]); ++ } ++ cpu_relax(); ++ } ++ } ++ } ++ if (result != QM_MCR_RESULT_OK && ++ result != QM_MCR_RESULT_PENDING) { ++ /* error */ ++ pr_err("qman_retire_fq failed on FQ 0x%x, result=0x%x\n", ++ fqid, result); ++ return -1; ++ } ++ if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { ++ /* ORL had no entries, no need to wait until the ++ ERNs come in */ ++ orl_empty = 1; ++ } ++ /* Retirement succeeded, check to see if FQ needs ++ to be drained */ ++ if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { ++ /* FQ is Not Empty, drain using volatile DQ commands */ ++ fq_empty = 0; ++ do { ++ const struct qm_dqrr_entry *dqrr = NULL; ++ u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); ++ qm_dqrr_vdqcr_set(portal[0], vdqcr); ++ ++ /* Wait for a dequeue to occur */ ++ while (dqrr == NULL) { ++ qm_dqrr_pvb_update(portal[0]); ++ dqrr = qm_dqrr_current(portal[0]); ++ if (!dqrr) ++ cpu_relax(); ++ } ++ /* Process the dequeues, making sure to ++ empty the ring completely */ ++ while (dqrr) { ++ if (be32_to_cpu(dqrr->fqid) == fqid && ++ dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) ++ fq_empty = 1; ++ qm_dqrr_cdc_consume_1ptr(portal[0], ++ dqrr, 0); ++ qm_dqrr_pvb_update(portal[0]); ++ qm_dqrr_next(portal[0]); ++ dqrr = qm_dqrr_current(portal[0]); ++ } ++ } while (fq_empty == 0); ++ } ++ for (i = 0; i < portal_count; i++) ++ qm_dqrr_sdqcr_set(portal[i], 0); ++ ++ /* Wait for the ORL to have been completely drained */ ++ while (orl_empty == 0) { ++ const struct qm_mr_entry *msg; ++ qm_mr_pvb_update(portal[0]); ++ msg = qm_mr_current(portal[0]); ++ while (msg) { ++ if ((msg->verb & QM_MR_VERB_TYPE_MASK) == ++ QM_MR_VERB_FQRL) ++ orl_empty = 1; ++ qm_mr_next(portal[0]); ++ qm_mr_cci_consume_to_current(portal[0]); ++ qm_mr_pvb_update(portal[0]); ++ msg = qm_mr_current(portal[0]); ++ } ++ cpu_relax(); ++ } ++ mcc = qm_mc_start(portal[0]); ++ mcc->alterfq.fqid = cpu_to_be32(fqid); ++ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS); ++ while (!(mcr = qm_mc_result(portal[0]))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_MCR_VERB_ALTER_OOS); ++ if (mcr->result != QM_MCR_RESULT_OK) { ++ pr_err("OOS after drain Failed on FQID 0x%x, result 0x%x\n", ++ fqid, mcr->result); ++ return -1; ++ } ++ return 0; ++ case QM_MCR_NP_STATE_RETIRED: ++ /* Send OOS Command */ ++ mcc = qm_mc_start(portal[0]); ++ mcc->alterfq.fqid = cpu_to_be32(fqid); ++ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS); ++ while (!(mcr = qm_mc_result(portal[0]))) ++ cpu_relax(); ++ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == ++ QM_MCR_VERB_ALTER_OOS); ++ if (mcr->result) { ++ pr_err("OOS Failed on FQID 0x%x\n", fqid); ++ return -1; ++ } ++ return 0; ++ } ++ return -1; ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_private.h +@@ -0,0 +1,398 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "dpa_sys.h" ++#include <linux/fsl_qman.h> ++#include <linux/iommu.h> ++ ++#if defined(CONFIG_FSL_PAMU) ++#include <asm/fsl_pamu_stash.h> ++#endif ++ ++#if !defined(CONFIG_FSL_QMAN_FQ_LOOKUP) && defined(CONFIG_PPC64) ++#error "_PPC64 requires _FSL_QMAN_FQ_LOOKUP" ++#endif ++ ++#define QBMAN_ANY_PORTAL_IDX 0xffffffff ++ /* ----------------- */ ++ /* Congestion Groups */ ++ /* ----------------- */ ++/* This wrapper represents a bit-array for the state of the 256 Qman congestion ++ * groups. Is also used as a *mask* for congestion groups, eg. so we ignore ++ * those that don't concern us. We harness the structure and accessor details ++ * already used in the management command to query congestion groups. */ ++struct qman_cgrs { ++ struct __qm_mcr_querycongestion q; ++}; ++static inline void qman_cgrs_init(struct qman_cgrs *c) ++{ ++ memset(c, 0, sizeof(*c)); ++} ++static inline void qman_cgrs_fill(struct qman_cgrs *c) ++{ ++ memset(c, 0xff, sizeof(*c)); ++} ++static inline int qman_cgrs_get(struct qman_cgrs *c, int num) ++{ ++ return QM_MCR_QUERYCONGESTION(&c->q, num); ++} ++static inline void qman_cgrs_set(struct qman_cgrs *c, int num) ++{ ++ c->q.__state[__CGR_WORD(num)] |= (0x80000000 >> __CGR_SHIFT(num)); ++} ++static inline void qman_cgrs_unset(struct qman_cgrs *c, int num) ++{ ++ c->q.__state[__CGR_WORD(num)] &= ~(0x80000000 >> __CGR_SHIFT(num)); ++} ++static inline int qman_cgrs_next(struct qman_cgrs *c, int num) ++{ ++ while ((++num < __CGR_NUM) && !qman_cgrs_get(c, num)) ++ ; ++ return num; ++} ++static inline void qman_cgrs_cp(struct qman_cgrs *dest, ++ const struct qman_cgrs *src) ++{ ++ *dest = *src; ++} ++static inline void qman_cgrs_and(struct qman_cgrs *dest, ++ const struct qman_cgrs *a, const struct qman_cgrs *b) ++{ ++ int ret; ++ u32 *_d = dest->q.__state; ++ const u32 *_a = a->q.__state; ++ const u32 *_b = b->q.__state; ++ for (ret = 0; ret < 8; ret++) ++ *(_d++) = *(_a++) & *(_b++); ++} ++static inline void qman_cgrs_xor(struct qman_cgrs *dest, ++ const struct qman_cgrs *a, const struct qman_cgrs *b) ++{ ++ int ret; ++ u32 *_d = dest->q.__state; ++ const u32 *_a = a->q.__state; ++ const u32 *_b = b->q.__state; ++ for (ret = 0; ret < 8; ret++) ++ *(_d++) = *(_a++) ^ *(_b++); ++} ++ ++ /* ----------------------- */ ++ /* CEETM Congestion Groups */ ++ /* ----------------------- */ ++/* This wrapper represents a bit-array for the state of the 512 Qman CEETM ++ * congestion groups. ++ */ ++struct qman_ccgrs { ++ struct __qm_mcr_querycongestion q[2]; ++}; ++static inline void qman_ccgrs_init(struct qman_ccgrs *c) ++{ ++ memset(c, 0, sizeof(*c)); ++} ++static inline void qman_ccgrs_fill(struct qman_ccgrs *c) ++{ ++ memset(c, 0xff, sizeof(*c)); ++} ++static inline int qman_ccgrs_get(struct qman_ccgrs *c, int num) ++{ ++ if (num < __CGR_NUM) ++ return QM_MCR_QUERYCONGESTION(&c->q[0], num); ++ else ++ return QM_MCR_QUERYCONGESTION(&c->q[1], (num - __CGR_NUM)); ++} ++static inline int qman_ccgrs_next(struct qman_ccgrs *c, int num) ++{ ++ while ((++num < __CGR_NUM) && !qman_ccgrs_get(c, num)) ++ ; ++ return num; ++} ++static inline void qman_ccgrs_cp(struct qman_ccgrs *dest, ++ const struct qman_ccgrs *src) ++{ ++ *dest = *src; ++} ++static inline void qman_ccgrs_and(struct qman_ccgrs *dest, ++ const struct qman_ccgrs *a, const struct qman_ccgrs *b) ++{ ++ int ret, i; ++ u32 *_d; ++ const u32 *_a, *_b; ++ for (i = 0; i < 2; i++) { ++ _d = dest->q[i].__state; ++ _a = a->q[i].__state; ++ _b = b->q[i].__state; ++ for (ret = 0; ret < 8; ret++) ++ *(_d++) = *(_a++) & *(_b++); ++ } ++} ++static inline void qman_ccgrs_xor(struct qman_ccgrs *dest, ++ const struct qman_ccgrs *a, const struct qman_ccgrs *b) ++{ ++ int ret, i; ++ u32 *_d; ++ const u32 *_a, *_b; ++ for (i = 0; i < 2; i++) { ++ _d = dest->q[i].__state; ++ _a = a->q[i].__state; ++ _b = b->q[i].__state; ++ for (ret = 0; ret < 8; ret++) ++ *(_d++) = *(_a++) ^ *(_b++); ++ } ++} ++ ++/* used by CCSR and portal interrupt code */ ++enum qm_isr_reg { ++ qm_isr_status = 0, ++ qm_isr_enable = 1, ++ qm_isr_disable = 2, ++ qm_isr_inhibit = 3 ++}; ++ ++struct qm_portal_config { ++ /* Corenet portal addresses; ++ * [0]==cache-enabled, [1]==cache-inhibited. */ ++ __iomem void *addr_virt[2]; ++ struct resource addr_phys[2]; ++ struct device dev; ++ struct iommu_domain *iommu_domain; ++ /* Allow these to be joined in lists */ ++ struct list_head list; ++ /* User-visible portal configuration settings */ ++ struct qman_portal_config public_cfg; ++ /* power management saved data */ ++ u32 saved_isdr; ++}; ++ ++/* Revision info (for errata and feature handling) */ ++#define QMAN_REV11 0x0101 ++#define QMAN_REV12 0x0102 ++#define QMAN_REV20 0x0200 ++#define QMAN_REV30 0x0300 ++#define QMAN_REV31 0x0301 ++#define QMAN_REV32 0x0302 ++ ++/* QMan REV_2 register contains the Cfg option */ ++#define QMAN_REV_CFG_0 0x0 ++#define QMAN_REV_CFG_1 0x1 ++#define QMAN_REV_CFG_2 0x2 ++#define QMAN_REV_CFG_3 0x3 ++ ++extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */ ++extern u8 qman_ip_cfg; ++extern u32 qman_clk; ++extern u16 qman_portal_max; ++ ++#ifdef CONFIG_FSL_QMAN_CONFIG ++/* Hooks from qman_driver.c to qman_config.c */ ++int qman_init_ccsr(struct device_node *node); ++void qman_liodn_fixup(u16 channel); ++int qman_set_sdest(u16 channel, unsigned int cpu_idx); ++size_t get_qman_fqd_size(void); ++#else ++static inline size_t get_qman_fqd_size(void) ++{ ++ return (PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ); ++} ++#endif ++ ++int qm_set_wpm(int wpm); ++int qm_get_wpm(int *wpm); ++ ++/* Hooks from qman_driver.c in to qman_high.c */ ++struct qman_portal *qman_create_portal( ++ struct qman_portal *portal, ++ const struct qm_portal_config *config, ++ const struct qman_cgrs *cgrs); ++ ++struct qman_portal *qman_create_affine_portal( ++ const struct qm_portal_config *config, ++ const struct qman_cgrs *cgrs); ++struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect, ++ int cpu); ++const struct qm_portal_config *qman_destroy_affine_portal(void); ++void qman_destroy_portal(struct qman_portal *qm); ++ ++/* Hooks from fsl_usdpaa.c to qman_driver.c */ ++struct qm_portal_config *qm_get_unused_portal(void); ++struct qm_portal_config *qm_get_unused_portal_idx(uint32_t idx); ++ ++void qm_put_unused_portal(struct qm_portal_config *pcfg); ++void qm_set_liodns(struct qm_portal_config *pcfg); ++ ++/* This CGR feature is supported by h/w and required by unit-tests and the ++ * debugfs hooks, so is implemented in the driver. However it allows an explicit ++ * corruption of h/w fields by s/w that are usually incorruptible (because the ++ * counters are usually maintained entirely within h/w). As such, we declare ++ * this API internally. */ ++int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, ++ struct qm_mcr_cgrtestwrite *result); ++ ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++/* If the fq object pointer is greater than the size of context_b field, ++ * than a lookup table is required. */ ++int qman_setup_fq_lookup_table(size_t num_entries); ++#endif ++ ++ ++/*************************************************/ ++/* QMan s/w corenet portal, low-level i/face */ ++/*************************************************/ ++ ++/* Note: most functions are only used by the high-level interface, so are ++ * inlined from qman_low.h. The stuff below is for use by other parts of the ++ * driver. */ ++ ++/* For qm_dqrr_sdqcr_set(); Choose one SOURCE. Choose one COUNT. Choose one ++ * dequeue TYPE. Choose TOKEN (8-bit). ++ * If SOURCE == CHANNELS, ++ * Choose CHANNELS_DEDICATED and/or CHANNELS_POOL(n). ++ * You can choose DEDICATED_PRECEDENCE if the portal channel should have ++ * priority. ++ * If SOURCE == SPECIFICWQ, ++ * Either select the work-queue ID with SPECIFICWQ_WQ(), or select the ++ * channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the ++ * work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the ++ * same value. ++ */ ++#define QM_SDQCR_SOURCE_CHANNELS 0x0 ++#define QM_SDQCR_SOURCE_SPECIFICWQ 0x40000000 ++#define QM_SDQCR_COUNT_EXACT1 0x0 ++#define QM_SDQCR_COUNT_UPTO3 0x20000000 ++#define QM_SDQCR_DEDICATED_PRECEDENCE 0x10000000 ++#define QM_SDQCR_TYPE_MASK 0x03000000 ++#define QM_SDQCR_TYPE_NULL 0x0 ++#define QM_SDQCR_TYPE_PRIO_QOS 0x01000000 ++#define QM_SDQCR_TYPE_ACTIVE_QOS 0x02000000 ++#define QM_SDQCR_TYPE_ACTIVE 0x03000000 ++#define QM_SDQCR_TOKEN_MASK 0x00ff0000 ++#define QM_SDQCR_TOKEN_SET(v) (((v) & 0xff) << 16) ++#define QM_SDQCR_TOKEN_GET(v) (((v) >> 16) & 0xff) ++#define QM_SDQCR_CHANNELS_DEDICATED 0x00008000 ++#define QM_SDQCR_SPECIFICWQ_MASK 0x000000f7 ++#define QM_SDQCR_SPECIFICWQ_DEDICATED 0x00000000 ++#define QM_SDQCR_SPECIFICWQ_POOL(n) ((n) << 4) ++#define QM_SDQCR_SPECIFICWQ_WQ(n) (n) ++ ++/* For qm_dqrr_vdqcr_set(): use FQID(n) to fill in the frame queue ID */ ++#define QM_VDQCR_FQID_MASK 0x00ffffff ++#define QM_VDQCR_FQID(n) ((n) & QM_VDQCR_FQID_MASK) ++ ++/* For qm_dqrr_pdqcr_set(); Choose one MODE. Choose one COUNT. ++ * If MODE==SCHEDULED ++ * Choose SCHEDULED_CHANNELS or SCHEDULED_SPECIFICWQ. Choose one dequeue TYPE. ++ * If CHANNELS, ++ * Choose CHANNELS_DEDICATED and/or CHANNELS_POOL() channels. ++ * You can choose DEDICATED_PRECEDENCE if the portal channel should have ++ * priority. ++ * If SPECIFICWQ, ++ * Either select the work-queue ID with SPECIFICWQ_WQ(), or select the ++ * channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the ++ * work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the ++ * same value. ++ * If MODE==UNSCHEDULED ++ * Choose FQID(). ++ */ ++#define QM_PDQCR_MODE_SCHEDULED 0x0 ++#define QM_PDQCR_MODE_UNSCHEDULED 0x80000000 ++#define QM_PDQCR_SCHEDULED_CHANNELS 0x0 ++#define QM_PDQCR_SCHEDULED_SPECIFICWQ 0x40000000 ++#define QM_PDQCR_COUNT_EXACT1 0x0 ++#define QM_PDQCR_COUNT_UPTO3 0x20000000 ++#define QM_PDQCR_DEDICATED_PRECEDENCE 0x10000000 ++#define QM_PDQCR_TYPE_MASK 0x03000000 ++#define QM_PDQCR_TYPE_NULL 0x0 ++#define QM_PDQCR_TYPE_PRIO_QOS 0x01000000 ++#define QM_PDQCR_TYPE_ACTIVE_QOS 0x02000000 ++#define QM_PDQCR_TYPE_ACTIVE 0x03000000 ++#define QM_PDQCR_CHANNELS_DEDICATED 0x00008000 ++#define QM_PDQCR_CHANNELS_POOL(n) (0x00008000 >> (n)) ++#define QM_PDQCR_SPECIFICWQ_MASK 0x000000f7 ++#define QM_PDQCR_SPECIFICWQ_DEDICATED 0x00000000 ++#define QM_PDQCR_SPECIFICWQ_POOL(n) ((n) << 4) ++#define QM_PDQCR_SPECIFICWQ_WQ(n) (n) ++#define QM_PDQCR_FQID(n) ((n) & 0xffffff) ++ ++/* Used by all portal interrupt registers except 'inhibit' ++ * Channels with frame availability ++ */ ++#define QM_PIRQ_DQAVAIL 0x0000ffff ++ ++/* The DQAVAIL interrupt fields break down into these bits; */ ++#define QM_DQAVAIL_PORTAL 0x8000 /* Portal channel */ ++#define QM_DQAVAIL_POOL(n) (0x8000 >> (n)) /* Pool channel, n==[1..15] */ ++#define QM_DQAVAIL_MASK 0xffff ++/* This mask contains all the "irqsource" bits visible to API users */ ++#define QM_PIRQ_VISIBLE (QM_PIRQ_SLOW | QM_PIRQ_DQRI) ++ ++/* These are qm_<reg>_<verb>(). So for example, qm_disable_write() means "write ++ * the disable register" rather than "disable the ability to write". */ ++#define qm_isr_status_read(qm) __qm_isr_read(qm, qm_isr_status) ++#define qm_isr_status_clear(qm, m) __qm_isr_write(qm, qm_isr_status, m) ++#define qm_isr_enable_read(qm) __qm_isr_read(qm, qm_isr_enable) ++#define qm_isr_enable_write(qm, v) __qm_isr_write(qm, qm_isr_enable, v) ++#define qm_isr_disable_read(qm) __qm_isr_read(qm, qm_isr_disable) ++#define qm_isr_disable_write(qm, v) __qm_isr_write(qm, qm_isr_disable, v) ++/* TODO: unfortunate name-clash here, reword? */ ++#define qm_isr_inhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 1) ++#define qm_isr_uninhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 0) ++ ++#ifdef CONFIG_FSL_QMAN_CONFIG ++int qman_have_ccsr(void); ++#else ++#define qman_have_ccsr 0 ++#endif ++ ++__init int qman_init(void); ++__init int qman_resource_init(void); ++ ++/* CEETM related */ ++#define QMAN_CEETM_MAX 2 ++extern u8 num_ceetms; ++extern struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX]; ++int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal); ++int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal); ++int qman_ceetm_set_prescaler(enum qm_dc_portal portal); ++int qman_ceetm_get_prescaler(u16 *pres); ++int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid, ++ struct qm_mcr_ceetm_cq_query *cq_query); ++int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query, ++ struct qm_mcr_ceetm_ccgr_query *response); ++int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num); ++ ++extern void *affine_portals[NR_CPUS]; ++const struct qm_portal_config *qman_get_qm_portal_config( ++ struct qman_portal *portal); ++ ++/* power management */ ++#ifdef CONFIG_SUSPEND ++void suspend_unused_qportal(void); ++void resume_unused_qportal(void); ++#endif +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_test.c +@@ -0,0 +1,57 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "qman_test.h" ++ ++MODULE_AUTHOR("Geoff Thorpe"); ++MODULE_LICENSE("Dual BSD/GPL"); ++MODULE_DESCRIPTION("Qman testing"); ++ ++static int test_init(void) ++{ ++ int loop = 1; ++ while (loop--) { ++#ifdef CONFIG_FSL_QMAN_TEST_STASH_POTATO ++ qman_test_hotpotato(); ++#endif ++#ifdef CONFIG_FSL_QMAN_TEST_HIGH ++ qman_test_high(); ++#endif ++ } ++ return 0; ++} ++ ++static void test_exit(void) ++{ ++} ++ ++module_init(test_init); ++module_exit(test_exit); +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_test.h +@@ -0,0 +1,45 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include <linux/kernel.h> ++#include <linux/errno.h> ++#include <linux/io.h> ++#include <linux/slab.h> ++#include <linux/module.h> ++#include <linux/interrupt.h> ++#include <linux/delay.h> ++#include <linux/sched.h> ++ ++#include <linux/fsl_qman.h> ++ ++void qman_test_hotpotato(void); ++void qman_test_high(void); ++ +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_test_high.c +@@ -0,0 +1,216 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "qman_test.h" ++ ++/*************/ ++/* constants */ ++/*************/ ++ ++#define CGR_ID 27 ++#define POOL_ID 2 ++#define FQ_FLAGS QMAN_FQ_FLAG_DYNAMIC_FQID ++#define NUM_ENQUEUES 10 ++#define NUM_PARTIAL 4 ++#define PORTAL_SDQCR (QM_SDQCR_SOURCE_CHANNELS | \ ++ QM_SDQCR_TYPE_PRIO_QOS | \ ++ QM_SDQCR_TOKEN_SET(0x98) | \ ++ QM_SDQCR_CHANNELS_DEDICATED | \ ++ QM_SDQCR_CHANNELS_POOL(POOL_ID)) ++#define PORTAL_OPAQUE ((void *)0xf00dbeef) ++#define VDQCR_FLAGS (QMAN_VOLATILE_FLAG_WAIT | QMAN_VOLATILE_FLAG_FINISH) ++ ++/*************************************/ ++/* Predeclarations (eg. for fq_base) */ ++/*************************************/ ++ ++static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *, ++ struct qman_fq *, ++ const struct qm_dqrr_entry *); ++static void cb_ern(struct qman_portal *, struct qman_fq *, ++ const struct qm_mr_entry *); ++static void cb_fqs(struct qman_portal *, struct qman_fq *, ++ const struct qm_mr_entry *); ++ ++/***************/ ++/* global vars */ ++/***************/ ++ ++static struct qm_fd fd, fd_dq; ++static struct qman_fq fq_base = { ++ .cb.dqrr = cb_dqrr, ++ .cb.ern = cb_ern, ++ .cb.fqs = cb_fqs ++}; ++static DECLARE_WAIT_QUEUE_HEAD(waitqueue); ++static int retire_complete, sdqcr_complete; ++ ++/**********************/ ++/* internal functions */ ++/**********************/ ++ ++/* Helpers for initialising and "incrementing" a frame descriptor */ ++static void fd_init(struct qm_fd *__fd) ++{ ++ qm_fd_addr_set64(__fd, 0xabdeadbeefLLU); ++ __fd->format = qm_fd_contig_big; ++ __fd->length29 = 0x0000ffff; ++ __fd->cmd = 0xfeedf00d; ++} ++ ++static void fd_inc(struct qm_fd *__fd) ++{ ++ u64 t = qm_fd_addr_get64(__fd); ++ int z = t >> 40; ++ t <<= 1; ++ if (z) ++ t |= 1; ++ qm_fd_addr_set64(__fd, t); ++ __fd->length29--; ++ __fd->cmd++; ++} ++ ++/* The only part of the 'fd' we can't memcmp() is the ppid */ ++static int fd_cmp(const struct qm_fd *a, const struct qm_fd *b) ++{ ++ int r = (qm_fd_addr_get64(a) == qm_fd_addr_get64(b)) ? 0 : -1; ++ if (!r) ++ r = a->format - b->format; ++ if (!r) ++ r = a->opaque - b->opaque; ++ if (!r) ++ r = a->cmd - b->cmd; ++ return r; ++} ++ ++/********/ ++/* test */ ++/********/ ++ ++static void do_enqueues(struct qman_fq *fq) ++{ ++ unsigned int loop; ++ for (loop = 0; loop < NUM_ENQUEUES; loop++) { ++ if (qman_enqueue(fq, &fd, QMAN_ENQUEUE_FLAG_WAIT | ++ (((loop + 1) == NUM_ENQUEUES) ? ++ QMAN_ENQUEUE_FLAG_WAIT_SYNC : 0))) ++ panic("qman_enqueue() failed\n"); ++ fd_inc(&fd); ++ } ++} ++ ++void qman_test_high(void) ++{ ++ unsigned int flags; ++ int res; ++ struct qman_fq *fq = &fq_base; ++ ++ pr_info("qman_test_high starting\n"); ++ fd_init(&fd); ++ fd_init(&fd_dq); ++ ++ /* Initialise (parked) FQ */ ++ if (qman_create_fq(0, FQ_FLAGS, fq)) ++ panic("qman_create_fq() failed\n"); ++ if (qman_init_fq(fq, QMAN_INITFQ_FLAG_LOCAL, NULL)) ++ panic("qman_init_fq() failed\n"); ++ ++ /* Do enqueues + VDQCR, twice. (Parked FQ) */ ++ do_enqueues(fq); ++ pr_info("VDQCR (till-empty);\n"); ++ if (qman_volatile_dequeue(fq, VDQCR_FLAGS, ++ QM_VDQCR_NUMFRAMES_TILLEMPTY)) ++ panic("qman_volatile_dequeue() failed\n"); ++ do_enqueues(fq); ++ pr_info("VDQCR (%d of %d);\n", NUM_PARTIAL, NUM_ENQUEUES); ++ if (qman_volatile_dequeue(fq, VDQCR_FLAGS, ++ QM_VDQCR_NUMFRAMES_SET(NUM_PARTIAL))) ++ panic("qman_volatile_dequeue() failed\n"); ++ pr_info("VDQCR (%d of %d);\n", NUM_ENQUEUES - NUM_PARTIAL, ++ NUM_ENQUEUES); ++ if (qman_volatile_dequeue(fq, VDQCR_FLAGS, ++ QM_VDQCR_NUMFRAMES_SET(NUM_ENQUEUES - NUM_PARTIAL))) ++ panic("qman_volatile_dequeue() failed\n"); ++ ++ do_enqueues(fq); ++ pr_info("scheduled dequeue (till-empty)\n"); ++ if (qman_schedule_fq(fq)) ++ panic("qman_schedule_fq() failed\n"); ++ wait_event(waitqueue, sdqcr_complete); ++ ++ /* Retire and OOS the FQ */ ++ res = qman_retire_fq(fq, &flags); ++ if (res < 0) ++ panic("qman_retire_fq() failed\n"); ++ wait_event(waitqueue, retire_complete); ++ if (flags & QMAN_FQ_STATE_BLOCKOOS) ++ panic("leaking frames\n"); ++ if (qman_oos_fq(fq)) ++ panic("qman_oos_fq() failed\n"); ++ qman_destroy_fq(fq, 0); ++ pr_info("qman_test_high finished\n"); ++} ++ ++static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *p, ++ struct qman_fq *fq, ++ const struct qm_dqrr_entry *dq) ++{ ++ if (fd_cmp(&fd_dq, &dq->fd)) { ++ pr_err("BADNESS: dequeued frame doesn't match;\n"); ++ pr_err("Expected 0x%llx, got 0x%llx\n", ++ (unsigned long long)fd_dq.length29, ++ (unsigned long long)dq->fd.length29); ++ BUG(); ++ } ++ fd_inc(&fd_dq); ++ if (!(dq->stat & QM_DQRR_STAT_UNSCHEDULED) && !fd_cmp(&fd_dq, &fd)) { ++ sdqcr_complete = 1; ++ wake_up(&waitqueue); ++ } ++ return qman_cb_dqrr_consume; ++} ++ ++static void cb_ern(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_mr_entry *msg) ++{ ++ panic("cb_ern() unimplemented"); ++} ++ ++static void cb_fqs(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_mr_entry *msg) ++{ ++ u8 verb = (msg->verb & QM_MR_VERB_TYPE_MASK); ++ if ((verb != QM_MR_VERB_FQRN) && (verb != QM_MR_VERB_FQRNI)) ++ panic("unexpected FQS message"); ++ pr_info("Retirement message received\n"); ++ retire_complete = 1; ++ wake_up(&waitqueue); ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_test_hotpotato.c +@@ -0,0 +1,499 @@ ++/* Copyright 2009-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include <linux/kthread.h> ++#include <linux/platform_device.h> ++#include <linux/dma-mapping.h> ++#include "qman_test.h" ++ ++/* Algorithm: ++ * ++ * Each cpu will have HP_PER_CPU "handlers" set up, each of which incorporates ++ * an rx/tx pair of FQ objects (both of which are stashed on dequeue). The ++ * organisation of FQIDs is such that the HP_PER_CPU*NUM_CPUS handlers will ++ * shuttle a "hot potato" frame around them such that every forwarding action ++ * moves it from one cpu to another. (The use of more than one handler per cpu ++ * is to allow enough handlers/FQs to truly test the significance of caching - ++ * ie. when cache-expiries are occurring.) ++ * ++ * The "hot potato" frame content will be HP_NUM_WORDS*4 bytes in size, and the ++ * first and last words of the frame data will undergo a transformation step on ++ * each forwarding action. To achieve this, each handler will be assigned a ++ * 32-bit "mixer", that is produced using a 32-bit LFSR. When a frame is ++ * received by a handler, the mixer of the expected sender is XOR'd into all ++ * words of the entire frame, which is then validated against the original ++ * values. Then, before forwarding, the entire frame is XOR'd with the mixer of ++ * the current handler. Apart from validating that the frame is taking the ++ * expected path, this also provides some quasi-realistic overheads to each ++ * forwarding action - dereferencing *all* the frame data, computation, and ++ * conditional branching. There is a "special" handler designated to act as the ++ * instigator of the test by creating an enqueuing the "hot potato" frame, and ++ * to determine when the test has completed by counting HP_LOOPS iterations. ++ * ++ * Init phases: ++ * ++ * 1. prepare each cpu's 'hp_cpu' struct using on_each_cpu(,,1) and link them ++ * into 'hp_cpu_list'. Specifically, set processor_id, allocate HP_PER_CPU ++ * handlers and link-list them (but do no other handler setup). ++ * ++ * 2. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each ++ * hp_cpu's 'iterator' to point to its first handler. With each loop, ++ * allocate rx/tx FQIDs and mixer values to the hp_cpu's iterator handler ++ * and advance the iterator for the next loop. This includes a final fixup, ++ * which connects the last handler to the first (and which is why phase 2 ++ * and 3 are separate). ++ * ++ * 3. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each ++ * hp_cpu's 'iterator' to point to its first handler. With each loop, ++ * initialise FQ objects and advance the iterator for the next loop. ++ * Moreover, do this initialisation on the cpu it applies to so that Rx FQ ++ * initialisation targets the correct cpu. ++ */ ++ ++/* helper to run something on all cpus (can't use on_each_cpu(), as that invokes ++ * the fn from irq context, which is too restrictive). */ ++struct bstrap { ++ void (*fn)(void); ++ atomic_t started; ++}; ++static int bstrap_fn(void *__bstrap) ++{ ++ struct bstrap *bstrap = __bstrap; ++ atomic_inc(&bstrap->started); ++ bstrap->fn(); ++ while (!kthread_should_stop()) ++ msleep(1); ++ return 0; ++} ++static int on_all_cpus(void (*fn)(void)) ++{ ++ int cpu; ++ for_each_cpu(cpu, cpu_online_mask) { ++ struct bstrap bstrap = { ++ .fn = fn, ++ .started = ATOMIC_INIT(0) ++ }; ++ struct task_struct *k = kthread_create(bstrap_fn, &bstrap, ++ "hotpotato%d", cpu); ++ int ret; ++ if (IS_ERR(k)) ++ return -ENOMEM; ++ kthread_bind(k, cpu); ++ wake_up_process(k); ++ /* If we call kthread_stop() before the "wake up" has had an ++ * effect, then the thread may exit with -EINTR without ever ++ * running the function. So poll until it's started before ++ * requesting it to stop. */ ++ while (!atomic_read(&bstrap.started)) ++ msleep(10); ++ ret = kthread_stop(k); ++ if (ret) ++ return ret; ++ } ++ return 0; ++} ++ ++struct hp_handler { ++ ++ /* The following data is stashed when 'rx' is dequeued; */ ++ /* -------------- */ ++ /* The Rx FQ, dequeues of which will stash the entire hp_handler */ ++ struct qman_fq rx; ++ /* The Tx FQ we should forward to */ ++ struct qman_fq tx; ++ /* The value we XOR post-dequeue, prior to validating */ ++ u32 rx_mixer; ++ /* The value we XOR pre-enqueue, after validating */ ++ u32 tx_mixer; ++ /* what the hotpotato address should be on dequeue */ ++ dma_addr_t addr; ++ u32 *frame_ptr; ++ ++ /* The following data isn't (necessarily) stashed on dequeue; */ ++ /* -------------- */ ++ u32 fqid_rx, fqid_tx; ++ /* list node for linking us into 'hp_cpu' */ ++ struct list_head node; ++ /* Just to check ... */ ++ unsigned int processor_id; ++} ____cacheline_aligned; ++ ++struct hp_cpu { ++ /* identify the cpu we run on; */ ++ unsigned int processor_id; ++ /* root node for the per-cpu list of handlers */ ++ struct list_head handlers; ++ /* list node for linking us into 'hp_cpu_list' */ ++ struct list_head node; ++ /* when repeatedly scanning 'hp_list', each time linking the n'th ++ * handlers together, this is used as per-cpu iterator state */ ++ struct hp_handler *iterator; ++}; ++ ++/* Each cpu has one of these */ ++static DEFINE_PER_CPU(struct hp_cpu, hp_cpus); ++ ++/* links together the hp_cpu structs, in first-come first-serve order. */ ++static LIST_HEAD(hp_cpu_list); ++static spinlock_t hp_lock = __SPIN_LOCK_UNLOCKED(hp_lock); ++ ++static unsigned int hp_cpu_list_length; ++ ++/* the "special" handler, that starts and terminates the test. */ ++static struct hp_handler *special_handler; ++static int loop_counter; ++ ++/* handlers are allocated out of this, so they're properly aligned. */ ++static struct kmem_cache *hp_handler_slab; ++ ++/* this is the frame data */ ++static void *__frame_ptr; ++static u32 *frame_ptr; ++static dma_addr_t frame_dma; ++ ++/* the main function waits on this */ ++static DECLARE_WAIT_QUEUE_HEAD(queue); ++ ++#define HP_PER_CPU 2 ++#define HP_LOOPS 8 ++/* 80 bytes, like a small ethernet frame, and bleeds into a second cacheline */ ++#define HP_NUM_WORDS 80 ++/* First word of the LFSR-based frame data */ ++#define HP_FIRST_WORD 0xabbaf00d ++ ++static inline u32 do_lfsr(u32 prev) ++{ ++ return (prev >> 1) ^ (-(prev & 1u) & 0xd0000001u); ++} ++ ++static void allocate_frame_data(void) ++{ ++ u32 lfsr = HP_FIRST_WORD; ++ int loop; ++ struct platform_device *pdev = platform_device_alloc("foobar", -1); ++ if (!pdev) ++ panic("platform_device_alloc() failed"); ++ if (platform_device_add(pdev)) ++ panic("platform_device_add() failed"); ++ __frame_ptr = kmalloc(4 * HP_NUM_WORDS, GFP_KERNEL); ++ if (!__frame_ptr) ++ panic("kmalloc() failed"); ++ frame_ptr = (void *)(((unsigned long)__frame_ptr + 63) & ++ ~(unsigned long)63); ++ for (loop = 0; loop < HP_NUM_WORDS; loop++) { ++ frame_ptr[loop] = lfsr; ++ lfsr = do_lfsr(lfsr); ++ } ++ frame_dma = dma_map_single(&pdev->dev, frame_ptr, 4 * HP_NUM_WORDS, ++ DMA_BIDIRECTIONAL); ++ platform_device_del(pdev); ++ platform_device_put(pdev); ++} ++ ++static void deallocate_frame_data(void) ++{ ++ kfree(__frame_ptr); ++} ++ ++static inline void process_frame_data(struct hp_handler *handler, ++ const struct qm_fd *fd) ++{ ++ u32 *p = handler->frame_ptr; ++ u32 lfsr = HP_FIRST_WORD; ++ int loop; ++ if (qm_fd_addr_get64(fd) != handler->addr) ++ panic("bad frame address"); ++ for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) { ++ *p ^= handler->rx_mixer; ++ if (*p != lfsr) ++ panic("corrupt frame data"); ++ *p ^= handler->tx_mixer; ++ lfsr = do_lfsr(lfsr); ++ } ++} ++ ++static enum qman_cb_dqrr_result normal_dqrr(struct qman_portal *portal, ++ struct qman_fq *fq, ++ const struct qm_dqrr_entry *dqrr) ++{ ++ struct hp_handler *handler = (struct hp_handler *)fq; ++ ++ process_frame_data(handler, &dqrr->fd); ++ if (qman_enqueue(&handler->tx, &dqrr->fd, 0)) ++ panic("qman_enqueue() failed"); ++ return qman_cb_dqrr_consume; ++} ++ ++static enum qman_cb_dqrr_result special_dqrr(struct qman_portal *portal, ++ struct qman_fq *fq, ++ const struct qm_dqrr_entry *dqrr) ++{ ++ struct hp_handler *handler = (struct hp_handler *)fq; ++ ++ process_frame_data(handler, &dqrr->fd); ++ if (++loop_counter < HP_LOOPS) { ++ if (qman_enqueue(&handler->tx, &dqrr->fd, 0)) ++ panic("qman_enqueue() failed"); ++ } else { ++ pr_info("Received final (%dth) frame\n", loop_counter); ++ wake_up(&queue); ++ } ++ return qman_cb_dqrr_consume; ++} ++ ++static void create_per_cpu_handlers(void) ++{ ++ struct hp_handler *handler; ++ int loop; ++ struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus); ++ ++ hp_cpu->processor_id = smp_processor_id(); ++ spin_lock(&hp_lock); ++ list_add_tail(&hp_cpu->node, &hp_cpu_list); ++ hp_cpu_list_length++; ++ spin_unlock(&hp_lock); ++ INIT_LIST_HEAD(&hp_cpu->handlers); ++ for (loop = 0; loop < HP_PER_CPU; loop++) { ++ handler = kmem_cache_alloc(hp_handler_slab, GFP_KERNEL); ++ if (!handler) ++ panic("kmem_cache_alloc() failed"); ++ handler->processor_id = hp_cpu->processor_id; ++ handler->addr = frame_dma; ++ handler->frame_ptr = frame_ptr; ++ list_add_tail(&handler->node, &hp_cpu->handlers); ++ } ++ put_cpu_var(hp_cpus); ++} ++ ++static void destroy_per_cpu_handlers(void) ++{ ++ struct list_head *loop, *tmp; ++ struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus); ++ ++ spin_lock(&hp_lock); ++ list_del(&hp_cpu->node); ++ spin_unlock(&hp_lock); ++ list_for_each_safe(loop, tmp, &hp_cpu->handlers) { ++ u32 flags; ++ struct hp_handler *handler = list_entry(loop, struct hp_handler, ++ node); ++ if (qman_retire_fq(&handler->rx, &flags)) ++ panic("qman_retire_fq(rx) failed"); ++ BUG_ON(flags & QMAN_FQ_STATE_BLOCKOOS); ++ if (qman_oos_fq(&handler->rx)) ++ panic("qman_oos_fq(rx) failed"); ++ qman_destroy_fq(&handler->rx, 0); ++ qman_destroy_fq(&handler->tx, 0); ++ qman_release_fqid(handler->fqid_rx); ++ list_del(&handler->node); ++ kmem_cache_free(hp_handler_slab, handler); ++ } ++ put_cpu_var(hp_cpus); ++} ++ ++static inline u8 num_cachelines(u32 offset) ++{ ++ u8 res = (offset + (L1_CACHE_BYTES - 1)) ++ / (L1_CACHE_BYTES); ++ if (res > 3) ++ return 3; ++ return res; ++} ++#define STASH_DATA_CL \ ++ num_cachelines(HP_NUM_WORDS * 4) ++#define STASH_CTX_CL \ ++ num_cachelines(offsetof(struct hp_handler, fqid_rx)) ++ ++static void init_handler(void *__handler) ++{ ++ struct qm_mcc_initfq opts; ++ struct hp_handler *handler = __handler; ++ BUG_ON(handler->processor_id != smp_processor_id()); ++ /* Set up rx */ ++ memset(&handler->rx, 0, sizeof(handler->rx)); ++ if (handler == special_handler) ++ handler->rx.cb.dqrr = special_dqrr; ++ else ++ handler->rx.cb.dqrr = normal_dqrr; ++ if (qman_create_fq(handler->fqid_rx, 0, &handler->rx)) ++ panic("qman_create_fq(rx) failed"); ++ memset(&opts, 0, sizeof(opts)); ++ opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_CONTEXTA; ++ opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING; ++ opts.fqd.context_a.stashing.data_cl = STASH_DATA_CL; ++ opts.fqd.context_a.stashing.context_cl = STASH_CTX_CL; ++ if (qman_init_fq(&handler->rx, QMAN_INITFQ_FLAG_SCHED | ++ QMAN_INITFQ_FLAG_LOCAL, &opts)) ++ panic("qman_init_fq(rx) failed"); ++ /* Set up tx */ ++ memset(&handler->tx, 0, sizeof(handler->tx)); ++ if (qman_create_fq(handler->fqid_tx, QMAN_FQ_FLAG_NO_MODIFY, ++ &handler->tx)) ++ panic("qman_create_fq(tx) failed"); ++} ++ ++static void init_phase2(void) ++{ ++ int loop; ++ u32 fqid = 0; ++ u32 lfsr = 0xdeadbeef; ++ struct hp_cpu *hp_cpu; ++ struct hp_handler *handler; ++ ++ for (loop = 0; loop < HP_PER_CPU; loop++) { ++ list_for_each_entry(hp_cpu, &hp_cpu_list, node) { ++ int ret; ++ if (!loop) ++ hp_cpu->iterator = list_first_entry( ++ &hp_cpu->handlers, ++ struct hp_handler, node); ++ else ++ hp_cpu->iterator = list_entry( ++ hp_cpu->iterator->node.next, ++ struct hp_handler, node); ++ /* Rx FQID is the previous handler's Tx FQID */ ++ hp_cpu->iterator->fqid_rx = fqid; ++ /* Allocate new FQID for Tx */ ++ ret = qman_alloc_fqid(&fqid); ++ if (ret) ++ panic("qman_alloc_fqid() failed"); ++ hp_cpu->iterator->fqid_tx = fqid; ++ /* Rx mixer is the previous handler's Tx mixer */ ++ hp_cpu->iterator->rx_mixer = lfsr; ++ /* Get new mixer for Tx */ ++ lfsr = do_lfsr(lfsr); ++ hp_cpu->iterator->tx_mixer = lfsr; ++ } ++ } ++ /* Fix up the first handler (fqid_rx==0, rx_mixer=0xdeadbeef) */ ++ hp_cpu = list_first_entry(&hp_cpu_list, struct hp_cpu, node); ++ handler = list_first_entry(&hp_cpu->handlers, struct hp_handler, node); ++ BUG_ON((handler->fqid_rx != 0) || (handler->rx_mixer != 0xdeadbeef)); ++ handler->fqid_rx = fqid; ++ handler->rx_mixer = lfsr; ++ /* and tag it as our "special" handler */ ++ special_handler = handler; ++} ++ ++static void init_phase3(void) ++{ ++ int loop; ++ struct hp_cpu *hp_cpu; ++ ++ for (loop = 0; loop < HP_PER_CPU; loop++) { ++ list_for_each_entry(hp_cpu, &hp_cpu_list, node) { ++ if (!loop) ++ hp_cpu->iterator = list_first_entry( ++ &hp_cpu->handlers, ++ struct hp_handler, node); ++ else ++ hp_cpu->iterator = list_entry( ++ hp_cpu->iterator->node.next, ++ struct hp_handler, node); ++ preempt_disable(); ++ if (hp_cpu->processor_id == smp_processor_id()) ++ init_handler(hp_cpu->iterator); ++ else ++ smp_call_function_single(hp_cpu->processor_id, ++ init_handler, hp_cpu->iterator, 1); ++ preempt_enable(); ++ } ++ } ++} ++ ++static void send_first_frame(void *ignore) ++{ ++ u32 *p = special_handler->frame_ptr; ++ u32 lfsr = HP_FIRST_WORD; ++ int loop; ++ struct qm_fd fd; ++ ++ BUG_ON(special_handler->processor_id != smp_processor_id()); ++ memset(&fd, 0, sizeof(fd)); ++ qm_fd_addr_set64(&fd, special_handler->addr); ++ fd.format = qm_fd_contig_big; ++ fd.length29 = HP_NUM_WORDS * 4; ++ for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) { ++ if (*p != lfsr) ++ panic("corrupt frame data"); ++ *p ^= special_handler->tx_mixer; ++ lfsr = do_lfsr(lfsr); ++ } ++ pr_info("Sending first frame\n"); ++ if (qman_enqueue(&special_handler->tx, &fd, 0)) ++ panic("qman_enqueue() failed"); ++} ++ ++void qman_test_hotpotato(void) ++{ ++ if (cpumask_weight(cpu_online_mask) < 2) { ++ pr_info("qman_test_hotpotato, skip - only 1 CPU\n"); ++ return; ++ } ++ ++ pr_info("qman_test_hotpotato starting\n"); ++ ++ hp_cpu_list_length = 0; ++ loop_counter = 0; ++ hp_handler_slab = kmem_cache_create("hp_handler_slab", ++ sizeof(struct hp_handler), L1_CACHE_BYTES, ++ SLAB_HWCACHE_ALIGN, NULL); ++ if (!hp_handler_slab) ++ panic("kmem_cache_create() failed"); ++ ++ allocate_frame_data(); ++ ++ /* Init phase 1 */ ++ pr_info("Creating %d handlers per cpu...\n", HP_PER_CPU); ++ if (on_all_cpus(create_per_cpu_handlers)) ++ panic("on_each_cpu() failed"); ++ pr_info("Number of cpus: %d, total of %d handlers\n", ++ hp_cpu_list_length, hp_cpu_list_length * HP_PER_CPU); ++ ++ init_phase2(); ++ ++ init_phase3(); ++ ++ preempt_disable(); ++ if (special_handler->processor_id == smp_processor_id()) ++ send_first_frame(NULL); ++ else ++ smp_call_function_single(special_handler->processor_id, ++ send_first_frame, NULL, 1); ++ preempt_enable(); ++ ++ wait_event(queue, loop_counter == HP_LOOPS); ++ deallocate_frame_data(); ++ if (on_all_cpus(destroy_per_cpu_handlers)) ++ panic("on_each_cpu() failed"); ++ kmem_cache_destroy(hp_handler_slab); ++ pr_info("qman_test_hotpotato finished\n"); ++} +--- /dev/null ++++ b/drivers/staging/fsl_qbman/qman_utility.c +@@ -0,0 +1,129 @@ ++/* Copyright 2008-2011 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#include "qman_private.h" ++ ++/* ----------------- */ ++/* --- FQID Pool --- */ ++ ++struct qman_fqid_pool { ++ /* Base and size of the FQID range */ ++ u32 fqid_base; ++ u32 total; ++ /* Number of FQIDs currently "allocated" */ ++ u32 used; ++ /* Allocation optimisation. When 'used<total', it is the index of an ++ * available FQID. Otherwise there are no available FQIDs, and this ++ * will be set when the next deallocation occurs. */ ++ u32 next; ++ /* A bit-field representation of the FQID range. */ ++ unsigned long *bits; ++}; ++ ++#define QLONG_BYTES sizeof(unsigned long) ++#define QLONG_BITS (QLONG_BYTES * 8) ++/* Number of 'longs' required for the given number of bits */ ++#define QNUM_LONGS(b) (((b) + QLONG_BITS - 1) / QLONG_BITS) ++/* Shorthand for the number of bytes of same (kmalloc, memset, etc) */ ++#define QNUM_BYTES(b) (QNUM_LONGS(b) * QLONG_BYTES) ++/* And in bits */ ++#define QNUM_BITS(b) (QNUM_LONGS(b) * QLONG_BITS) ++ ++struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num) ++{ ++ struct qman_fqid_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); ++ unsigned int i; ++ ++ BUG_ON(!num); ++ if (!pool) ++ return NULL; ++ pool->fqid_base = fqid_start; ++ pool->total = num; ++ pool->used = 0; ++ pool->next = 0; ++ pool->bits = kzalloc(QNUM_BYTES(num), GFP_KERNEL); ++ if (!pool->bits) { ++ kfree(pool); ++ return NULL; ++ } ++ /* If num is not an even multiple of QLONG_BITS (or even 8, for ++ * byte-oriented searching) then we fill the trailing bits with 1, to ++ * make them look allocated (permanently). */ ++ for (i = num + 1; i < QNUM_BITS(num); i++) ++ set_bit(i, pool->bits); ++ return pool; ++} ++EXPORT_SYMBOL(qman_fqid_pool_create); ++ ++int qman_fqid_pool_destroy(struct qman_fqid_pool *pool) ++{ ++ int ret = pool->used; ++ kfree(pool->bits); ++ kfree(pool); ++ return ret; ++} ++EXPORT_SYMBOL(qman_fqid_pool_destroy); ++ ++int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid) ++{ ++ int ret; ++ if (pool->used == pool->total) ++ return -ENOMEM; ++ *fqid = pool->fqid_base + pool->next; ++ ret = test_and_set_bit(pool->next, pool->bits); ++ BUG_ON(ret); ++ if (++pool->used == pool->total) ++ return 0; ++ pool->next = find_next_zero_bit(pool->bits, pool->total, pool->next); ++ if (pool->next >= pool->total) ++ pool->next = find_first_zero_bit(pool->bits, pool->total); ++ BUG_ON(pool->next >= pool->total); ++ return 0; ++} ++EXPORT_SYMBOL(qman_fqid_pool_alloc); ++ ++void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid) ++{ ++ int ret; ++ ++ fqid -= pool->fqid_base; ++ ret = test_and_clear_bit(fqid, pool->bits); ++ BUG_ON(!ret); ++ if (pool->used-- == pool->total) ++ pool->next = fqid; ++} ++EXPORT_SYMBOL(qman_fqid_pool_free); ++ ++u32 qman_fqid_pool_used(struct qman_fqid_pool *pool) ++{ ++ return pool->used; ++} ++EXPORT_SYMBOL(qman_fqid_pool_used); +--- /dev/null ++++ b/include/linux/fsl_bman.h +@@ -0,0 +1,532 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef FSL_BMAN_H ++#define FSL_BMAN_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* Last updated for v00.79 of the BG */ ++ ++/* Portal processing (interrupt) sources */ ++#define BM_PIRQ_RCRI 0x00000002 /* RCR Ring (below threshold) */ ++#define BM_PIRQ_BSCN 0x00000001 /* Buffer depletion State Change */ ++ ++/* This wrapper represents a bit-array for the depletion state of the 64 Bman ++ * buffer pools. */ ++struct bman_depletion { ++ u32 __state[2]; ++}; ++#define BMAN_DEPLETION_EMPTY { { 0x00000000, 0x00000000 } } ++#define BMAN_DEPLETION_FULL { { 0xffffffff, 0xffffffff } } ++#define __bmdep_word(x) ((x) >> 5) ++#define __bmdep_shift(x) ((x) & 0x1f) ++#define __bmdep_bit(x) (0x80000000 >> __bmdep_shift(x)) ++static inline void bman_depletion_init(struct bman_depletion *c) ++{ ++ c->__state[0] = c->__state[1] = 0; ++} ++static inline void bman_depletion_fill(struct bman_depletion *c) ++{ ++ c->__state[0] = c->__state[1] = ~0; ++} ++static inline int bman_depletion_get(const struct bman_depletion *c, u8 bpid) ++{ ++ return c->__state[__bmdep_word(bpid)] & __bmdep_bit(bpid); ++} ++static inline void bman_depletion_set(struct bman_depletion *c, u8 bpid) ++{ ++ c->__state[__bmdep_word(bpid)] |= __bmdep_bit(bpid); ++} ++static inline void bman_depletion_unset(struct bman_depletion *c, u8 bpid) ++{ ++ c->__state[__bmdep_word(bpid)] &= ~__bmdep_bit(bpid); ++} ++ ++/* ------------------------------------------------------- */ ++/* --- Bman data structures (and associated constants) --- */ ++ ++/* Represents s/w corenet portal mapped data structures */ ++struct bm_rcr_entry; /* RCR (Release Command Ring) entries */ ++struct bm_mc_command; /* MC (Management Command) command */ ++struct bm_mc_result; /* MC result */ ++ ++/* Code-reduction, define a wrapper for 48-bit buffers. In cases where a buffer ++ * pool id specific to this buffer is needed (BM_RCR_VERB_CMD_BPID_MULTI, ++ * BM_MCC_VERB_ACQUIRE), the 'bpid' field is used. */ ++struct bm_buffer { ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 __reserved1; ++ u8 bpid; ++ u16 hi; /* High 16-bits of 48-bit address */ ++ u32 lo; /* Low 32-bits of 48-bit address */ ++#else ++ u32 lo; ++ u16 hi; ++ u8 bpid; ++ u8 __reserved; ++#endif ++ }; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u64 __notaddress:16; ++ u64 addr:48; ++#else ++ u64 addr:48; ++ u64 __notaddress:16; ++#endif ++ }; ++ u64 opaque; ++ }; ++} __aligned(8); ++static inline u64 bm_buffer_get64(const struct bm_buffer *buf) ++{ ++ return buf->addr; ++} ++static inline dma_addr_t bm_buf_addr(const struct bm_buffer *buf) ++{ ++ return (dma_addr_t)buf->addr; ++} ++/* Macro, so we compile better if 'v' isn't always 64-bit */ ++#define bm_buffer_set64(buf, v) \ ++ do { \ ++ struct bm_buffer *__buf931 = (buf); \ ++ __buf931->hi = upper_32_bits(v); \ ++ __buf931->lo = lower_32_bits(v); \ ++ } while (0) ++ ++/* See 1.5.3.5.4: "Release Command" */ ++struct bm_rcr_entry { ++ union { ++ struct { ++ u8 __dont_write_directly__verb; ++ u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */ ++ u8 __reserved1[62]; ++ }; ++ struct bm_buffer bufs[8]; ++ }; ++} __packed; ++#define BM_RCR_VERB_VBIT 0x80 ++#define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */ ++#define BM_RCR_VERB_CMD_BPID_SINGLE 0x20 ++#define BM_RCR_VERB_CMD_BPID_MULTI 0x30 ++#define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */ ++ ++/* See 1.5.3.1: "Acquire Command" */ ++/* See 1.5.3.2: "Query Command" */ ++struct bm_mcc_acquire { ++ u8 bpid; ++ u8 __reserved1[62]; ++} __packed; ++struct bm_mcc_query { ++ u8 __reserved2[63]; ++} __packed; ++struct bm_mc_command { ++ u8 __dont_write_directly__verb; ++ union { ++ struct bm_mcc_acquire acquire; ++ struct bm_mcc_query query; ++ }; ++} __packed; ++#define BM_MCC_VERB_VBIT 0x80 ++#define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */ ++#define BM_MCC_VERB_CMD_ACQUIRE 0x10 ++#define BM_MCC_VERB_CMD_QUERY 0x40 ++#define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */ ++ ++/* See 1.5.3.3: "Acquire Response" */ ++/* See 1.5.3.4: "Query Response" */ ++struct bm_pool_state { ++ u8 __reserved1[32]; ++ /* "availability state" and "depletion state" */ ++ struct { ++ u8 __reserved1[8]; ++ /* Access using bman_depletion_***() */ ++ struct bman_depletion state; ++ } as, ds; ++}; ++struct bm_mc_result { ++ union { ++ struct { ++ u8 verb; ++ u8 __reserved1[63]; ++ }; ++ union { ++ struct { ++ u8 __reserved1; ++ u8 bpid; ++ u8 __reserved2[62]; ++ }; ++ struct bm_buffer bufs[8]; ++ } acquire; ++ struct bm_pool_state query; ++ }; ++} __packed; ++#define BM_MCR_VERB_VBIT 0x80 ++#define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK ++#define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE ++#define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY ++#define BM_MCR_VERB_CMD_ERR_INVALID 0x60 ++#define BM_MCR_VERB_CMD_ERR_ECC 0x70 ++#define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */ ++/* Determine the "availability state" of pool 'p' from a query result 'r' */ ++#define BM_MCR_QUERY_AVAILABILITY(r, p) \ ++ bman_depletion_get(&r->query.as.state, p) ++/* Determine the "depletion state" of pool 'p' from a query result 'r' */ ++#define BM_MCR_QUERY_DEPLETION(r, p) \ ++ bman_depletion_get(&r->query.ds.state, p) ++ ++/*******************************************************************/ ++/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */ ++/*******************************************************************/ ++ ++ /* Portal and Buffer Pools */ ++ /* ----------------------- */ ++/* Represents a managed portal */ ++struct bman_portal; ++ ++/* This object type represents Bman buffer pools. */ ++struct bman_pool; ++ ++struct bman_portal_config { ++ /* This is used for any "core-affine" portals, ie. default portals ++ * associated to the corresponding cpu. -1 implies that there is no core ++ * affinity configured. */ ++ int cpu; ++ /* portal interrupt line */ ++ int irq; ++ /* the unique index of this portal */ ++ u32 index; ++ /* Is this portal shared? (If so, it has coarser locking and demuxes ++ * processing on behalf of other CPUs.) */ ++ int is_shared; ++ /* These are the buffer pool IDs that may be used via this portal. */ ++ struct bman_depletion mask; ++}; ++ ++/* This callback type is used when handling pool depletion entry/exit. The ++ * 'cb_ctx' value is the opaque value associated with the pool object in ++ * bman_new_pool(). 'depleted' is non-zero on depletion-entry, and zero on ++ * depletion-exit. */ ++typedef void (*bman_cb_depletion)(struct bman_portal *bm, ++ struct bman_pool *pool, void *cb_ctx, int depleted); ++ ++/* This struct specifies parameters for a bman_pool object. */ ++struct bman_pool_params { ++ /* index of the buffer pool to encapsulate (0-63), ignored if ++ * BMAN_POOL_FLAG_DYNAMIC_BPID is set. */ ++ u32 bpid; ++ /* bit-mask of BMAN_POOL_FLAG_*** options */ ++ u32 flags; ++ /* depletion-entry/exit callback, if BMAN_POOL_FLAG_DEPLETION is set */ ++ bman_cb_depletion cb; ++ /* opaque user value passed as a parameter to 'cb' */ ++ void *cb_ctx; ++ /* depletion-entry/exit thresholds, if BMAN_POOL_FLAG_THRESH is set. NB: ++ * this is only allowed if BMAN_POOL_FLAG_DYNAMIC_BPID is used *and* ++ * when run in the control plane (which controls Bman CCSR). This array ++ * matches the definition of bm_pool_set(). */ ++ u32 thresholds[4]; ++}; ++ ++/* Flags to bman_new_pool() */ ++#define BMAN_POOL_FLAG_NO_RELEASE 0x00000001 /* can't release to pool */ ++#define BMAN_POOL_FLAG_ONLY_RELEASE 0x00000002 /* can only release to pool */ ++#define BMAN_POOL_FLAG_DEPLETION 0x00000004 /* track depletion entry/exit */ ++#define BMAN_POOL_FLAG_DYNAMIC_BPID 0x00000008 /* (de)allocate bpid */ ++#define BMAN_POOL_FLAG_THRESH 0x00000010 /* set depletion thresholds */ ++#define BMAN_POOL_FLAG_STOCKPILE 0x00000020 /* stockpile to reduce hw ops */ ++ ++/* Flags to bman_release() */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++#define BMAN_RELEASE_FLAG_WAIT 0x00000001 /* wait if RCR is full */ ++#define BMAN_RELEASE_FLAG_WAIT_INT 0x00000002 /* if we wait, interruptible? */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++#define BMAN_RELEASE_FLAG_WAIT_SYNC 0x00000004 /* if wait, until consumed? */ ++#endif ++#endif ++#define BMAN_RELEASE_FLAG_NOW 0x00000008 /* issue immediate release */ ++ ++/* Flags to bman_acquire() */ ++#define BMAN_ACQUIRE_FLAG_STOCKPILE 0x00000001 /* no hw op, stockpile only */ ++ ++ /* Portal Management */ ++ /* ----------------- */ ++/** ++ * bman_get_portal_config - get portal configuration settings ++ * ++ * This returns a read-only view of the current cpu's affine portal settings. ++ */ ++const struct bman_portal_config *bman_get_portal_config(void); ++ ++/** ++ * bman_irqsource_get - return the portal work that is interrupt-driven ++ * ++ * Returns a bitmask of BM_PIRQ_**I processing sources that are currently ++ * enabled for interrupt handling on the current cpu's affine portal. These ++ * sources will trigger the portal interrupt and the interrupt handler (or a ++ * tasklet/bottom-half it defers to) will perform the corresponding processing ++ * work. The bman_poll_***() functions will only process sources that are not in ++ * this bitmask. If the current CPU is sharing a portal hosted on another CPU, ++ * this always returns zero. ++ */ ++u32 bman_irqsource_get(void); ++ ++/** ++ * bman_irqsource_add - add processing sources to be interrupt-driven ++ * @bits: bitmask of BM_PIRQ_**I processing sources ++ * ++ * Adds processing sources that should be interrupt-driven (rather than ++ * processed via bman_poll_***() functions). Returns zero for success, or ++ * -EINVAL if the current CPU is sharing a portal hosted on another CPU. */ ++int bman_irqsource_add(u32 bits); ++ ++/** ++ * bman_irqsource_remove - remove processing sources from being interrupt-driven ++ * @bits: bitmask of BM_PIRQ_**I processing sources ++ * ++ * Removes processing sources from being interrupt-driven, so that they will ++ * instead be processed via bman_poll_***() functions. Returns zero for success, ++ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU. */ ++int bman_irqsource_remove(u32 bits); ++ ++/** ++ * bman_affine_cpus - return a mask of cpus that have affine portals ++ */ ++const cpumask_t *bman_affine_cpus(void); ++ ++/** ++ * bman_poll_slow - process anything that isn't interrupt-driven. ++ * ++ * This function does any portal processing that isn't interrupt-driven. If the ++ * current CPU is sharing a portal hosted on another CPU, this function will ++ * return -EINVAL, otherwise the return value is a bitmask of BM_PIRQ_* sources ++ * indicating what interrupt sources were actually processed by the call. ++ * ++ * NB, unlike the legacy wrapper bman_poll(), this function will ++ * deterministically check for the presence of portal processing work and do it, ++ * which implies some latency even if there's nothing to do. The bman_poll() ++ * wrapper on the other hand (like the qman_poll() wrapper) attenuates this by ++ * checking for (and doing) portal processing infrequently. Ie. such that ++ * qman_poll() and bman_poll() can be called from core-processing loops. Use ++ * bman_poll_slow() when you yourself are deciding when to incur the overhead of ++ * processing. ++ */ ++u32 bman_poll_slow(void); ++ ++/** ++ * bman_poll - process anything that isn't interrupt-driven. ++ * ++ * Dispatcher logic on a cpu can use this to trigger any maintenance of the ++ * affine portal. This function does whatever processing is not triggered by ++ * interrupts. This is a legacy wrapper that can be used in core-processing ++ * loops but mitigates the performance overhead of portal processing by ++ * adaptively bypassing true portal processing most of the time. (Processing is ++ * done once every 10 calls if the previous processing revealed that work needed ++ * to be done, or once very 1000 calls if the previous processing revealed no ++ * work needed doing.) If you wish to control this yourself, call ++ * bman_poll_slow() instead, which always checks for portal processing work. ++ */ ++void bman_poll(void); ++ ++/** ++ * bman_rcr_is_empty - Determine if portal's RCR is empty ++ * ++ * For use in situations where a cpu-affine caller needs to determine when all ++ * releases for the local portal have been processed by Bman but can't use the ++ * BMAN_RELEASE_FLAG_WAIT_SYNC flag to do this from the final bman_release(). ++ * The function forces tracking of RCR consumption (which normally doesn't ++ * happen until release processing needs to find space to put new release ++ * commands), and returns zero if the ring still has unprocessed entries, ++ * non-zero if it is empty. ++ */ ++int bman_rcr_is_empty(void); ++ ++/** ++ * bman_alloc_bpid_range - Allocate a contiguous range of BPIDs ++ * @result: is set by the API to the base BPID of the allocated range ++ * @count: the number of BPIDs required ++ * @align: required alignment of the allocated range ++ * @partial: non-zero if the API can return fewer than @count BPIDs ++ * ++ * Returns the number of buffer pools allocated, or a negative error code. If ++ * @partial is non zero, the allocation request may return a smaller range of ++ * BPs than requested (though alignment will be as requested). If @partial is ++ * zero, the return value will either be 'count' or negative. ++ */ ++int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial); ++static inline int bman_alloc_bpid(u32 *result) ++{ ++ int ret = bman_alloc_bpid_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++ ++/** ++ * bman_release_bpid_range - Release the specified range of buffer pool IDs ++ * @bpid: the base BPID of the range to deallocate ++ * @count: the number of BPIDs in the range ++ * ++ * This function can also be used to seed the allocator with ranges of BPIDs ++ * that it can subsequently allocate from. ++ */ ++void bman_release_bpid_range(u32 bpid, unsigned int count); ++static inline void bman_release_bpid(u32 bpid) ++{ ++ bman_release_bpid_range(bpid, 1); ++} ++ ++int bman_reserve_bpid_range(u32 bpid, unsigned int count); ++static inline int bman_reserve_bpid(u32 bpid) ++{ ++ return bman_reserve_bpid_range(bpid, 1); ++} ++ ++void bman_seed_bpid_range(u32 bpid, unsigned int count); ++ ++ ++int bman_shutdown_pool(u32 bpid); ++ ++ /* Pool management */ ++ /* --------------- */ ++/** ++ * bman_new_pool - Allocates a Buffer Pool object ++ * @params: parameters specifying the buffer pool ID and behaviour ++ * ++ * Creates a pool object for the given @params. A portal and the depletion ++ * callback field of @params are only used if the BMAN_POOL_FLAG_DEPLETION flag ++ * is set. NB, the fields from @params are copied into the new pool object, so ++ * the structure provided by the caller can be released or reused after the ++ * function returns. ++ */ ++struct bman_pool *bman_new_pool(const struct bman_pool_params *params); ++ ++/** ++ * bman_free_pool - Deallocates a Buffer Pool object ++ * @pool: the pool object to release ++ * ++ */ ++void bman_free_pool(struct bman_pool *pool); ++ ++/** ++ * bman_get_params - Returns a pool object's parameters. ++ * @pool: the pool object ++ * ++ * The returned pointer refers to state within the pool object so must not be ++ * modified and can no longer be read once the pool object is destroyed. ++ */ ++const struct bman_pool_params *bman_get_params(const struct bman_pool *pool); ++ ++/** ++ * bman_release - Release buffer(s) to the buffer pool ++ * @pool: the buffer pool object to release to ++ * @bufs: an array of buffers to release ++ * @num: the number of buffers in @bufs (1-8) ++ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options ++ * ++ * Adds the given buffers to RCR entries. If the portal @p was created with the ++ * "COMPACT" flag, then it will be using a compaction algorithm to improve ++ * utilisation of RCR. As such, these buffers may join an existing ring entry ++ * and/or it may not be issued right away so as to allow future releases to join ++ * the same ring entry. Use the BMAN_RELEASE_FLAG_NOW flag to override this ++ * behaviour by committing the RCR entry (or entries) right away. If the RCR ++ * ring is full, the function will return -EBUSY unless BMAN_RELEASE_FLAG_WAIT ++ * is selected, in which case it will sleep waiting for space to become ++ * available in RCR. If the function receives a signal before such time (and ++ * BMAN_RELEASE_FLAG_WAIT_INT is set), the function returns -EINTR. Otherwise, ++ * it returns zero. ++ */ ++int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, ++ u32 flags); ++ ++/** ++ * bman_acquire - Acquire buffer(s) from a buffer pool ++ * @pool: the buffer pool object to acquire from ++ * @bufs: array for storing the acquired buffers ++ * @num: the number of buffers desired (@bufs is at least this big) ++ * ++ * Issues an "Acquire" command via the portal's management command interface. ++ * The return value will be the number of buffers obtained from the pool, or a ++ * negative error code if a h/w error or pool starvation was encountered. In ++ * the latter case, the content of @bufs is undefined. ++ */ ++int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, ++ u32 flags); ++ ++/** ++ * bman_flush_stockpile - Flush stockpile buffer(s) to the buffer pool ++ * @pool: the buffer pool object the stockpile belongs ++ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options ++ * ++ * Adds stockpile buffers to RCR entries until the stockpile is empty. ++ * The return value will be a negative error code if a h/w error occurred. ++ * If BMAN_RELEASE_FLAG_NOW flag is passed and RCR ring is full, ++ * -EAGAIN will be returned. ++ */ ++int bman_flush_stockpile(struct bman_pool *pool, u32 flags); ++ ++/** ++ * bman_query_pools - Query all buffer pool states ++ * @state: storage for the queried availability and depletion states ++ */ ++int bman_query_pools(struct bm_pool_state *state); ++ ++#ifdef CONFIG_FSL_BMAN_CONFIG ++/** ++ * bman_query_free_buffers - Query how many free buffers are in buffer pool ++ * @pool: the buffer pool object to query ++ * ++ * Return the number of the free buffers ++ */ ++u32 bman_query_free_buffers(struct bman_pool *pool); ++ ++/** ++ * bman_update_pool_thresholds - Change the buffer pool's depletion thresholds ++ * @pool: the buffer pool object to which the thresholds will be set ++ * @thresholds: the new thresholds ++ */ ++int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds); ++#endif ++ ++/** ++ * The below bman_p_***() variant might be called in a situation that the cpu ++ * which the portal affine to is not online yet. ++ * @bman_portal specifies which portal the API will use. ++*/ ++int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits); ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* FSL_BMAN_H */ +--- /dev/null ++++ b/include/linux/fsl_qman.h +@@ -0,0 +1,3889 @@ ++/* Copyright 2008-2012 Freescale Semiconductor, Inc. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are met: ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above copyright ++ * notice, this list of conditions and the following disclaimer in the ++ * documentation and/or other materials provided with the distribution. ++ * * Neither the name of Freescale Semiconductor nor the ++ * names of its contributors may be used to endorse or promote products ++ * derived from this software without specific prior written permission. ++ * ++ * ++ * ALTERNATIVELY, this software may be distributed under the terms of the ++ * GNU General Public License ("GPL") as published by the Free Software ++ * Foundation, either version 2 of that License or (at your option) any ++ * later version. ++ * ++ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY ++ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ++ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ++ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY ++ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ++ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ++ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ++ */ ++ ++#ifndef FSL_QMAN_H ++#define FSL_QMAN_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++/* Last updated for v00.800 of the BG */ ++ ++/* Hardware constants */ ++#define QM_CHANNEL_SWPORTAL0 0 ++#define QMAN_CHANNEL_POOL1 0x21 ++#define QMAN_CHANNEL_CAAM 0x80 ++#define QMAN_CHANNEL_PME 0xa0 ++#define QMAN_CHANNEL_POOL1_REV3 0x401 ++#define QMAN_CHANNEL_CAAM_REV3 0x840 ++#define QMAN_CHANNEL_PME_REV3 0x860 ++#define QMAN_CHANNEL_DCE 0x8a0 ++#define QMAN_CHANNEL_DCE_QMANREV312 0x880 ++extern u16 qm_channel_pool1; ++extern u16 qm_channel_caam; ++extern u16 qm_channel_pme; ++extern u16 qm_channel_dce; ++enum qm_dc_portal { ++ qm_dc_portal_fman0 = 0, ++ qm_dc_portal_fman1 = 1, ++ qm_dc_portal_caam = 2, ++ qm_dc_portal_pme = 3, ++ qm_dc_portal_rman = 4, ++ qm_dc_portal_dce = 5 ++}; ++ ++/* Portal processing (interrupt) sources */ ++#define QM_PIRQ_CCSCI 0x00200000 /* CEETM Congestion State Change */ ++#define QM_PIRQ_CSCI 0x00100000 /* Congestion State Change */ ++#define QM_PIRQ_EQCI 0x00080000 /* Enqueue Command Committed */ ++#define QM_PIRQ_EQRI 0x00040000 /* EQCR Ring (below threshold) */ ++#define QM_PIRQ_DQRI 0x00020000 /* DQRR Ring (non-empty) */ ++#define QM_PIRQ_MRI 0x00010000 /* MR Ring (non-empty) */ ++/* This mask contains all the interrupt sources that need handling except DQRI, ++ * ie. that if present should trigger slow-path processing. */ ++#define QM_PIRQ_SLOW (QM_PIRQ_CSCI | QM_PIRQ_EQCI | QM_PIRQ_EQRI | \ ++ QM_PIRQ_MRI | QM_PIRQ_CCSCI) ++ ++/* --- Clock speed --- */ ++/* A qman driver instance may or may not know the current qman clock speed. ++ * However, certain CEETM calculations may not be possible if this is not known. ++ * The 'set' function will only succeed (return zero) if the driver did not ++ * already know the clock speed. Likewise, the 'get' function will only succeed ++ * if the driver does know the clock speed (either because it knew when booting, ++ * or was told via 'set'). In cases where software is running on a driver ++ * instance that does not know the clock speed (eg. on a hypervised data-plane), ++ * and the user can obtain the current qman clock speed by other means (eg. from ++ * a message sent from the control-plane), then the 'set' function can be used ++ * to enable rate-calculations in a driver where it would otherwise not be ++ * possible. */ ++int qm_get_clock(u64 *clock_hz); ++int qm_set_clock(u64 clock_hz); ++ ++/* For qman_static_dequeue_*** APIs */ ++#define QM_SDQCR_CHANNELS_POOL_MASK 0x00007fff ++/* for n in [1,15] */ ++#define QM_SDQCR_CHANNELS_POOL(n) (0x00008000 >> (n)) ++/* for conversion from n of qm_channel */ ++static inline u32 QM_SDQCR_CHANNELS_POOL_CONV(u16 channel) ++{ ++ return QM_SDQCR_CHANNELS_POOL(channel + 1 - qm_channel_pool1); ++} ++ ++/* For qman_volatile_dequeue(); Choose one PRECEDENCE. EXACT is optional. Use ++ * NUMFRAMES(n) (6-bit) or NUMFRAMES_TILLEMPTY to fill in the frame-count. Use ++ * FQID(n) to fill in the frame queue ID. */ ++#define QM_VDQCR_PRECEDENCE_VDQCR 0x0 ++#define QM_VDQCR_PRECEDENCE_SDQCR 0x80000000 ++#define QM_VDQCR_EXACT 0x40000000 ++#define QM_VDQCR_NUMFRAMES_MASK 0x3f000000 ++#define QM_VDQCR_NUMFRAMES_SET(n) (((n) & 0x3f) << 24) ++#define QM_VDQCR_NUMFRAMES_GET(n) (((n) >> 24) & 0x3f) ++#define QM_VDQCR_NUMFRAMES_TILLEMPTY QM_VDQCR_NUMFRAMES_SET(0) ++ ++ ++/* ------------------------------------------------------- */ ++/* --- Qman data structures (and associated constants) --- */ ++ ++/* Represents s/w corenet portal mapped data structures */ ++struct qm_eqcr_entry; /* EQCR (EnQueue Command Ring) entries */ ++struct qm_dqrr_entry; /* DQRR (DeQueue Response Ring) entries */ ++struct qm_mr_entry; /* MR (Message Ring) entries */ ++struct qm_mc_command; /* MC (Management Command) command */ ++struct qm_mc_result; /* MC result */ ++ ++/* See David Lapp's "Frame formats" document, "dpateam", Jan 07, 2008 */ ++#define QM_FD_FORMAT_SG 0x4 ++#define QM_FD_FORMAT_LONG 0x2 ++#define QM_FD_FORMAT_COMPOUND 0x1 ++enum qm_fd_format { ++ /* 'contig' implies a contiguous buffer, whereas 'sg' implies a ++ * scatter-gather table. 'big' implies a 29-bit length with no offset ++ * field, otherwise length is 20-bit and offset is 9-bit. 'compound' ++ * implies a s/g-like table, where each entry itself represents a frame ++ * (contiguous or scatter-gather) and the 29-bit "length" is ++ * interpreted purely for congestion calculations, ie. a "congestion ++ * weight". */ ++ qm_fd_contig = 0, ++ qm_fd_contig_big = QM_FD_FORMAT_LONG, ++ qm_fd_sg = QM_FD_FORMAT_SG, ++ qm_fd_sg_big = QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG, ++ qm_fd_compound = QM_FD_FORMAT_COMPOUND ++}; ++ ++/* Capitalised versions are un-typed but can be used in static expressions */ ++#define QM_FD_CONTIG 0 ++#define QM_FD_CONTIG_BIG QM_FD_FORMAT_LONG ++#define QM_FD_SG QM_FD_FORMAT_SG ++#define QM_FD_SG_BIG (QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG) ++#define QM_FD_COMPOUND QM_FD_FORMAT_COMPOUND ++ ++/* See 1.5.1.1: "Frame Descriptor (FD)" */ ++struct qm_fd { ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 dd:2; /* dynamic debug */ ++ u8 liodn_offset:6; ++ u8 bpid:8; /* Buffer Pool ID */ ++ u8 eliodn_offset:4; ++ u8 __reserved:4; ++ u8 addr_hi; /* high 8-bits of 40-bit address */ ++ u32 addr_lo; /* low 32-bits of 40-bit address */ ++#else ++ u8 liodn_offset:6; ++ u8 dd:2; /* dynamic debug */ ++ u8 bpid:8; /* Buffer Pool ID */ ++ u8 __reserved:4; ++ u8 eliodn_offset:4; ++ u8 addr_hi; /* high 8-bits of 40-bit address */ ++ u32 addr_lo; /* low 32-bits of 40-bit address */ ++#endif ++ }; ++ struct { ++ u64 __notaddress:24; ++ /* More efficient address accessor */ ++ u64 addr:40; ++ }; ++ u64 opaque_addr; ++ }; ++ /* The 'format' field indicates the interpretation of the remaining 29 ++ * bits of the 32-bit word. For packing reasons, it is duplicated in the ++ * other union elements. Note, union'd structs are difficult to use with ++ * static initialisation under gcc, in which case use the "opaque" form ++ * with one of the macros. */ ++ union { ++ /* For easier/faster copying of this part of the fd (eg. from a ++ * DQRR entry to an EQCR entry) copy 'opaque' */ ++ u32 opaque; ++ /* If 'format' is _contig or _sg, 20b length and 9b offset */ ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ enum qm_fd_format format:3; ++ u16 offset:9; ++ u32 length20:20; ++#else ++ u32 length20:20; ++ u16 offset:9; ++ enum qm_fd_format format:3; ++#endif ++ }; ++ /* If 'format' is _contig_big or _sg_big, 29b length */ ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ enum qm_fd_format _format1:3; ++ u32 length29:29; ++#else ++ u32 length29:29; ++ enum qm_fd_format _format1:3; ++#endif ++ }; ++ /* If 'format' is _compound, 29b "congestion weight" */ ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ enum qm_fd_format _format2:3; ++ u32 cong_weight:29; ++#else ++ u32 cong_weight:29; ++ enum qm_fd_format _format2:3; ++#endif ++ }; ++ }; ++ union { ++ u32 cmd; ++ u32 status; ++ }; ++} __aligned(8); ++#define QM_FD_DD_NULL 0x00 ++#define QM_FD_PID_MASK 0x3f ++static inline u64 qm_fd_addr_get64(const struct qm_fd *fd) ++{ ++ return fd->addr; ++} ++ ++static inline dma_addr_t qm_fd_addr(const struct qm_fd *fd) ++{ ++ return (dma_addr_t)fd->addr; ++} ++/* Macro, so we compile better if 'v' isn't always 64-bit */ ++#define qm_fd_addr_set64(fd, v) \ ++ do { \ ++ struct qm_fd *__fd931 = (fd); \ ++ __fd931->addr = v; \ ++ } while (0) ++ ++/* For static initialisation of FDs (which is complicated by the use of unions ++ * in "struct qm_fd"), use the following macros. Note that; ++ * - 'dd', 'pid' and 'bpid' are ignored because there's no static initialisation ++ * use-case), ++ * - use capitalised QM_FD_*** formats for static initialisation. ++ */ ++#define QM_FD_FMT_20(cmd, addr_hi, addr_lo, fmt, off, len) \ ++ { 0, 0, 0, 0, 0, addr_hi, addr_lo, \ ++ { (((fmt)&0x7) << 29) | (((off)&0x1ff) << 20) | ((len)&0xfffff) }, \ ++ { cmd } } ++#define QM_FD_FMT_29(cmd, addr_hi, addr_lo, fmt, len) \ ++ { 0, 0, 0, 0, 0, addr_hi, addr_lo, \ ++ { (((fmt)&0x7) << 29) | ((len)&0x1fffffff) }, \ ++ { cmd } } ++ ++/* See 2.2.1.3 Multi-Core Datapath Acceleration Architecture */ ++#define QM_SG_OFFSET_MASK 0x1FFF ++struct qm_sg_entry { ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 __reserved1[3]; ++ u8 addr_hi; /* high 8-bits of 40-bit address */ ++ u32 addr_lo; /* low 32-bits of 40-bit address */ ++#else ++ u32 addr_lo; /* low 32-bits of 40-bit address */ ++ u8 addr_hi; /* high 8-bits of 40-bit address */ ++ u8 __reserved1[3]; ++#endif ++ }; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u64 __notaddress:24; ++ u64 addr:40; ++#else ++ u64 addr:40; ++ u64 __notaddress:24; ++#endif ++ }; ++ u64 opaque; ++ }; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u32 extension:1; /* Extension bit */ ++ u32 final:1; /* Final bit */ ++ u32 length:30; ++#else ++ u32 length:30; ++ u32 final:1; /* Final bit */ ++ u32 extension:1; /* Extension bit */ ++#endif ++ }; ++ u32 sgt_efl; ++ }; ++ u8 __reserved2; ++ u8 bpid; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 __reserved3:3; ++ u16 offset:13; ++#else ++ u16 offset:13; ++ u16 __reserved3:3; ++#endif ++ }; ++ u16 opaque_offset; ++ }; ++} __packed; ++union qm_sg_efl { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u32 extension:1; /* Extension bit */ ++ u32 final:1; /* Final bit */ ++ u32 length:30; ++#else ++ u32 length:30; ++ u32 final:1; /* Final bit */ ++ u32 extension:1; /* Extension bit */ ++#endif ++ }; ++ u32 efl; ++}; ++static inline u64 qm_sg_entry_get64(const struct qm_sg_entry *sg) ++{ ++ return be64_to_cpu(sg->opaque); ++} ++static inline dma_addr_t qm_sg_addr(const struct qm_sg_entry *sg) ++{ ++ return (dma_addr_t)be64_to_cpu(sg->opaque); ++} ++static inline u8 qm_sg_entry_get_ext(const struct qm_sg_entry *sg) ++{ ++ union qm_sg_efl u; ++ ++ u.efl = be32_to_cpu(sg->sgt_efl); ++ return u.extension; ++} ++static inline u8 qm_sg_entry_get_final(const struct qm_sg_entry *sg) ++{ ++ union qm_sg_efl u; ++ ++ u.efl = be32_to_cpu(sg->sgt_efl); ++ return u.final; ++} ++static inline u32 qm_sg_entry_get_len(const struct qm_sg_entry *sg) ++{ ++ union qm_sg_efl u; ++ ++ u.efl = be32_to_cpu(sg->sgt_efl); ++ return u.length; ++} ++static inline u8 qm_sg_entry_get_bpid(const struct qm_sg_entry *sg) ++{ ++ return sg->bpid; ++} ++static inline u16 qm_sg_entry_get_offset(const struct qm_sg_entry *sg) ++{ ++ u32 opaque_offset = be16_to_cpu(sg->opaque_offset); ++ ++ return opaque_offset & 0x1fff; ++} ++ ++/* Macro, so we compile better if 'v' isn't always 64-bit */ ++#define qm_sg_entry_set64(sg, v) \ ++ do { \ ++ struct qm_sg_entry *__sg931 = (sg); \ ++ __sg931->opaque = cpu_to_be64(v); \ ++ } while (0) ++#define qm_sg_entry_set_ext(sg, v) \ ++ do { \ ++ union qm_sg_efl __u932; \ ++ __u932.efl = be32_to_cpu((sg)->sgt_efl); \ ++ __u932.extension = v; \ ++ (sg)->sgt_efl = cpu_to_be32(__u932.efl); \ ++ } while (0) ++#define qm_sg_entry_set_final(sg, v) \ ++ do { \ ++ union qm_sg_efl __u933; \ ++ __u933.efl = be32_to_cpu((sg)->sgt_efl); \ ++ __u933.final = v; \ ++ (sg)->sgt_efl = cpu_to_be32(__u933.efl); \ ++ } while (0) ++#define qm_sg_entry_set_len(sg, v) \ ++ do { \ ++ union qm_sg_efl __u934; \ ++ __u934.efl = be32_to_cpu((sg)->sgt_efl); \ ++ __u934.length = v; \ ++ (sg)->sgt_efl = cpu_to_be32(__u934.efl); \ ++ } while (0) ++#define qm_sg_entry_set_bpid(sg, v) \ ++ do { \ ++ struct qm_sg_entry *__u935 = (sg); \ ++ __u935->bpid = v; \ ++ } while (0) ++#define qm_sg_entry_set_offset(sg, v) \ ++ do { \ ++ struct qm_sg_entry *__u936 = (sg); \ ++ __u936->opaque_offset = cpu_to_be16(v); \ ++ } while (0) ++ ++/* See 1.5.8.1: "Enqueue Command" */ ++struct qm_eqcr_entry { ++ u8 __dont_write_directly__verb; ++ u8 dca; ++ u16 seqnum; ++ u32 orp; /* 24-bit */ ++ u32 fqid; /* 24-bit */ ++ u32 tag; ++ struct qm_fd fd; ++ u8 __reserved3[32]; ++} __packed; ++#define QM_EQCR_VERB_VBIT 0x80 ++#define QM_EQCR_VERB_CMD_MASK 0x61 /* but only one value; */ ++#define QM_EQCR_VERB_CMD_ENQUEUE 0x01 ++#define QM_EQCR_VERB_COLOUR_MASK 0x18 /* 4 possible values; */ ++#define QM_EQCR_VERB_COLOUR_GREEN 0x00 ++#define QM_EQCR_VERB_COLOUR_YELLOW 0x08 ++#define QM_EQCR_VERB_COLOUR_RED 0x10 ++#define QM_EQCR_VERB_COLOUR_OVERRIDE 0x18 ++#define QM_EQCR_VERB_INTERRUPT 0x04 /* on command consumption */ ++#define QM_EQCR_VERB_ORP 0x02 /* enable order restoration */ ++#define QM_EQCR_DCA_ENABLE 0x80 ++#define QM_EQCR_DCA_PARK 0x40 ++#define QM_EQCR_DCA_IDXMASK 0x0f /* "DQRR::idx" goes here */ ++#define QM_EQCR_SEQNUM_NESN 0x8000 /* Advance NESN */ ++#define QM_EQCR_SEQNUM_NLIS 0x4000 /* More fragments to come */ ++#define QM_EQCR_SEQNUM_SEQMASK 0x3fff /* sequence number goes here */ ++#define QM_EQCR_FQID_NULL 0 /* eg. for an ORP seqnum hole */ ++ ++/* See 1.5.8.2: "Frame Dequeue Response" */ ++struct qm_dqrr_entry { ++ u8 verb; ++ u8 stat; ++ u16 seqnum; /* 15-bit */ ++ u8 tok; ++ u8 __reserved2[3]; ++ u32 fqid; /* 24-bit */ ++ u32 contextB; ++ struct qm_fd fd; ++ u8 __reserved4[32]; ++}; ++#define QM_DQRR_VERB_VBIT 0x80 ++#define QM_DQRR_VERB_MASK 0x7f /* where the verb contains; */ ++#define QM_DQRR_VERB_FRAME_DEQUEUE 0x60 /* "this format" */ ++#define QM_DQRR_STAT_FQ_EMPTY 0x80 /* FQ empty */ ++#define QM_DQRR_STAT_FQ_HELDACTIVE 0x40 /* FQ held active */ ++#define QM_DQRR_STAT_FQ_FORCEELIGIBLE 0x20 /* FQ was force-eligible'd */ ++#define QM_DQRR_STAT_FD_VALID 0x10 /* has a non-NULL FD */ ++#define QM_DQRR_STAT_UNSCHEDULED 0x02 /* Unscheduled dequeue */ ++#define QM_DQRR_STAT_DQCR_EXPIRED 0x01 /* VDQCR or PDQCR expired*/ ++ ++/* See 1.5.8.3: "ERN Message Response" */ ++/* See 1.5.8.4: "FQ State Change Notification" */ ++struct qm_mr_entry { ++ u8 verb; ++ union { ++ struct { ++ u8 dca; ++ u16 seqnum; ++ u8 rc; /* Rejection Code */ ++ u32 orp:24; ++ u32 fqid; /* 24-bit */ ++ u32 tag; ++ struct qm_fd fd; ++ } __packed ern; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */ ++ u8 __reserved1:3; ++ enum qm_dc_portal portal:3; ++#else ++ enum qm_dc_portal portal:3; ++ u8 __reserved1:3; ++ u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */ ++#endif ++ u16 __reserved2; ++ u8 rc; /* Rejection Code */ ++ u32 __reserved3:24; ++ u32 fqid; /* 24-bit */ ++ u32 tag; ++ struct qm_fd fd; ++ } __packed dcern; ++ struct { ++ u8 fqs; /* Frame Queue Status */ ++ u8 __reserved1[6]; ++ u32 fqid; /* 24-bit */ ++ u32 contextB; ++ u8 __reserved2[16]; ++ } __packed fq; /* FQRN/FQRNI/FQRL/FQPN */ ++ }; ++ u8 __reserved2[32]; ++} __packed; ++#define QM_MR_VERB_VBIT 0x80 ++/* The "ern" VERB bits match QM_EQCR_VERB_*** so aren't reproduced here. ERNs ++ * originating from direct-connect portals ("dcern") use 0x20 as a verb which ++ * would be invalid as a s/w enqueue verb. A s/w ERN can be distinguished from ++ * the other MR types by noting if the 0x20 bit is unset. */ ++#define QM_MR_VERB_TYPE_MASK 0x27 ++#define QM_MR_VERB_DC_ERN 0x20 ++#define QM_MR_VERB_FQRN 0x21 ++#define QM_MR_VERB_FQRNI 0x22 ++#define QM_MR_VERB_FQRL 0x23 ++#define QM_MR_VERB_FQPN 0x24 ++#define QM_MR_RC_MASK 0xf0 /* contains one of; */ ++#define QM_MR_RC_CGR_TAILDROP 0x00 ++#define QM_MR_RC_WRED 0x10 ++#define QM_MR_RC_ERROR 0x20 ++#define QM_MR_RC_ORPWINDOW_EARLY 0x30 ++#define QM_MR_RC_ORPWINDOW_LATE 0x40 ++#define QM_MR_RC_FQ_TAILDROP 0x50 ++#define QM_MR_RC_ORPWINDOW_RETIRED 0x60 ++#define QM_MR_RC_ORP_ZERO 0x70 ++#define QM_MR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */ ++#define QM_MR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */ ++#define QM_MR_DCERN_COLOUR_GREEN 0x00 ++#define QM_MR_DCERN_COLOUR_YELLOW 0x01 ++#define QM_MR_DCERN_COLOUR_RED 0x02 ++#define QM_MR_DCERN_COLOUR_OVERRIDE 0x03 ++ ++/* An identical structure of FQD fields is present in the "Init FQ" command and ++ * the "Query FQ" result, it's suctioned out into the "struct qm_fqd" type. ++ * Within that, the 'stashing' and 'taildrop' pieces are also factored out, the ++ * latter has two inlines to assist with converting to/from the mant+exp ++ * representation. */ ++struct qm_fqd_stashing { ++ /* See QM_STASHING_EXCL_<...> */ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 exclusive; ++ u8 __reserved1:2; ++ /* Numbers of cachelines */ ++ u8 annotation_cl:2; ++ u8 data_cl:2; ++ u8 context_cl:2; ++#else ++ u8 context_cl:2; ++ u8 data_cl:2; ++ u8 annotation_cl:2; ++ u8 __reserved1:2; ++ u8 exclusive; ++#endif ++} __packed; ++struct qm_fqd_taildrop { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 __reserved1:3; ++ u16 mant:8; ++ u16 exp:5; ++#else ++ u16 exp:5; ++ u16 mant:8; ++ u16 __reserved1:3; ++#endif ++} __packed; ++struct qm_fqd_oac { ++ /* See QM_OAC_<...> */ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 oac:2; /* "Overhead Accounting Control" */ ++ u8 __reserved1:6; ++#else ++ u8 __reserved1:6; ++ u8 oac:2; /* "Overhead Accounting Control" */ ++#endif ++ /* Two's-complement value (-128 to +127) */ ++ signed char oal; /* "Overhead Accounting Length" */ ++} __packed; ++struct qm_fqd { ++ union { ++ u8 orpc; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 __reserved1:2; ++ u8 orprws:3; ++ u8 oa:1; ++ u8 olws:2; ++#else ++ u8 olws:2; ++ u8 oa:1; ++ u8 orprws:3; ++ u8 __reserved1:2; ++#endif ++ } __packed; ++ }; ++ u8 cgid; ++ u16 fq_ctrl; /* See QM_FQCTRL_<...> */ ++ union { ++ u16 dest_wq; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 channel:13; /* qm_channel */ ++ u16 wq:3; ++#else ++ u16 wq:3; ++ u16 channel:13; /* qm_channel */ ++#endif ++ } __packed dest; ++ }; ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 __reserved2:1; ++ u16 ics_cred:15; ++#else ++ u16 __reserved2:1; ++ u16 ics_cred:15; ++#endif ++ /* For "Initialize Frame Queue" commands, the write-enable mask ++ * determines whether 'td' or 'oac_init' is observed. For query ++ * commands, this field is always 'td', and 'oac_query' (below) reflects ++ * the Overhead ACcounting values. */ ++ union { ++ struct qm_fqd_taildrop td; ++ struct qm_fqd_oac oac_init; ++ }; ++ u32 context_b; ++ union { ++ /* Treat it as 64-bit opaque */ ++ u64 opaque; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u32 hi; ++ u32 lo; ++#else ++ u32 lo; ++ u32 hi; ++#endif ++ }; ++ /* Treat it as s/w portal stashing config */ ++ /* See 1.5.6.7.1: "FQD Context_A field used for [...] */ ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ struct qm_fqd_stashing stashing; ++ /* 48-bit address of FQ context to ++ * stash, must be cacheline-aligned */ ++ u16 context_hi; ++ u32 context_lo; ++#else ++ u32 context_lo; ++ u16 context_hi; ++ struct qm_fqd_stashing stashing; ++#endif ++ } __packed; ++ } context_a; ++ struct qm_fqd_oac oac_query; ++} __packed; ++/* 64-bit converters for context_hi/lo */ ++static inline u64 qm_fqd_stashing_get64(const struct qm_fqd *fqd) ++{ ++ return ((u64)fqd->context_a.context_hi << 32) | ++ (u64)fqd->context_a.context_lo; ++} ++static inline dma_addr_t qm_fqd_stashing_addr(const struct qm_fqd *fqd) ++{ ++ return (dma_addr_t)qm_fqd_stashing_get64(fqd); ++} ++static inline u64 qm_fqd_context_a_get64(const struct qm_fqd *fqd) ++{ ++ return ((u64)fqd->context_a.hi << 32) | ++ (u64)fqd->context_a.lo; ++} ++/* Macro, so we compile better when 'v' isn't necessarily 64-bit */ ++#define qm_fqd_stashing_set64(fqd, v) \ ++ do { \ ++ struct qm_fqd *__fqd931 = (fqd); \ ++ __fqd931->context_a.context_hi = upper_32_bits(v); \ ++ __fqd931->context_a.context_lo = lower_32_bits(v); \ ++ } while (0) ++#define qm_fqd_context_a_set64(fqd, v) \ ++ do { \ ++ struct qm_fqd *__fqd931 = (fqd); \ ++ __fqd931->context_a.hi = upper_32_bits(v); \ ++ __fqd931->context_a.lo = lower_32_bits(v); \ ++ } while (0) ++/* convert a threshold value into mant+exp representation */ ++static inline int qm_fqd_taildrop_set(struct qm_fqd_taildrop *td, u32 val, ++ int roundup) ++{ ++ u32 e = 0; ++ int oddbit = 0; ++ if (val > 0xe0000000) ++ return -ERANGE; ++ while (val > 0xff) { ++ oddbit = val & 1; ++ val >>= 1; ++ e++; ++ if (roundup && oddbit) ++ val++; ++ } ++ td->exp = e; ++ td->mant = val; ++ return 0; ++} ++/* and the other direction */ ++static inline u32 qm_fqd_taildrop_get(const struct qm_fqd_taildrop *td) ++{ ++ return (u32)td->mant << td->exp; ++} ++ ++/* See 1.5.2.2: "Frame Queue Descriptor (FQD)" */ ++/* Frame Queue Descriptor (FQD) field 'fq_ctrl' uses these constants */ ++#define QM_FQCTRL_MASK 0x07ff /* 'fq_ctrl' flags; */ ++#define QM_FQCTRL_CGE 0x0400 /* Congestion Group Enable */ ++#define QM_FQCTRL_TDE 0x0200 /* Tail-Drop Enable */ ++#define QM_FQCTRL_ORP 0x0100 /* ORP Enable */ ++#define QM_FQCTRL_CTXASTASHING 0x0080 /* Context-A stashing */ ++#define QM_FQCTRL_CPCSTASH 0x0040 /* CPC Stash Enable */ ++#define QM_FQCTRL_FORCESFDR 0x0008 /* High-priority SFDRs */ ++#define QM_FQCTRL_AVOIDBLOCK 0x0004 /* Don't block active */ ++#define QM_FQCTRL_HOLDACTIVE 0x0002 /* Hold active in portal */ ++#define QM_FQCTRL_PREFERINCACHE 0x0001 /* Aggressively cache FQD */ ++#define QM_FQCTRL_LOCKINCACHE QM_FQCTRL_PREFERINCACHE /* older naming */ ++ ++/* See 1.5.6.7.1: "FQD Context_A field used for [...] */ ++/* Frame Queue Descriptor (FQD) field 'CONTEXT_A' uses these constants */ ++#define QM_STASHING_EXCL_ANNOTATION 0x04 ++#define QM_STASHING_EXCL_DATA 0x02 ++#define QM_STASHING_EXCL_CTX 0x01 ++ ++/* See 1.5.5.3: "Intra Class Scheduling" */ ++/* FQD field 'OAC' (Overhead ACcounting) uses these constants */ ++#define QM_OAC_ICS 0x2 /* Accounting for Intra-Class Scheduling */ ++#define QM_OAC_CG 0x1 /* Accounting for Congestion Groups */ ++ ++/* See 1.5.8.4: "FQ State Change Notification" */ ++/* This struct represents the 32-bit "WR_PARM_[GYR]" parameters in CGR fields ++ * and associated commands/responses. The WRED parameters are calculated from ++ * these fields as follows; ++ * MaxTH = MA * (2 ^ Mn) ++ * Slope = SA / (2 ^ Sn) ++ * MaxP = 4 * (Pn + 1) ++ */ ++struct qm_cgr_wr_parm { ++ union { ++ u32 word; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u32 MA:8; ++ u32 Mn:5; ++ u32 SA:7; /* must be between 64-127 */ ++ u32 Sn:6; ++ u32 Pn:6; ++#else ++ u32 Pn:6; ++ u32 Sn:6; ++ u32 SA:7; /* must be between 64-127 */ ++ u32 Mn:5; ++ u32 MA:8; ++#endif ++ } __packed; ++ }; ++} __packed; ++/* This struct represents the 13-bit "CS_THRES" CGR field. In the corresponding ++ * management commands, this is padded to a 16-bit structure field, so that's ++ * how we represent it here. The congestion state threshold is calculated from ++ * these fields as follows; ++ * CS threshold = TA * (2 ^ Tn) ++ */ ++struct qm_cgr_cs_thres { ++ union { ++ u16 hword; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 __reserved:3; ++ u16 TA:8; ++ u16 Tn:5; ++#else ++ u16 Tn:5; ++ u16 TA:8; ++ u16 __reserved:3; ++#endif ++ } __packed; ++ }; ++} __packed; ++/* This identical structure of CGR fields is present in the "Init/Modify CGR" ++ * commands and the "Query CGR" result. It's suctioned out here into its own ++ * struct. */ ++struct __qm_mc_cgr { ++ struct qm_cgr_wr_parm wr_parm_g; ++ struct qm_cgr_wr_parm wr_parm_y; ++ struct qm_cgr_wr_parm wr_parm_r; ++ u8 wr_en_g; /* boolean, use QM_CGR_EN */ ++ u8 wr_en_y; /* boolean, use QM_CGR_EN */ ++ u8 wr_en_r; /* boolean, use QM_CGR_EN */ ++ u8 cscn_en; /* boolean, use QM_CGR_EN */ ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */ ++ u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */ ++#else ++ u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */ ++ u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */ ++#endif ++ }; ++ u32 cscn_targ; /* use QM_CGR_TARG_* */ ++ }; ++ u8 cstd_en; /* boolean, use QM_CGR_EN */ ++ u8 cs; /* boolean, only used in query response */ ++ union { ++ /* use qm_cgr_cs_thres_set64() */ ++ struct qm_cgr_cs_thres cs_thres; ++ u16 __cs_thres; ++ }; ++ u8 mode; /* QMAN_CGR_MODE_FRAME not supported in rev1.0 */ ++} __packed; ++#define QM_CGR_EN 0x01 /* For wr_en_*, cscn_en, cstd_en */ ++#define QM_CGR_TARG_UDP_CTRL_WRITE_BIT 0x8000 /* value written to portal bit*/ ++#define QM_CGR_TARG_UDP_CTRL_DCP 0x4000 /* 0: SWP, 1: DCP */ ++#define QM_CGR_TARG_PORTAL(n) (0x80000000 >> (n)) /* s/w portal, 0-9 */ ++#define QM_CGR_TARG_FMAN0 0x00200000 /* direct-connect portal: fman0 */ ++#define QM_CGR_TARG_FMAN1 0x00100000 /* : fman1 */ ++/* Convert CGR thresholds to/from "cs_thres" format */ ++static inline u64 qm_cgr_cs_thres_get64(const struct qm_cgr_cs_thres *th) ++{ ++ return (u64)th->TA << th->Tn; ++} ++static inline int qm_cgr_cs_thres_set64(struct qm_cgr_cs_thres *th, u64 val, ++ int roundup) ++{ ++ u32 e = 0; ++ int oddbit = 0; ++ while (val > 0xff) { ++ oddbit = val & 1; ++ val >>= 1; ++ e++; ++ if (roundup && oddbit) ++ val++; ++ } ++ th->Tn = e; ++ th->TA = val; ++ return 0; ++} ++ ++/* See 1.5.8.5.1: "Initialize FQ" */ ++/* See 1.5.8.5.2: "Query FQ" */ ++/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */ ++/* See 1.5.8.5.4: "Alter FQ State Commands " */ ++/* See 1.5.8.6.1: "Initialize/Modify CGR" */ ++/* See 1.5.8.6.2: "CGR Test Write" */ ++/* See 1.5.8.6.3: "Query CGR" */ ++/* See 1.5.8.6.4: "Query Congestion Group State" */ ++struct qm_mcc_initfq { ++ u8 __reserved1; ++ u16 we_mask; /* Write Enable Mask */ ++ u32 fqid; /* 24-bit */ ++ u16 count; /* Initialises 'count+1' FQDs */ ++ struct qm_fqd fqd; /* the FQD fields go here */ ++ u8 __reserved3[30]; ++} __packed; ++struct qm_mcc_queryfq { ++ u8 __reserved1[3]; ++ u32 fqid; /* 24-bit */ ++ u8 __reserved2[56]; ++} __packed; ++struct qm_mcc_queryfq_np { ++ u8 __reserved1[3]; ++ u32 fqid; /* 24-bit */ ++ u8 __reserved2[56]; ++} __packed; ++struct qm_mcc_alterfq { ++ u8 __reserved1[3]; ++ u32 fqid; /* 24-bit */ ++ u8 __reserved2; ++ u8 count; /* number of consecutive FQID */ ++ u8 __reserved3[10]; ++ u32 context_b; /* frame queue context b */ ++ u8 __reserved4[40]; ++} __packed; ++struct qm_mcc_initcgr { ++ u8 __reserved1; ++ u16 we_mask; /* Write Enable Mask */ ++ struct __qm_mc_cgr cgr; /* CGR fields */ ++ u8 __reserved2[2]; ++ u8 cgid; ++ u8 __reserved4[32]; ++} __packed; ++struct qm_mcc_cgrtestwrite { ++ u8 __reserved1[2]; ++ u8 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ ++ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ ++ u8 __reserved2[23]; ++ u8 cgid; ++ u8 __reserved3[32]; ++} __packed; ++struct qm_mcc_querycgr { ++ u8 __reserved1[30]; ++ u8 cgid; ++ u8 __reserved2[32]; ++} __packed; ++struct qm_mcc_querycongestion { ++ u8 __reserved[63]; ++} __packed; ++struct qm_mcc_querywq { ++ u8 __reserved; ++ /* select channel if verb != QUERYWQ_DEDICATED */ ++ union { ++ u16 channel_wq; /* ignores wq (3 lsbits) */ ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 id:13; /* qm_channel */ ++ u16 __reserved1:3; ++#else ++ u16 __reserved1:3; ++ u16 id:13; /* qm_channel */ ++#endif ++ } __packed channel; ++ }; ++ u8 __reserved2[60]; ++} __packed; ++ ++struct qm_mcc_ceetm_lfqmt_config { ++ u8 __reserved1[4]; ++ u32 lfqid:24; ++ u8 __reserved2[2]; ++ u16 cqid; ++ u8 __reserved3[2]; ++ u16 dctidx; ++ u8 __reserved4[48]; ++} __packed; ++ ++struct qm_mcc_ceetm_lfqmt_query { ++ u8 __reserved1[4]; ++ u32 lfqid:24; ++ u8 __reserved2[56]; ++} __packed; ++ ++struct qm_mcc_ceetm_cq_config { ++ u8 __reserved1; ++ u16 cqid; ++ u8 dcpid; ++ u8 __reserved2; ++ u16 ccgid; ++ u8 __reserved3[56]; ++} __packed; ++ ++struct qm_mcc_ceetm_cq_query { ++ u8 __reserved1; ++ u16 cqid; ++ u8 dcpid; ++ u8 __reserved2[59]; ++} __packed; ++ ++struct qm_mcc_ceetm_dct_config { ++ u8 __reserved1; ++ u16 dctidx; ++ u8 dcpid; ++ u8 __reserved2[15]; ++ u32 context_b; ++ u64 context_a; ++ u8 __reserved3[32]; ++} __packed; ++ ++struct qm_mcc_ceetm_dct_query { ++ u8 __reserved1; ++ u16 dctidx; ++ u8 dcpid; ++ u8 __reserved2[59]; ++} __packed; ++ ++struct qm_mcc_ceetm_class_scheduler_config { ++ u8 __reserved1; ++ u16 cqcid; ++ u8 dcpid; ++ u8 __reserved2[6]; ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 gpc_reserved:1; ++ u8 gpc_combine_flag:1; ++ u8 gpc_prio_b:3; ++ u8 gpc_prio_a:3; ++#else ++ u8 gpc_prio_a:3; ++ u8 gpc_prio_b:3; ++ u8 gpc_combine_flag:1; ++ u8 gpc_reserved:1; ++#endif ++ u16 crem; ++ u16 erem; ++ u8 w[8]; ++ u8 __reserved3[40]; ++} __packed; ++ ++struct qm_mcc_ceetm_class_scheduler_query { ++ u8 __reserved1; ++ u16 cqcid; ++ u8 dcpid; ++ u8 __reserved2[59]; ++} __packed; ++ ++#define CEETM_COMMAND_CHANNEL_MAPPING (0 << 12) ++#define CEETM_COMMAND_SP_MAPPING (1 << 12) ++#define CEETM_COMMAND_CHANNEL_SHAPER (2 << 12) ++#define CEETM_COMMAND_LNI_SHAPER (3 << 12) ++#define CEETM_COMMAND_TCFC (4 << 12) ++ ++#define CEETM_CCGRID_MASK 0x01FF ++#define CEETM_CCGR_CM_CONFIGURE (0 << 14) ++#define CEETM_CCGR_DN_CONFIGURE (1 << 14) ++#define CEETM_CCGR_TEST_WRITE (2 << 14) ++#define CEETM_CCGR_CM_QUERY (0 << 14) ++#define CEETM_CCGR_DN_QUERY (1 << 14) ++#define CEETM_CCGR_DN_QUERY_FLUSH (2 << 14) ++#define CEETM_QUERY_CONGESTION_STATE (3 << 14) ++ ++struct qm_mcc_ceetm_mapping_shaper_tcfc_config { ++ u8 __reserved1; ++ u16 cid; ++ u8 dcpid; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 map_shaped:1; ++ u8 map_reserved:4; ++ u8 map_lni_id:3; ++#else ++ u8 map_lni_id:3; ++ u8 map_reserved:4; ++ u8 map_shaped:1; ++#endif ++ u8 __reserved2[58]; ++ } __packed channel_mapping; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 map_reserved:5; ++ u8 map_lni_id:3; ++#else ++ u8 map_lni_id:3; ++ u8 map_reserved:5; ++#endif ++ u8 __reserved2[58]; ++ } __packed sp_mapping; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 cpl:1; ++ u8 cpl_reserved:2; ++ u8 oal:5; ++#else ++ u8 oal:5; ++ u8 cpl_reserved:2; ++ u8 cpl:1; ++#endif ++ u32 crtcr:24; ++ u32 ertcr:24; ++ u16 crtbl; ++ u16 ertbl; ++ u8 mps; /* This will be hardcoded by driver with 60 */ ++ u8 __reserved2[47]; ++ } __packed shaper_config; ++ struct { ++ u8 __reserved2[11]; ++ u64 lnitcfcc; ++ u8 __reserved3[40]; ++ } __packed tcfc_config; ++ }; ++} __packed; ++ ++struct qm_mcc_ceetm_mapping_shaper_tcfc_query { ++ u8 __reserved1; ++ u16 cid; ++ u8 dcpid; ++ u8 __reserved2[59]; ++} __packed; ++ ++struct qm_mcc_ceetm_ccgr_config { ++ u8 __reserved1; ++ u16 ccgrid; ++ u8 dcpid; ++ u8 __reserved2; ++ u16 we_mask; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 ctl_reserved:1; ++ u8 ctl_wr_en_g:1; ++ u8 ctl_wr_en_y:1; ++ u8 ctl_wr_en_r:1; ++ u8 ctl_td_en:1; ++ u8 ctl_td_mode:1; ++ u8 ctl_cscn_en:1; ++ u8 ctl_mode:1; ++#else ++ u8 ctl_mode:1; ++ u8 ctl_cscn_en:1; ++ u8 ctl_td_mode:1; ++ u8 ctl_td_en:1; ++ u8 ctl_wr_en_r:1; ++ u8 ctl_wr_en_y:1; ++ u8 ctl_wr_en_g:1; ++ u8 ctl_reserved:1; ++#endif ++ u8 cdv; ++ u16 cscn_tupd; ++ u8 oal; ++ u8 __reserved3; ++ struct qm_cgr_cs_thres cs_thres; ++ struct qm_cgr_cs_thres cs_thres_x; ++ struct qm_cgr_cs_thres td_thres; ++ struct qm_cgr_wr_parm wr_parm_g; ++ struct qm_cgr_wr_parm wr_parm_y; ++ struct qm_cgr_wr_parm wr_parm_r; ++ } __packed cm_config; ++ struct { ++ u8 dnc; ++ u8 dn0; ++ u8 dn1; ++ u64 dnba:40; ++ u8 __reserved3[2]; ++ u16 dnth_0; ++ u8 __reserved4[2]; ++ u16 dnth_1; ++ u8 __reserved5[8]; ++ } __packed dn_config; ++ struct { ++ u8 __reserved3[3]; ++ u64 i_cnt:40; ++ u8 __reserved4[16]; ++ } __packed test_write; ++ }; ++ u8 __reserved5[32]; ++} __packed; ++ ++struct qm_mcc_ceetm_ccgr_query { ++ u8 __reserved1; ++ u16 ccgrid; ++ u8 dcpid; ++ u8 __reserved2[59]; ++} __packed; ++ ++struct qm_mcc_ceetm_cq_peek_pop_xsfdrread { ++ u8 __reserved1; ++ u16 cqid; ++ u8 dcpid; ++ u8 ct; ++ u16 xsfdr; ++ u8 __reserved2[56]; ++} __packed; ++ ++#define CEETM_QUERY_DEQUEUE_STATISTICS 0x00 ++#define CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS 0x01 ++#define CEETM_WRITE_DEQUEUE_STATISTICS 0x02 ++#define CEETM_QUERY_REJECT_STATISTICS 0x03 ++#define CEETM_QUERY_REJECT_CLEAR_STATISTICS 0x04 ++#define CEETM_WRITE_REJECT_STATISTICS 0x05 ++struct qm_mcc_ceetm_statistics_query_write { ++ u8 __reserved1; ++ u16 cid; ++ u8 dcpid; ++ u8 ct; ++ u8 __reserved2[13]; ++ u64 frm_cnt:40; ++ u8 __reserved3[2]; ++ u64 byte_cnt:48; ++ u8 __reserved[32]; ++} __packed; ++ ++struct qm_mc_command { ++ u8 __dont_write_directly__verb; ++ union { ++ struct qm_mcc_initfq initfq; ++ struct qm_mcc_queryfq queryfq; ++ struct qm_mcc_queryfq_np queryfq_np; ++ struct qm_mcc_alterfq alterfq; ++ struct qm_mcc_initcgr initcgr; ++ struct qm_mcc_cgrtestwrite cgrtestwrite; ++ struct qm_mcc_querycgr querycgr; ++ struct qm_mcc_querycongestion querycongestion; ++ struct qm_mcc_querywq querywq; ++ struct qm_mcc_ceetm_lfqmt_config lfqmt_config; ++ struct qm_mcc_ceetm_lfqmt_query lfqmt_query; ++ struct qm_mcc_ceetm_cq_config cq_config; ++ struct qm_mcc_ceetm_cq_query cq_query; ++ struct qm_mcc_ceetm_dct_config dct_config; ++ struct qm_mcc_ceetm_dct_query dct_query; ++ struct qm_mcc_ceetm_class_scheduler_config csch_config; ++ struct qm_mcc_ceetm_class_scheduler_query csch_query; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_config mst_config; ++ struct qm_mcc_ceetm_mapping_shaper_tcfc_query mst_query; ++ struct qm_mcc_ceetm_ccgr_config ccgr_config; ++ struct qm_mcc_ceetm_ccgr_query ccgr_query; ++ struct qm_mcc_ceetm_cq_peek_pop_xsfdrread cq_ppxr; ++ struct qm_mcc_ceetm_statistics_query_write stats_query_write; ++ }; ++} __packed; ++#define QM_MCC_VERB_VBIT 0x80 ++#define QM_MCC_VERB_MASK 0x7f /* where the verb contains; */ ++#define QM_MCC_VERB_INITFQ_PARKED 0x40 ++#define QM_MCC_VERB_INITFQ_SCHED 0x41 ++#define QM_MCC_VERB_QUERYFQ 0x44 ++#define QM_MCC_VERB_QUERYFQ_NP 0x45 /* "non-programmable" fields */ ++#define QM_MCC_VERB_QUERYWQ 0x46 ++#define QM_MCC_VERB_QUERYWQ_DEDICATED 0x47 ++#define QM_MCC_VERB_ALTER_SCHED 0x48 /* Schedule FQ */ ++#define QM_MCC_VERB_ALTER_FE 0x49 /* Force Eligible FQ */ ++#define QM_MCC_VERB_ALTER_RETIRE 0x4a /* Retire FQ */ ++#define QM_MCC_VERB_ALTER_OOS 0x4b /* Take FQ out of service */ ++#define QM_MCC_VERB_ALTER_FQXON 0x4d /* FQ XON */ ++#define QM_MCC_VERB_ALTER_FQXOFF 0x4e /* FQ XOFF */ ++#define QM_MCC_VERB_INITCGR 0x50 ++#define QM_MCC_VERB_MODIFYCGR 0x51 ++#define QM_MCC_VERB_CGRTESTWRITE 0x52 ++#define QM_MCC_VERB_QUERYCGR 0x58 ++#define QM_MCC_VERB_QUERYCONGESTION 0x59 ++/* INITFQ-specific flags */ ++#define QM_INITFQ_WE_MASK 0x01ff /* 'Write Enable' flags; */ ++#define QM_INITFQ_WE_OAC 0x0100 ++#define QM_INITFQ_WE_ORPC 0x0080 ++#define QM_INITFQ_WE_CGID 0x0040 ++#define QM_INITFQ_WE_FQCTRL 0x0020 ++#define QM_INITFQ_WE_DESTWQ 0x0010 ++#define QM_INITFQ_WE_ICSCRED 0x0008 ++#define QM_INITFQ_WE_TDTHRESH 0x0004 ++#define QM_INITFQ_WE_CONTEXTB 0x0002 ++#define QM_INITFQ_WE_CONTEXTA 0x0001 ++/* INITCGR/MODIFYCGR-specific flags */ ++#define QM_CGR_WE_MASK 0x07ff /* 'Write Enable Mask'; */ ++#define QM_CGR_WE_WR_PARM_G 0x0400 ++#define QM_CGR_WE_WR_PARM_Y 0x0200 ++#define QM_CGR_WE_WR_PARM_R 0x0100 ++#define QM_CGR_WE_WR_EN_G 0x0080 ++#define QM_CGR_WE_WR_EN_Y 0x0040 ++#define QM_CGR_WE_WR_EN_R 0x0020 ++#define QM_CGR_WE_CSCN_EN 0x0010 ++#define QM_CGR_WE_CSCN_TARG 0x0008 ++#define QM_CGR_WE_CSTD_EN 0x0004 ++#define QM_CGR_WE_CS_THRES 0x0002 ++#define QM_CGR_WE_MODE 0x0001 ++ ++/* See 1.5.9.7 CEETM Management Commands */ ++#define QM_CEETM_VERB_LFQMT_CONFIG 0x70 ++#define QM_CEETM_VERB_LFQMT_QUERY 0x71 ++#define QM_CEETM_VERB_CQ_CONFIG 0x72 ++#define QM_CEETM_VERB_CQ_QUERY 0x73 ++#define QM_CEETM_VERB_DCT_CONFIG 0x74 ++#define QM_CEETM_VERB_DCT_QUERY 0x75 ++#define QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG 0x76 ++#define QM_CEETM_VERB_CLASS_SCHEDULER_QUERY 0x77 ++#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG 0x78 ++#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY 0x79 ++#define QM_CEETM_VERB_CCGR_CONFIG 0x7A ++#define QM_CEETM_VERB_CCGR_QUERY 0x7B ++#define QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD 0x7C ++#define QM_CEETM_VERB_STATISTICS_QUERY_WRITE 0x7D ++ ++/* See 1.5.8.5.1: "Initialize FQ" */ ++/* See 1.5.8.5.2: "Query FQ" */ ++/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */ ++/* See 1.5.8.5.4: "Alter FQ State Commands " */ ++/* See 1.5.8.6.1: "Initialize/Modify CGR" */ ++/* See 1.5.8.6.2: "CGR Test Write" */ ++/* See 1.5.8.6.3: "Query CGR" */ ++/* See 1.5.8.6.4: "Query Congestion Group State" */ ++struct qm_mcr_initfq { ++ u8 __reserved1[62]; ++} __packed; ++struct qm_mcr_queryfq { ++ u8 __reserved1[8]; ++ struct qm_fqd fqd; /* the FQD fields are here */ ++ u8 __reserved2[30]; ++} __packed; ++struct qm_mcr_queryfq_np { ++ u8 __reserved1; ++ u8 state; /* QM_MCR_NP_STATE_*** */ ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 __reserved2; ++ u32 fqd_link:24; ++ u16 __reserved3:2; ++ u16 odp_seq:14; ++ u16 __reserved4:2; ++ u16 orp_nesn:14; ++ u16 __reserved5:1; ++ u16 orp_ea_hseq:15; ++ u16 __reserved6:1; ++ u16 orp_ea_tseq:15; ++ u8 __reserved7; ++ u32 orp_ea_hptr:24; ++ u8 __reserved8; ++ u32 orp_ea_tptr:24; ++ u8 __reserved9; ++ u32 pfdr_hptr:24; ++ u8 __reserved10; ++ u32 pfdr_tptr:24; ++ u8 __reserved11[5]; ++ u8 __reserved12:7; ++ u8 is:1; ++ u16 ics_surp; ++ u32 byte_cnt; ++ u8 __reserved13; ++ u32 frm_cnt:24; ++ u32 __reserved14; ++ u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ ++ u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ ++ u16 __reserved15; ++ u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ ++ u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ ++ u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ ++#else ++ u8 __reserved2; ++ u32 fqd_link:24; ++ ++ u16 odp_seq:14; ++ u16 __reserved3:2; ++ ++ u16 orp_nesn:14; ++ u16 __reserved4:2; ++ ++ u16 orp_ea_hseq:15; ++ u16 __reserved5:1; ++ ++ u16 orp_ea_tseq:15; ++ u16 __reserved6:1; ++ ++ u8 __reserved7; ++ u32 orp_ea_hptr:24; ++ ++ u8 __reserved8; ++ u32 orp_ea_tptr:24; ++ ++ u8 __reserved9; ++ u32 pfdr_hptr:24; ++ ++ u8 __reserved10; ++ u32 pfdr_tptr:24; ++ ++ u8 __reserved11[5]; ++ u8 is:1; ++ u8 __reserved12:7; ++ u16 ics_surp; ++ u32 byte_cnt; ++ u8 __reserved13; ++ u32 frm_cnt:24; ++ u32 __reserved14; ++ u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ ++ u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ ++ u16 __reserved15; ++ u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ ++ u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ ++ u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ ++#endif ++} __packed; ++ ++ ++struct qm_mcr_alterfq { ++ u8 fqs; /* Frame Queue Status */ ++ u8 __reserved1[61]; ++} __packed; ++struct qm_mcr_initcgr { ++ u8 __reserved1[62]; ++} __packed; ++struct qm_mcr_cgrtestwrite { ++ u16 __reserved1; ++ struct __qm_mc_cgr cgr; /* CGR fields */ ++ u8 __reserved2[3]; ++ u32 __reserved3:24; ++ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ ++ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ ++ u32 __reserved4:24; ++ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ ++ u32 a_bcnt_lo; /* low 32-bits of 40-bit */ ++ u16 lgt; /* Last Group Tick */ ++ u16 wr_prob_g; ++ u16 wr_prob_y; ++ u16 wr_prob_r; ++ u8 __reserved5[8]; ++} __packed; ++struct qm_mcr_querycgr { ++ u16 __reserved1; ++ struct __qm_mc_cgr cgr; /* CGR fields */ ++ u8 __reserved2[3]; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u32 __reserved3:24; ++ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ ++ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ ++#else ++ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ ++ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ ++ u32 __reserved3:24; ++#endif ++ }; ++ u64 i_bcnt; ++ }; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u32 __reserved4:24; ++ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ ++ u32 a_bcnt_lo; /* low 32-bits of 40-bit */ ++#else ++ u32 a_bcnt_lo; /* low 32-bits of 40-bit */ ++ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ ++ u32 __reserved4:24; ++#endif ++ }; ++ u64 a_bcnt; ++ }; ++ union { ++ u32 cscn_targ_swp[4]; ++ u8 __reserved5[16]; ++ }; ++} __packed; ++static inline u64 qm_mcr_querycgr_i_get64(const struct qm_mcr_querycgr *q) ++{ ++ return be64_to_cpu(q->i_bcnt); ++} ++static inline u64 qm_mcr_querycgr_a_get64(const struct qm_mcr_querycgr *q) ++{ ++ return be64_to_cpu(q->a_bcnt); ++} ++static inline u64 qm_mcr_cgrtestwrite_i_get64( ++ const struct qm_mcr_cgrtestwrite *q) ++{ ++ return be64_to_cpu(((u64)q->i_bcnt_hi << 32) | (u64)q->i_bcnt_lo); ++} ++static inline u64 qm_mcr_cgrtestwrite_a_get64( ++ const struct qm_mcr_cgrtestwrite *q) ++{ ++ return be64_to_cpu(((u64)q->a_bcnt_hi << 32) | (u64)q->a_bcnt_lo); ++} ++/* Macro, so we compile better if 'v' isn't always 64-bit */ ++#define qm_mcr_querycgr_i_set64(q, v) \ ++ do { \ ++ struct qm_mcr_querycgr *__q931 = (fd); \ ++ __q931->i_bcnt_hi = upper_32_bits(v); \ ++ __q931->i_bcnt_lo = lower_32_bits(v); \ ++ } while (0) ++#define qm_mcr_querycgr_a_set64(q, v) \ ++ do { \ ++ struct qm_mcr_querycgr *__q931 = (fd); \ ++ __q931->a_bcnt_hi = upper_32_bits(v); \ ++ __q931->a_bcnt_lo = lower_32_bits(v); \ ++ } while (0) ++struct __qm_mcr_querycongestion { ++ u32 __state[8]; ++}; ++struct qm_mcr_querycongestion { ++ u8 __reserved[30]; ++ /* Access this struct using QM_MCR_QUERYCONGESTION() */ ++ struct __qm_mcr_querycongestion state; ++} __packed; ++struct qm_mcr_querywq { ++ union { ++ u16 channel_wq; /* ignores wq (3 lsbits) */ ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u16 id:13; /* qm_channel */ ++ u16 __reserved:3; ++#else ++ u16 __reserved:3; ++ u16 id:13; /* qm_channel */ ++#endif ++ } __packed channel; ++ }; ++ u8 __reserved[28]; ++ u32 wq_len[8]; ++} __packed; ++ ++/* QMAN CEETM Management Command Response */ ++struct qm_mcr_ceetm_lfqmt_config { ++ u8 __reserved1[62]; ++} __packed; ++struct qm_mcr_ceetm_lfqmt_query { ++ u8 __reserved1[8]; ++ u16 cqid; ++ u8 __reserved2[2]; ++ u16 dctidx; ++ u8 __reserved3[2]; ++ u16 ccgid; ++ u8 __reserved4[44]; ++} __packed; ++ ++struct qm_mcr_ceetm_cq_config { ++ u8 __reserved1[62]; ++} __packed; ++ ++struct qm_mcr_ceetm_cq_query { ++ u8 __reserved1[4]; ++ u16 ccgid; ++ u16 state; ++ u32 pfdr_hptr:24; ++ u32 pfdr_tptr:24; ++ u16 od1_xsfdr; ++ u16 od2_xsfdr; ++ u16 od3_xsfdr; ++ u16 od4_xsfdr; ++ u16 od5_xsfdr; ++ u16 od6_xsfdr; ++ u16 ra1_xsfdr; ++ u16 ra2_xsfdr; ++ u8 __reserved2; ++ u32 frm_cnt:24; ++ u8 __reserved333[28]; ++} __packed; ++ ++struct qm_mcr_ceetm_dct_config { ++ u8 __reserved1[62]; ++} __packed; ++ ++struct qm_mcr_ceetm_dct_query { ++ u8 __reserved1[18]; ++ u32 context_b; ++ u64 context_a; ++ u8 __reserved2[32]; ++} __packed; ++ ++struct qm_mcr_ceetm_class_scheduler_config { ++ u8 __reserved1[62]; ++} __packed; ++ ++struct qm_mcr_ceetm_class_scheduler_query { ++ u8 __reserved1[9]; ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 gpc_reserved:1; ++ u8 gpc_combine_flag:1; ++ u8 gpc_prio_b:3; ++ u8 gpc_prio_a:3; ++#else ++ u8 gpc_prio_a:3; ++ u8 gpc_prio_b:3; ++ u8 gpc_combine_flag:1; ++ u8 gpc_reserved:1; ++#endif ++ u16 crem; ++ u16 erem; ++ u8 w[8]; ++ u8 __reserved2[5]; ++ u32 wbfslist:24; ++ u32 d8; ++ u32 d9; ++ u32 d10; ++ u32 d11; ++ u32 d12; ++ u32 d13; ++ u32 d14; ++ u32 d15; ++} __packed; ++ ++struct qm_mcr_ceetm_mapping_shaper_tcfc_config { ++ u16 cid; ++ u8 __reserved2[60]; ++} __packed; ++ ++struct qm_mcr_ceetm_mapping_shaper_tcfc_query { ++ u16 cid; ++ u8 __reserved1; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 map_shaped:1; ++ u8 map_reserved:4; ++ u8 map_lni_id:3; ++#else ++ u8 map_lni_id:3; ++ u8 map_reserved:4; ++ u8 map_shaped:1; ++#endif ++ u8 __reserved2[58]; ++ } __packed channel_mapping_query; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 map_reserved:5; ++ u8 map_lni_id:3; ++#else ++ u8 map_lni_id:3; ++ u8 map_reserved:5; ++#endif ++ u8 __reserved2[58]; ++ } __packed sp_mapping_query; ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 cpl:1; ++ u8 cpl_reserved:2; ++ u8 oal:5; ++#else ++ u8 oal:5; ++ u8 cpl_reserved:2; ++ u8 cpl:1; ++#endif ++ u32 crtcr:24; ++ u32 ertcr:24; ++ u16 crtbl; ++ u16 ertbl; ++ u8 mps; ++ u8 __reserved2[15]; ++ u32 crat; ++ u32 erat; ++ u8 __reserved3[24]; ++ } __packed shaper_query; ++ struct { ++ u8 __reserved1[11]; ++ u64 lnitcfcc; ++ u8 __reserved3[40]; ++ } __packed tcfc_query; ++ }; ++} __packed; ++ ++struct qm_mcr_ceetm_ccgr_config { ++ u8 __reserved1[46]; ++ union { ++ u8 __reserved2[8]; ++ struct { ++ u16 timestamp; ++ u16 wr_porb_g; ++ u16 wr_prob_y; ++ u16 wr_prob_r; ++ } __packed test_write; ++ }; ++ u8 __reserved3[8]; ++} __packed; ++ ++struct qm_mcr_ceetm_ccgr_query { ++ u8 __reserved1[6]; ++ union { ++ struct { ++#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ ++ u8 ctl_reserved:1; ++ u8 ctl_wr_en_g:1; ++ u8 ctl_wr_en_y:1; ++ u8 ctl_wr_en_r:1; ++ u8 ctl_td_en:1; ++ u8 ctl_td_mode:1; ++ u8 ctl_cscn_en:1; ++ u8 ctl_mode:1; ++#else ++ u8 ctl_mode:1; ++ u8 ctl_cscn_en:1; ++ u8 ctl_td_mode:1; ++ u8 ctl_td_en:1; ++ u8 ctl_wr_en_r:1; ++ u8 ctl_wr_en_y:1; ++ u8 ctl_wr_en_g:1; ++ u8 ctl_reserved:1; ++#endif ++ u8 cdv; ++ u8 __reserved2[2]; ++ u8 oal; ++ u8 __reserved3; ++ struct qm_cgr_cs_thres cs_thres; ++ struct qm_cgr_cs_thres cs_thres_x; ++ struct qm_cgr_cs_thres td_thres; ++ struct qm_cgr_wr_parm wr_parm_g; ++ struct qm_cgr_wr_parm wr_parm_y; ++ struct qm_cgr_wr_parm wr_parm_r; ++ u16 cscn_targ_dcp; ++ u8 dcp_lsn; ++ u64 i_cnt:40; ++ u8 __reserved4[3]; ++ u64 a_cnt:40; ++ u32 cscn_targ_swp[4]; ++ } __packed cm_query; ++ struct { ++ u8 dnc; ++ u8 dn0; ++ u8 dn1; ++ u64 dnba:40; ++ u8 __reserved2[2]; ++ u16 dnth_0; ++ u8 __reserved3[2]; ++ u16 dnth_1; ++ u8 __reserved4[10]; ++ u16 dnacc_0; ++ u8 __reserved5[2]; ++ u16 dnacc_1; ++ u8 __reserved6[24]; ++ } __packed dn_query; ++ struct { ++ u8 __reserved2[24]; ++ struct __qm_mcr_querycongestion state; ++ } __packed congestion_state; ++ ++ }; ++} __packed; ++ ++struct qm_mcr_ceetm_cq_peek_pop_xsfdrread { ++ u8 stat; ++ u8 __reserved1[11]; ++ u16 dctidx; ++ struct qm_fd fd; ++ u8 __reserved2[32]; ++} __packed; ++ ++struct qm_mcr_ceetm_statistics_query { ++ u8 __reserved1[17]; ++ u64 frm_cnt:40; ++ u8 __reserved2[2]; ++ u64 byte_cnt:48; ++ u8 __reserved3[32]; ++} __packed; ++ ++struct qm_mc_result { ++ u8 verb; ++ u8 result; ++ union { ++ struct qm_mcr_initfq initfq; ++ struct qm_mcr_queryfq queryfq; ++ struct qm_mcr_queryfq_np queryfq_np; ++ struct qm_mcr_alterfq alterfq; ++ struct qm_mcr_initcgr initcgr; ++ struct qm_mcr_cgrtestwrite cgrtestwrite; ++ struct qm_mcr_querycgr querycgr; ++ struct qm_mcr_querycongestion querycongestion; ++ struct qm_mcr_querywq querywq; ++ struct qm_mcr_ceetm_lfqmt_config lfqmt_config; ++ struct qm_mcr_ceetm_lfqmt_query lfqmt_query; ++ struct qm_mcr_ceetm_cq_config cq_config; ++ struct qm_mcr_ceetm_cq_query cq_query; ++ struct qm_mcr_ceetm_dct_config dct_config; ++ struct qm_mcr_ceetm_dct_query dct_query; ++ struct qm_mcr_ceetm_class_scheduler_config csch_config; ++ struct qm_mcr_ceetm_class_scheduler_query csch_query; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_config mst_config; ++ struct qm_mcr_ceetm_mapping_shaper_tcfc_query mst_query; ++ struct qm_mcr_ceetm_ccgr_config ccgr_config; ++ struct qm_mcr_ceetm_ccgr_query ccgr_query; ++ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread cq_ppxr; ++ struct qm_mcr_ceetm_statistics_query stats_query; ++ }; ++} __packed; ++ ++#define QM_MCR_VERB_RRID 0x80 ++#define QM_MCR_VERB_MASK QM_MCC_VERB_MASK ++#define QM_MCR_VERB_INITFQ_PARKED QM_MCC_VERB_INITFQ_PARKED ++#define QM_MCR_VERB_INITFQ_SCHED QM_MCC_VERB_INITFQ_SCHED ++#define QM_MCR_VERB_QUERYFQ QM_MCC_VERB_QUERYFQ ++#define QM_MCR_VERB_QUERYFQ_NP QM_MCC_VERB_QUERYFQ_NP ++#define QM_MCR_VERB_QUERYWQ QM_MCC_VERB_QUERYWQ ++#define QM_MCR_VERB_QUERYWQ_DEDICATED QM_MCC_VERB_QUERYWQ_DEDICATED ++#define QM_MCR_VERB_ALTER_SCHED QM_MCC_VERB_ALTER_SCHED ++#define QM_MCR_VERB_ALTER_FE QM_MCC_VERB_ALTER_FE ++#define QM_MCR_VERB_ALTER_RETIRE QM_MCC_VERB_ALTER_RETIRE ++#define QM_MCR_VERB_ALTER_OOS QM_MCC_VERB_ALTER_OOS ++#define QM_MCR_RESULT_NULL 0x00 ++#define QM_MCR_RESULT_OK 0xf0 ++#define QM_MCR_RESULT_ERR_FQID 0xf1 ++#define QM_MCR_RESULT_ERR_FQSTATE 0xf2 ++#define QM_MCR_RESULT_ERR_NOTEMPTY 0xf3 /* OOS fails if FQ is !empty */ ++#define QM_MCR_RESULT_ERR_BADCHANNEL 0xf4 ++#define QM_MCR_RESULT_PENDING 0xf8 ++#define QM_MCR_RESULT_ERR_BADCOMMAND 0xff ++#define QM_MCR_NP_STATE_FE 0x10 ++#define QM_MCR_NP_STATE_R 0x08 ++#define QM_MCR_NP_STATE_MASK 0x07 /* Reads FQD::STATE; */ ++#define QM_MCR_NP_STATE_OOS 0x00 ++#define QM_MCR_NP_STATE_RETIRED 0x01 ++#define QM_MCR_NP_STATE_TEN_SCHED 0x02 ++#define QM_MCR_NP_STATE_TRU_SCHED 0x03 ++#define QM_MCR_NP_STATE_PARKED 0x04 ++#define QM_MCR_NP_STATE_ACTIVE 0x05 ++#define QM_MCR_NP_PTR_MASK 0x07ff /* for RA[12] & OD[123] */ ++#define QM_MCR_NP_RA1_NRA(v) (((v) >> 14) & 0x3) /* FQD::NRA */ ++#define QM_MCR_NP_RA2_IT(v) (((v) >> 14) & 0x1) /* FQD::IT */ ++#define QM_MCR_NP_OD1_NOD(v) (((v) >> 14) & 0x3) /* FQD::NOD */ ++#define QM_MCR_NP_OD3_NPC(v) (((v) >> 14) & 0x3) /* FQD::NPC */ ++#define QM_MCR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */ ++#define QM_MCR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */ ++/* This extracts the state for congestion group 'n' from a query response. ++ * Eg. ++ * u8 cgr = [...]; ++ * struct qm_mc_result *res = [...]; ++ * printf("congestion group %d congestion state: %d\n", cgr, ++ * QM_MCR_QUERYCONGESTION(&res->querycongestion.state, cgr)); ++ */ ++#define __CGR_WORD(num) (num >> 5) ++#define __CGR_SHIFT(num) (num & 0x1f) ++#define __CGR_NUM (sizeof(struct __qm_mcr_querycongestion) << 3) ++static inline int QM_MCR_QUERYCONGESTION(struct __qm_mcr_querycongestion *p, ++ u8 cgr) ++{ ++ return be32_to_cpu(p->__state[__CGR_WORD(cgr)]) & ++ (0x80000000 >> __CGR_SHIFT(cgr)); ++} ++ ++ ++/*********************/ ++/* Utility interface */ ++/*********************/ ++ ++/* Represents an allocator over a range of FQIDs. NB, accesses are not locked, ++ * spinlock them yourself if needed. */ ++struct qman_fqid_pool; ++ ++/* Create/destroy a FQID pool, num must be a multiple of 32. NB, _destroy() ++ * always succeeds, but returns non-zero if there were "leaked" FQID ++ * allocations. */ ++struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num); ++int qman_fqid_pool_destroy(struct qman_fqid_pool *pool); ++/* Alloc/free a FQID from the range. _alloc() returns zero for success. */ ++int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid); ++void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid); ++u32 qman_fqid_pool_used(struct qman_fqid_pool *pool); ++ ++/*******************************************************************/ ++/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */ ++/*******************************************************************/ ++ ++ /* Portal and Frame Queues */ ++ /* ----------------------- */ ++/* Represents a managed portal */ ++struct qman_portal; ++ ++/* This object type represents Qman frame queue descriptors (FQD), it is ++ * cacheline-aligned, and initialised by qman_create_fq(). The structure is ++ * defined further down. */ ++struct qman_fq; ++ ++/* This object type represents a Qman congestion group, it is defined further ++ * down. */ ++struct qman_cgr; ++ ++struct qman_portal_config { ++ /* If the caller enables DQRR stashing (and thus wishes to operate the ++ * portal from only one cpu), this is the logical CPU that the portal ++ * will stash to. Whether stashing is enabled or not, this setting is ++ * also used for any "core-affine" portals, ie. default portals ++ * associated to the corresponding cpu. -1 implies that there is no core ++ * affinity configured. */ ++ int cpu; ++ /* portal interrupt line */ ++ int irq; ++ /* the unique index of this portal */ ++ u32 index; ++ /* Is this portal shared? (If so, it has coarser locking and demuxes ++ * processing on behalf of other CPUs.) */ ++ int is_shared; ++ /* The portal's dedicated channel id, use this value for initialising ++ * frame queues to target this portal when scheduled. */ ++ u16 channel; ++ /* A mask of which pool channels this portal has dequeue access to ++ * (using QM_SDQCR_CHANNELS_POOL(n) for the bitmask) */ ++ u32 pools; ++}; ++ ++/* This enum, and the callback type that returns it, are used when handling ++ * dequeued frames via DQRR. Note that for "null" callbacks registered with the ++ * portal object (for handling dequeues that do not demux because contextB is ++ * NULL), the return value *MUST* be qman_cb_dqrr_consume. */ ++enum qman_cb_dqrr_result { ++ /* DQRR entry can be consumed */ ++ qman_cb_dqrr_consume, ++ /* Like _consume, but requests parking - FQ must be held-active */ ++ qman_cb_dqrr_park, ++ /* Does not consume, for DCA mode only. This allows out-of-order ++ * consumes by explicit calls to qman_dca() and/or the use of implicit ++ * DCA via EQCR entries. */ ++ qman_cb_dqrr_defer, ++ /* Stop processing without consuming this ring entry. Exits the current ++ * qman_poll_dqrr() or interrupt-handling, as appropriate. If within an ++ * interrupt handler, the callback would typically call ++ * qman_irqsource_remove(QM_PIRQ_DQRI) before returning this value, ++ * otherwise the interrupt will reassert immediately. */ ++ qman_cb_dqrr_stop, ++ /* Like qman_cb_dqrr_stop, but consumes the current entry. */ ++ qman_cb_dqrr_consume_stop ++}; ++typedef enum qman_cb_dqrr_result (*qman_cb_dqrr)(struct qman_portal *qm, ++ struct qman_fq *fq, ++ const struct qm_dqrr_entry *dqrr); ++ ++/* This callback type is used when handling ERNs, FQRNs and FQRLs via MR. They ++ * are always consumed after the callback returns. */ ++typedef void (*qman_cb_mr)(struct qman_portal *qm, struct qman_fq *fq, ++ const struct qm_mr_entry *msg); ++ ++/* This callback type is used when handling DCP ERNs */ ++typedef void (*qman_cb_dc_ern)(struct qman_portal *qm, ++ const struct qm_mr_entry *msg); ++ ++/* s/w-visible states. Ie. tentatively scheduled + truly scheduled + active + ++ * held-active + held-suspended are just "sched". Things like "retired" will not ++ * be assumed until it is complete (ie. QMAN_FQ_STATE_CHANGING is set until ++ * then, to indicate it's completing and to gate attempts to retry the retire ++ * command). Note, park commands do not set QMAN_FQ_STATE_CHANGING because it's ++ * technically impossible in the case of enqueue DCAs (which refer to DQRR ring ++ * index rather than the FQ that ring entry corresponds to), so repeated park ++ * commands are allowed (if you're silly enough to try) but won't change FQ ++ * state, and the resulting park notifications move FQs from "sched" to ++ * "parked". */ ++enum qman_fq_state { ++ qman_fq_state_oos, ++ qman_fq_state_parked, ++ qman_fq_state_sched, ++ qman_fq_state_retired ++}; ++ ++/* Frame queue objects (struct qman_fq) are stored within memory passed to ++ * qman_create_fq(), as this allows stashing of caller-provided demux callback ++ * pointers at no extra cost to stashing of (driver-internal) FQ state. If the ++ * caller wishes to add per-FQ state and have it benefit from dequeue-stashing, ++ * they should; ++ * ++ * (a) extend the qman_fq structure with their state; eg. ++ * ++ * // myfq is allocated and driver_fq callbacks filled in; ++ * struct my_fq { ++ * struct qman_fq base; ++ * int an_extra_field; ++ * [ ... add other fields to be associated with each FQ ...] ++ * } *myfq = some_my_fq_allocator(); ++ * struct qman_fq *fq = qman_create_fq(fqid, flags, &myfq->base); ++ * ++ * // in a dequeue callback, access extra fields from 'fq' via a cast; ++ * struct my_fq *myfq = (struct my_fq *)fq; ++ * do_something_with(myfq->an_extra_field); ++ * [...] ++ * ++ * (b) when and if configuring the FQ for context stashing, specify how ever ++ * many cachelines are required to stash 'struct my_fq', to accelerate not ++ * only the Qman driver but the callback as well. ++ */ ++ ++struct qman_fq_cb { ++ qman_cb_dqrr dqrr; /* for dequeued frames */ ++ qman_cb_mr ern; /* for s/w ERNs */ ++ qman_cb_mr fqs; /* frame-queue state changes*/ ++}; ++ ++struct qman_fq { ++ /* Caller of qman_create_fq() provides these demux callbacks */ ++ struct qman_fq_cb cb; ++ /* These are internal to the driver, don't touch. In particular, they ++ * may change, be removed, or extended (so you shouldn't rely on ++ * sizeof(qman_fq) being a constant). */ ++ spinlock_t fqlock; ++ u32 fqid; ++ volatile unsigned long flags; ++ enum qman_fq_state state; ++ int cgr_groupid; ++ struct rb_node node; ++#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP ++ u32 key; ++#endif ++}; ++ ++/* This callback type is used when handling congestion group entry/exit. ++ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit. */ ++typedef void (*qman_cb_cgr)(struct qman_portal *qm, ++ struct qman_cgr *cgr, int congested); ++ ++struct qman_cgr { ++ /* Set these prior to qman_create_cgr() */ ++ u32 cgrid; /* 0..255, but u32 to allow specials like -1, 256, etc.*/ ++ qman_cb_cgr cb; ++ /* These are private to the driver */ ++ u16 chan; /* portal channel this object is created on */ ++ struct list_head node; ++}; ++ ++/* Flags to qman_create_fq() */ ++#define QMAN_FQ_FLAG_NO_ENQUEUE 0x00000001 /* can't enqueue */ ++#define QMAN_FQ_FLAG_NO_MODIFY 0x00000002 /* can only enqueue */ ++#define QMAN_FQ_FLAG_TO_DCPORTAL 0x00000004 /* consumed by CAAM/PME/Fman */ ++#define QMAN_FQ_FLAG_LOCKED 0x00000008 /* multi-core locking */ ++#define QMAN_FQ_FLAG_AS_IS 0x00000010 /* query h/w state */ ++#define QMAN_FQ_FLAG_DYNAMIC_FQID 0x00000020 /* (de)allocate fqid */ ++ ++/* Flags to qman_destroy_fq() */ ++#define QMAN_FQ_DESTROY_PARKED 0x00000001 /* FQ can be parked or OOS */ ++ ++/* Flags from qman_fq_state() */ ++#define QMAN_FQ_STATE_CHANGING 0x80000000 /* 'state' is changing */ ++#define QMAN_FQ_STATE_NE 0x40000000 /* retired FQ isn't empty */ ++#define QMAN_FQ_STATE_ORL 0x20000000 /* retired FQ has ORL */ ++#define QMAN_FQ_STATE_BLOCKOOS 0xe0000000 /* if any are set, no OOS */ ++#define QMAN_FQ_STATE_CGR_EN 0x10000000 /* CGR enabled */ ++#define QMAN_FQ_STATE_VDQCR 0x08000000 /* being volatile dequeued */ ++ ++/* Flags to qman_init_fq() */ ++#define QMAN_INITFQ_FLAG_SCHED 0x00000001 /* schedule rather than park */ ++#define QMAN_INITFQ_FLAG_LOCAL 0x00000004 /* set dest portal */ ++ ++/* Flags to qman_volatile_dequeue() */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++#define QMAN_VOLATILE_FLAG_WAIT 0x00000001 /* wait if VDQCR is in use */ ++#define QMAN_VOLATILE_FLAG_WAIT_INT 0x00000002 /* if wait, interruptible? */ ++#define QMAN_VOLATILE_FLAG_FINISH 0x00000004 /* wait till VDQCR completes */ ++#endif ++ ++/* Flags to qman_enqueue(). NB, the strange numbering is to align with hardware, ++ * bit-wise. (NB: the PME API is sensitive to these precise numberings too, so ++ * any change here should be audited in PME.) */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT ++#define QMAN_ENQUEUE_FLAG_WAIT 0x00010000 /* wait if EQCR is full */ ++#define QMAN_ENQUEUE_FLAG_WAIT_INT 0x00020000 /* if wait, interruptible? */ ++#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC ++#define QMAN_ENQUEUE_FLAG_WAIT_SYNC 0x00000004 /* if wait, until consumed? */ ++#endif ++#endif ++#define QMAN_ENQUEUE_FLAG_WATCH_CGR 0x00080000 /* watch congestion state */ ++#define QMAN_ENQUEUE_FLAG_DCA 0x00008000 /* perform enqueue-DCA */ ++#define QMAN_ENQUEUE_FLAG_DCA_PARK 0x00004000 /* If DCA, requests park */ ++#define QMAN_ENQUEUE_FLAG_DCA_PTR(p) /* If DCA, p is DQRR entry */ \ ++ (((u32)(p) << 2) & 0x00000f00) ++#define QMAN_ENQUEUE_FLAG_C_GREEN 0x00000000 /* choose one C_*** flag */ ++#define QMAN_ENQUEUE_FLAG_C_YELLOW 0x00000008 ++#define QMAN_ENQUEUE_FLAG_C_RED 0x00000010 ++#define QMAN_ENQUEUE_FLAG_C_OVERRIDE 0x00000018 ++/* For the ORP-specific qman_enqueue_orp() variant; ++ * - this flag indicates "Not Last In Sequence", ie. all but the final fragment ++ * of a frame. */ ++#define QMAN_ENQUEUE_FLAG_NLIS 0x01000000 ++/* - this flag performs no enqueue but fills in an ORP sequence number that ++ * would otherwise block it (eg. if a frame has been dropped). */ ++#define QMAN_ENQUEUE_FLAG_HOLE 0x02000000 ++/* - this flag performs no enqueue but advances NESN to the given sequence ++ * number. */ ++#define QMAN_ENQUEUE_FLAG_NESN 0x04000000 ++ ++/* Flags to qman_modify_cgr() */ ++#define QMAN_CGR_FLAG_USE_INIT 0x00000001 ++#define QMAN_CGR_MODE_FRAME 0x00000001 ++ ++ /* Portal Management */ ++ /* ----------------- */ ++/** ++ * qman_get_portal_config - get portal configuration settings ++ * ++ * This returns a read-only view of the current cpu's affine portal settings. ++ */ ++const struct qman_portal_config *qman_get_portal_config(void); ++ ++/** ++ * qman_irqsource_get - return the portal work that is interrupt-driven ++ * ++ * Returns a bitmask of QM_PIRQ_**I processing sources that are currently ++ * enabled for interrupt handling on the current cpu's affine portal. These ++ * sources will trigger the portal interrupt and the interrupt handler (or a ++ * tasklet/bottom-half it defers to) will perform the corresponding processing ++ * work. The qman_poll_***() functions will only process sources that are not in ++ * this bitmask. If the current CPU is sharing a portal hosted on another CPU, ++ * this always returns zero. ++ */ ++u32 qman_irqsource_get(void); ++ ++/** ++ * qman_irqsource_add - add processing sources to be interrupt-driven ++ * @bits: bitmask of QM_PIRQ_**I processing sources ++ * ++ * Adds processing sources that should be interrupt-driven (rather than ++ * processed via qman_poll_***() functions). Returns zero for success, or ++ * -EINVAL if the current CPU is sharing a portal hosted on another CPU. ++ */ ++int qman_irqsource_add(u32 bits); ++ ++/** ++ * qman_irqsource_remove - remove processing sources from being interrupt-driven ++ * @bits: bitmask of QM_PIRQ_**I processing sources ++ * ++ * Removes processing sources from being interrupt-driven, so that they will ++ * instead be processed via qman_poll_***() functions. Returns zero for success, ++ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU. ++ */ ++int qman_irqsource_remove(u32 bits); ++ ++/** ++ * qman_affine_cpus - return a mask of cpus that have affine portals ++ */ ++const cpumask_t *qman_affine_cpus(void); ++ ++/** ++ * qman_affine_channel - return the channel ID of an portal ++ * @cpu: the cpu whose affine portal is the subject of the query ++ * ++ * If @cpu is -1, the affine portal for the current CPU will be used. It is a ++ * bug to call this function for any value of @cpu (other than -1) that is not a ++ * member of the mask returned from qman_affine_cpus(). ++ */ ++u16 qman_affine_channel(int cpu); ++ ++/** ++ * qman_get_affine_portal - return the portal pointer affine to cpu ++ * @cpu: the cpu whose affine portal is the subject of the query ++ * ++ */ ++void *qman_get_affine_portal(int cpu); ++ ++/** ++ * qman_poll_dqrr - process DQRR (fast-path) entries ++ * @limit: the maximum number of DQRR entries to process ++ * ++ * Use of this function requires that DQRR processing not be interrupt-driven. ++ * Ie. the value returned by qman_irqsource_get() should not include ++ * QM_PIRQ_DQRI. If the current CPU is sharing a portal hosted on another CPU, ++ * this function will return -EINVAL, otherwise the return value is >=0 and ++ * represents the number of DQRR entries processed. ++ */ ++int qman_poll_dqrr(unsigned int limit); ++ ++/** ++ * qman_poll_slow - process anything (except DQRR) that isn't interrupt-driven. ++ * ++ * This function does any portal processing that isn't interrupt-driven. If the ++ * current CPU is sharing a portal hosted on another CPU, this function will ++ * return (u32)-1, otherwise the return value is a bitmask of QM_PIRQ_* sources ++ * indicating what interrupt sources were actually processed by the call. ++ */ ++u32 qman_poll_slow(void); ++ ++/** ++ * qman_poll - legacy wrapper for qman_poll_dqrr() and qman_poll_slow() ++ * ++ * Dispatcher logic on a cpu can use this to trigger any maintenance of the ++ * affine portal. There are two classes of portal processing in question; ++ * fast-path (which involves demuxing dequeue ring (DQRR) entries and tracking ++ * enqueue ring (EQCR) consumption), and slow-path (which involves EQCR ++ * thresholds, congestion state changes, etc). This function does whatever ++ * processing is not triggered by interrupts. ++ * ++ * Note, if DQRR and some slow-path processing are poll-driven (rather than ++ * interrupt-driven) then this function uses a heuristic to determine how often ++ * to run slow-path processing - as slow-path processing introduces at least a ++ * minimum latency each time it is run, whereas fast-path (DQRR) processing is ++ * close to zero-cost if there is no work to be done. Applications can tune this ++ * behaviour themselves by using qman_poll_dqrr() and qman_poll_slow() directly ++ * rather than going via this wrapper. ++ */ ++void qman_poll(void); ++ ++/** ++ * qman_stop_dequeues - Stop h/w dequeuing to the s/w portal ++ * ++ * Disables DQRR processing of the portal. This is reference-counted, so ++ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to ++ * truly re-enable dequeuing. ++ */ ++void qman_stop_dequeues(void); ++ ++/** ++ * qman_start_dequeues - (Re)start h/w dequeuing to the s/w portal ++ * ++ * Enables DQRR processing of the portal. This is reference-counted, so ++ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to ++ * truly re-enable dequeuing. ++ */ ++void qman_start_dequeues(void); ++ ++/** ++ * qman_static_dequeue_add - Add pool channels to the portal SDQCR ++ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n) ++ * ++ * Adds a set of pool channels to the portal's static dequeue command register ++ * (SDQCR). The requested pools are limited to those the portal has dequeue ++ * access to. ++ */ ++void qman_static_dequeue_add(u32 pools); ++ ++/** ++ * qman_static_dequeue_del - Remove pool channels from the portal SDQCR ++ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n) ++ * ++ * Removes a set of pool channels from the portal's static dequeue command ++ * register (SDQCR). The requested pools are limited to those the portal has ++ * dequeue access to. ++ */ ++void qman_static_dequeue_del(u32 pools); ++ ++/** ++ * qman_static_dequeue_get - return the portal's current SDQCR ++ * ++ * Returns the portal's current static dequeue command register (SDQCR). The ++ * entire register is returned, so if only the currently-enabled pool channels ++ * are desired, mask the return value with QM_SDQCR_CHANNELS_POOL_MASK. ++ */ ++u32 qman_static_dequeue_get(void); ++ ++/** ++ * qman_dca - Perform a Discrete Consumption Acknowledgement ++ * @dq: the DQRR entry to be consumed ++ * @park_request: indicates whether the held-active @fq should be parked ++ * ++ * Only allowed in DCA-mode portals, for DQRR entries whose handler callback had ++ * previously returned 'qman_cb_dqrr_defer'. NB, as with the other APIs, this ++ * does not take a 'portal' argument but implies the core affine portal from the ++ * cpu that is currently executing the function. For reasons of locking, this ++ * function must be called from the same CPU as that which processed the DQRR ++ * entry in the first place. ++ */ ++void qman_dca(struct qm_dqrr_entry *dq, int park_request); ++ ++/** ++ * qman_eqcr_is_empty - Determine if portal's EQCR is empty ++ * ++ * For use in situations where a cpu-affine caller needs to determine when all ++ * enqueues for the local portal have been processed by Qman but can't use the ++ * QMAN_ENQUEUE_FLAG_WAIT_SYNC flag to do this from the final qman_enqueue(). ++ * The function forces tracking of EQCR consumption (which normally doesn't ++ * happen until enqueue processing needs to find space to put new enqueue ++ * commands), and returns zero if the ring still has unprocessed entries, ++ * non-zero if it is empty. ++ */ ++int qman_eqcr_is_empty(void); ++ ++/** ++ * qman_set_dc_ern - Set the handler for DCP enqueue rejection notifications ++ * @handler: callback for processing DCP ERNs ++ * @affine: whether this handler is specific to the locally affine portal ++ * ++ * If a hardware block's interface to Qman (ie. its direct-connect portal, or ++ * DCP) is configured not to receive enqueue rejections, then any enqueues ++ * through that DCP that are rejected will be sent to a given software portal. ++ * If @affine is non-zero, then this handler will only be used for DCP ERNs ++ * received on the portal affine to the current CPU. If multiple CPUs share a ++ * portal and they all call this function, they will be setting the handler for ++ * the same portal! If @affine is zero, then this handler will be global to all ++ * portals handled by this instance of the driver. Only those portals that do ++ * not have their own affine handler will use the global handler. ++ */ ++void qman_set_dc_ern(qman_cb_dc_ern handler, int affine); ++ ++ /* FQ management */ ++ /* ------------- */ ++/** ++ * qman_create_fq - Allocates a FQ ++ * @fqid: the index of the FQD to encapsulate, must be "Out of Service" ++ * @flags: bit-mask of QMAN_FQ_FLAG_*** options ++ * @fq: memory for storing the 'fq', with callbacks filled in ++ * ++ * Creates a frame queue object for the given @fqid, unless the ++ * QMAN_FQ_FLAG_DYNAMIC_FQID flag is set in @flags, in which case a FQID is ++ * dynamically allocated (or the function fails if none are available). Once ++ * created, the caller should not touch the memory at 'fq' except as extended to ++ * adjacent memory for user-defined fields (see the definition of "struct ++ * qman_fq" for more info). NO_MODIFY is only intended for enqueuing to ++ * pre-existing frame-queues that aren't to be otherwise interfered with, it ++ * prevents all other modifications to the frame queue. The TO_DCPORTAL flag ++ * causes the driver to honour any contextB modifications requested in the ++ * qm_init_fq() API, as this indicates the frame queue will be consumed by a ++ * direct-connect portal (PME, CAAM, or Fman). When frame queues are consumed by ++ * software portals, the contextB field is controlled by the driver and can't be ++ * modified by the caller. If the AS_IS flag is specified, management commands ++ * will be used on portal @p to query state for frame queue @fqid and construct ++ * a frame queue object based on that, rather than assuming/requiring that it be ++ * Out of Service. ++ */ ++int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq); ++ ++/** ++ * qman_destroy_fq - Deallocates a FQ ++ * @fq: the frame queue object to release ++ * @flags: bit-mask of QMAN_FQ_FREE_*** options ++ * ++ * The memory for this frame queue object ('fq' provided in qman_create_fq()) is ++ * not deallocated but the caller regains ownership, to do with as desired. The ++ * FQ must be in the 'out-of-service' state unless the QMAN_FQ_FREE_PARKED flag ++ * is specified, in which case it may also be in the 'parked' state. ++ */ ++void qman_destroy_fq(struct qman_fq *fq, u32 flags); ++ ++/** ++ * qman_fq_fqid - Queries the frame queue ID of a FQ object ++ * @fq: the frame queue object to query ++ */ ++u32 qman_fq_fqid(struct qman_fq *fq); ++ ++/** ++ * qman_fq_state - Queries the state of a FQ object ++ * @fq: the frame queue object to query ++ * @state: pointer to state enum to return the FQ scheduling state ++ * @flags: pointer to state flags to receive QMAN_FQ_STATE_*** bitmask ++ * ++ * Queries the state of the FQ object, without performing any h/w commands. ++ * This captures the state, as seen by the driver, at the time the function ++ * executes. ++ */ ++void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags); ++ ++/** ++ * qman_init_fq - Initialises FQ fields, leaves the FQ "parked" or "scheduled" ++ * @fq: the frame queue object to modify, must be 'parked' or new. ++ * @flags: bit-mask of QMAN_INITFQ_FLAG_*** options ++ * @opts: the FQ-modification settings, as defined in the low-level API ++ * ++ * The @opts parameter comes from the low-level portal API. Select ++ * QMAN_INITFQ_FLAG_SCHED in @flags to cause the frame queue to be scheduled ++ * rather than parked. NB, @opts can be NULL. ++ * ++ * Note that some fields and options within @opts may be ignored or overwritten ++ * by the driver; ++ * 1. the 'count' and 'fqid' fields are always ignored (this operation only ++ * affects one frame queue: @fq). ++ * 2. the QM_INITFQ_WE_CONTEXTB option of the 'we_mask' field and the associated ++ * 'fqd' structure's 'context_b' field are sometimes overwritten; ++ * - if @fq was not created with QMAN_FQ_FLAG_TO_DCPORTAL, then context_b is ++ * initialised to a value used by the driver for demux. ++ * - if context_b is initialised for demux, so is context_a in case stashing ++ * is requested (see item 4). ++ * (So caller control of context_b is only possible for TO_DCPORTAL frame queue ++ * objects.) ++ * 3. if @flags contains QMAN_INITFQ_FLAG_LOCAL, the 'fqd' structure's ++ * 'dest::channel' field will be overwritten to match the portal used to issue ++ * the command. If the WE_DESTWQ write-enable bit had already been set by the ++ * caller, the channel workqueue will be left as-is, otherwise the write-enable ++ * bit is set and the workqueue is set to a default of 4. If the "LOCAL" flag ++ * isn't set, the destination channel/workqueue fields and the write-enable bit ++ * are left as-is. ++ * 4. if the driver overwrites context_a/b for demux, then if ++ * QM_INITFQ_WE_CONTEXTA is set, the driver will only overwrite ++ * context_a.address fields and will leave the stashing fields provided by the ++ * user alone, otherwise it will zero out the context_a.stashing fields. ++ */ ++int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts); ++ ++/** ++ * qman_schedule_fq - Schedules a FQ ++ * @fq: the frame queue object to schedule, must be 'parked' ++ * ++ * Schedules the frame queue, which must be Parked, which takes it to ++ * Tentatively-Scheduled or Truly-Scheduled depending on its fill-level. ++ */ ++int qman_schedule_fq(struct qman_fq *fq); ++ ++/** ++ * qman_retire_fq - Retires a FQ ++ * @fq: the frame queue object to retire ++ * @flags: FQ flags (as per qman_fq_state) if retirement completes immediately ++ * ++ * Retires the frame queue. This returns zero if it succeeds immediately, +1 if ++ * the retirement was started asynchronously, otherwise it returns negative for ++ * failure. When this function returns zero, @flags is set to indicate whether ++ * the retired FQ is empty and/or whether it has any ORL fragments (to show up ++ * as ERNs). Otherwise the corresponding flags will be known when a subsequent ++ * FQRN message shows up on the portal's message ring. ++ * ++ * NB, if the retirement is asynchronous (the FQ was in the Truly Scheduled or ++ * Active state), the completion will be via the message ring as a FQRN - but ++ * the corresponding callback may occur before this function returns!! Ie. the ++ * caller should be prepared to accept the callback as the function is called, ++ * not only once it has returned. ++ */ ++int qman_retire_fq(struct qman_fq *fq, u32 *flags); ++ ++/** ++ * qman_oos_fq - Puts a FQ "out of service" ++ * @fq: the frame queue object to be put out-of-service, must be 'retired' ++ * ++ * The frame queue must be retired and empty, and if any order restoration list ++ * was released as ERNs at the time of retirement, they must all be consumed. ++ */ ++int qman_oos_fq(struct qman_fq *fq); ++ ++/** ++ * qman_fq_flow_control - Set the XON/XOFF state of a FQ ++ * @fq: the frame queue object to be set to XON/XOFF state, must not be 'oos', ++ * or 'retired' or 'parked' state ++ * @xon: boolean to set fq in XON or XOFF state ++ * ++ * The frame should be in Tentatively Scheduled state or Truly Schedule sate, ++ * otherwise the IFSI interrupt will be asserted. ++ */ ++int qman_fq_flow_control(struct qman_fq *fq, int xon); ++ ++/** ++ * qman_query_fq - Queries FQD fields (via h/w query command) ++ * @fq: the frame queue object to be queried ++ * @fqd: storage for the queried FQD fields ++ */ ++int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd); ++ ++/** ++ * qman_query_fq_np - Queries non-programmable FQD fields ++ * @fq: the frame queue object to be queried ++ * @np: storage for the queried FQD fields ++ */ ++int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np); ++ ++/** ++ * qman_query_wq - Queries work queue lengths ++ * @query_dedicated: If non-zero, query length of WQs in the channel dedicated ++ * to this software portal. Otherwise, query length of WQs in a ++ * channel specified in wq. ++ * @wq: storage for the queried WQs lengths. Also specified the channel to ++ * to query if query_dedicated is zero. ++ */ ++int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq); ++ ++/** ++ * qman_volatile_dequeue - Issue a volatile dequeue command ++ * @fq: the frame queue object to dequeue from ++ * @flags: a bit-mask of QMAN_VOLATILE_FLAG_*** options ++ * @vdqcr: bit mask of QM_VDQCR_*** options, as per qm_dqrr_vdqcr_set() ++ * ++ * Attempts to lock access to the portal's VDQCR volatile dequeue functionality. ++ * The function will block and sleep if QMAN_VOLATILE_FLAG_WAIT is specified and ++ * the VDQCR is already in use, otherwise returns non-zero for failure. If ++ * QMAN_VOLATILE_FLAG_FINISH is specified, the function will only return once ++ * the VDQCR command has finished executing (ie. once the callback for the last ++ * DQRR entry resulting from the VDQCR command has been called). If not using ++ * the FINISH flag, completion can be determined either by detecting the ++ * presence of the QM_DQRR_STAT_UNSCHEDULED and QM_DQRR_STAT_DQCR_EXPIRED bits ++ * in the "stat" field of the "struct qm_dqrr_entry" passed to the FQ's dequeue ++ * callback, or by waiting for the QMAN_FQ_STATE_VDQCR bit to disappear from the ++ * "flags" retrieved from qman_fq_state(). ++ */ ++int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr); ++ ++/** ++ * qman_enqueue - Enqueue a frame to a frame queue ++ * @fq: the frame queue object to enqueue to ++ * @fd: a descriptor of the frame to be enqueued ++ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options ++ * ++ * Fills an entry in the EQCR of portal @qm to enqueue the frame described by ++ * @fd. The descriptor details are copied from @fd to the EQCR entry, the 'pid' ++ * field is ignored. The return value is non-zero on error, such as ring full ++ * (and FLAG_WAIT not specified), congestion avoidance (FLAG_WATCH_CGR ++ * specified), etc. If the ring is full and FLAG_WAIT is specified, this ++ * function will block. If FLAG_INTERRUPT is set, the EQCI bit of the portal ++ * interrupt will assert when Qman consumes the EQCR entry (subject to "status ++ * disable", "enable", and "inhibit" registers). If FLAG_DCA is set, Qman will ++ * perform an implied "discrete consumption acknowledgement" on the dequeue ++ * ring's (DQRR) entry, at the ring index specified by the FLAG_DCA_IDX(x) ++ * macro. (As an alternative to issuing explicit DCA actions on DQRR entries, ++ * this implicit DCA can delay the release of a "held active" frame queue ++ * corresponding to a DQRR entry until Qman consumes the EQCR entry - providing ++ * order-preservation semantics in packet-forwarding scenarios.) If FLAG_DCA is ++ * set, then FLAG_DCA_PARK can also be set to imply that the DQRR consumption ++ * acknowledgement should "park request" the "held active" frame queue. Ie. ++ * when the portal eventually releases that frame queue, it will be left in the ++ * Parked state rather than Tentatively Scheduled or Truly Scheduled. If the ++ * portal is watching congestion groups, the QMAN_ENQUEUE_FLAG_WATCH_CGR flag ++ * is requested, and the FQ is a member of a congestion group, then this ++ * function returns -EAGAIN if the congestion group is currently congested. ++ * Note, this does not eliminate ERNs, as the async interface means we can be ++ * sending enqueue commands to an un-congested FQ that becomes congested before ++ * the enqueue commands are processed, but it does minimise needless thrashing ++ * of an already busy hardware resource by throttling many of the to-be-dropped ++ * enqueues "at the source". ++ */ ++int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags); ++ ++typedef int (*qman_cb_precommit) (void *arg); ++/** ++ * qman_enqueue_precommit - Enqueue a frame to a frame queue and call cb ++ * @fq: the frame queue object to enqueue to ++ * @fd: a descriptor of the frame to be enqueued ++ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options ++ * @cb: user supplied callback function to invoke before writing commit verb. ++ * @cb_arg: callback function argument ++ * ++ * This is similar to qman_enqueue except that it will invoke a user supplied ++ * callback function just before writng the commit verb. This is useful ++ * when the user want to do something *just before* enqueuing the request and ++ * the enqueue can't fail. ++ */ ++int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd, ++ u32 flags, qman_cb_precommit cb, void *cb_arg); ++ ++/** ++ * qman_enqueue_orp - Enqueue a frame to a frame queue using an ORP ++ * @fq: the frame queue object to enqueue to ++ * @fd: a descriptor of the frame to be enqueued ++ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options ++ * @orp: the frame queue object used as an order restoration point. ++ * @orp_seqnum: the sequence number of this frame in the order restoration path ++ * ++ * Similar to qman_enqueue(), but with the addition of an Order Restoration ++ * Point (@orp) and corresponding sequence number (@orp_seqnum) for this ++ * enqueue operation to employ order restoration. Each frame queue object acts ++ * as an Order Definition Point (ODP) by providing each frame dequeued from it ++ * with an incrementing sequence number, this value is generally ignored unless ++ * that sequence of dequeued frames will need order restoration later. Each ++ * frame queue object also encapsulates an Order Restoration Point (ORP), which ++ * is a re-assembly context for re-ordering frames relative to their sequence ++ * numbers as they are enqueued. The ORP does not have to be within the frame ++ * queue that receives the enqueued frame, in fact it is usually the frame ++ * queue from which the frames were originally dequeued. For the purposes of ++ * order restoration, multiple frames (or "fragments") can be enqueued for a ++ * single sequence number by setting the QMAN_ENQUEUE_FLAG_NLIS flag for all ++ * enqueues except the final fragment of a given sequence number. Ordering ++ * between sequence numbers is guaranteed, even if fragments of different ++ * sequence numbers are interlaced with one another. Fragments of the same ++ * sequence number will retain the order in which they are enqueued. If no ++ * enqueue is to performed, QMAN_ENQUEUE_FLAG_HOLE indicates that the given ++ * sequence number is to be "skipped" by the ORP logic (eg. if a frame has been ++ * dropped from a sequence), or QMAN_ENQUEUE_FLAG_NESN indicates that the given ++ * sequence number should become the ORP's "Next Expected Sequence Number". ++ * ++ * Side note: a frame queue object can be used purely as an ORP, without ++ * carrying any frames at all. Care should be taken not to deallocate a frame ++ * queue object that is being actively used as an ORP, as a future allocation ++ * of the frame queue object may start using the internal ORP before the ++ * previous use has finished. ++ */ ++int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, ++ struct qman_fq *orp, u16 orp_seqnum); ++ ++/** ++ * qman_alloc_fqid_range - Allocate a contiguous range of FQIDs ++ * @result: is set by the API to the base FQID of the allocated range ++ * @count: the number of FQIDs required ++ * @align: required alignment of the allocated range ++ * @partial: non-zero if the API can return fewer than @count FQIDs ++ * ++ * Returns the number of frame queues allocated, or a negative error code. If ++ * @partial is non zero, the allocation request may return a smaller range of ++ * FQs than requested (though alignment will be as requested). If @partial is ++ * zero, the return value will either be 'count' or negative. ++ */ ++int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial); ++static inline int qman_alloc_fqid(u32 *result) ++{ ++ int ret = qman_alloc_fqid_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++ ++/** ++ * qman_release_fqid_range - Release the specified range of frame queue IDs ++ * @fqid: the base FQID of the range to deallocate ++ * @count: the number of FQIDs in the range ++ * ++ * This function can also be used to seed the allocator with ranges of FQIDs ++ * that it can subsequently allocate from. ++ */ ++void qman_release_fqid_range(u32 fqid, unsigned int count); ++static inline void qman_release_fqid(u32 fqid) ++{ ++ qman_release_fqid_range(fqid, 1); ++} ++ ++void qman_seed_fqid_range(u32 fqid, unsigned int count); ++ ++ ++int qman_shutdown_fq(u32 fqid); ++ ++/** ++ * qman_reserve_fqid_range - Reserve the specified range of frame queue IDs ++ * @fqid: the base FQID of the range to deallocate ++ * @count: the number of FQIDs in the range ++ */ ++int qman_reserve_fqid_range(u32 fqid, unsigned int count); ++static inline int qman_reserve_fqid(u32 fqid) ++{ ++ return qman_reserve_fqid_range(fqid, 1); ++} ++ ++ /* Pool-channel management */ ++ /* ----------------------- */ ++/** ++ * qman_alloc_pool_range - Allocate a contiguous range of pool-channel IDs ++ * @result: is set by the API to the base pool-channel ID of the allocated range ++ * @count: the number of pool-channel IDs required ++ * @align: required alignment of the allocated range ++ * @partial: non-zero if the API can return fewer than @count ++ * ++ * Returns the number of pool-channel IDs allocated, or a negative error code. ++ * If @partial is non zero, the allocation request may return a smaller range of ++ * than requested (though alignment will be as requested). If @partial is zero, ++ * the return value will either be 'count' or negative. ++ */ ++int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial); ++static inline int qman_alloc_pool(u32 *result) ++{ ++ int ret = qman_alloc_pool_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++ ++/** ++ * qman_release_pool_range - Release the specified range of pool-channel IDs ++ * @id: the base pool-channel ID of the range to deallocate ++ * @count: the number of pool-channel IDs in the range ++ */ ++void qman_release_pool_range(u32 id, unsigned int count); ++static inline void qman_release_pool(u32 id) ++{ ++ qman_release_pool_range(id, 1); ++} ++ ++/** ++ * qman_reserve_pool_range - Reserve the specified range of pool-channel IDs ++ * @id: the base pool-channel ID of the range to reserve ++ * @count: the number of pool-channel IDs in the range ++ */ ++int qman_reserve_pool_range(u32 id, unsigned int count); ++static inline int qman_reserve_pool(u32 id) ++{ ++ return qman_reserve_pool_range(id, 1); ++} ++ ++void qman_seed_pool_range(u32 id, unsigned int count); ++ ++ /* CGR management */ ++ /* -------------- */ ++/** ++ * qman_create_cgr - Register a congestion group object ++ * @cgr: the 'cgr' object, with fields filled in ++ * @flags: QMAN_CGR_FLAG_* values ++ * @opts: optional state of CGR settings ++ * ++ * Registers this object to receiving congestion entry/exit callbacks on the ++ * portal affine to the cpu portal on which this API is executed. If opts is ++ * NULL then only the callback (cgr->cb) function is registered. If @flags ++ * contains QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset ++ * any unspecified parameters) will be used rather than a modify hw hardware ++ * (which only modifies the specified parameters). ++ */ ++int qman_create_cgr(struct qman_cgr *cgr, u32 flags, ++ struct qm_mcc_initcgr *opts); ++ ++/** ++ * qman_create_cgr_to_dcp - Register a congestion group object to DCP portal ++ * @cgr: the 'cgr' object, with fields filled in ++ * @flags: QMAN_CGR_FLAG_* values ++ * @dcp_portal: the DCP portal to which the cgr object is registered. ++ * @opts: optional state of CGR settings ++ * ++ */ ++int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, ++ struct qm_mcc_initcgr *opts); ++ ++/** ++ * qman_delete_cgr - Deregisters a congestion group object ++ * @cgr: the 'cgr' object to deregister ++ * ++ * "Unplugs" this CGR object from the portal affine to the cpu on which this API ++ * is executed. This must be excuted on the same affine portal on which it was ++ * created. ++ */ ++int qman_delete_cgr(struct qman_cgr *cgr); ++ ++/** ++ * qman_delete_cgr_safe - Deregisters a congestion group object from any CPU ++ * @cgr: the 'cgr' object to deregister ++ * ++ * This will select the proper CPU and run there qman_delete_cgr(). ++ */ ++void qman_delete_cgr_safe(struct qman_cgr *cgr); ++ ++/** ++ * qman_modify_cgr - Modify CGR fields ++ * @cgr: the 'cgr' object to modify ++ * @flags: QMAN_CGR_FLAG_* values ++ * @opts: the CGR-modification settings ++ * ++ * The @opts parameter comes from the low-level portal API, and can be NULL. ++ * Note that some fields and options within @opts may be ignored or overwritten ++ * by the driver, in particular the 'cgrid' field is ignored (this operation ++ * only affects the given CGR object). If @flags contains ++ * QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset any ++ * unspecified parameters) will be used rather than a modify hw hardware (which ++ * only modifies the specified parameters). ++ */ ++int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, ++ struct qm_mcc_initcgr *opts); ++ ++/** ++* qman_query_cgr - Queries CGR fields ++* @cgr: the 'cgr' object to query ++* @result: storage for the queried congestion group record ++*/ ++int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *result); ++ ++/** ++ * qman_query_congestion - Queries the state of all congestion groups ++ * @congestion: storage for the queried state of all congestion groups ++ */ ++int qman_query_congestion(struct qm_mcr_querycongestion *congestion); ++ ++/** ++ * qman_alloc_cgrid_range - Allocate a contiguous range of CGR IDs ++ * @result: is set by the API to the base CGR ID of the allocated range ++ * @count: the number of CGR IDs required ++ * @align: required alignment of the allocated range ++ * @partial: non-zero if the API can return fewer than @count ++ * ++ * Returns the number of CGR IDs allocated, or a negative error code. ++ * If @partial is non zero, the allocation request may return a smaller range of ++ * than requested (though alignment will be as requested). If @partial is zero, ++ * the return value will either be 'count' or negative. ++ */ ++int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial); ++static inline int qman_alloc_cgrid(u32 *result) ++{ ++ int ret = qman_alloc_cgrid_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++ ++/** ++ * qman_release_cgrid_range - Release the specified range of CGR IDs ++ * @id: the base CGR ID of the range to deallocate ++ * @count: the number of CGR IDs in the range ++ */ ++void qman_release_cgrid_range(u32 id, unsigned int count); ++static inline void qman_release_cgrid(u32 id) ++{ ++ qman_release_cgrid_range(id, 1); ++} ++ ++/** ++ * qman_reserve_cgrid_range - Reserve the specified range of CGR ID ++ * @id: the base CGR ID of the range to reserve ++ * @count: the number of CGR IDs in the range ++ */ ++int qman_reserve_cgrid_range(u32 id, unsigned int count); ++static inline int qman_reserve_cgrid(u32 id) ++{ ++ return qman_reserve_cgrid_range(id, 1); ++} ++ ++void qman_seed_cgrid_range(u32 id, unsigned int count); ++ ++ ++ /* Helpers */ ++ /* ------- */ ++/** ++ * qman_poll_fq_for_init - Check if an FQ has been initialised from OOS ++ * @fqid: the FQID that will be initialised by other s/w ++ * ++ * In many situations, a FQID is provided for communication between s/w ++ * entities, and whilst the consumer is responsible for initialising and ++ * scheduling the FQ, the producer(s) generally create a wrapper FQ object using ++ * and only call qman_enqueue() (no FQ initialisation, scheduling, etc). Ie; ++ * qman_create_fq(..., QMAN_FQ_FLAG_NO_MODIFY, ...); ++ * However, data can not be enqueued to the FQ until it is initialised out of ++ * the OOS state - this function polls for that condition. It is particularly ++ * useful for users of IPC functions - each endpoint's Rx FQ is the other ++ * endpoint's Tx FQ, so each side can initialise and schedule their Rx FQ object ++ * and then use this API on the (NO_MODIFY) Tx FQ object in order to ++ * synchronise. The function returns zero for success, +1 if the FQ is still in ++ * the OOS state, or negative if there was an error. ++ */ ++static inline int qman_poll_fq_for_init(struct qman_fq *fq) ++{ ++ struct qm_mcr_queryfq_np np; ++ int err; ++ err = qman_query_fq_np(fq, &np); ++ if (err) ++ return err; ++ if ((np.state & QM_MCR_NP_STATE_MASK) == QM_MCR_NP_STATE_OOS) ++ return 1; ++ return 0; ++} ++ ++ /* -------------- */ ++ /* CEETM :: types */ ++ /* -------------- */ ++/** ++ * Token Rate Structure ++ * Shaping rates are based on a "credit" system and a pre-configured h/w ++ * internal timer. The following type represents a shaper "rate" parameter as a ++ * fractional number of "tokens". Here's how it works. This (fractional) number ++ * of tokens is added to the shaper's "credit" every time the h/w timer elapses ++ * (up to a limit which is set by another shaper parameter). Every time a frame ++ * is enqueued through a shaper, the shaper deducts as many tokens as there are ++ * bytes of data in the enqueued frame. A shaper will not allow itself to ++ * enqueue any frames if its token count is negative. As such; ++ * ++ * The rate at which data is enqueued is limited by the ++ * rate at which tokens are added. ++ * ++ * Therefore if the user knows the period between these h/w timer updates in ++ * seconds, they can calculate the maximum traffic rate of the shaper (in ++ * bytes-per-second) from the token rate. And vice versa, they can calculate ++ * the token rate to use in order to achieve a given traffic rate. ++ */ ++struct qm_ceetm_rate { ++ /* The token rate is; whole + (fraction/8192) */ ++ u32 whole:11; /* 0..2047 */ ++ u32 fraction:13; /* 0..8191 */ ++}; ++ ++struct qm_ceetm_weight_code { ++ /* The weight code is; 5 msbits + 3 lsbits */ ++ u8 y:5; ++ u8 x:3; ++}; ++ ++struct qm_ceetm { ++ unsigned int idx; ++ struct list_head sub_portals; ++ struct list_head lnis; ++ unsigned int sp_range[2]; ++ unsigned int lni_range[2]; ++}; ++ ++struct qm_ceetm_sp { ++ struct list_head node; ++ unsigned int idx; ++ unsigned int dcp_idx; ++ int is_claimed; ++ struct qm_ceetm_lni *lni; ++}; ++ ++/* Logical Network Interface */ ++struct qm_ceetm_lni { ++ struct list_head node; ++ unsigned int idx; ++ unsigned int dcp_idx; ++ int is_claimed; ++ struct qm_ceetm_sp *sp; ++ struct list_head channels; ++ int shaper_enable; ++ int shaper_couple; ++ int oal; ++ struct qm_ceetm_rate cr_token_rate; ++ struct qm_ceetm_rate er_token_rate; ++ u16 cr_token_bucket_limit; ++ u16 er_token_bucket_limit; ++}; ++ ++/* Class Queue Channel */ ++struct qm_ceetm_channel { ++ struct list_head node; ++ unsigned int idx; ++ unsigned int lni_idx; ++ unsigned int dcp_idx; ++ struct list_head class_queues; ++ struct list_head ccgs; ++ u8 shaper_enable; ++ u8 shaper_couple; ++ struct qm_ceetm_rate cr_token_rate; ++ struct qm_ceetm_rate er_token_rate; ++ u16 cr_token_bucket_limit; ++ u16 er_token_bucket_limit; ++}; ++ ++struct qm_ceetm_ccg; ++ ++/* This callback type is used when handling congestion entry/exit. The ++ * 'cb_ctx' value is the opaque value associated with ccg object. ++ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit. ++ */ ++typedef void (*qman_cb_ccgr)(struct qm_ceetm_ccg *ccg, void *cb_ctx, ++ int congested); ++ ++/* Class Congestion Group */ ++struct qm_ceetm_ccg { ++ struct qm_ceetm_channel *parent; ++ struct list_head node; ++ struct list_head cb_node; ++ qman_cb_ccgr cb; ++ void *cb_ctx; ++ unsigned int idx; ++}; ++ ++/* Class Queue */ ++struct qm_ceetm_cq { ++ struct qm_ceetm_channel *parent; ++ struct qm_ceetm_ccg *ccg; ++ struct list_head node; ++ unsigned int idx; ++ int is_claimed; ++ struct list_head bound_lfqids; ++ struct list_head binding_node; ++}; ++ ++/* Logical Frame Queue */ ++struct qm_ceetm_lfq { ++ struct qm_ceetm_channel *parent; ++ struct list_head node; ++ unsigned int idx; ++ unsigned int dctidx; ++ u64 context_a; ++ u32 context_b; ++ qman_cb_mr ern; ++}; ++ ++/** ++ * qman_ceetm_bps2tokenrate - Given a desired rate 'bps' measured in bps ++ * (ie. bits-per-second), compute the 'token_rate' fraction that best ++ * approximates that rate. ++ * @bps: the desired shaper rate in bps. ++ * @token_rate: the output token rate computed with the given kbps. ++ * @rounding: dictates how to round if an exact conversion is not possible; if ++ * it is negative then 'token_rate' will round down to the highest value that ++ * does not exceed the desired rate, if it is positive then 'token_rate' will ++ * round up to the lowest value that is greater than or equal to the desired ++ * rate, and if it is zero then it will round to the nearest approximation, ++ * whether that be up or down. ++ * ++ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available. ++ */ ++int qman_ceetm_bps2tokenrate(u64 bps, ++ struct qm_ceetm_rate *token_rate, ++ int rounding); ++ ++/** ++ * qman_ceetm_tokenrate2bps - Given a 'token_rate', compute the ++ * corresponding number of 'bps'. ++ * @token_rate: the input desired token_rate fraction. ++ * @bps: the output shaper rate in bps computed with the give token rate. ++ * @rounding: has the same semantics as the previous function. ++ * ++ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available. ++ */ ++int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, ++ u64 *bps, ++ int rounding); ++ ++int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align, ++ int partial); ++static inline int qman_alloc_ceetm0_channel(u32 *result) ++{ ++ int ret = qman_alloc_ceetm0_channel_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++void qman_release_ceetm0_channel_range(u32 channelid, u32 count); ++static inline void qman_release_ceetm0_channelid(u32 channelid) ++{ ++ qman_release_ceetm0_channel_range(channelid, 1); ++} ++ ++int qman_reserve_ceetm0_channel_range(u32 channelid, u32 count); ++static inline int qman_reserve_ceetm0_channelid(u32 channelid) ++{ ++ return qman_reserve_ceetm0_channel_range(channelid, 1); ++} ++ ++void qman_seed_ceetm0_channel_range(u32 channelid, u32 count); ++ ++ ++int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align, ++ int partial); ++static inline int qman_alloc_ceetm1_channel(u32 *result) ++{ ++ int ret = qman_alloc_ceetm1_channel_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++void qman_release_ceetm1_channel_range(u32 channelid, u32 count); ++static inline void qman_release_ceetm1_channelid(u32 channelid) ++{ ++ qman_release_ceetm1_channel_range(channelid, 1); ++} ++int qman_reserve_ceetm1_channel_range(u32 channelid, u32 count); ++static inline int qman_reserve_ceetm1_channelid(u32 channelid) ++{ ++ return qman_reserve_ceetm1_channel_range(channelid, 1); ++} ++ ++void qman_seed_ceetm1_channel_range(u32 channelid, u32 count); ++ ++ ++int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align, ++ int partial); ++static inline int qman_alloc_ceetm0_lfqid(u32 *result) ++{ ++ int ret = qman_alloc_ceetm0_lfqid_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count); ++static inline void qman_release_ceetm0_lfqid(u32 lfqid) ++{ ++ qman_release_ceetm0_lfqid_range(lfqid, 1); ++} ++int qman_reserve_ceetm0_lfqid_range(u32 lfqid, u32 count); ++static inline int qman_reserve_ceetm0_lfqid(u32 lfqid) ++{ ++ return qman_reserve_ceetm0_lfqid_range(lfqid, 1); ++} ++ ++void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count); ++ ++ ++int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align, ++ int partial); ++static inline int qman_alloc_ceetm1_lfqid(u32 *result) ++{ ++ int ret = qman_alloc_ceetm1_lfqid_range(result, 1, 0, 0); ++ return (ret > 0) ? 0 : ret; ++} ++void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count); ++static inline void qman_release_ceetm1_lfqid(u32 lfqid) ++{ ++ qman_release_ceetm1_lfqid_range(lfqid, 1); ++} ++int qman_reserve_ceetm1_lfqid_range(u32 lfqid, u32 count); ++static inline int qman_reserve_ceetm1_lfqid(u32 lfqid) ++{ ++ return qman_reserve_ceetm1_lfqid_range(lfqid, 1); ++} ++ ++void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count); ++ ++ ++ /* ----------------------------- */ ++ /* CEETM :: sub-portals */ ++ /* ----------------------------- */ ++ ++/** ++ * qman_ceetm_claim_sp - Claims the given sub-portal, provided it is available ++ * to us and configured for traffic-management. ++ * @sp: the returned sub-portal object, if successful. ++ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM ++ * instance), ++ * @sp_idx" is the desired sub-portal index from 0 to 15. ++ * ++ * Returns zero for success, or -ENODEV if the sub-portal is in use, or -EINVAL ++ * if the sp_idx is out of range. ++ * ++ * Note that if there are multiple driver domains (eg. a linux kernel versus ++ * user-space drivers in USDPAA, or multiple guests running under a hypervisor) ++ * then a sub-portal may be accessible by more than one instance of a qman ++ * driver and so it may be claimed multiple times. If this is the case, it is ++ * up to the system architect to prevent conflicting configuration actions ++ * coming from the different driver domains. The qman drivers do not have any ++ * behind-the-scenes coordination to prevent this from happening. ++ */ ++int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, ++ enum qm_dc_portal dcp_idx, ++ unsigned int sp_idx); ++ ++/** ++ * qman_ceetm_sp_release - Releases a previously claimed sub-portal. ++ * @sp: the sub-portal to be released. ++ * ++ * Returns 0 for success, or -EBUSY for failure if the dependencies are not ++ * released. ++ */ ++int qman_ceetm_sp_release(struct qm_ceetm_sp *sp); ++ ++ /* ----------------------------------- */ ++ /* CEETM :: logical network interfaces */ ++ /* ----------------------------------- */ ++ ++/** ++ * qman_ceetm_lni_claim - Claims an unclaimed LNI. ++ * @lni: the returned LNI object, if successful. ++ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM ++ * instance) ++ * @lni_idx: is the desired LNI index. ++ * ++ * Returns zero for success, or -EINVAL on failure, which will happen if the LNI ++ * is not available or has already been claimed (and not yet successfully ++ * released), or lni_dix is out of range. ++ * ++ * Note that there may be multiple driver domains (or instances) that need to ++ * transmit out the same LNI, so this claim is only guaranteeing exclusivity ++ * within the domain of the driver being called. See qman_ceetm_sp_claim() and ++ * qman_ceetm_sp_get_lni() for more information. ++ */ ++int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, ++ enum qm_dc_portal dcp_id, ++ unsigned int lni_idx); ++ ++/** ++ * qman_ceetm_lni_releaes - Releases a previously claimed LNI. ++ * @lni: the lni needs to be released. ++ * ++ * This will only succeed if all dependent objects have been released. ++ * Returns zero for success, or -EBUSY if the dependencies are not released. ++ */ ++int qman_ceetm_lni_release(struct qm_ceetm_lni *lni); ++ ++/** ++ * qman_ceetm_sp_set_lni ++ * qman_ceetm_sp_get_lni - Set/get the LNI that the sub-portal is currently ++ * mapped to. ++ * @sp: the given sub-portal. ++ * @lni(in "set"function): the LNI object which the sp will be mappaed to. ++ * @lni_idx(in "get" function): the LNI index which the sp is mapped to. ++ * ++ * Returns zero for success, or -EINVAL for the "set" function when this sp-lni ++ * mapping has been set, or configure mapping command returns error, and ++ * -EINVAL for "get" function when this sp-lni mapping is not set or the query ++ * mapping command returns error. ++ * ++ * This may be useful in situations where multiple driver domains have access ++ * to the same sub-portals in order to all be able to transmit out the same ++ * physical interface (perhaps they're on different IP addresses or VPNs, so ++ * Fman is splitting Rx traffic and here we need to converge Tx traffic). In ++ * that case, a control-plane is likely to use qman_ceetm_lni_claim() followed ++ * by qman_ceetm_sp_set_lni() to configure the sub-portal, and other domains ++ * are likely to use qman_ceetm_sp_get_lni() followed by qman_ceetm_lni_claim() ++ * in order to determine the LNI that the control-plane had assigned. This is ++ * why the "get" returns an index, whereas the "set" takes an (already claimed) ++ * LNI object. ++ */ ++int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, ++ struct qm_ceetm_lni *lni); ++int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, ++ unsigned int *lni_idx); ++ ++/** ++ * qman_ceetm_lni_enable_shaper ++ * qman_ceetm_lni_disable_shaper - Enables/disables shaping on the LNI. ++ * @lni: the given LNI. ++ * @coupled: indicates whether CR and ER are coupled. ++ * @oal: the overhead accounting length which is added to the actual length of ++ * each frame when performing shaper calculations. ++ * ++ * When the number of (unused) committed-rate tokens reach the committed-rate ++ * token limit, 'coupled' indicates whether surplus tokens should be added to ++ * the excess-rate token count (up to the excess-rate token limit). ++ * When LNI is claimed, the shaper is disabled by default. The enable function ++ * will turn on this shaper for this lni. ++ * Whenever a claimed LNI is first enabled for shaping, its committed and ++ * excess token rates and limits are zero, so will need to be changed to do ++ * anything useful. The shaper can subsequently be enabled/disabled without ++ * resetting the shaping parameters, but the shaping parameters will be reset ++ * when the LNI is released. ++ * ++ * Returns zero for success, or errno for "enable" function in the cases as: ++ * a) -EINVAL if the shaper is already enabled, ++ * b) -EIO if the configure shaper command returns error. ++ * For "disable" function, returns: ++ * a) -EINVAL if the shaper is has already disabled. ++ * b) -EIO if calling configure shaper command returns error. ++ */ ++int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled, ++ int oal); ++int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni); ++ ++/** ++ * qman_ceetm_lni_is_shaper_enabled - Check LNI shaper status ++ * @lni: the give LNI ++ */ ++int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni); ++ ++/** ++ * qman_ceetm_lni_set_commit_rate ++ * qman_ceetm_lni_get_commit_rate ++ * qman_ceetm_lni_set_excess_rate ++ * qman_ceetm_lni_get_excess_rate - Set/get the shaper CR/ER token rate and ++ * token limit for the given LNI. ++ * @lni: the given LNI. ++ * @token_rate: the desired token rate for "set" fuction, or the token rate of ++ * the LNI queried by "get" function. ++ * @token_limit: the desired token bucket limit for "set" function, or the token ++ * limit of the given LNI queried by "get" function. ++ * ++ * Returns zero for success. The "set" function returns -EINVAL if the given ++ * LNI is unshapped or -EIO if the configure shaper command returns error. ++ * The "get" function returns -EINVAL if the token rate or the token limit is ++ * not set or the query command returns error. ++ */ ++int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit); ++int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit); ++int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit); ++int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit); ++/** ++ * qman_ceetm_lni_set_commit_rate_bps ++ * qman_ceetm_lni_get_commit_rate_bps ++ * qman_ceetm_lni_set_excess_rate_bps ++ * qman_ceetm_lni_get_excess_rate_bps - Set/get the shaper CR/ER rate ++ * and token limit for the given LNI. ++ * @lni: the given LNI. ++ * @bps: the desired shaping rate in bps for "set" fuction, or the shaping rate ++ * of the LNI queried by "get" function. ++ * @token_limit: the desired token bucket limit for "set" function, or the token ++ * limit of the given LNI queried by "get" function. ++ * ++ * Returns zero for success. The "set" function returns -EINVAL if the given ++ * LNI is unshapped or -EIO if the configure shaper command returns error. ++ * The "get" function returns -EINVAL if the token rate or the token limit is ++ * not set or the query command returns error. ++ */ ++int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni, ++ u64 bps, ++ u16 token_limit); ++int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni, ++ u64 *bps, u16 *token_limit); ++int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni, ++ u64 bps, ++ u16 token_limit); ++int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni, ++ u64 *bps, u16 *token_limit); ++ ++/** ++ * qman_ceetm_lni_set_tcfcc ++ * qman_ceetm_lni_get_tcfcc - Configure/query "Traffic Class Flow Control". ++ * @lni: the given LNI. ++ * @cq_level: is between 0 and 15, representing individual class queue levels ++ * (CQ0 to CQ7 for every channel) and grouped class queue levels (CQ8 to CQ15 ++ * for every channel). ++ * @traffic_class: is between 0 and 7 when associating a given class queue level ++ * to a traffic class, or -1 when disabling traffic class flow control for this ++ * class queue level. ++ * ++ * Return zero for success, or -EINVAL if the cq_level or traffic_class is out ++ * of range as indicated above, or -EIO if the configure/query tcfcc command ++ * returns error. ++ * ++ * Refer to the section of QMan CEETM traffic class flow control in the ++ * Reference Manual. ++ */ ++int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni, ++ unsigned int cq_level, ++ int traffic_class); ++int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni, ++ unsigned int cq_level, ++ int *traffic_class); ++ ++ /* ----------------------------- */ ++ /* CEETM :: class queue channels */ ++ /* ----------------------------- */ ++ ++/** ++ * qman_ceetm_channel_claim - Claims an unclaimed CQ channel that is mapped to ++ * the given LNI. ++ * @channel: the returned class queue channel object, if successful. ++ * @lni: the LNI that the channel belongs to. ++ * ++ * Channels are always initially "unshaped". ++ * ++ * Return zero for success, or -ENODEV if there is no channel available(all 32 ++ * channels are claimed) or -EINVAL if the channel mapping command returns ++ * error. ++ */ ++int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel, ++ struct qm_ceetm_lni *lni); ++ ++/** ++ * qman_ceetm_channel_release - Releases a previously claimed CQ channel. ++ * @channel: the channel needs to be released. ++ * ++ * Returns zero for success, or -EBUSY if the dependencies are still in use. ++ * ++ * Note any shaping of the channel will be cleared to leave it in an unshaped ++ * state. ++ */ ++int qman_ceetm_channel_release(struct qm_ceetm_channel *channel); ++ ++/** ++ * qman_ceetm_channel_enable_shaper ++ * qman_ceetm_channel_disable_shaper - Enables/disables shaping on the channel. ++ * @channel: the given channel. ++ * @coupled: indicates whether surplus CR tokens should be added to the ++ * excess-rate token count (up to the excess-rate token limit) when the number ++ * of (unused) committed-rate tokens reach the committed_rate token limit. ++ * ++ * Whenever a claimed channel is first enabled for shaping, its committed and ++ * excess token rates and limits are zero, so will need to be changed to do ++ * anything useful. The shaper can subsequently be enabled/disabled without ++ * resetting the shaping parameters, but the shaping parameters will be reset ++ * when the channel is released. ++ * ++ * Return 0 for success, or -EINVAL for failure, in the case that the channel ++ * shaper has been enabled/disabled or the management command returns error. ++ */ ++int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel, ++ int coupled); ++int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel); ++ ++/** ++ * qman_ceetm_channel_is_shaper_enabled - Check channel shaper status. ++ * @channel: the give channel. ++ */ ++int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel); ++ ++/** ++ * qman_ceetm_channel_set_commit_rate ++ * qman_ceetm_channel_get_commit_rate ++ * qman_ceetm_channel_set_excess_rate ++ * qman_ceetm_channel_get_excess_rate - Set/get channel CR/ER shaper parameters. ++ * @channel: the given channel. ++ * @token_rate: the desired token rate for "set" function, or the queried token ++ * rate for "get" function. ++ * @token_limit: the desired token limit for "set" function, or the queried ++ * token limit for "get" function. ++ * ++ * Return zero for success. The "set" function returns -EINVAL if the channel ++ * is unshaped, or -EIO if the configure shapper command returns error. The ++ * "get" function returns -EINVAL if token rate of token limit is not set, or ++ * the query shaper command returns error. ++ */ ++int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit); ++int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit); ++int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel, ++ const struct qm_ceetm_rate *token_rate, ++ u16 token_limit); ++int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel, ++ struct qm_ceetm_rate *token_rate, ++ u16 *token_limit); ++/** ++ * qman_ceetm_channel_set_commit_rate_bps ++ * qman_ceetm_channel_get_commit_rate_bps ++ * qman_ceetm_channel_set_excess_rate_bps ++ * qman_ceetm_channel_get_excess_rate_bps - Set/get channel CR/ER shaper ++ * parameters. ++ * @channel: the given channel. ++ * @token_rate: the desired shaper rate in bps for "set" function, or the ++ * shaper rate in bps for "get" function. ++ * @token_limit: the desired token limit for "set" function, or the queried ++ * token limit for "get" function. ++ * ++ * Return zero for success. The "set" function returns -EINVAL if the channel ++ * is unshaped, or -EIO if the configure shapper command returns error. The ++ * "get" function returns -EINVAL if token rate of token limit is not set, or ++ * the query shaper command returns error. ++ */ ++int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel, ++ u64 bps, u16 token_limit); ++int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel, ++ u64 *bps, u16 *token_limit); ++int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel, ++ u64 bps, u16 token_limit); ++int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel, ++ u64 *bps, u16 *token_limit); ++ ++/** ++ * qman_ceetm_channel_set_weight ++ * qman_ceetm_channel_get_weight - Set/get the weight for unshaped channel ++ * @channel: the given channel. ++ * @token_limit: the desired token limit as the weight of the unshaped channel ++ * for "set" function, or the queried token limit for "get" function. ++ * ++ * The algorithm of unshaped fair queuing (uFQ) is used for unshaped channel. ++ * It allows the unshaped channels to be included in the CR time eligible list, ++ * and thus use the configured CR token limit value as their fair queuing ++ * weight. ++ * ++ * Return zero for success, or -EINVAL if the channel is a shaped channel or ++ * the management command returns error. ++ */ ++int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel, ++ u16 token_limit); ++int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel, ++ u16 *token_limit); ++ ++/** ++ * qman_ceetm_channel_set_group ++ * qman_ceetm_channel_get_group - Set/get the grouping of the class scheduler. ++ * @channel: the given channel. ++ * @group_b: indicates whether there is group B in this channel. ++ * @prio_a: the priority of group A. ++ * @prio_b: the priority of group B. ++ * ++ * There are 8 individual class queues (CQ0-CQ7), and 8 grouped class queues ++ * (CQ8-CQ15). If 'group_b' is zero, then all the grouped class queues are in ++ * group A, otherwise they are split into group A (CQ8-11) and group B ++ * (CQ12-C15). The individual class queues and the group(s) are in strict ++ * priority order relative to each other. Within the group(s), the scheduling ++ * is not strict priority order, but the result of scheduling within a group ++ * is in strict priority order relative to the other class queues in the ++ * channel. 'prio_a' and 'prio_b' control the priority order of the groups ++ * relative to the individual class queues, and take values from 0-7. Eg. if ++ * 'group_b' is non-zero, 'prio_a' is 2 and 'prio_b' is 6, then the strict ++ * priority order would be; ++ * CQ0, CQ1, CQ2, GROUPA, CQ3, CQ4, CQ5, CQ6, GROUPB, CQ7 ++ * ++ * Return 0 for success. For "set" function, returns -EINVAL if prio_a or ++ * prio_b are out of the range 0 - 7 (priority of group A or group B can not ++ * be 0, CQ0 is always the highest class queue in this channel.), or -EIO if ++ * the configure scheduler command returns error. For "get" function, return ++ * -EINVAL if the query scheduler command returns error. ++ */ ++int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, ++ int group_b, ++ unsigned int prio_a, ++ unsigned int prio_b); ++int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, ++ int *group_b, ++ unsigned int *prio_a, ++ unsigned int *prio_b); ++ ++/** ++ * qman_ceetm_channel_set_group_cr_eligibility ++ * qman_ceetm_channel_set_group_er_eligibility - Set channel group eligibility ++ * @channel: the given channel object ++ * @group_b: indicates whether there is group B in this channel. ++ * @cre: the commit rate eligibility, 1 for enable, 0 for disable. ++ * ++ * Return zero for success, or -EINVAL if eligibility setting fails. ++*/ ++int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel ++ *channel, int group_b, int cre); ++int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel ++ *channel, int group_b, int ere); ++ ++/** ++ * qman_ceetm_channel_set_cq_cr_eligibility ++ * qman_ceetm_channel_set_cq_er_eligibility - Set channel cq eligibility ++ * @channel: the given channel object ++ * @idx: is from 0 to 7 (representing CQ0 to CQ7). ++ * @cre: the commit rate eligibility, 1 for enable, 0 for disable. ++ * ++ * Return zero for success, or -EINVAL if eligibility setting fails. ++*/ ++int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel, ++ unsigned int idx, int cre); ++int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel, ++ unsigned int idx, int ere); ++ ++ /* --------------------- */ ++ /* CEETM :: class queues */ ++ /* --------------------- */ ++ ++/** ++ * qman_ceetm_cq_claim - Claims an individual class queue. ++ * @cq: the returned class queue object, if successful. ++ * @channel: the class queue channel. ++ * @idx: is from 0 to 7 (representing CQ0 to CQ7). ++ * @ccg: represents the class congestion group that this class queue should be ++ * subscribed to, or NULL if no congestion group membership is desired. ++ * ++ * Returns zero for success, or -EINVAL if @idx is out of range 0 - 7 or ++ * if this class queue has been claimed, or configure class queue command ++ * returns error, or returns -ENOMEM if allocating CQ memory fails. ++ */ ++int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq, ++ struct qm_ceetm_channel *channel, ++ unsigned int idx, ++ struct qm_ceetm_ccg *ccg); ++ ++/** ++ * qman_ceetm_cq_claim_A - Claims a class queue group A. ++ * @cq: the returned class queue object, if successful. ++ * @channel: the class queue channel. ++ * @idx: is from 8 to 15 if only group A exits, otherwise, it is from 8 to 11. ++ * @ccg: represents the class congestion group that this class queue should be ++ * subscribed to, or NULL if no congestion group membership is desired. ++ * ++ * Return zero for success, or -EINVAL if @idx is out the range or if ++ * this class queue has been claimed or configure class queue command returns ++ * error, or returns -ENOMEM if allocating CQ memory fails. ++ */ ++int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq, ++ struct qm_ceetm_channel *channel, ++ unsigned int idx, ++ struct qm_ceetm_ccg *ccg); ++ ++/** ++ * qman_ceetm_cq_claim_B - Claims a class queue group B. ++ * @cq: the returned class queue object, if successful. ++ * @channel: the class queue channel. ++ * @idx: is from 0 to 3 (CQ12 to CQ15). ++ * @ccg: represents the class congestion group that this class queue should be ++ * subscribed to, or NULL if no congestion group membership is desired. ++ * ++ * Return zero for success, or -EINVAL if @idx is out the range or if ++ * this class queue has been claimed or configure class queue command returns ++ * error, or returns -ENOMEM if allocating CQ memory fails. ++ */ ++int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq, ++ struct qm_ceetm_channel *channel, ++ unsigned int idx, ++ struct qm_ceetm_ccg *ccg); ++ ++/** ++ * qman_ceetm_cq_release - Releases a previously claimed class queue. ++ * @cq: The class queue to be released. ++ * ++ * Return zero for success, or -EBUSY if the dependent objects (eg. logical ++ * FQIDs) have not been released. ++ */ ++int qman_ceetm_cq_release(struct qm_ceetm_cq *cq); ++ ++/** ++ * qman_ceetm_set_queue_weight ++ * qman_ceetm_get_queue_weight - Configure/query the weight of a grouped class ++ * queue. ++ * @cq: the given class queue. ++ * @weight_code: the desired weight code to set for the given class queue for ++ * "set" function or the queired weight code for "get" function. ++ * ++ * Grouped class queues have a default weight code of zero, which corresponds to ++ * a scheduler weighting of 1. This function can be used to modify a grouped ++ * class queue to another weight, (Use the helpers qman_ceetm_wbfs2ratio() ++ * and qman_ceetm_ratio2wbfs() to convert between these 'weight_code' values ++ * and the corresponding sharing weight.) ++ * ++ * Returns zero for success, or -EIO if the configure weight command returns ++ * error for "set" function, or -EINVAL if the query command returns ++ * error for "get" function. ++ * See section "CEETM Weighted Scheduling among Grouped Classes" in Reference ++ * Manual for weight and weight code. ++ */ ++int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq, ++ struct qm_ceetm_weight_code *weight_code); ++int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq, ++ struct qm_ceetm_weight_code *weight_code); ++ ++/** ++ * qman_ceetm_set_queue_weight_in_ratio ++ * qman_ceetm_get_queue_weight_in_ratio - Configure/query the weight of a ++ * grouped class queue. ++ * @cq: the given class queue. ++ * @ratio: the weight in ratio. It should be the real ratio number multiplied ++ * by 100 to get rid of fraction. ++ * ++ * Returns zero for success, or -EIO if the configure weight command returns ++ * error for "set" function, or -EINVAL if the query command returns ++ * error for "get" function. ++ */ ++int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio); ++int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio); ++ ++/* Weights are encoded using a pseudo-exponential scheme. The weight codes 0, ++ * 32, 64, [...] correspond to weights of 1, 2, 4, [...]. The weights ++ * corresponding to intermediate weight codes are calculated using linear ++ * interpolation on the inverted values. Or put another way, the inverse weights ++ * for each 32nd weight code are 1, 1/2, 1/4, [...], and so the intervals ++ * between these are divided linearly into 32 intermediate values, the inverses ++ * of which form the remaining weight codes. ++ * ++ * The Weighted Bandwidth Fair Scheduling (WBFS) algorithm provides a form of ++ * scheduling within a group of class queues (group A or B). Weights are used to ++ * normalise the class queues to an underlying BFS algorithm where all class ++ * queues are assumed to require "equal bandwidth". So the weights referred to ++ * by the weight codes act as divisors on the size of frames being enqueued. Ie. ++ * one class queue in a group is assigned a weight of 2 whilst the other class ++ * queues in the group keep the default weight of 1, then the WBFS scheduler ++ * will effectively treat all frames enqueued on the weight-2 class queue as ++ * having half the number of bytes they really have. Ie. if all other things are ++ * equal, that class queue would get twice as much bytes-per-second bandwidth as ++ * the others. So weights should be chosen to provide bandwidth ratios between ++ * members of the same class queue group. These weights have no bearing on ++ * behaviour outside that group's WBFS mechanism though. ++ */ ++ ++/** ++ * qman_ceetm_wbfs2ratio - Given a weight code ('wbfs'), an accurate fractional ++ * representation of the corresponding weight is given (in order to not lose ++ * any precision). ++ * @weight_code: The given weight code in WBFS. ++ * @numerator: the numerator part of the weight computed by the weight code. ++ * @denominator: the denominator part of the weight computed by the weight code ++ * ++ * Returns zero for success or -EINVAL if the given weight code is illegal. ++ */ ++int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code, ++ u32 *numerator, ++ u32 *denominator); ++/** ++ * qman_ceetm_ratio2wbfs - Given a weight, find the nearest possible weight code ++ * If the user needs to know how close this is, convert the resulting weight ++ * code back to a weight and compare. ++ * @numerator: numerator part of the given weight. ++ * @denominator: denominator part of the given weight. ++ * @weight_code: the weight code computed from the given weight. ++ * ++ * Returns zero for success, or -ERANGE if "numerator/denominator" is outside ++ * the range of weights. ++ */ ++int qman_ceetm_ratio2wbfs(u32 numerator, ++ u32 denominator, ++ struct qm_ceetm_weight_code *weight_code, ++ int rounding); ++ ++#define QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER 0x1 ++/** ++ * qman_ceetm_cq_get_dequeue_statistics - Get the statistics provided by CEETM ++ * CQ counters. ++ * @cq: the given CQ object. ++ * @flags: indicates whether the statistics counter will be cleared after query. ++ * @frame_count: The number of the frames that have been counted since the ++ * counter was cleared last time. ++ * @byte_count: the number of bytes in all frames that have been counted. ++ * ++ * Return zero for success or -EINVAL if query statistics command returns error. ++ * ++ */ ++int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags, ++ u64 *frame_count, u64 *byte_count); ++ ++/** ++ * qman_ceetm_drain_cq - drain the CQ till it is empty. ++ * @cq: the give CQ object. ++ * Return 0 for success or -EINVAL for unsuccessful command to empty CQ. ++ */ ++int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq); ++ ++ /* ---------------------- */ ++ /* CEETM :: logical FQIDs */ ++ /* ---------------------- */ ++/** ++ * qman_ceetm_lfq_claim - Claims an unused logical FQID, associates it with ++ * the given class queue. ++ * @lfq: the returned lfq object, if successful. ++ * @cq: the class queue which needs to claim a LFQID. ++ * ++ * Return zero for success, or -ENODEV if no LFQID is available or -ENOMEM if ++ * allocating memory for lfq fails, or -EINVAL if configuring LFQMT fails. ++ */ ++int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq, ++ struct qm_ceetm_cq *cq); ++ ++/** ++ * qman_ceetm_lfq_release - Releases a previously claimed logical FQID. ++ * @lfq: the lfq to be released. ++ * ++ * Return zero for success. ++ */ ++int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq); ++ ++/** ++ * qman_ceetm_lfq_set_context ++ * qman_ceetm_lfq_get_context - Set/get the context_a/context_b pair to the ++ * "dequeue context table" associated with the logical FQID. ++ * @lfq: the given logical FQ object. ++ * @context_a: contextA of the dequeue context. ++ * @context_b: contextB of the dequeue context. ++ * ++ * Returns zero for success, or -EINVAL if there is error to set/get the ++ * context pair. ++ */ ++int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, ++ u64 context_a, ++ u32 context_b); ++int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, ++ u64 *context_a, ++ u32 *context_b); ++ ++/** ++ * qman_ceetm_create_fq - Initialise a FQ object for the LFQ. ++ * @lfq: the given logic fq. ++ * @fq: the fq object created for the given logic fq. ++ * ++ * The FQ object can be used in qman_enqueue() and qman_enqueue_orp() APIs to ++ * target a logical FQID (and the class queue it is associated with). ++ * Note that this FQ object can only be used for enqueues, and ++ * in the case of qman_enqueue_orp() it can not be used as the 'orp' parameter, ++ * only as 'fq'. This FQ object can not (and shouldn't) be destroyed, it is only ++ * valid as long as the underlying 'lfq' remains claimed. It is the user's ++ * responsibility to ensure that the underlying 'lfq' is not released until any ++ * enqueues to this FQ object have completed. The only field the user needs to ++ * fill in is fq->cb.ern, as that enqueue rejection handler is the callback that ++ * could conceivably be called on this FQ object. This API can be called ++ * multiple times to create multiple FQ objects referring to the same logical ++ * FQID, and any enqueue rejections will respect the callback of the object that ++ * issued the enqueue (and will identify the object via the parameter passed to ++ * the callback too). There is no 'flags' parameter to this API as there is for ++ * qman_create_fq() - the created FQ object behaves as though qman_create_fq() ++ * had been called with the single flag QMAN_FQ_FLAG_NO_MODIFY. ++ * ++ * Returns 0 for success. ++ */ ++int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq); ++ ++ /* -------------------------------- */ ++ /* CEETM :: class congestion groups */ ++ /* -------------------------------- */ ++ ++/** ++ * qman_ceetm_ccg_claim - Claims an unused CCG. ++ * @ccg: the returned CCG object, if successful. ++ * @channel: the given class queue channel ++ * @cscn: the callback function of this CCG. ++ * @cb_ctx: the corresponding context to be used used if state change ++ * notifications are later enabled for this CCG. ++ * ++ * The congestion group is local to the given class queue channel, so only ++ * class queues within the channel can be associated with that congestion group. ++ * The association of class queues to congestion groups occurs when the class ++ * queues are claimed, see qman_ceetm_cq_claim() and related functions. ++ * Congestion groups are in a "zero" state when initially claimed, and they are ++ * returned to that state when released. ++ * ++ * Return zero for success, or -EINVAL if no CCG in the channel is available. ++ */ ++int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg, ++ struct qm_ceetm_channel *channel, ++ unsigned int idx, ++ void (*cscn)(struct qm_ceetm_ccg *, ++ void *cb_ctx, ++ int congested), ++ void *cb_ctx); ++ ++/** ++ * qman_ceetm_ccg_release - Releases a previously claimed CCG. ++ * @ccg: the given ccg. ++ * ++ * Returns zero for success, or -EBUSY if the given ccg's dependent objects ++ * (class queues that are associated with the CCG) have not been released. ++ */ ++int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg); ++ ++/* This struct is used to specify attributes for a CCG. The 'we_mask' field ++ * controls which CCG attributes are to be updated, and the remainder specify ++ * the values for those attributes. A CCG counts either frames or the bytes ++ * within those frames, but not both ('mode'). A CCG can optionally cause ++ * enqueues to be rejected, due to tail-drop or WRED, or both (they are ++ * independent options, 'td_en' and 'wr_en_g,wr_en_y,wr_en_r'). Tail-drop can be ++ * level-triggered due to a single threshold ('td_thres') or edge-triggered due ++ * to a "congestion state", but not both ('td_mode'). Congestion state has ++ * distinct entry and exit thresholds ('cs_thres_in' and 'cs_thres_out'), and ++ * notifications can be sent to software the CCG goes in to and out of this ++ * congested state ('cscn_en'). */ ++struct qm_ceetm_ccg_params { ++ /* Boolean fields together in a single bitfield struct */ ++ struct { ++ /* Whether to count bytes or frames. 1==frames */ ++ u8 mode:1; ++ /* En/disable tail-drop. 1==enable */ ++ u8 td_en:1; ++ /* Tail-drop on congestion-state or threshold. 1=threshold */ ++ u8 td_mode:1; ++ /* Generate congestion state change notifications. 1==enable */ ++ u8 cscn_en:1; ++ /* Enable WRED rejections (per colour). 1==enable */ ++ u8 wr_en_g:1; ++ u8 wr_en_y:1; ++ u8 wr_en_r:1; ++ } __packed; ++ /* Tail-drop threshold. See qm_cgr_thres_[gs]et64(). */ ++ struct qm_cgr_cs_thres td_thres; ++ /* Congestion state thresholds, for entry and exit. */ ++ struct qm_cgr_cs_thres cs_thres_in; ++ struct qm_cgr_cs_thres cs_thres_out; ++ /* Overhead accounting length. Per-packet "tax", from -128 to +127 */ ++ signed char oal; ++ /* Congestion state change notification for DCP portal, virtual CCGID*/ ++ /* WRED parameters. */ ++ struct qm_cgr_wr_parm wr_parm_g; ++ struct qm_cgr_wr_parm wr_parm_y; ++ struct qm_cgr_wr_parm wr_parm_r; ++}; ++/* Bits used in 'we_mask' to qman_ceetm_ccg_set(), controls which attributes of ++ * the CCGR are to be updated. */ ++#define QM_CCGR_WE_MODE 0x0001 /* mode (bytes/frames) */ ++#define QM_CCGR_WE_CS_THRES_IN 0x0002 /* congestion state entry threshold */ ++#define QM_CCGR_WE_TD_EN 0x0004 /* congestion state tail-drop enable */ ++#define QM_CCGR_WE_CSCN_TUPD 0x0008 /* CSCN target update */ ++#define QM_CCGR_WE_CSCN_EN 0x0010 /* congestion notification enable */ ++#define QM_CCGR_WE_WR_EN_R 0x0020 /* WRED enable - red */ ++#define QM_CCGR_WE_WR_EN_Y 0x0040 /* WRED enable - yellow */ ++#define QM_CCGR_WE_WR_EN_G 0x0080 /* WRED enable - green */ ++#define QM_CCGR_WE_WR_PARM_R 0x0100 /* WRED parameters - red */ ++#define QM_CCGR_WE_WR_PARM_Y 0x0200 /* WRED parameters - yellow */ ++#define QM_CCGR_WE_WR_PARM_G 0x0400 /* WRED parameters - green */ ++#define QM_CCGR_WE_OAL 0x0800 /* overhead accounting length */ ++#define QM_CCGR_WE_CS_THRES_OUT 0x1000 /* congestion state exit threshold */ ++#define QM_CCGR_WE_TD_THRES 0x2000 /* tail-drop threshold */ ++#define QM_CCGR_WE_TD_MODE 0x4000 /* tail-drop mode (state/threshold) */ ++#define QM_CCGR_WE_CDV 0x8000 /* cdv */ ++ ++/** ++ * qman_ceetm_ccg_set ++ * qman_ceetm_ccg_get - Configure/query a subset of CCG attributes. ++ * @ccg: the given CCG object. ++ * @we_mask: the write enable mask. ++ * @params: the parameters setting for this ccg ++ * ++ * Return 0 for success, or -EIO if configure ccg command returns error for ++ * "set" function, or -EINVAL if query ccg command returns error for "get" ++ * function. ++ */ ++int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, ++ u16 we_mask, ++ const struct qm_ceetm_ccg_params *params); ++int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg, ++ struct qm_ceetm_ccg_params *params); ++ ++/** qman_ceetm_cscn_swp_set - Add or remove a software portal from the target ++ * mask. ++ * qman_ceetm_cscn_swp_get - Query whether a given software portal index is ++ * in the cscn target mask. ++ * @ccg: the give CCG object. ++ * @swp_idx: the index of the software portal. ++ * @cscn_enabled: 1: Set the swp to be cscn target. 0: remove the swp from ++ * the target mask. ++ * @we_mask: the write enable mask. ++ * @params: the parameters setting for this ccg ++ * ++ * Return 0 for success, or -EINVAL if command in set/get function fails. ++ */ ++int qman_ceetm_cscn_swp_set(struct qm_ceetm_ccg *ccg, ++ u16 swp_idx, ++ unsigned int cscn_enabled, ++ u16 we_mask, ++ const struct qm_ceetm_ccg_params *params); ++int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg, ++ u16 swp_idx, ++ unsigned int *cscn_enabled); ++ ++/** qman_ceetm_cscn_dcp_set - Add or remove a direct connect portal from the\ ++ * target mask. ++ * qman_ceetm_cscn_swp_get - Query whether a given direct connect portal index ++ * is in the cscn target mask. ++ * @ccg: the give CCG object. ++ * @dcp_idx: the index of the direct connect portal. ++ * @vcgid: congestion state change notification for dcp portal, virtual CGID. ++ * @cscn_enabled: 1: Set the dcp to be cscn target. 0: remove the dcp from ++ * the target mask. ++ * @we_mask: the write enable mask. ++ * @params: the parameters setting for this ccg ++ * ++ * Return 0 for success, or -EINVAL if command in set/get function fails. ++ */ ++int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg, ++ u16 dcp_idx, ++ u8 vcgid, ++ unsigned int cscn_enabled, ++ u16 we_mask, ++ const struct qm_ceetm_ccg_params *params); ++int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg, ++ u16 dcp_idx, ++ u8 *vcgid, ++ unsigned int *cscn_enabled); ++ ++/** ++ * qman_ceetm_ccg_get_reject_statistics - Get the statistics provided by ++ * CEETM CCG counters. ++ * @ccg: the given CCG object. ++ * @flags: indicates whether the statistics counter will be cleared after query. ++ * @frame_count: The number of the frames that have been counted since the ++ * counter was cleared last time. ++ * @byte_count: the number of bytes in all frames that have been counted. ++ * ++ * Return zero for success or -EINVAL if query statistics command returns error. ++ * ++ */ ++int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags, ++ u64 *frame_count, u64 *byte_count); ++ ++/** ++ * qman_ceetm_query_lfqmt - Query the logical frame queue mapping table ++ * @lfqid: Logical Frame Queue ID ++ * @lfqmt_query: Results of the query command ++ * ++ * Returns zero for success or -EIO if the query command returns error. ++ * ++ */ ++int qman_ceetm_query_lfqmt(int lfqid, ++ struct qm_mcr_ceetm_lfqmt_query *lfqmt_query); ++ ++/** ++ * qman_ceetm_query_write_statistics - Query (and optionally write) statistics ++ * @cid: Target ID (CQID or CCGRID) ++ * @dcp_idx: CEETM portal ID ++ * @command_type: One of the following: ++ * 0 = Query dequeue statistics. CID carries the CQID to be queried. ++ * 1 = Query and clear dequeue statistics. CID carries the CQID to be queried ++ * 2 = Write dequeue statistics. CID carries the CQID to be written. ++ * 3 = Query reject statistics. CID carries the CCGRID to be queried. ++ * 4 = Query and clear reject statistics. CID carries the CCGRID to be queried ++ * 5 = Write reject statistics. CID carries the CCGRID to be written ++ * @frame_count: Frame count value to be written if this is a write command ++ * @byte_count: Bytes count value to be written if this is a write command ++ * ++ * Returns zero for success or -EIO if the query command returns error. ++ */ ++int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx, ++ u16 command_type, u64 frame_count, ++ u64 byte_count); ++ ++/** ++ * qman_set_wpm - Set waterfall power management ++ * ++ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm. ++ * ++ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not ++ * accessible. ++ */ ++int qman_set_wpm(int wpm_enable); ++ ++/** ++ * qman_get_swp - Query the waterfall power management setting ++ * ++ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm. ++ * ++ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not ++ * accessible. ++ */ ++int qman_get_wpm(int *wpm_enable); ++ ++/* The below qman_p_***() variants might be called in a migration situation ++ * (e.g. cpu hotplug). They are used to continue accessing the portal that ++ * execution was affine to prior to migration. ++ * @qman_portal specifies which portal the APIs will use. ++*/ ++const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal ++ *p); ++int qman_p_irqsource_add(struct qman_portal *p, u32 bits); ++int qman_p_irqsource_remove(struct qman_portal *p, u32 bits); ++int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit); ++u32 qman_p_poll_slow(struct qman_portal *p); ++void qman_p_poll(struct qman_portal *p); ++void qman_p_stop_dequeues(struct qman_portal *p); ++void qman_p_start_dequeues(struct qman_portal *p); ++void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools); ++void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools); ++u32 qman_p_static_dequeue_get(struct qman_portal *p); ++void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq, ++ int park_request); ++int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq, ++ u32 flags __maybe_unused, u32 vdqcr); ++int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_fd *fd, u32 flags); ++int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_fd *fd, u32 flags, ++ struct qman_fq *orp, u16 orp_seqnum); ++int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq, ++ const struct qm_fd *fd, u32 flags, ++ qman_cb_precommit cb, void *cb_arg); ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* FSL_QMAN_H */ +--- /dev/null ++++ b/include/linux/fsl_usdpaa.h +@@ -0,0 +1,372 @@ ++/* Copyright 2011-2012 Freescale Semiconductor, Inc. ++ * ++ * This file is licensed under the terms of the GNU General Public License ++ * version 2. This program is licensed "as is" without any warranty of any ++ * kind, whether express or implied. ++ */ ++ ++#ifndef FSL_USDPAA_H ++#define FSL_USDPAA_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include <linux/uaccess.h> ++#include <linux/ioctl.h> ++#include <linux/fsl_qman.h> /* For "enum qm_channel" */ ++#include <linux/compat.h> ++ ++#ifdef CONFIG_FSL_USDPAA ++ ++/******************************/ ++/* Allocation of resource IDs */ ++/******************************/ ++ ++/* This enum is used to distinguish between the type of underlying object being ++ * manipulated. */ ++enum usdpaa_id_type { ++ usdpaa_id_fqid, ++ usdpaa_id_bpid, ++ usdpaa_id_qpool, ++ usdpaa_id_cgrid, ++ usdpaa_id_ceetm0_lfqid, ++ usdpaa_id_ceetm0_channelid, ++ usdpaa_id_ceetm1_lfqid, ++ usdpaa_id_ceetm1_channelid, ++ usdpaa_id_max /* <-- not a valid type, represents the number of types */ ++}; ++#define USDPAA_IOCTL_MAGIC 'u' ++struct usdpaa_ioctl_id_alloc { ++ uint32_t base; /* Return value, the start of the allocated range */ ++ enum usdpaa_id_type id_type; /* what kind of resource(s) to allocate */ ++ uint32_t num; /* how many IDs to allocate (and return value) */ ++ uint32_t align; /* must be a power of 2, 0 is treated like 1 */ ++ int partial; /* whether to allow less than 'num' */ ++}; ++struct usdpaa_ioctl_id_release { ++ /* Input; */ ++ enum usdpaa_id_type id_type; ++ uint32_t base; ++ uint32_t num; ++}; ++struct usdpaa_ioctl_id_reserve { ++ enum usdpaa_id_type id_type; ++ uint32_t base; ++ uint32_t num; ++}; ++ ++ ++/* ioctl() commands */ ++#define USDPAA_IOCTL_ID_ALLOC \ ++ _IOWR(USDPAA_IOCTL_MAGIC, 0x01, struct usdpaa_ioctl_id_alloc) ++#define USDPAA_IOCTL_ID_RELEASE \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x02, struct usdpaa_ioctl_id_release) ++#define USDPAA_IOCTL_ID_RESERVE \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x0A, struct usdpaa_ioctl_id_reserve) ++ ++/**********************/ ++/* Mapping DMA memory */ ++/**********************/ ++ ++/* Maximum length for a map name, including NULL-terminator */ ++#define USDPAA_DMA_NAME_MAX 16 ++/* Flags for requesting DMA maps. Maps are private+unnamed or sharable+named. ++ * For a sharable and named map, specify _SHARED (whether creating one or ++ * binding to an existing one). If _SHARED is specified and _CREATE is not, then ++ * the mapping must already exist. If _SHARED and _CREATE are specified and the ++ * mapping doesn't already exist, it will be created. If _SHARED and _CREATE are ++ * specified and the mapping already exists, the mapping will fail unless _LAZY ++ * is specified. When mapping to a pre-existing sharable map, the length must be ++ * an exact match. Lengths must be a power-of-4 multiple of page size. ++ * ++ * Note that this does not actually map the memory to user-space, that is done ++ * by a subsequent mmap() using the page offset returned from this ioctl(). The ++ * ioctl() is what gives the process permission to do this, and a page-offset ++ * with which to do so. ++ */ ++#define USDPAA_DMA_FLAG_SHARE 0x01 ++#define USDPAA_DMA_FLAG_CREATE 0x02 ++#define USDPAA_DMA_FLAG_LAZY 0x04 ++#define USDPAA_DMA_FLAG_RDONLY 0x08 ++struct usdpaa_ioctl_dma_map { ++ /* Output parameters - virtual and physical addresses */ ++ void *ptr; ++ uint64_t phys_addr; ++ /* Input parameter, the length of the region to be created (or if ++ * mapping an existing region, this must match it). Must be a power-of-4 ++ * multiple of page size. */ ++ uint64_t len; ++ /* Input parameter, the USDPAA_DMA_FLAG_* settings. */ ++ uint32_t flags; ++ /* If _FLAG_SHARE is specified, the name of the region to be created (or ++ * of the existing mapping to use). */ ++ char name[USDPAA_DMA_NAME_MAX]; ++ /* If this ioctl() creates the mapping, this is an input parameter ++ * stating whether the region supports locking. If mapping an existing ++ * region, this is a return value indicating the same thing. */ ++ int has_locking; ++ /* In the case of a successful map with _CREATE and _LAZY, this return ++ * value indicates whether we created the mapped region or whether it ++ * already existed. */ ++ int did_create; ++}; ++ ++#ifdef CONFIG_COMPAT ++struct usdpaa_ioctl_dma_map_compat { ++ /* Output parameters - virtual and physical addresses */ ++ compat_uptr_t ptr; ++ uint64_t phys_addr; ++ /* Input parameter, the length of the region to be created (or if ++ * mapping an existing region, this must match it). Must be a power-of-4 ++ * multiple of page size. */ ++ uint64_t len; ++ /* Input parameter, the USDPAA_DMA_FLAG_* settings. */ ++ uint32_t flags; ++ /* If _FLAG_SHARE is specified, the name of the region to be created (or ++ * of the existing mapping to use). */ ++ char name[USDPAA_DMA_NAME_MAX]; ++ /* If this ioctl() creates the mapping, this is an input parameter ++ * stating whether the region supports locking. If mapping an existing ++ * region, this is a return value indicating the same thing. */ ++ int has_locking; ++ /* In the case of a successful map with _CREATE and _LAZY, this return ++ * value indicates whether we created the mapped region or whether it ++ * already existed. */ ++ int did_create; ++}; ++ ++#define USDPAA_IOCTL_DMA_MAP_COMPAT \ ++ _IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map_compat) ++#endif ++ ++ ++#define USDPAA_IOCTL_DMA_MAP \ ++ _IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map) ++/* munmap() does not remove the DMA map, just the user-space mapping to it. ++ * This ioctl will do both (though you can munmap() before calling the ioctl ++ * too). */ ++#define USDPAA_IOCTL_DMA_UNMAP \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x04, unsigned char) ++/* We implement a cross-process locking scheme per DMA map. Call this ioctl() ++ * with a mmap()'d address, and the process will (interruptible) sleep if the ++ * lock is already held by another process. Process destruction will ++ * automatically clean up any held locks. */ ++#define USDPAA_IOCTL_DMA_LOCK \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x05, unsigned char) ++#define USDPAA_IOCTL_DMA_UNLOCK \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x06, unsigned char) ++ ++/***************************************/ ++/* Mapping and using QMan/BMan portals */ ++/***************************************/ ++enum usdpaa_portal_type { ++ usdpaa_portal_qman, ++ usdpaa_portal_bman, ++}; ++ ++#define QBMAN_ANY_PORTAL_IDX 0xffffffff ++ ++struct usdpaa_ioctl_portal_map { ++ /* Input parameter, is a qman or bman portal required. */ ++ ++ enum usdpaa_portal_type type; ++ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX ++ for don't care. The portal index will be populated by the ++ driver when the ioctl() successfully completes */ ++ uint32_t index; ++ ++ /* Return value if the map succeeds, this gives the mapped ++ * cache-inhibited (cinh) and cache-enabled (cena) addresses. */ ++ struct usdpaa_portal_map { ++ void *cinh; ++ void *cena; ++ } addr; ++ /* Qman-specific return values */ ++ uint16_t channel; ++ uint32_t pools; ++}; ++ ++#ifdef CONFIG_COMPAT ++struct compat_usdpaa_ioctl_portal_map { ++ /* Input parameter, is a qman or bman portal required. */ ++ enum usdpaa_portal_type type; ++ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX ++ for don't care. The portal index will be populated by the ++ driver when the ioctl() successfully completes */ ++ uint32_t index; ++ /* Return value if the map succeeds, this gives the mapped ++ * cache-inhibited (cinh) and cache-enabled (cena) addresses. */ ++ struct usdpaa_portal_map_compat { ++ compat_uptr_t cinh; ++ compat_uptr_t cena; ++ } addr; ++ /* Qman-specific return values */ ++ uint16_t channel; ++ uint32_t pools; ++}; ++#define USDPAA_IOCTL_PORTAL_MAP_COMPAT \ ++ _IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct compat_usdpaa_ioctl_portal_map) ++#define USDPAA_IOCTL_PORTAL_UNMAP_COMPAT \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map_compat) ++#endif ++ ++#define USDPAA_IOCTL_PORTAL_MAP \ ++ _IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct usdpaa_ioctl_portal_map) ++#define USDPAA_IOCTL_PORTAL_UNMAP \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map) ++ ++struct usdpaa_ioctl_irq_map { ++ enum usdpaa_portal_type type; /* Type of portal to map */ ++ int fd; /* File descriptor that contains the portal */ ++ void *portal_cinh; /* Cache inhibited area to identify the portal */ ++}; ++ ++#define USDPAA_IOCTL_PORTAL_IRQ_MAP \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x09, struct usdpaa_ioctl_irq_map) ++ ++#ifdef CONFIG_COMPAT ++ ++struct compat_ioctl_irq_map { ++ enum usdpaa_portal_type type; /* Type of portal to map */ ++ compat_int_t fd; /* File descriptor that contains the portal */ ++ compat_uptr_t portal_cinh; /* Used identify the portal */}; ++ ++#define USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT \ ++ _IOW(USDPAA_IOCTL_MAGIC, 0x09, struct compat_ioctl_irq_map) ++#endif ++ ++/* ioctl to query the amount of DMA memory used in the system */ ++struct usdpaa_ioctl_dma_used { ++ uint64_t free_bytes; ++ uint64_t total_bytes; ++}; ++#define USDPAA_IOCTL_DMA_USED \ ++ _IOR(USDPAA_IOCTL_MAGIC, 0x0B, struct usdpaa_ioctl_dma_used) ++ ++/* ioctl to allocate a raw portal */ ++struct usdpaa_ioctl_raw_portal { ++ /* inputs */ ++ enum usdpaa_portal_type type; /* Type of portal to allocate */ ++ ++ /* set to non zero to turn on stashing */ ++ uint8_t enable_stash; ++ /* Stashing attributes for the portal */ ++ uint32_t cpu; ++ uint32_t cache; ++ uint32_t window; ++ ++ /* Specifies the stash request queue this portal should use */ ++ uint8_t sdest; ++ ++ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX ++ * for don't care. The portal index will be populated by the ++ * driver when the ioctl() successfully completes */ ++ uint32_t index; ++ ++ /* outputs */ ++ uint64_t cinh; ++ uint64_t cena; ++}; ++ ++#define USDPAA_IOCTL_ALLOC_RAW_PORTAL \ ++ _IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct usdpaa_ioctl_raw_portal) ++ ++#define USDPAA_IOCTL_FREE_RAW_PORTAL \ ++ _IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct usdpaa_ioctl_raw_portal) ++ ++#ifdef CONFIG_COMPAT ++ ++struct compat_ioctl_raw_portal { ++ /* inputs */ ++ enum usdpaa_portal_type type; /* Type of portal to allocate */ ++ ++ /* set to non zero to turn on stashing */ ++ uint8_t enable_stash; ++ /* Stashing attributes for the portal */ ++ uint32_t cpu; ++ uint32_t cache; ++ uint32_t window; ++ /* Specifies the stash request queue this portal should use */ ++ uint8_t sdest; ++ ++ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX ++ * for don't care. The portal index will be populated by the ++ * driver when the ioctl() successfully completes */ ++ uint32_t index; ++ ++ /* outputs */ ++ uint64_t cinh; ++ uint64_t cena; ++}; ++ ++#define USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT \ ++ _IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct compat_ioctl_raw_portal) ++ ++#define USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT \ ++ _IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct compat_ioctl_raw_portal) ++ ++#endif ++ ++#ifdef __KERNEL__ ++ ++/* Early-boot hook */ ++int __init fsl_usdpaa_init_early(void); ++ ++/* Fault-handling in arch/powerpc/mm/mem.c gives USDPAA an opportunity to detect ++ * faults within its ranges via this hook. */ ++int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size); ++ ++#endif /* __KERNEL__ */ ++ ++#endif /* CONFIG_FSL_USDPAA */ ++ ++#ifdef __KERNEL__ ++/* This interface is needed in a few places and though it's not specific to ++ * USDPAA as such, creating a new header for it doesn't make any sense. The ++ * qbman kernel driver implements this interface and uses it as the backend for ++ * both the FQID and BPID allocators. The fsl_usdpaa driver also uses this ++ * interface for tracking per-process allocations handed out to user-space. */ ++struct dpa_alloc { ++ struct list_head free; ++ spinlock_t lock; ++ struct list_head used; ++}; ++#define DECLARE_DPA_ALLOC(name) \ ++ struct dpa_alloc name = { \ ++ .free = { \ ++ .prev = &name.free, \ ++ .next = &name.free \ ++ }, \ ++ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ ++ .used = { \ ++ .prev = &name.used, \ ++ .next = &name.used \ ++ } \ ++ } ++static inline void dpa_alloc_init(struct dpa_alloc *alloc) ++{ ++ INIT_LIST_HEAD(&alloc->free); ++ INIT_LIST_HEAD(&alloc->used); ++ spin_lock_init(&alloc->lock); ++} ++int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align, ++ int partial); ++void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count); ++void dpa_alloc_seed(struct dpa_alloc *alloc, u32 fqid, u32 count); ++ ++/* Like 'new' but specifies the desired range, returns -ENOMEM if the entire ++ * desired range is not available, or 0 for success. */ ++int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base_id, u32 count); ++/* Pops and returns contiguous ranges from the allocator. Returns -ENOMEM when ++ * 'alloc' is empty. */ ++int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count); ++/* Returns 1 if the specified id is alloced, 0 otherwise */ ++int dpa_alloc_check(struct dpa_alloc *list, u32 id); ++#endif /* __KERNEL__ */ ++ ++#ifdef __cplusplus ++} ++#endif ++ ++#endif /* FSL_USDPAA_H */ |