diff options
Diffstat (limited to 'target/linux/layerscape/patches-5.4/701-net-0007-fsl_qbman-SDK-DPAA-1.x-QBMan-drivers.patch')
| -rw-r--r-- | target/linux/layerscape/patches-5.4/701-net-0007-fsl_qbman-SDK-DPAA-1.x-QBMan-drivers.patch | 24880 |
1 files changed, 0 insertions, 24880 deletions
diff --git a/target/linux/layerscape/patches-5.4/701-net-0007-fsl_qbman-SDK-DPAA-1.x-QBMan-drivers.patch b/target/linux/layerscape/patches-5.4/701-net-0007-fsl_qbman-SDK-DPAA-1.x-QBMan-drivers.patch deleted file mode 100644 index b9e7667065..0000000000 --- a/target/linux/layerscape/patches-5.4/701-net-0007-fsl_qbman-SDK-DPAA-1.x-QBMan-drivers.patch +++ /dev/null @@ -1,24880 +0,0 @@ -From 772438d01bf57bc8939f53c3101a323fc774428f Mon Sep 17 00:00:00 2001 -From: Madalin Bucur <madalin.bucur@nxp.com> -Date: Wed, 10 May 2017 16:30:12 +0300 -Subject: [PATCH] fsl_qbman: SDK DPAA 1.x QBMan drivers - -Signed-off-by: Roy Pledge <roy.pledge@nxp.com> -Signed-off-by: Madalin Bucur <madalin.bucur@nxp.com> ---- - drivers/staging/fsl_qbman/Kconfig | 228 + - drivers/staging/fsl_qbman/Makefile | 28 + - drivers/staging/fsl_qbman/bman_config.c | 720 +++ - drivers/staging/fsl_qbman/bman_debugfs.c | 119 + - drivers/staging/fsl_qbman/bman_driver.c | 559 +++ - drivers/staging/fsl_qbman/bman_high.c | 1145 +++++ - drivers/staging/fsl_qbman/bman_low.h | 565 +++ - drivers/staging/fsl_qbman/bman_private.h | 166 + - drivers/staging/fsl_qbman/bman_test.c | 56 + - drivers/staging/fsl_qbman/bman_test.h | 44 + - drivers/staging/fsl_qbman/bman_test_high.c | 183 + - drivers/staging/fsl_qbman/bman_test_thresh.c | 196 + - drivers/staging/fsl_qbman/dpa_alloc.c | 706 +++ - drivers/staging/fsl_qbman/dpa_sys.h | 259 ++ - drivers/staging/fsl_qbman/dpa_sys_arm.h | 95 + - drivers/staging/fsl_qbman/dpa_sys_arm64.h | 102 + - drivers/staging/fsl_qbman/dpa_sys_ppc32.h | 70 + - drivers/staging/fsl_qbman/dpa_sys_ppc64.h | 79 + - drivers/staging/fsl_qbman/fsl_usdpaa.c | 1984 ++++++++ - drivers/staging/fsl_qbman/fsl_usdpaa_irq.c | 289 ++ - drivers/staging/fsl_qbman/qbman_driver.c | 88 + - drivers/staging/fsl_qbman/qman_config.c | 1224 +++++ - drivers/staging/fsl_qbman/qman_debugfs.c | 1594 +++++++ - drivers/staging/fsl_qbman/qman_driver.c | 961 ++++ - drivers/staging/fsl_qbman/qman_high.c | 5669 +++++++++++++++++++++++ - drivers/staging/fsl_qbman/qman_low.h | 1427 ++++++ - drivers/staging/fsl_qbman/qman_private.h | 398 ++ - drivers/staging/fsl_qbman/qman_test.c | 57 + - drivers/staging/fsl_qbman/qman_test.h | 45 + - drivers/staging/fsl_qbman/qman_test_high.c | 216 + - drivers/staging/fsl_qbman/qman_test_hotpotato.c | 502 ++ - drivers/staging/fsl_qbman/qman_utility.c | 129 + - include/linux/fsl_bman.h | 532 +++ - include/linux/fsl_qman.h | 3888 ++++++++++++++++ - include/linux/fsl_usdpaa.h | 372 ++ - 35 files changed, 24695 insertions(+) - create mode 100644 drivers/staging/fsl_qbman/Kconfig - create mode 100644 drivers/staging/fsl_qbman/Makefile - create mode 100644 drivers/staging/fsl_qbman/bman_config.c - create mode 100644 drivers/staging/fsl_qbman/bman_debugfs.c - create mode 100644 drivers/staging/fsl_qbman/bman_driver.c - create mode 100644 drivers/staging/fsl_qbman/bman_high.c - create mode 100644 drivers/staging/fsl_qbman/bman_low.h - create mode 100644 drivers/staging/fsl_qbman/bman_private.h - create mode 100644 drivers/staging/fsl_qbman/bman_test.c - create mode 100644 drivers/staging/fsl_qbman/bman_test.h - create mode 100644 drivers/staging/fsl_qbman/bman_test_high.c - create mode 100644 drivers/staging/fsl_qbman/bman_test_thresh.c - create mode 100644 drivers/staging/fsl_qbman/dpa_alloc.c - create mode 100644 drivers/staging/fsl_qbman/dpa_sys.h - create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm.h - create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm64.h - create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc32.h - create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc64.h - create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa.c - create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa_irq.c - create mode 100644 drivers/staging/fsl_qbman/qbman_driver.c - create mode 100644 drivers/staging/fsl_qbman/qman_config.c - create mode 100644 drivers/staging/fsl_qbman/qman_debugfs.c - create mode 100644 drivers/staging/fsl_qbman/qman_driver.c - create mode 100644 drivers/staging/fsl_qbman/qman_high.c - create mode 100644 drivers/staging/fsl_qbman/qman_low.h - create mode 100644 drivers/staging/fsl_qbman/qman_private.h - create mode 100644 drivers/staging/fsl_qbman/qman_test.c - create mode 100644 drivers/staging/fsl_qbman/qman_test.h - create mode 100644 drivers/staging/fsl_qbman/qman_test_high.c - create mode 100644 drivers/staging/fsl_qbman/qman_test_hotpotato.c - create mode 100644 drivers/staging/fsl_qbman/qman_utility.c - create mode 100644 include/linux/fsl_bman.h - create mode 100644 include/linux/fsl_qman.h - create mode 100644 include/linux/fsl_usdpaa.h - ---- /dev/null -+++ b/drivers/staging/fsl_qbman/Kconfig -@@ -0,0 +1,228 @@ -+config FSL_SDK_DPA -+ bool "Freescale Datapath Queue and Buffer management" -+ depends on !FSL_DPAA -+ select FSL_QMAN_FQ_LOOKUP if PPC64 -+ select FSL_QMAN_FQ_LOOKUP if ARM64 -+ -+ -+menu "Freescale Datapath QMan/BMan options" -+ depends on FSL_SDK_DPA -+ -+config FSL_DPA_CHECKING -+ bool "additional driver checking" -+ default n -+ ---help--- -+ Compiles in additional checks to sanity-check the drivers and any -+ use of it by other code. Not recommended for performance. -+ -+config FSL_DPA_CAN_WAIT -+ bool -+ default y -+ -+config FSL_DPA_CAN_WAIT_SYNC -+ bool -+ default y -+ -+config FSL_DPA_PIRQ_FAST -+ bool -+ default y -+ -+config FSL_DPA_PIRQ_SLOW -+ bool -+ default y -+ -+config FSL_DPA_PORTAL_SHARE -+ bool -+ default y -+ -+config FSL_SDK_BMAN -+ bool "Freescale Buffer Manager (BMan) support" -+ default y -+ -+if FSL_SDK_BMAN -+ -+config FSL_BMAN_CONFIG -+ bool "BMan device management" -+ default y -+ ---help--- -+ If this linux image is running natively, you need this option. If this -+ linux image is running as a guest OS under the hypervisor, only one -+ guest OS ("the control plane") needs this option. -+ -+config FSL_BMAN_TEST -+ tristate "BMan self-tests" -+ default n -+ ---help--- -+ This option compiles self-test code for BMan. -+ -+config FSL_BMAN_TEST_HIGH -+ bool "BMan high-level self-test" -+ depends on FSL_BMAN_TEST -+ default y -+ ---help--- -+ This requires the presence of cpu-affine portals, and performs -+ high-level API testing with them (whichever portal(s) are affine to -+ the cpu(s) the test executes on). -+ -+config FSL_BMAN_TEST_THRESH -+ bool "BMan threshold test" -+ depends on FSL_BMAN_TEST -+ default y -+ ---help--- -+ Multi-threaded (SMP) test of BMan pool depletion. A pool is seeded -+ before multiple threads (one per cpu) create pool objects to track -+ depletion state changes. The pool is then drained to empty by a -+ "drainer" thread, and the other threads that they observe exactly -+ the depletion state changes that are expected. -+ -+config FSL_BMAN_DEBUGFS -+ tristate "BMan debugfs interface" -+ depends on DEBUG_FS -+ default y -+ ---help--- -+ This option compiles debugfs code for BMan. -+ -+endif # FSL_SDK_BMAN -+ -+config FSL_SDK_QMAN -+ bool "Freescale Queue Manager (QMan) support" -+ default y -+ -+if FSL_SDK_QMAN -+ -+config FSL_QMAN_POLL_LIMIT -+ int -+ default 32 -+ -+config FSL_QMAN_CONFIG -+ bool "QMan device management" -+ default y -+ ---help--- -+ If this linux image is running natively, you need this option. If this -+ linux image is running as a guest OS under the hypervisor, only one -+ guest OS ("the control plane") needs this option. -+ -+config FSL_QMAN_TEST -+ tristate "QMan self-tests" -+ default n -+ ---help--- -+ This option compiles self-test code for QMan. -+ -+config FSL_QMAN_TEST_STASH_POTATO -+ bool "QMan 'hot potato' data-stashing self-test" -+ depends on FSL_QMAN_TEST -+ default y -+ ---help--- -+ This performs a "hot potato" style test enqueuing/dequeuing a frame -+ across a series of FQs scheduled to different portals (and cpus), with -+ DQRR, data and context stashing always on. -+ -+config FSL_QMAN_TEST_HIGH -+ bool "QMan high-level self-test" -+ depends on FSL_QMAN_TEST -+ default y -+ ---help--- -+ This requires the presence of cpu-affine portals, and performs -+ high-level API testing with them (whichever portal(s) are affine to -+ the cpu(s) the test executes on). -+ -+config FSL_QMAN_DEBUGFS -+ tristate "QMan debugfs interface" -+ depends on DEBUG_FS -+ default y -+ ---help--- -+ This option compiles debugfs code for QMan. -+ -+# H/w settings that can be hard-coded for now. -+config FSL_QMAN_FQD_SZ -+ int "size of Frame Queue Descriptor region" -+ default 10 -+ ---help--- -+ This is the size of the FQD region defined as: PAGE_SIZE * (2^value) -+ ex: 10 => PAGE_SIZE * (2^10) -+ Note: Default device-trees now require minimum Kconfig setting of 10. -+ -+config FSL_QMAN_PFDR_SZ -+ int "size of the PFDR pool" -+ default 13 -+ ---help--- -+ This is the size of the PFDR pool defined as: PAGE_SIZE * (2^value) -+ ex: 13 => PAGE_SIZE * (2^13) -+ -+# Corenet initiator settings. Stash request queues are 4-deep to match cores' -+# ability to snart. Stash priority is 3, other priorities are 2. -+config FSL_QMAN_CI_SCHED_CFG_SRCCIV -+ int -+ depends on FSL_QMAN_CONFIG -+ default 4 -+config FSL_QMAN_CI_SCHED_CFG_SRQ_W -+ int -+ depends on FSL_QMAN_CONFIG -+ default 3 -+config FSL_QMAN_CI_SCHED_CFG_RW_W -+ int -+ depends on FSL_QMAN_CONFIG -+ default 2 -+config FSL_QMAN_CI_SCHED_CFG_BMAN_W -+ int -+ depends on FSL_QMAN_CONFIG -+ default 2 -+ -+# portal interrupt settings -+config FSL_QMAN_PIRQ_DQRR_ITHRESH -+ int -+ default 12 -+config FSL_QMAN_PIRQ_MR_ITHRESH -+ int -+ default 4 -+config FSL_QMAN_PIRQ_IPERIOD -+ int -+ default 100 -+ -+# 64 bit kernel support -+config FSL_QMAN_FQ_LOOKUP -+ bool -+ default n -+ -+config QMAN_CEETM_UPDATE_PERIOD -+ int "Token update period for shaping, in nanoseconds" -+ default 1000 -+ ---help--- -+ Traffic shaping works by performing token calculations (using -+ credits) on shaper instances periodically. This update period -+ sets the granularity for how often those token rate credit -+ updates are performed, and thus determines the accuracy and -+ range of traffic rates that can be configured by users. The -+ reference manual recommends a 1 microsecond period as providing -+ a good balance between granularity and range. -+ -+ Unless you know what you are doing, leave this value at its default. -+ -+config FSL_QMAN_INIT_TIMEOUT -+ int "timeout for qman init stage, in seconds" -+ default 10 -+ ---help--- -+ The timeout setting to quit the initialization loop for non-control -+ partition in case the control partition fails to boot-up. -+ -+endif # FSL_SDK_QMAN -+ -+config FSL_USDPAA -+ bool "Freescale USDPAA process driver" -+ depends on FSL_SDK_DPA -+ default y -+ ---help--- -+ This driver provides user-space access to kernel-managed -+ resource interfaces for USDPAA applications, on the assumption -+ that each process will open this device once. Specifically, this -+ device exposes functionality that would be awkward if exposed -+ via the portal devices - ie. this device exposes functionality -+ that is inherently process-wide rather than portal-specific. -+ This device is necessary for obtaining access to DMA memory and -+ for allocation of Qman and Bman resources. In short, if you wish -+ to use USDPAA applications, you need this. -+ -+ If unsure, say Y. -+ -+ -+endmenu ---- /dev/null -+++ b/drivers/staging/fsl_qbman/Makefile -@@ -0,0 +1,28 @@ -+subdir-ccflags-y := -Werror -+ -+# Common -+obj-$(CONFIG_FSL_SDK_DPA) += dpa_alloc.o -+obj-$(CONFIG_FSL_SDK_DPA) += qbman_driver.o -+ -+# Bman -+obj-$(CONFIG_FSL_SDK_BMAN) += bman_high.o -+obj-$(CONFIG_FSL_BMAN_CONFIG) += bman_config.o bman_driver.o -+obj-$(CONFIG_FSL_BMAN_TEST) += bman_tester.o -+obj-$(CONFIG_FSL_BMAN_DEBUGFS) += bman_debugfs_interface.o -+bman_tester-y = bman_test.o -+bman_tester-$(CONFIG_FSL_BMAN_TEST_HIGH) += bman_test_high.o -+bman_tester-$(CONFIG_FSL_BMAN_TEST_THRESH) += bman_test_thresh.o -+bman_debugfs_interface-y = bman_debugfs.o -+ -+# Qman -+obj-$(CONFIG_FSL_SDK_QMAN) += qman_high.o qman_utility.o -+obj-$(CONFIG_FSL_QMAN_CONFIG) += qman_config.o qman_driver.o -+obj-$(CONFIG_FSL_QMAN_TEST) += qman_tester.o -+qman_tester-y = qman_test.o -+qman_tester-$(CONFIG_FSL_QMAN_TEST_STASH_POTATO) += qman_test_hotpotato.o -+qman_tester-$(CONFIG_FSL_QMAN_TEST_HIGH) += qman_test_high.o -+obj-$(CONFIG_FSL_QMAN_DEBUGFS) += qman_debugfs_interface.o -+qman_debugfs_interface-y = qman_debugfs.o -+ -+# USDPAA -+obj-$(CONFIG_FSL_USDPAA) += fsl_usdpaa.o fsl_usdpaa_irq.o ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_config.c -@@ -0,0 +1,720 @@ -+/* Copyright (c) 2009-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <asm/cacheflush.h> -+#include "bman_private.h" -+#include <linux/of_reserved_mem.h> -+ -+/* Last updated for v00.79 of the BG */ -+ -+struct bman; -+ -+/* Register offsets */ -+#define REG_POOL_SWDET(n) (0x0000 + ((n) * 0x04)) -+#define REG_POOL_HWDET(n) (0x0100 + ((n) * 0x04)) -+#define REG_POOL_SWDXT(n) (0x0200 + ((n) * 0x04)) -+#define REG_POOL_HWDXT(n) (0x0300 + ((n) * 0x04)) -+#define REG_POOL_CONTENT(n) (0x0600 + ((n) * 0x04)) -+#define REG_FBPR_FPC 0x0800 -+#define REG_STATE_IDLE 0x960 -+#define REG_STATE_STOP 0x964 -+#define REG_ECSR 0x0a00 -+#define REG_ECIR 0x0a04 -+#define REG_EADR 0x0a08 -+#define REG_EDATA(n) (0x0a10 + ((n) * 0x04)) -+#define REG_SBEC(n) (0x0a80 + ((n) * 0x04)) -+#define REG_IP_REV_1 0x0bf8 -+#define REG_IP_REV_2 0x0bfc -+#define REG_FBPR_BARE 0x0c00 -+#define REG_FBPR_BAR 0x0c04 -+#define REG_FBPR_AR 0x0c10 -+#define REG_SRCIDR 0x0d04 -+#define REG_LIODNR 0x0d08 -+#define REG_ERR_ISR 0x0e00 /* + "enum bm_isr_reg" */ -+ -+/* Used by all error interrupt registers except 'inhibit' */ -+#define BM_EIRQ_IVCI 0x00000010 /* Invalid Command Verb */ -+#define BM_EIRQ_FLWI 0x00000008 /* FBPR Low Watermark */ -+#define BM_EIRQ_MBEI 0x00000004 /* Multi-bit ECC Error */ -+#define BM_EIRQ_SBEI 0x00000002 /* Single-bit ECC Error */ -+#define BM_EIRQ_BSCN 0x00000001 /* pool State Change Notification */ -+ -+/* BMAN_ECIR valid error bit */ -+#define PORTAL_ECSR_ERR (BM_EIRQ_IVCI) -+ -+union bman_ecir { -+ u32 ecir_raw; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 __reserved1:4; -+ u32 portal_num:4; -+ u32 __reserved2:12; -+ u32 numb:4; -+ u32 __reserved3:2; -+ u32 pid:6; -+#else -+ u32 pid:6; -+ u32 __reserved3:2; -+ u32 numb:4; -+ u32 __reserved2:12; -+ u32 portal_num:4; -+ u32 __reserved1:4; -+#endif -+ } __packed info; -+}; -+ -+union bman_eadr { -+ u32 eadr_raw; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 __reserved1:5; -+ u32 memid:3; -+ u32 __reserved2:14; -+ u32 eadr:10; -+#else -+ u32 eadr:10; -+ u32 __reserved2:14; -+ u32 memid:3; -+ u32 __reserved1:5; -+#endif -+ } __packed info; -+}; -+ -+struct bman_hwerr_txt { -+ u32 mask; -+ const char *txt; -+}; -+ -+#define BMAN_HWE_TXT(a, b) { .mask = BM_EIRQ_##a, .txt = b } -+ -+static const struct bman_hwerr_txt bman_hwerr_txts[] = { -+ BMAN_HWE_TXT(IVCI, "Invalid Command Verb"), -+ BMAN_HWE_TXT(FLWI, "FBPR Low Watermark"), -+ BMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"), -+ BMAN_HWE_TXT(SBEI, "Single-bit ECC Error"), -+ BMAN_HWE_TXT(BSCN, "Pool State Change Notification"), -+}; -+#define BMAN_HWE_COUNT (sizeof(bman_hwerr_txts)/sizeof(struct bman_hwerr_txt)) -+ -+struct bman_error_info_mdata { -+ u16 addr_mask; -+ u16 bits; -+ const char *txt; -+}; -+ -+#define BMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c} -+static const struct bman_error_info_mdata error_mdata[] = { -+ BMAN_ERR_MDATA(0x03FF, 192, "Stockpile memory"), -+ BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 1"), -+ BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 2"), -+}; -+#define BMAN_ERR_MDATA_COUNT \ -+ (sizeof(error_mdata)/sizeof(struct bman_error_info_mdata)) -+ -+/* Add this in Kconfig */ -+#define BMAN_ERRS_TO_UNENABLE (BM_EIRQ_FLWI) -+ -+/** -+ * bm_err_isr_<reg>_<verb> - Manipulate global interrupt registers -+ * @v: for accessors that write values, this is the 32-bit value -+ * -+ * Manipulates BMAN_ERR_ISR, BMAN_ERR_IER, BMAN_ERR_ISDR, BMAN_ERR_IIR. All -+ * manipulations except bm_err_isr_[un]inhibit() use 32-bit masks composed of -+ * the BM_EIRQ_*** definitions. Note that "bm_err_isr_enable_write" means -+ * "write the enable register" rather than "enable the write register"! -+ */ -+#define bm_err_isr_status_read(bm) \ -+ __bm_err_isr_read(bm, bm_isr_status) -+#define bm_err_isr_status_clear(bm, m) \ -+ __bm_err_isr_write(bm, bm_isr_status, m) -+#define bm_err_isr_enable_read(bm) \ -+ __bm_err_isr_read(bm, bm_isr_enable) -+#define bm_err_isr_enable_write(bm, v) \ -+ __bm_err_isr_write(bm, bm_isr_enable, v) -+#define bm_err_isr_disable_read(bm) \ -+ __bm_err_isr_read(bm, bm_isr_disable) -+#define bm_err_isr_disable_write(bm, v) \ -+ __bm_err_isr_write(bm, bm_isr_disable, v) -+#define bm_err_isr_inhibit(bm) \ -+ __bm_err_isr_write(bm, bm_isr_inhibit, 1) -+#define bm_err_isr_uninhibit(bm) \ -+ __bm_err_isr_write(bm, bm_isr_inhibit, 0) -+ -+/* -+ * TODO: unimplemented registers -+ * -+ * BMAN_POOLk_SDCNT, BMAN_POOLk_HDCNT, BMAN_FULT, -+ * BMAN_VLDPL, BMAN_EECC, BMAN_SBET, BMAN_EINJ -+ */ -+ -+/* Encapsulate "struct bman *" as a cast of the register space address. */ -+ -+static struct bman *bm_create(void *regs) -+{ -+ return (struct bman *)regs; -+} -+ -+static inline u32 __bm_in(struct bman *bm, u32 offset) -+{ -+ return in_be32((void *)bm + offset); -+} -+static inline void __bm_out(struct bman *bm, u32 offset, u32 val) -+{ -+ out_be32((void *)bm + offset, val); -+} -+#define bm_in(reg) __bm_in(bm, REG_##reg) -+#define bm_out(reg, val) __bm_out(bm, REG_##reg, val) -+ -+static u32 __bm_err_isr_read(struct bman *bm, enum bm_isr_reg n) -+{ -+ return __bm_in(bm, REG_ERR_ISR + (n << 2)); -+} -+ -+static void __bm_err_isr_write(struct bman *bm, enum bm_isr_reg n, u32 val) -+{ -+ __bm_out(bm, REG_ERR_ISR + (n << 2), val); -+} -+ -+static void bm_get_version(struct bman *bm, u16 *id, u8 *major, u8 *minor) -+{ -+ u32 v = bm_in(IP_REV_1); -+ *id = (v >> 16); -+ *major = (v >> 8) & 0xff; -+ *minor = v & 0xff; -+} -+ -+static u32 __generate_thresh(u32 val, int roundup) -+{ -+ u32 e = 0; /* co-efficient, exponent */ -+ int oddbit = 0; -+ while (val > 0xff) { -+ oddbit = val & 1; -+ val >>= 1; -+ e++; -+ if (roundup && oddbit) -+ val++; -+ } -+ DPA_ASSERT(e < 0x10); -+ return val | (e << 8); -+} -+ -+static void bm_set_pool(struct bman *bm, u8 pool, u32 swdet, u32 swdxt, -+ u32 hwdet, u32 hwdxt) -+{ -+ DPA_ASSERT(pool < bman_pool_max); -+ bm_out(POOL_SWDET(pool), __generate_thresh(swdet, 0)); -+ bm_out(POOL_SWDXT(pool), __generate_thresh(swdxt, 1)); -+ bm_out(POOL_HWDET(pool), __generate_thresh(hwdet, 0)); -+ bm_out(POOL_HWDXT(pool), __generate_thresh(hwdxt, 1)); -+} -+ -+static void bm_set_memory(struct bman *bm, u64 ba, int prio, u32 size) -+{ -+ u32 exp = ilog2(size); -+ /* choke if size isn't within range */ -+ DPA_ASSERT((size >= 4096) && (size <= 1073741824) && -+ is_power_of_2(size)); -+ /* choke if '[e]ba' has lower-alignment than 'size' */ -+ DPA_ASSERT(!(ba & (size - 1))); -+ bm_out(FBPR_BARE, upper_32_bits(ba)); -+ bm_out(FBPR_BAR, lower_32_bits(ba)); -+ bm_out(FBPR_AR, (prio ? 0x40000000 : 0) | (exp - 1)); -+} -+ -+/*****************/ -+/* Config driver */ -+/*****************/ -+ -+/* TODO: Kconfig these? */ -+#define DEFAULT_FBPR_SZ (PAGE_SIZE << 12) -+ -+/* We support only one of these. */ -+static struct bman *bm; -+static struct device_node *bm_node; -+ -+/* And this state belongs to 'bm'. It is set during fsl_bman_init(), but used -+ * during bman_init_ccsr(). */ -+static dma_addr_t fbpr_a; -+static size_t fbpr_sz = DEFAULT_FBPR_SZ; -+ -+static int bman_fbpr(struct reserved_mem *rmem) -+{ -+ fbpr_a = rmem->base; -+ fbpr_sz = rmem->size; -+ -+ WARN_ON(!(fbpr_a && fbpr_sz)); -+ -+ return 0; -+} -+RESERVEDMEM_OF_DECLARE(bman_fbpr, "fsl,bman-fbpr", bman_fbpr); -+ -+static int __init fsl_bman_init(struct device_node *node) -+{ -+ struct resource res; -+ u32 __iomem *regs; -+ const char *s; -+ int ret, standby = 0; -+ u16 id; -+ u8 major, minor; -+ -+ ret = of_address_to_resource(node, 0, &res); -+ if (ret) { -+ pr_err("Can't get %s property 'reg'\n", -+ node->full_name); -+ return ret; -+ } -+ s = of_get_property(node, "fsl,hv-claimable", &ret); -+ if (s && !strcmp(s, "standby")) -+ standby = 1; -+ /* Global configuration */ -+ regs = ioremap(res.start, res.end - res.start + 1); -+ bm = bm_create(regs); -+ BUG_ON(!bm); -+ bm_node = node; -+ bm_get_version(bm, &id, &major, &minor); -+ pr_info("Bman ver:%04x,%02x,%02x\n", id, major, minor); -+ if ((major == 1) && (minor == 0)) { -+ bman_ip_rev = BMAN_REV10; -+ bman_pool_max = 64; -+ } else if ((major == 2) && (minor == 0)) { -+ bman_ip_rev = BMAN_REV20; -+ bman_pool_max = 8; -+ } else if ((major == 2) && (minor == 1)) { -+ bman_ip_rev = BMAN_REV21; -+ bman_pool_max = 64; -+ } else { -+ pr_warn("unknown Bman version, default to rev1.0\n"); -+ } -+ -+ if (standby) { -+ pr_info(" -> in standby mode\n"); -+ return 0; -+ } -+ return 0; -+} -+ -+int bman_have_ccsr(void) -+{ -+ return bm ? 1 : 0; -+} -+ -+int bm_pool_set(u32 bpid, const u32 *thresholds) -+{ -+ if (!bm) -+ return -ENODEV; -+ bm_set_pool(bm, bpid, thresholds[0], -+ thresholds[1], thresholds[2], -+ thresholds[3]); -+ return 0; -+} -+EXPORT_SYMBOL(bm_pool_set); -+ -+__init int bman_init_early(void) -+{ -+ struct device_node *dn; -+ int ret; -+ -+ for_each_compatible_node(dn, NULL, "fsl,bman") { -+ if (bm) -+ pr_err("%s: only one 'fsl,bman' allowed\n", -+ dn->full_name); -+ else { -+ if (!of_device_is_available(dn)) -+ continue; -+ -+ ret = fsl_bman_init(dn); -+ BUG_ON(ret); -+ } -+ } -+ return 0; -+} -+postcore_initcall_sync(bman_init_early); -+ -+ -+static void log_edata_bits(u32 bit_count) -+{ -+ u32 i, j, mask = 0xffffffff; -+ -+ pr_warn("Bman ErrInt, EDATA:\n"); -+ i = bit_count/32; -+ if (bit_count%32) { -+ i++; -+ mask = ~(mask << bit_count%32); -+ } -+ j = 16-i; -+ pr_warn(" 0x%08x\n", bm_in(EDATA(j)) & mask); -+ j++; -+ for (; j < 16; j++) -+ pr_warn(" 0x%08x\n", bm_in(EDATA(j))); -+} -+ -+static void log_additional_error_info(u32 isr_val, u32 ecsr_val) -+{ -+ union bman_ecir ecir_val; -+ union bman_eadr eadr_val; -+ -+ ecir_val.ecir_raw = bm_in(ECIR); -+ /* Is portal info valid */ -+ if (ecsr_val & PORTAL_ECSR_ERR) { -+ pr_warn("Bman ErrInt: SWP id %d, numb %d, pid %d\n", -+ ecir_val.info.portal_num, ecir_val.info.numb, -+ ecir_val.info.pid); -+ } -+ if (ecsr_val & (BM_EIRQ_SBEI|BM_EIRQ_MBEI)) { -+ eadr_val.eadr_raw = bm_in(EADR); -+ pr_warn("Bman ErrInt: EADR Memory: %s, 0x%x\n", -+ error_mdata[eadr_val.info.memid].txt, -+ error_mdata[eadr_val.info.memid].addr_mask -+ & eadr_val.info.eadr); -+ log_edata_bits(error_mdata[eadr_val.info.memid].bits); -+ } -+} -+ -+/* Bman interrupt handler */ -+static irqreturn_t bman_isr(int irq, void *ptr) -+{ -+ u32 isr_val, ier_val, ecsr_val, isr_mask, i; -+ -+ ier_val = bm_err_isr_enable_read(bm); -+ isr_val = bm_err_isr_status_read(bm); -+ ecsr_val = bm_in(ECSR); -+ isr_mask = isr_val & ier_val; -+ -+ if (!isr_mask) -+ return IRQ_NONE; -+ for (i = 0; i < BMAN_HWE_COUNT; i++) { -+ if (bman_hwerr_txts[i].mask & isr_mask) { -+ pr_warn("Bman ErrInt: %s\n", bman_hwerr_txts[i].txt); -+ if (bman_hwerr_txts[i].mask & ecsr_val) { -+ log_additional_error_info(isr_mask, ecsr_val); -+ /* Re-arm error capture registers */ -+ bm_out(ECSR, ecsr_val); -+ } -+ if (bman_hwerr_txts[i].mask & BMAN_ERRS_TO_UNENABLE) { -+ pr_devel("Bman un-enabling error 0x%x\n", -+ bman_hwerr_txts[i].mask); -+ ier_val &= ~bman_hwerr_txts[i].mask; -+ bm_err_isr_enable_write(bm, ier_val); -+ } -+ } -+ } -+ bm_err_isr_status_clear(bm, isr_val); -+ return IRQ_HANDLED; -+} -+ -+static int __bind_irq(void) -+{ -+ int ret, err_irq; -+ -+ err_irq = of_irq_to_resource(bm_node, 0, NULL); -+ if (err_irq == 0) { -+ pr_info("Can't get %s property '%s'\n", bm_node->full_name, -+ "interrupts"); -+ return -ENODEV; -+ } -+ ret = request_irq(err_irq, bman_isr, IRQF_SHARED, "bman-err", bm_node); -+ if (ret) { -+ pr_err("request_irq() failed %d for '%s'\n", ret, -+ bm_node->full_name); -+ return -ENODEV; -+ } -+ /* Disable Buffer Pool State Change */ -+ bm_err_isr_disable_write(bm, BM_EIRQ_BSCN); -+ /* Write-to-clear any stale bits, (eg. starvation being asserted prior -+ * to resource allocation during driver init). */ -+ bm_err_isr_status_clear(bm, 0xffffffff); -+ /* Enable Error Interrupts */ -+ bm_err_isr_enable_write(bm, 0xffffffff); -+ return 0; -+} -+ -+int bman_init_ccsr(struct device_node *node) -+{ -+ int ret; -+ if (!bman_have_ccsr()) -+ return 0; -+ if (node != bm_node) -+ return -EINVAL; -+ /* FBPR memory */ -+ bm_set_memory(bm, fbpr_a, 0, fbpr_sz); -+ pr_info("bman-fbpr addr %pad size 0x%zx\n", &fbpr_a, fbpr_sz); -+ -+ ret = __bind_irq(); -+ if (ret) -+ return ret; -+ return 0; -+} -+ -+u32 bm_pool_free_buffers(u32 bpid) -+{ -+ return bm_in(POOL_CONTENT(bpid)); -+} -+ -+#ifdef CONFIG_SYSFS -+ -+#define DRV_NAME "fsl-bman" -+#define SBEC_MAX_ID 1 -+#define SBEC_MIN_ID 0 -+ -+static ssize_t show_fbpr_fpc(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(FBPR_FPC)); -+}; -+ -+static ssize_t show_pool_count(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ u32 data; -+ int i; -+ -+ if (!sscanf(dev_attr->attr.name, "%d", &i) || (i >= bman_pool_max)) -+ return -EINVAL; -+ data = bm_in(POOL_CONTENT(i)); -+ return snprintf(buf, PAGE_SIZE, "%d\n", data); -+}; -+ -+static ssize_t show_err_isr(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ return snprintf(buf, PAGE_SIZE, "0x%08x\n", bm_in(ERR_ISR)); -+}; -+ -+static ssize_t show_sbec(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ int i; -+ -+ if (!sscanf(dev_attr->attr.name, "sbec_%d", &i)) -+ return -EINVAL; -+ if (i < SBEC_MIN_ID || i > SBEC_MAX_ID) -+ return -EINVAL; -+ return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(SBEC(i))); -+}; -+ -+static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL); -+static DEVICE_ATTR(fbpr_fpc, S_IRUSR, show_fbpr_fpc, NULL); -+ -+/* Didn't use DEVICE_ATTR as 64 of this would be required. -+ * Initialize them when needed. */ -+static char *name_attrs_pool_count; /* "xx" + null-terminator */ -+static struct device_attribute *dev_attr_buffer_pool_count; -+ -+static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL); -+ -+static struct attribute *bman_dev_attributes[] = { -+ &dev_attr_fbpr_fpc.attr, -+ &dev_attr_err_isr.attr, -+ NULL -+}; -+ -+static struct attribute *bman_dev_ecr_attributes[] = { -+ &dev_attr_sbec_0.attr, -+ &dev_attr_sbec_1.attr, -+ NULL -+}; -+ -+static struct attribute **bman_dev_pool_count_attributes; -+ -+ -+/* root level */ -+static const struct attribute_group bman_dev_attr_grp = { -+ .name = NULL, -+ .attrs = bman_dev_attributes -+}; -+static const struct attribute_group bman_dev_ecr_grp = { -+ .name = "error_capture", -+ .attrs = bman_dev_ecr_attributes -+}; -+static struct attribute_group bman_dev_pool_countent_grp = { -+ .name = "pool_count", -+}; -+ -+static int of_fsl_bman_remove(struct platform_device *ofdev) -+{ -+ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp); -+ return 0; -+}; -+ -+static int of_fsl_bman_probe(struct platform_device *ofdev) -+{ -+ int ret, i; -+ -+ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_attr_grp); -+ if (ret) -+ goto done; -+ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_ecr_grp); -+ if (ret) -+ goto del_group_0; -+ -+ name_attrs_pool_count = kmalloc(sizeof(char) * bman_pool_max * 3, -+ GFP_KERNEL); -+ if (!name_attrs_pool_count) { -+ pr_err("Can't alloc name_attrs_pool_count\n"); -+ goto del_group_1; -+ } -+ -+ dev_attr_buffer_pool_count = kmalloc(sizeof(struct device_attribute) * -+ bman_pool_max, GFP_KERNEL); -+ if (!dev_attr_buffer_pool_count) { -+ pr_err("Can't alloc dev_attr-buffer_pool_count\n"); -+ goto del_group_2; -+ } -+ -+ bman_dev_pool_count_attributes = kmalloc(sizeof(struct attribute *) * -+ (bman_pool_max + 1), GFP_KERNEL); -+ if (!bman_dev_pool_count_attributes) { -+ pr_err("can't alloc bman_dev_pool_count_attributes\n"); -+ goto del_group_3; -+ } -+ -+ for (i = 0; i < bman_pool_max; i++) { -+ ret = scnprintf((name_attrs_pool_count + i * 3), 3, "%d", i); -+ if (!ret) -+ goto del_group_4; -+ dev_attr_buffer_pool_count[i].attr.name = -+ (name_attrs_pool_count + i * 3); -+ dev_attr_buffer_pool_count[i].attr.mode = S_IRUSR; -+ dev_attr_buffer_pool_count[i].show = show_pool_count; -+ bman_dev_pool_count_attributes[i] = -+ &dev_attr_buffer_pool_count[i].attr; -+ sysfs_attr_init(bman_dev_pool_count_attributes[i]); -+ } -+ bman_dev_pool_count_attributes[bman_pool_max] = NULL; -+ -+ bman_dev_pool_countent_grp.attrs = bman_dev_pool_count_attributes; -+ -+ ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_pool_countent_grp); -+ if (ret) -+ goto del_group_4; -+ -+ goto done; -+ -+del_group_4: -+ kfree(bman_dev_pool_count_attributes); -+del_group_3: -+ kfree(dev_attr_buffer_pool_count); -+del_group_2: -+ kfree(name_attrs_pool_count); -+del_group_1: -+ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_ecr_grp); -+del_group_0: -+ sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp); -+done: -+ if (ret) -+ dev_err(&ofdev->dev, -+ "Cannot create dev attributes ret=%d\n", ret); -+ return ret; -+}; -+ -+static struct of_device_id of_fsl_bman_ids[] = { -+ { -+ .compatible = "fsl,bman", -+ }, -+ {} -+}; -+MODULE_DEVICE_TABLE(of, of_fsl_bman_ids); -+ -+#ifdef CONFIG_SUSPEND -+static u32 saved_isdr; -+ -+static int bman_pm_suspend_noirq(struct device *dev) -+{ -+ uint32_t idle_state; -+ -+ suspend_unused_bportal(); -+ /* save isdr, disable all, clear isr */ -+ saved_isdr = bm_err_isr_disable_read(bm); -+ bm_err_isr_disable_write(bm, 0xffffffff); -+ bm_err_isr_status_clear(bm, 0xffffffff); -+ -+ if (bman_ip_rev < BMAN_REV21) { -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Bman version doesn't have STATE_IDLE\n"); -+#endif -+ return 0; -+ } -+ idle_state = bm_in(STATE_IDLE); -+ if (!(idle_state & 0x1)) { -+ pr_err("Bman not idle 0x%x aborting\n", idle_state); -+ bm_err_isr_disable_write(bm, saved_isdr); -+ resume_unused_bportal(); -+ return -EBUSY; -+ } -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Bman suspend code, IDLE_STAT = 0x%x\n", idle_state); -+#endif -+ return 0; -+} -+ -+static int bman_pm_resume_noirq(struct device *dev) -+{ -+ /* restore isdr */ -+ bm_err_isr_disable_write(bm, saved_isdr); -+ resume_unused_bportal(); -+ return 0; -+} -+#else -+#define bman_pm_suspend_noirq NULL -+#define bman_pm_resume_noirq NULL -+#endif -+ -+static const struct dev_pm_ops bman_pm_ops = { -+ .suspend_noirq = bman_pm_suspend_noirq, -+ .resume_noirq = bman_pm_resume_noirq, -+}; -+ -+static struct platform_driver of_fsl_bman_driver = { -+ .driver = { -+ .owner = THIS_MODULE, -+ .name = DRV_NAME, -+ .of_match_table = of_fsl_bman_ids, -+ .pm = &bman_pm_ops, -+ }, -+ .probe = of_fsl_bman_probe, -+ .remove = of_fsl_bman_remove, -+}; -+ -+static int bman_ctrl_init(void) -+{ -+ return platform_driver_register(&of_fsl_bman_driver); -+} -+ -+static void bman_ctrl_exit(void) -+{ -+ platform_driver_unregister(&of_fsl_bman_driver); -+} -+ -+module_init(bman_ctrl_init); -+module_exit(bman_ctrl_exit); -+ -+#endif /* CONFIG_SYSFS */ ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_debugfs.c -@@ -0,0 +1,119 @@ -+/* Copyright 2010-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+#include <linux/module.h> -+#include <linux/fsl_bman.h> -+#include <linux/debugfs.h> -+#include <linux/seq_file.h> -+#include <linux/uaccess.h> -+ -+static struct dentry *dfs_root; /* debugfs root directory */ -+ -+/******************************************************************************* -+ * Query Buffer Pool State -+ ******************************************************************************/ -+static int query_bp_state_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct bm_pool_state state; -+ int i, j; -+ u32 mask; -+ -+ memset(&state, 0, sizeof(struct bm_pool_state)); -+ ret = bman_query_pools(&state); -+ if (ret) { -+ seq_printf(file, "Error %d\n", ret); -+ return 0; -+ } -+ seq_puts(file, "bp_id free_buffers_avail bp_depleted\n"); -+ for (i = 0; i < 2; i++) { -+ mask = 0x80000000; -+ for (j = 0; j < 32; j++) { -+ seq_printf(file, -+ " %-2u %-3s %-3s\n", -+ (i*32)+j, -+ (state.as.state.__state[i] & mask) ? "no" : "yes", -+ (state.ds.state.__state[i] & mask) ? "yes" : "no"); -+ mask >>= 1; -+ } -+ } -+ return 0; -+} -+ -+static int query_bp_state_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, query_bp_state_show, NULL); -+} -+ -+static const struct file_operations query_bp_state_fops = { -+ .owner = THIS_MODULE, -+ .open = query_bp_state_open, -+ .read = seq_read, -+ .release = single_release, -+}; -+ -+static int __init bman_debugfs_module_init(void) -+{ -+ int ret = 0; -+ struct dentry *d; -+ -+ dfs_root = debugfs_create_dir("bman", NULL); -+ -+ if (dfs_root == NULL) { -+ ret = -ENOMEM; -+ pr_err("Cannot create bman debugfs dir\n"); -+ goto _return; -+ } -+ d = debugfs_create_file("query_bp_state", -+ S_IRUGO, -+ dfs_root, -+ NULL, -+ &query_bp_state_fops); -+ if (d == NULL) { -+ ret = -ENOMEM; -+ pr_err("Cannot create query_bp_state\n"); -+ goto _return; -+ } -+ return 0; -+ -+_return: -+ debugfs_remove_recursive(dfs_root); -+ return ret; -+} -+ -+static void __exit bman_debugfs_module_exit(void) -+{ -+ debugfs_remove_recursive(dfs_root); -+} -+ -+ -+module_init(bman_debugfs_module_init); -+module_exit(bman_debugfs_module_exit); -+MODULE_LICENSE("Dual BSD/GPL"); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_driver.c -@@ -0,0 +1,559 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+#include "bman_low.h" -+#ifdef CONFIG_HOTPLUG_CPU -+#include <linux/cpu.h> -+#endif -+/* -+ * Global variables of the max portal/pool number this bman version supported -+ */ -+u16 bman_ip_rev; -+EXPORT_SYMBOL(bman_ip_rev); -+u16 bman_pool_max; -+EXPORT_SYMBOL(bman_pool_max); -+static u16 bman_portal_max; -+ -+/* After initialising cpus that own shared portal configs, we cache the -+ * resulting portals (ie. not just the configs) in this array. Then we -+ * initialise slave cpus that don't have their own portals, redirecting them to -+ * portals from this cache in a round-robin assignment. */ -+static struct bman_portal *shared_portals[NR_CPUS]; -+static int num_shared_portals; -+static int shared_portals_idx; -+static LIST_HEAD(unused_pcfgs); -+static DEFINE_SPINLOCK(unused_pcfgs_lock); -+static void *affine_bportals[NR_CPUS]; -+ -+static int __init fsl_bpool_init(struct device_node *node) -+{ -+ int ret; -+ u32 *thresh, *bpid = (u32 *)of_get_property(node, "fsl,bpid", &ret); -+ if (!bpid || (ret != 4)) { -+ pr_err("Can't get %s property 'fsl,bpid'\n", node->full_name); -+ return -ENODEV; -+ } -+ thresh = (u32 *)of_get_property(node, "fsl,bpool-thresholds", &ret); -+ if (thresh) { -+ if (ret != 16) { -+ pr_err("Invalid %s property '%s'\n", -+ node->full_name, "fsl,bpool-thresholds"); -+ return -ENODEV; -+ } -+ } -+ if (thresh) { -+#ifdef CONFIG_FSL_BMAN_CONFIG -+ ret = bm_pool_set(be32_to_cpu(*bpid), thresh); -+ if (ret) -+ pr_err("No CCSR node for %s property '%s'\n", -+ node->full_name, "fsl,bpool-thresholds"); -+ return ret; -+#else -+ pr_err("Ignoring %s property '%s', no CCSR support\n", -+ node->full_name, "fsl,bpool-thresholds"); -+#endif -+ } -+ return 0; -+} -+ -+static int __init fsl_bpid_range_init(struct device_node *node) -+{ -+ int ret; -+ u32 *range = (u32 *)of_get_property(node, "fsl,bpid-range", &ret); -+ if (!range) { -+ pr_err("No 'fsl,bpid-range' property in node %s\n", -+ node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err("'fsl,bpid-range' is not a 2-cell range in node %s\n", -+ node->full_name); -+ return -EINVAL; -+ } -+ bman_seed_bpid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ pr_info("Bman: BPID allocator includes range %d:%d\n", -+ be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ return 0; -+} -+ -+static struct bm_portal_config * __init parse_pcfg(struct device_node *node) -+{ -+ struct bm_portal_config *pcfg; -+ const u32 *index; -+ int irq, ret; -+ resource_size_t len; -+ -+ pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL); -+ if (!pcfg) { -+ pr_err("can't allocate portal config"); -+ return NULL; -+ } -+ -+ if (of_device_is_compatible(node, "fsl,bman-portal-1.0") || -+ of_device_is_compatible(node, "fsl,bman-portal-1.0.0")) { -+ bman_ip_rev = BMAN_REV10; -+ bman_pool_max = 64; -+ bman_portal_max = 10; -+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.0") || -+ of_device_is_compatible(node, "fsl,bman-portal-2.0.8")) { -+ bman_ip_rev = BMAN_REV20; -+ bman_pool_max = 8; -+ bman_portal_max = 3; -+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.0")) { -+ bman_ip_rev = BMAN_REV21; -+ bman_pool_max = 64; -+ bman_portal_max = 50; -+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.1")) { -+ bman_ip_rev = BMAN_REV21; -+ bman_pool_max = 64; -+ bman_portal_max = 25; -+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.2")) { -+ bman_ip_rev = BMAN_REV21; -+ bman_pool_max = 64; -+ bman_portal_max = 18; -+ } else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.3")) { -+ bman_ip_rev = BMAN_REV21; -+ bman_pool_max = 64; -+ bman_portal_max = 10; -+ } else { -+ pr_warn("unknown BMan version in portal node," -+ "default to rev1.0\n"); -+ bman_ip_rev = BMAN_REV10; -+ bman_pool_max = 64; -+ bman_portal_max = 10; -+ } -+ -+ ret = of_address_to_resource(node, DPA_PORTAL_CE, -+ &pcfg->addr_phys[DPA_PORTAL_CE]); -+ if (ret) { -+ pr_err("Can't get %s property 'reg::CE'\n", node->full_name); -+ goto err; -+ } -+ ret = of_address_to_resource(node, DPA_PORTAL_CI, -+ &pcfg->addr_phys[DPA_PORTAL_CI]); -+ if (ret) { -+ pr_err("Can't get %s property 'reg::CI'\n", node->full_name); -+ goto err; -+ } -+ -+ index = of_get_property(node, "cell-index", &ret); -+ if (!index || (ret != 4)) { -+ pr_err("Can't get %s property '%s'\n", node->full_name, -+ "cell-index"); -+ goto err; -+ } -+ if (be32_to_cpu(*index) >= bman_portal_max) { -+ pr_err("BMan portal cell index %d out of range, max %d\n", -+ be32_to_cpu(*index), bman_portal_max); -+ goto err; -+ } -+ -+ pcfg->public_cfg.cpu = -1; -+ -+ irq = irq_of_parse_and_map(node, 0); -+ if (irq == 0) { -+ pr_err("Can't get %s property 'interrupts'\n", node->full_name); -+ goto err; -+ } -+ pcfg->public_cfg.irq = irq; -+ pcfg->public_cfg.index = be32_to_cpu(*index); -+ bman_depletion_fill(&pcfg->public_cfg.mask); -+ -+ len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]); -+ if (len != (unsigned long)len) -+ goto err; -+ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns( -+ pcfg->addr_phys[DPA_PORTAL_CE].start, -+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CE])); -+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap( -+ pcfg->addr_phys[DPA_PORTAL_CI].start, -+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI])); -+ -+#else -+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot( -+ pcfg->addr_phys[DPA_PORTAL_CE].start, -+ (unsigned long)len, -+ 0); -+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot( -+ pcfg->addr_phys[DPA_PORTAL_CI].start, -+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]), -+ _PAGE_GUARDED | _PAGE_NO_CACHE); -+#endif -+ /* disable bp depletion */ -+ __raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(0)); -+ __raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(1)); -+ return pcfg; -+err: -+ kfree(pcfg); -+ return NULL; -+} -+ -+static struct bm_portal_config *get_pcfg(struct list_head *list) -+{ -+ struct bm_portal_config *pcfg; -+ if (list_empty(list)) -+ return NULL; -+ pcfg = list_entry(list->prev, struct bm_portal_config, list); -+ list_del(&pcfg->list); -+ return pcfg; -+} -+ -+static struct bm_portal_config *get_pcfg_idx(struct list_head *list, -+ uint32_t idx) -+{ -+ struct bm_portal_config *pcfg; -+ if (list_empty(list)) -+ return NULL; -+ list_for_each_entry(pcfg, list, list) { -+ if (pcfg->public_cfg.index == idx) { -+ list_del(&pcfg->list); -+ return pcfg; -+ } -+ } -+ return NULL; -+} -+ -+struct bm_portal_config *bm_get_unused_portal(void) -+{ -+ return bm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX); -+} -+ -+struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx) -+{ -+ struct bm_portal_config *ret; -+ spin_lock(&unused_pcfgs_lock); -+ if (idx == QBMAN_ANY_PORTAL_IDX) -+ ret = get_pcfg(&unused_pcfgs); -+ else -+ ret = get_pcfg_idx(&unused_pcfgs, idx); -+ spin_unlock(&unused_pcfgs_lock); -+ return ret; -+} -+ -+void bm_put_unused_portal(struct bm_portal_config *pcfg) -+{ -+ spin_lock(&unused_pcfgs_lock); -+ list_add(&pcfg->list, &unused_pcfgs); -+ spin_unlock(&unused_pcfgs_lock); -+} -+ -+static struct bman_portal *init_pcfg(struct bm_portal_config *pcfg) -+{ -+ struct bman_portal *p; -+ p = bman_create_affine_portal(pcfg); -+ if (p) { -+#ifdef CONFIG_FSL_DPA_PIRQ_SLOW -+ bman_p_irqsource_add(p, BM_PIRQ_RCRI | BM_PIRQ_BSCN); -+#endif -+ pr_info("Bman portal %sinitialised, cpu %d\n", -+ pcfg->public_cfg.is_shared ? "(shared) " : "", -+ pcfg->public_cfg.cpu); -+ affine_bportals[pcfg->public_cfg.cpu] = p; -+ } else -+ pr_crit("Bman portal failure on cpu %d\n", -+ pcfg->public_cfg.cpu); -+ return p; -+} -+ -+static void init_slave(int cpu) -+{ -+ struct bman_portal *p; -+ p = bman_create_affine_slave(shared_portals[shared_portals_idx++], cpu); -+ if (!p) -+ pr_err("Bman slave portal failure on cpu %d\n", cpu); -+ else -+ pr_info("Bman portal %sinitialised, cpu %d\n", "(slave) ", cpu); -+ if (shared_portals_idx >= num_shared_portals) -+ shared_portals_idx = 0; -+ affine_bportals[cpu] = p; -+} -+ -+/* Bootarg "bportals=[...]" has the same syntax as "qportals=", and so the -+ * parsing is in dpa_sys.h. The syntax is a comma-separated list of indexes -+ * and/or ranges of indexes, with each being optionally prefixed by "s" to -+ * explicitly mark it or them for sharing. -+ * Eg; -+ * bportals=s0,1-3,s4 -+ * means that cpus 1,2,3 get "unshared" portals, cpus 0 and 4 get "shared" -+ * portals, and any remaining cpus share the portals that are assigned to cpus 0 -+ * or 4, selected in a round-robin fashion. (In this example, cpu 5 would share -+ * cpu 0's portal, cpu 6 would share cpu4's portal, and cpu 7 would share cpu -+ * 0's portal.) */ -+static struct cpumask want_unshared __initdata; /* cpus requested without "s" */ -+static struct cpumask want_shared __initdata; /* cpus requested with "s" */ -+ -+static int __init parse_bportals(char *str) -+{ -+ return parse_portals_bootarg(str, &want_shared, &want_unshared, -+ "bportals"); -+} -+__setup("bportals=", parse_bportals); -+ -+static int bman_offline_cpu(unsigned int cpu) -+{ -+ struct bman_portal *p; -+ const struct bm_portal_config *pcfg; -+ p = (struct bman_portal *)affine_bportals[cpu]; -+ if (p) { -+ pcfg = bman_get_bm_portal_config(p); -+ if (pcfg) -+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0)); -+ } -+ return 0; -+} -+ -+#ifdef CONFIG_HOTPLUG_CPU -+static int bman_online_cpu(unsigned int cpu) -+{ -+ struct bman_portal *p; -+ const struct bm_portal_config *pcfg; -+ p = (struct bman_portal *)affine_bportals[cpu]; -+ if (p) { -+ pcfg = bman_get_bm_portal_config(p); -+ if (pcfg) -+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu)); -+ } -+ return 0; -+} -+#endif /* CONFIG_HOTPLUG_CPU */ -+ -+/* Initialise the Bman driver. The meat of this function deals with portals. The -+ * following describes the flow of portal-handling, the code "steps" refer to -+ * this description; -+ * 1. Portal configs are parsed from the device-tree into 'unused_pcfgs', with -+ * ::cpu==-1. Regions and interrupts are mapped (but interrupts are not -+ * bound). -+ * 2. The "want_shared" and "want_unshared" lists (as filled by the -+ * "bportals=[...]" bootarg) are processed, allocating portals and assigning -+ * them to cpus, placing them in the relevant list and setting ::cpu as -+ * appropriate. If no "bportals" bootarg was present, the defaut is to try to -+ * assign portals to all online cpus at the time of driver initialisation. -+ * Any failure to allocate portals (when parsing the "want" lists or when -+ * using default behaviour) will be silently tolerated (the "fixup" logic in -+ * step 3 will determine what happens in this case). -+ * 3. Do fixups relative to cpu_online_mask(). If no portals are marked for -+ * sharing and sharing is required (because not all cpus have been assigned -+ * portals), then one portal will marked for sharing. Conversely if no -+ * sharing is required, any portals marked for sharing will not be shared. It -+ * may be that sharing occurs when it wasn't expected, if portal allocation -+ * failed to honour all the requested assignments (including the default -+ * assignments if no bootarg is present). -+ * 4. Unshared portals are initialised on their respective cpus. -+ * 5. Shared portals are initialised on their respective cpus. -+ * 6. Each remaining cpu is initialised to slave to one of the shared portals, -+ * which are selected in a round-robin fashion. -+ * Any portal configs left unused are available for USDPAA allocation. -+ */ -+__init int bman_init(void) -+{ -+ struct cpumask slave_cpus; -+ struct cpumask unshared_cpus = *cpu_none_mask; -+ struct cpumask shared_cpus = *cpu_none_mask; -+ LIST_HEAD(unshared_pcfgs); -+ LIST_HEAD(shared_pcfgs); -+ struct device_node *dn; -+ struct bm_portal_config *pcfg; -+ struct bman_portal *p; -+ int cpu, ret; -+ struct cpumask offline_cpus; -+ -+ /* Initialise the Bman (CCSR) device */ -+ for_each_compatible_node(dn, NULL, "fsl,bman") { -+ if (!bman_init_ccsr(dn)) -+ pr_info("Bman err interrupt handler present\n"); -+ else -+ pr_err("Bman CCSR setup failed\n"); -+ } -+ /* Initialise any declared buffer pools */ -+ for_each_compatible_node(dn, NULL, "fsl,bpool") { -+ ret = fsl_bpool_init(dn); -+ if (ret) -+ return ret; -+ } -+ /* Step 1. See comments at the beginning of the file. */ -+ for_each_compatible_node(dn, NULL, "fsl,bman-portal") { -+ if (!of_device_is_available(dn)) -+ continue; -+ pcfg = parse_pcfg(dn); -+ if (pcfg) -+ list_add_tail(&pcfg->list, &unused_pcfgs); -+ } -+ /* Step 2. */ -+ for_each_possible_cpu(cpu) { -+ if (cpumask_test_cpu(cpu, &want_shared)) { -+ pcfg = get_pcfg(&unused_pcfgs); -+ if (!pcfg) -+ break; -+ pcfg->public_cfg.cpu = cpu; -+ list_add_tail(&pcfg->list, &shared_pcfgs); -+ cpumask_set_cpu(cpu, &shared_cpus); -+ } -+ if (cpumask_test_cpu(cpu, &want_unshared)) { -+ if (cpumask_test_cpu(cpu, &shared_cpus)) -+ continue; -+ pcfg = get_pcfg(&unused_pcfgs); -+ if (!pcfg) -+ break; -+ pcfg->public_cfg.cpu = cpu; -+ list_add_tail(&pcfg->list, &unshared_pcfgs); -+ cpumask_set_cpu(cpu, &unshared_cpus); -+ } -+ } -+ if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) { -+ /* Default, give an unshared portal to each online cpu */ -+ for_each_online_cpu(cpu) { -+ pcfg = get_pcfg(&unused_pcfgs); -+ if (!pcfg) -+ break; -+ pcfg->public_cfg.cpu = cpu; -+ list_add_tail(&pcfg->list, &unshared_pcfgs); -+ cpumask_set_cpu(cpu, &unshared_cpus); -+ } -+ } -+ /* Step 3. */ -+ cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus); -+ cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus); -+ if (cpumask_empty(&slave_cpus)) { -+ /* No sharing required */ -+ if (!list_empty(&shared_pcfgs)) { -+ /* Migrate "shared" to "unshared" */ -+ cpumask_or(&unshared_cpus, &unshared_cpus, -+ &shared_cpus); -+ cpumask_clear(&shared_cpus); -+ list_splice_tail(&shared_pcfgs, &unshared_pcfgs); -+ INIT_LIST_HEAD(&shared_pcfgs); -+ } -+ } else { -+ /* Sharing required */ -+ if (list_empty(&shared_pcfgs)) { -+ /* Migrate one "unshared" to "shared" */ -+ pcfg = get_pcfg(&unshared_pcfgs); -+ if (!pcfg) { -+ pr_crit("No BMan portals available!\n"); -+ return 0; -+ } -+ cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus); -+ cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus); -+ list_add_tail(&pcfg->list, &shared_pcfgs); -+ } -+ } -+ /* Step 4. */ -+ list_for_each_entry(pcfg, &unshared_pcfgs, list) { -+ pcfg->public_cfg.is_shared = 0; -+ p = init_pcfg(pcfg); -+ if (!p) { -+ pr_crit("Unable to initialize bman portal\n"); -+ return 0; -+ } -+ } -+ /* Step 5. */ -+ list_for_each_entry(pcfg, &shared_pcfgs, list) { -+ pcfg->public_cfg.is_shared = 1; -+ p = init_pcfg(pcfg); -+ if (p) -+ shared_portals[num_shared_portals++] = p; -+ } -+ /* Step 6. */ -+ if (!cpumask_empty(&slave_cpus)) -+ for_each_cpu(cpu, &slave_cpus) -+ init_slave(cpu); -+ pr_info("Bman portals initialised\n"); -+ cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask); -+ for_each_cpu(cpu, &offline_cpus) -+ bman_offline_cpu(cpu); -+#ifdef CONFIG_HOTPLUG_CPU -+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, -+ "soc/qbman_portal:online", -+ bman_online_cpu, bman_offline_cpu); -+ if (ret < 0) { -+ pr_err("bman: failed to register hotplug callbacks.\n"); -+ return 0; -+ } -+#endif -+ return 0; -+} -+ -+__init int bman_resource_init(void) -+{ -+ struct device_node *dn; -+ int ret; -+ -+ /* Initialise BPID allocation ranges */ -+ for_each_compatible_node(dn, NULL, "fsl,bpid-range") { -+ ret = fsl_bpid_range_init(dn); -+ if (ret) -+ return ret; -+ } -+ return 0; -+} -+ -+#ifdef CONFIG_SUSPEND -+void suspend_unused_bportal(void) -+{ -+ struct bm_portal_config *pcfg; -+ -+ if (list_empty(&unused_pcfgs)) -+ return; -+ -+ list_for_each_entry(pcfg, &unused_pcfgs, list) { -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Need to save bportal %d\n", pcfg->public_cfg.index); -+#endif -+ /* save isdr, disable all via isdr, clear isr */ -+ pcfg->saved_isdr = -+ __raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); -+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + -+ 0xe08); -+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + -+ 0xe00); -+ } -+ return; -+} -+ -+void resume_unused_bportal(void) -+{ -+ struct bm_portal_config *pcfg; -+ -+ if (list_empty(&unused_pcfgs)) -+ return; -+ -+ list_for_each_entry(pcfg, &unused_pcfgs, list) { -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Need to resume bportal %d\n", pcfg->public_cfg.index); -+#endif -+ /* restore isdr */ -+ __raw_writel(pcfg->saved_isdr, -+ pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); -+ } -+ return; -+} -+#endif ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_high.c -@@ -0,0 +1,1145 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "bman_low.h" -+ -+/* Compilation constants */ -+#define RCR_THRESH 2 /* reread h/w CI when running out of space */ -+#define IRQNAME "BMan portal %d" -+#define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */ -+ -+struct bman_portal { -+ struct bm_portal p; -+ /* 2-element array. pools[0] is mask, pools[1] is snapshot. */ -+ struct bman_depletion *pools; -+ int thresh_set; -+ unsigned long irq_sources; -+ u32 slowpoll; /* only used when interrupts are off */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */ -+#endif -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ raw_spinlock_t sharing_lock; /* only used if is_shared */ -+ int is_shared; -+ struct bman_portal *sharing_redirect; -+#endif -+ /* When the cpu-affine portal is activated, this is non-NULL */ -+ const struct bm_portal_config *config; -+ /* This is needed for power management */ -+ struct platform_device *pdev; -+ /* 64-entry hash-table of pool objects that are tracking depletion -+ * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so -+ * we're not fussy about cache-misses and so forth - whereas the above -+ * members should all fit in one cacheline. -+ * BTW, with 64 entries in the hash table and 64 buffer pools to track, -+ * you'll never guess the hash-function ... */ -+ struct bman_pool *cb[64]; -+ char irqname[MAX_IRQNAME]; -+ /* Track if the portal was alloced by the driver */ -+ u8 alloced; -+ /* power management data */ -+ u32 save_isdr; -+}; -+ -+/* For an explanation of the locking, redirection, or affine-portal logic, -+ * please consult the Qman driver for details. This is the same, only simpler -+ * (no fiddly Qman-specific bits.) */ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+#define PORTAL_IRQ_LOCK(p, irqflags) \ -+ do { \ -+ if ((p)->is_shared) \ -+ raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \ -+ else \ -+ local_irq_save(irqflags); \ -+ } while (0) -+#define PORTAL_IRQ_UNLOCK(p, irqflags) \ -+ do { \ -+ if ((p)->is_shared) \ -+ raw_spin_unlock_irqrestore(&(p)->sharing_lock, \ -+ irqflags); \ -+ else \ -+ local_irq_restore(irqflags); \ -+ } while (0) -+#else -+#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags) -+#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags) -+#endif -+ -+static cpumask_t affine_mask; -+static DEFINE_SPINLOCK(affine_mask_lock); -+static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal); -+static inline struct bman_portal *get_raw_affine_portal(void) -+{ -+ return &get_cpu_var(bman_affine_portal); -+} -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+static inline struct bman_portal *get_affine_portal(void) -+{ -+ struct bman_portal *p = get_raw_affine_portal(); -+ if (p->sharing_redirect) -+ return p->sharing_redirect; -+ return p; -+} -+#else -+#define get_affine_portal() get_raw_affine_portal() -+#endif -+static inline void put_affine_portal(void) -+{ -+ put_cpu_var(bman_affine_portal); -+} -+static inline struct bman_portal *get_poll_portal(void) -+{ -+ return &get_cpu_var(bman_affine_portal); -+} -+#define put_poll_portal() -+ -+/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be -+ * more than one such object per Bman buffer pool, eg. if different users of the -+ * pool are operating via different portals. */ -+struct bman_pool { -+ struct bman_pool_params params; -+ /* Used for hash-table admin when using depletion notifications. */ -+ struct bman_portal *portal; -+ struct bman_pool *next; -+ /* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */ -+ struct bm_buffer *sp; -+ unsigned int sp_fill; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ atomic_t in_use; -+#endif -+}; -+ -+/* (De)Registration of depletion notification callbacks */ -+static void depletion_link(struct bman_portal *portal, struct bman_pool *pool) -+{ -+ __maybe_unused unsigned long irqflags; -+ pool->portal = portal; -+ PORTAL_IRQ_LOCK(portal, irqflags); -+ pool->next = portal->cb[pool->params.bpid]; -+ portal->cb[pool->params.bpid] = pool; -+ if (!pool->next) -+ /* First object for that bpid on this portal, enable the BSCN -+ * mask bit. */ -+ bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1); -+ PORTAL_IRQ_UNLOCK(portal, irqflags); -+} -+static void depletion_unlink(struct bman_pool *pool) -+{ -+ struct bman_pool *it, *last = NULL; -+ struct bman_pool **base = &pool->portal->cb[pool->params.bpid]; -+ __maybe_unused unsigned long irqflags; -+ PORTAL_IRQ_LOCK(pool->portal, irqflags); -+ it = *base; /* <-- gotcha, don't do this prior to the irq_save */ -+ while (it != pool) { -+ last = it; -+ it = it->next; -+ } -+ if (!last) -+ *base = pool->next; -+ else -+ last->next = pool->next; -+ if (!last && !pool->next) { -+ /* Last object for that bpid on this portal, disable the BSCN -+ * mask bit. */ -+ bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0); -+ /* And "forget" that we last saw this pool as depleted */ -+ bman_depletion_unset(&pool->portal->pools[1], -+ pool->params.bpid); -+ } -+ PORTAL_IRQ_UNLOCK(pool->portal, irqflags); -+} -+ -+/* In the case that the application's core loop calls qman_poll() and -+ * bman_poll(), we ought to balance how often we incur the overheads of the -+ * slow-path poll. We'll use two decrementer sources. The idle decrementer -+ * constant is used when the last slow-poll detected no work to do, and the busy -+ * decrementer constant when the last slow-poll had work to do. */ -+#define SLOW_POLL_IDLE 1000 -+#define SLOW_POLL_BUSY 10 -+static u32 __poll_portal_slow(struct bman_portal *p, u32 is); -+ -+/* Portal interrupt handler */ -+static irqreturn_t portal_isr(__always_unused int irq, void *ptr) -+{ -+ struct bman_portal *p = ptr; -+ u32 clear = p->irq_sources; -+ u32 is = bm_isr_status_read(&p->p) & p->irq_sources; -+ clear |= __poll_portal_slow(p, is); -+ bm_isr_status_clear(&p->p, clear); -+ return IRQ_HANDLED; -+} -+ -+#ifdef CONFIG_SUSPEND -+static int _bman_portal_suspend_noirq(struct device *dev) -+{ -+ struct bman_portal *p = (struct bman_portal *)dev->platform_data; -+#ifdef CONFIG_PM_DEBUG -+ struct platform_device *pdev = to_platform_device(dev); -+#endif -+ p->save_isdr = bm_isr_disable_read(&p->p); -+ bm_isr_disable_write(&p->p, 0xffffffff); -+ bm_isr_status_clear(&p->p, 0xffffffff); -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Suspend for %s\n", pdev->name); -+#endif -+ return 0; -+} -+ -+static int _bman_portal_resume_noirq(struct device *dev) -+{ -+ struct bman_portal *p = (struct bman_portal *)dev->platform_data; -+ -+ /* restore isdr */ -+ bm_isr_disable_write(&p->p, p->save_isdr); -+ return 0; -+} -+#else -+#define _bman_portal_suspend_noirq NULL -+#define _bman_portal_resume_noirq NULL -+#endif -+ -+struct dev_pm_domain bman_portal_device_pm_domain = { -+ .ops = { -+ USE_PLATFORM_PM_SLEEP_OPS -+ .suspend_noirq = _bman_portal_suspend_noirq, -+ .resume_noirq = _bman_portal_resume_noirq, -+ } -+}; -+ -+struct bman_portal *bman_create_portal( -+ struct bman_portal *portal, -+ const struct bm_portal_config *config) -+{ -+ struct bm_portal *__p; -+ const struct bman_depletion *pools = &config->public_cfg.mask; -+ int ret; -+ u8 bpid = 0; -+ char buf[16]; -+ -+ if (!portal) { -+ portal = kmalloc(sizeof(*portal), GFP_KERNEL); -+ if (!portal) -+ return portal; -+ portal->alloced = 1; -+ } else -+ portal->alloced = 0; -+ -+ __p = &portal->p; -+ -+ /* prep the low-level portal struct with the mapped addresses from the -+ * config, everything that follows depends on it and "config" is more -+ * for (de)reference... */ -+ __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; -+ __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; -+ if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) { -+ pr_err("Bman RCR initialisation failed\n"); -+ goto fail_rcr; -+ } -+ if (bm_mc_init(__p)) { -+ pr_err("Bman MC initialisation failed\n"); -+ goto fail_mc; -+ } -+ if (bm_isr_init(__p)) { -+ pr_err("Bman ISR initialisation failed\n"); -+ goto fail_isr; -+ } -+ portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL); -+ if (!portal->pools) -+ goto fail_pools; -+ portal->pools[0] = *pools; -+ bman_depletion_init(portal->pools + 1); -+ while (bpid < bman_pool_max) { -+ /* Default to all BPIDs disabled, we enable as required at -+ * run-time. */ -+ bm_isr_bscn_mask(__p, bpid, 0); -+ bpid++; -+ } -+ portal->slowpoll = 0; -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ portal->rcri_owned = NULL; -+#endif -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ raw_spin_lock_init(&portal->sharing_lock); -+ portal->is_shared = config->public_cfg.is_shared; -+ portal->sharing_redirect = NULL; -+#endif -+ sprintf(buf, "bportal-%u", config->public_cfg.index); -+ portal->pdev = platform_device_alloc(buf, -1); -+ if (!portal->pdev) -+ goto fail_devalloc; -+ portal->pdev->dev.pm_domain = &bman_portal_device_pm_domain; -+ portal->pdev->dev.platform_data = portal; -+ ret = platform_device_add(portal->pdev); -+ if (ret) -+ goto fail_devadd; -+ memset(&portal->cb, 0, sizeof(portal->cb)); -+ /* Write-to-clear any stale interrupt status bits */ -+ bm_isr_disable_write(__p, 0xffffffff); -+ portal->irq_sources = 0; -+ bm_isr_enable_write(__p, portal->irq_sources); -+ bm_isr_status_clear(__p, 0xffffffff); -+ snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu); -+ if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname, -+ portal)) { -+ pr_err("request_irq() failed\n"); -+ goto fail_irq; -+ } -+ if ((config->public_cfg.cpu != -1) && -+ irq_can_set_affinity(config->public_cfg.irq) && -+ irq_set_affinity(config->public_cfg.irq, -+ cpumask_of(config->public_cfg.cpu))) { -+ pr_err("irq_set_affinity() failed %s\n", portal->irqname); -+ goto fail_affinity; -+ } -+ -+ /* Need RCR to be empty before continuing */ -+ ret = bm_rcr_get_fill(__p); -+ if (ret) { -+ pr_err("Bman RCR unclean\n"); -+ goto fail_rcr_empty; -+ } -+ /* Success */ -+ portal->config = config; -+ -+ bm_isr_disable_write(__p, 0); -+ bm_isr_uninhibit(__p); -+ return portal; -+fail_rcr_empty: -+fail_affinity: -+ free_irq(config->public_cfg.irq, portal); -+fail_irq: -+ platform_device_del(portal->pdev); -+fail_devadd: -+ platform_device_put(portal->pdev); -+fail_devalloc: -+ kfree(portal->pools); -+fail_pools: -+ bm_isr_finish(__p); -+fail_isr: -+ bm_mc_finish(__p); -+fail_mc: -+ bm_rcr_finish(__p); -+fail_rcr: -+ if (portal->alloced) -+ kfree(portal); -+ return NULL; -+} -+ -+struct bman_portal *bman_create_affine_portal( -+ const struct bm_portal_config *config) -+{ -+ struct bman_portal *portal; -+ -+ portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu); -+ portal = bman_create_portal(portal, config); -+ if (portal) { -+ spin_lock(&affine_mask_lock); -+ cpumask_set_cpu(config->public_cfg.cpu, &affine_mask); -+ spin_unlock(&affine_mask_lock); -+ } -+ return portal; -+} -+ -+ -+struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect, -+ int cpu) -+{ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ struct bman_portal *p; -+ p = &per_cpu(bman_affine_portal, cpu); -+ BUG_ON(p->config); -+ BUG_ON(p->is_shared); -+ BUG_ON(!redirect->config->public_cfg.is_shared); -+ p->irq_sources = 0; -+ p->sharing_redirect = redirect; -+ return p; -+#else -+ BUG(); -+ return NULL; -+#endif -+} -+ -+void bman_destroy_portal(struct bman_portal *bm) -+{ -+ const struct bm_portal_config *pcfg; -+ pcfg = bm->config; -+ bm_rcr_cce_update(&bm->p); -+ bm_rcr_cce_update(&bm->p); -+ -+ free_irq(pcfg->public_cfg.irq, bm); -+ -+ kfree(bm->pools); -+ bm_isr_finish(&bm->p); -+ bm_mc_finish(&bm->p); -+ bm_rcr_finish(&bm->p); -+ bm->config = NULL; -+ if (bm->alloced) -+ kfree(bm); -+} -+ -+const struct bm_portal_config *bman_destroy_affine_portal(void) -+{ -+ struct bman_portal *bm = get_raw_affine_portal(); -+ const struct bm_portal_config *pcfg; -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (bm->sharing_redirect) { -+ bm->sharing_redirect = NULL; -+ put_affine_portal(); -+ return NULL; -+ } -+ bm->is_shared = 0; -+#endif -+ pcfg = bm->config; -+ bman_destroy_portal(bm); -+ spin_lock(&affine_mask_lock); -+ cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask); -+ spin_unlock(&affine_mask_lock); -+ put_affine_portal(); -+ return pcfg; -+} -+ -+/* When release logic waits on available RCR space, we need a global waitqueue -+ * in the case of "affine" use (as the waits wake on different cpus which means -+ * different portals - so we can't wait on any per-portal waitqueue). */ -+static DECLARE_WAIT_QUEUE_HEAD(affine_queue); -+ -+static u32 __poll_portal_slow(struct bman_portal *p, u32 is) -+{ -+ struct bman_depletion tmp; -+ u32 ret = is; -+ -+ /* There is a gotcha to be aware of. If we do the query before clearing -+ * the status register, we may miss state changes that occur between the -+ * two. If we write to clear the status register before the query, the -+ * cache-enabled query command may overtake the status register write -+ * unless we use a heavyweight sync (which we don't want). Instead, we -+ * write-to-clear the status register then *read it back* before doing -+ * the query, hence the odd while loop with the 'is' accumulation. */ -+ if (is & BM_PIRQ_BSCN) { -+ struct bm_mc_result *mcr; -+ __maybe_unused unsigned long irqflags; -+ unsigned int i, j; -+ u32 __is; -+ bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); -+ while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) { -+ is |= __is; -+ bm_isr_status_clear(&p->p, BM_PIRQ_BSCN); -+ } -+ is &= ~BM_PIRQ_BSCN; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ bm_mc_start(&p->p); -+ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); -+ while (!(mcr = bm_mc_result(&p->p))) -+ cpu_relax(); -+ tmp = mcr->query.ds.state; -+ tmp.__state[0] = be32_to_cpu(tmp.__state[0]); -+ tmp.__state[1] = be32_to_cpu(tmp.__state[1]); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ for (i = 0; i < 2; i++) { -+ int idx = i * 32; -+ /* tmp is a mask of currently-depleted pools. -+ * pools[0] is mask of those we care about. -+ * pools[1] is our previous view (we only want to -+ * be told about changes). */ -+ tmp.__state[i] &= p->pools[0].__state[i]; -+ if (tmp.__state[i] == p->pools[1].__state[i]) -+ /* fast-path, nothing to see, move along */ -+ continue; -+ for (j = 0; j <= 31; j++, idx++) { -+ struct bman_pool *pool = p->cb[idx]; -+ int b4 = bman_depletion_get(&p->pools[1], idx); -+ int af = bman_depletion_get(&tmp, idx); -+ if (b4 == af) -+ continue; -+ while (pool) { -+ pool->params.cb(p, pool, -+ pool->params.cb_ctx, af); -+ pool = pool->next; -+ } -+ } -+ } -+ p->pools[1] = tmp; -+ } -+ -+ if (is & BM_PIRQ_RCRI) { -+ __maybe_unused unsigned long irqflags; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ bm_rcr_cce_update(&p->p); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ /* If waiting for sync, we only cancel the interrupt threshold -+ * when the ring utilisation hits zero. */ -+ if (p->rcri_owned) { -+ if (!bm_rcr_get_fill(&p->p)) { -+ p->rcri_owned = NULL; -+ bm_rcr_set_ithresh(&p->p, 0); -+ } -+ } else -+#endif -+ bm_rcr_set_ithresh(&p->p, 0); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ wake_up(&affine_queue); -+ bm_isr_status_clear(&p->p, BM_PIRQ_RCRI); -+ is &= ~BM_PIRQ_RCRI; -+ } -+ -+ /* There should be no status register bits left undefined */ -+ DPA_ASSERT(!is); -+ return ret; -+} -+ -+const struct bman_portal_config *bman_get_portal_config(void) -+{ -+ struct bman_portal *p = get_affine_portal(); -+ const struct bman_portal_config *ret = &p->config->public_cfg; -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(bman_get_portal_config); -+ -+u32 bman_irqsource_get(void) -+{ -+ struct bman_portal *p = get_raw_affine_portal(); -+ u32 ret = p->irq_sources & BM_PIRQ_VISIBLE; -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(bman_irqsource_get); -+ -+int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits) -+{ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (p->sharing_redirect) -+ return -EINVAL; -+ else -+#endif -+ { -+ __maybe_unused unsigned long irqflags; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources); -+ bm_isr_enable_write(&p->p, p->irq_sources); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ } -+ return 0; -+} -+EXPORT_SYMBOL(bman_p_irqsource_add); -+ -+int bman_irqsource_add(__maybe_unused u32 bits) -+{ -+ struct bman_portal *p = get_raw_affine_portal(); -+ int ret = 0; -+ ret = bman_p_irqsource_add(p, bits); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(bman_irqsource_add); -+ -+int bman_irqsource_remove(u32 bits) -+{ -+ struct bman_portal *p = get_raw_affine_portal(); -+ __maybe_unused unsigned long irqflags; -+ u32 ier; -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (p->sharing_redirect) { -+ put_affine_portal(); -+ return -EINVAL; -+ } -+#endif -+ /* Our interrupt handler only processes+clears status register bits that -+ * are in p->irq_sources. As we're trimming that mask, if one of them -+ * were to assert in the status register just before we remove it from -+ * the enable register, there would be an interrupt-storm when we -+ * release the IRQ lock. So we wait for the enable register update to -+ * take effect in h/w (by reading it back) and then clear all other bits -+ * in the status register. Ie. we clear them from ISR once it's certain -+ * IER won't allow them to reassert. */ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ bits &= BM_PIRQ_VISIBLE; -+ clear_bits(bits, &p->irq_sources); -+ bm_isr_enable_write(&p->p, p->irq_sources); -+ ier = bm_isr_enable_read(&p->p); -+ /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a -+ * data-dependency, ie. to protect against re-ordering. */ -+ bm_isr_status_clear(&p->p, ~ier); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return 0; -+} -+EXPORT_SYMBOL(bman_irqsource_remove); -+ -+const cpumask_t *bman_affine_cpus(void) -+{ -+ return &affine_mask; -+} -+EXPORT_SYMBOL(bman_affine_cpus); -+ -+u32 bman_poll_slow(void) -+{ -+ struct bman_portal *p = get_poll_portal(); -+ u32 ret; -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (unlikely(p->sharing_redirect)) -+ ret = (u32)-1; -+ else -+#endif -+ { -+ u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; -+ ret = __poll_portal_slow(p, is); -+ bm_isr_status_clear(&p->p, ret); -+ } -+ put_poll_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(bman_poll_slow); -+ -+/* Legacy wrapper */ -+void bman_poll(void) -+{ -+ struct bman_portal *p = get_poll_portal(); -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (unlikely(p->sharing_redirect)) -+ goto done; -+#endif -+ if (!(p->slowpoll--)) { -+ u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources; -+ u32 active = __poll_portal_slow(p, is); -+ if (active) -+ p->slowpoll = SLOW_POLL_BUSY; -+ else -+ p->slowpoll = SLOW_POLL_IDLE; -+ } -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+done: -+#endif -+ put_poll_portal(); -+} -+EXPORT_SYMBOL(bman_poll); -+ -+static const u32 zero_thresholds[4] = {0, 0, 0, 0}; -+ -+struct bman_pool *bman_new_pool(const struct bman_pool_params *params) -+{ -+ struct bman_pool *pool = NULL; -+ u32 bpid; -+ -+ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) { -+ int ret = bman_alloc_bpid(&bpid); -+ if (ret) -+ return NULL; -+ } else { -+ if (params->bpid >= bman_pool_max) -+ return NULL; -+ bpid = params->bpid; -+ } -+#ifdef CONFIG_FSL_BMAN_CONFIG -+ if (params->flags & BMAN_POOL_FLAG_THRESH) { -+ int ret = bm_pool_set(bpid, params->thresholds); -+ if (ret) -+ goto err; -+ } -+#else -+ if (params->flags & BMAN_POOL_FLAG_THRESH) -+ goto err; -+#endif -+ pool = kmalloc(sizeof(*pool), GFP_KERNEL); -+ if (!pool) -+ goto err; -+ pool->sp = NULL; -+ pool->sp_fill = 0; -+ pool->params = *params; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ atomic_set(&pool->in_use, 1); -+#endif -+ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) -+ pool->params.bpid = bpid; -+ if (params->flags & BMAN_POOL_FLAG_STOCKPILE) { -+ pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ, -+ GFP_KERNEL); -+ if (!pool->sp) -+ goto err; -+ } -+ if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) { -+ struct bman_portal *p = get_affine_portal(); -+ if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) { -+ pr_err("Depletion events disabled for bpid %d\n", bpid); -+ goto err; -+ } -+ depletion_link(p, pool); -+ put_affine_portal(); -+ } -+ return pool; -+err: -+#ifdef CONFIG_FSL_BMAN_CONFIG -+ if (params->flags & BMAN_POOL_FLAG_THRESH) -+ bm_pool_set(bpid, zero_thresholds); -+#endif -+ if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) -+ bman_release_bpid(bpid); -+ if (pool) { -+ kfree(pool->sp); -+ kfree(pool); -+ } -+ return NULL; -+} -+EXPORT_SYMBOL(bman_new_pool); -+ -+void bman_free_pool(struct bman_pool *pool) -+{ -+#ifdef CONFIG_FSL_BMAN_CONFIG -+ if (pool->params.flags & BMAN_POOL_FLAG_THRESH) -+ bm_pool_set(pool->params.bpid, zero_thresholds); -+#endif -+ if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) -+ depletion_unlink(pool); -+ if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) { -+ if (pool->sp_fill) -+ pr_err("Stockpile not flushed, has %u in bpid %u.\n", -+ pool->sp_fill, pool->params.bpid); -+ kfree(pool->sp); -+ pool->sp = NULL; -+ pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE; -+ } -+ if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID) -+ bman_release_bpid(pool->params.bpid); -+ kfree(pool); -+} -+EXPORT_SYMBOL(bman_free_pool); -+ -+const struct bman_pool_params *bman_get_params(const struct bman_pool *pool) -+{ -+ return &pool->params; -+} -+EXPORT_SYMBOL(bman_get_params); -+ -+static noinline void update_rcr_ci(struct bman_portal *p, u8 avail) -+{ -+ if (avail) -+ bm_rcr_cce_prefetch(&p->p); -+ else -+ bm_rcr_cce_update(&p->p); -+} -+ -+int bman_rcr_is_empty(void) -+{ -+ __maybe_unused unsigned long irqflags; -+ struct bman_portal *p = get_affine_portal(); -+ u8 avail; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ update_rcr_ci(p, 0); -+ avail = bm_rcr_get_fill(&p->p); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return avail == 0; -+} -+EXPORT_SYMBOL(bman_rcr_is_empty); -+ -+static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p, -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ __maybe_unused struct bman_pool *pool, -+#endif -+ __maybe_unused unsigned long *irqflags, -+ __maybe_unused u32 flags) -+{ -+ struct bm_rcr_entry *r; -+ u8 avail; -+ -+ *p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(*p, (*irqflags)); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && -+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { -+ if ((*p)->rcri_owned) { -+ PORTAL_IRQ_UNLOCK(*p, (*irqflags)); -+ put_affine_portal(); -+ return NULL; -+ } -+ (*p)->rcri_owned = pool; -+ } -+#endif -+ avail = bm_rcr_get_avail(&(*p)->p); -+ if (avail < 2) -+ update_rcr_ci(*p, avail); -+ r = bm_rcr_start(&(*p)->p); -+ if (unlikely(!r)) { -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && -+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) -+ (*p)->rcri_owned = NULL; -+#endif -+ PORTAL_IRQ_UNLOCK(*p, (*irqflags)); -+ put_affine_portal(); -+ } -+ return r; -+} -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p, -+ struct bman_pool *pool, -+ __maybe_unused unsigned long *irqflags, -+ u32 flags) -+{ -+ struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags); -+ if (!rcr) -+ bm_rcr_set_ithresh(&(*p)->p, 1); -+ return rcr; -+} -+ -+static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p, -+ struct bman_pool *pool, -+ __maybe_unused unsigned long *irqflags, -+ u32 flags) -+{ -+ struct bm_rcr_entry *rcr; -+#ifndef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ pool = NULL; -+#endif -+ if (flags & BMAN_RELEASE_FLAG_WAIT_INT) -+ /* NB: return NULL if signal occurs before completion. Signal -+ * can occur during return. Caller must check for signal */ -+ wait_event_interruptible(affine_queue, -+ (rcr = __wait_rel_start(p, pool, irqflags, flags))); -+ else -+ wait_event(affine_queue, -+ (rcr = __wait_rel_start(p, pool, irqflags, flags))); -+ return rcr; -+} -+#endif -+ -+static inline int __bman_release(struct bman_pool *pool, -+ const struct bm_buffer *bufs, u8 num, u32 flags) -+{ -+ struct bman_portal *p; -+ struct bm_rcr_entry *r; -+ __maybe_unused unsigned long irqflags; -+ u32 i = num - 1; -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & BMAN_RELEASE_FLAG_WAIT) -+ r = wait_rel_start(&p, pool, &irqflags, flags); -+ else -+ r = try_rel_start(&p, pool, &irqflags, flags); -+#else -+ r = try_rel_start(&p, &irqflags, flags); -+#endif -+ if (!r) -+ return -EBUSY; -+ /* We can copy all but the first entry, as this can trigger badness -+ * with the valid-bit. Use the overlay to mask the verb byte. */ -+ r->bufs[0].opaque = -+ ((cpu_to_be64((bufs[0].opaque | -+ ((u64)pool->params.bpid<<48)) -+ & 0x00ffffffffffffff))); -+ if (i) { -+ for (i = 1; i < num; i++) -+ r->bufs[i].opaque = -+ cpu_to_be64(bufs[i].opaque); -+ } -+ -+ bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE | -+ (num & BM_RCR_VERB_BUFCOUNT_MASK)); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ /* if we wish to sync we need to set the threshold after h/w sees the -+ * new ring entry. As we're mixing cache-enabled and cache-inhibited -+ * accesses, this requires a heavy-weight sync. */ -+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && -+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { -+ hwsync(); -+ bm_rcr_set_ithresh(&p->p, 1); -+ } -+#endif -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) && -+ (flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) { -+ if (flags & BMAN_RELEASE_FLAG_WAIT_INT) -+ /* NB: return success even if signal occurs before -+ * condition is true. pvb_commit guarantees success */ -+ wait_event_interruptible(affine_queue, -+ (p->rcri_owned != pool)); -+ else -+ wait_event(affine_queue, (p->rcri_owned != pool)); -+ } -+#endif -+ return 0; -+} -+ -+int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, -+ u32 flags) -+{ -+ int ret; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (!num || (num > 8)) -+ return -EINVAL; -+ if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE) -+ return -EINVAL; -+#endif -+ /* Without stockpile, this API is a pass-through to the h/w operation */ -+ if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) -+ return __bman_release(pool, bufs, num, flags); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (!atomic_dec_and_test(&pool->in_use)) { -+ pr_crit("Parallel attempts to enter bman_released() detected."); -+ panic("only one instance of bman_released/acquired allowed"); -+ } -+#endif -+ /* Two movements of buffers are possible, and can occur in either order. -+ * A: moving buffers from the caller to the stockpile. -+ * B: moving buffers from the stockpile to hardware. -+ * Order 1: if there is already enough space in the stockpile for A -+ * then we want to do A first, and only do B if we trigger the -+ * stockpile-high threshold. -+ * Order 2: if there is not enough space in the stockpile for A, then -+ * we want to do B first, then do A if B had succeeded. However in this -+ * case B is dependent on how many buffers the user needs to release, -+ * not the stockpile-high threshold. -+ * Due to the different handling of B between the two cases, putting A -+ * and B in a while() loop would require quite obscure logic, so handle -+ * the different sequences explicitly. */ -+ if ((pool->sp_fill + num) <= BMAN_STOCKPILE_SZ) { -+ /* Order 1: do A */ -+ copy_words(pool->sp + pool->sp_fill, bufs, -+ sizeof(struct bm_buffer) * num); -+ pool->sp_fill += num; -+ /* do B relative to STOCKPILE_HIGH */ -+ while (pool->sp_fill >= BMAN_STOCKPILE_HIGH) { -+ ret = __bman_release(pool, -+ pool->sp + (pool->sp_fill - 8), 8, -+ flags); -+ if (ret >= 0) -+ pool->sp_fill -= 8; -+ } -+ } else { -+ /* Order 2: do B relative to 'num' */ -+ do { -+ ret = __bman_release(pool, -+ pool->sp + (pool->sp_fill - 8), 8, -+ flags); -+ if (ret < 0) -+ /* failure */ -+ goto release_done; -+ pool->sp_fill -= 8; -+ } while ((pool->sp_fill + num) > BMAN_STOCKPILE_SZ); -+ /* do A */ -+ copy_words(pool->sp + pool->sp_fill, bufs, -+ sizeof(struct bm_buffer) * num); -+ pool->sp_fill += num; -+ } -+ /* success */ -+ ret = 0; -+release_done: -+#ifdef CONFIG_FSL_DPA_CHECKING -+ atomic_inc(&pool->in_use); -+#endif -+ return ret; -+} -+EXPORT_SYMBOL(bman_release); -+ -+static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, -+ u8 num) -+{ -+ struct bman_portal *p = get_affine_portal(); -+ struct bm_mc_command *mcc; -+ struct bm_mc_result *mcr; -+ __maybe_unused unsigned long irqflags; -+ int ret, i; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ mcc = bm_mc_start(&p->p); -+ mcc->acquire.bpid = pool->params.bpid; -+ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | -+ (num & BM_MCC_VERB_ACQUIRE_BUFCOUNT)); -+ while (!(mcr = bm_mc_result(&p->p))) -+ cpu_relax(); -+ ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT; -+ if (bufs) { -+ for (i = 0; i < num; i++) -+ bufs[i].opaque = -+ be64_to_cpu(mcr->acquire.bufs[i].opaque); -+ } -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (ret != num) -+ ret = -ENOMEM; -+ return ret; -+} -+ -+int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, -+ u32 flags) -+{ -+ int ret; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (!num || (num > 8)) -+ return -EINVAL; -+ if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE) -+ return -EINVAL; -+#endif -+ /* Without stockpile, this API is a pass-through to the h/w operation */ -+ if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE)) -+ return __bman_acquire(pool, bufs, num); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (!atomic_dec_and_test(&pool->in_use)) { -+ pr_crit("Parallel attempts to enter bman_acquire() detected."); -+ panic("only one instance of bman_released/acquired allowed"); -+ } -+#endif -+ /* Two movements of buffers are possible, and can occur in either order. -+ * A: moving buffers from stockpile to the caller. -+ * B: moving buffers from hardware to the stockpile. -+ * Order 1: if there are already enough buffers in the stockpile for A -+ * then we want to do A first, and only do B if we trigger the -+ * stockpile-low threshold. -+ * Order 2: if there are not enough buffers in the stockpile for A, -+ * then we want to do B first, then do A if B had succeeded. However in -+ * this case B is dependent on how many buffers the user needs, not the -+ * stockpile-low threshold. -+ * Due to the different handling of B between the two cases, putting A -+ * and B in a while() loop would require quite obscure logic, so handle -+ * the different sequences explicitly. */ -+ if (num <= pool->sp_fill) { -+ /* Order 1: do A */ -+ copy_words(bufs, pool->sp + (pool->sp_fill - num), -+ sizeof(struct bm_buffer) * num); -+ pool->sp_fill -= num; -+ /* do B relative to STOCKPILE_LOW */ -+ while (pool->sp_fill <= BMAN_STOCKPILE_LOW) { -+ ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8); -+ if (ret < 0) -+ ret = __bman_acquire(pool, -+ pool->sp + pool->sp_fill, 1); -+ if (ret < 0) -+ break; -+ pool->sp_fill += ret; -+ } -+ } else { -+ /* Order 2: do B relative to 'num' */ -+ do { -+ ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8); -+ if (ret < 0) -+ ret = __bman_acquire(pool, -+ pool->sp + pool->sp_fill, 1); -+ if (ret < 0) -+ /* failure */ -+ goto acquire_done; -+ pool->sp_fill += ret; -+ } while (pool->sp_fill < num); -+ /* do A */ -+ copy_words(bufs, pool->sp + (pool->sp_fill - num), -+ sizeof(struct bm_buffer) * num); -+ pool->sp_fill -= num; -+ } -+ /* success */ -+ ret = num; -+acquire_done: -+#ifdef CONFIG_FSL_DPA_CHECKING -+ atomic_inc(&pool->in_use); -+#endif -+ return ret; -+} -+EXPORT_SYMBOL(bman_acquire); -+ -+int bman_flush_stockpile(struct bman_pool *pool, u32 flags) -+{ -+ u8 num; -+ int ret; -+ -+ while (pool->sp_fill) { -+ num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill); -+ ret = __bman_release(pool, pool->sp + (pool->sp_fill - num), -+ num, flags); -+ if (ret) -+ return ret; -+ pool->sp_fill -= num; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(bman_flush_stockpile); -+ -+int bman_query_pools(struct bm_pool_state *state) -+{ -+ struct bman_portal *p = get_affine_portal(); -+ struct bm_mc_result *mcr; -+ __maybe_unused unsigned long irqflags; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ bm_mc_start(&p->p); -+ bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY); -+ while (!(mcr = bm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY); -+ *state = mcr->query; -+ state->as.state.__state[0] = be32_to_cpu(state->as.state.__state[0]); -+ state->as.state.__state[1] = be32_to_cpu(state->as.state.__state[1]); -+ state->ds.state.__state[0] = be32_to_cpu(state->ds.state.__state[0]); -+ state->ds.state.__state[1] = be32_to_cpu(state->ds.state.__state[1]); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return 0; -+} -+EXPORT_SYMBOL(bman_query_pools); -+ -+#ifdef CONFIG_FSL_BMAN_CONFIG -+u32 bman_query_free_buffers(struct bman_pool *pool) -+{ -+ return bm_pool_free_buffers(pool->params.bpid); -+} -+EXPORT_SYMBOL(bman_query_free_buffers); -+ -+int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds) -+{ -+ u32 bpid; -+ -+ bpid = bman_get_params(pool)->bpid; -+ -+ return bm_pool_set(bpid, thresholds); -+} -+EXPORT_SYMBOL(bman_update_pool_thresholds); -+#endif -+ -+int bman_shutdown_pool(u32 bpid) -+{ -+ struct bman_portal *p = get_affine_portal(); -+ __maybe_unused unsigned long irqflags; -+ int ret; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ ret = bm_shutdown_pool(&p->p, bpid); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(bman_shutdown_pool); -+ -+const struct bm_portal_config *bman_get_bm_portal_config( -+ struct bman_portal *portal) -+{ -+ return portal->sharing_redirect ? NULL : portal->config; -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_low.h -@@ -0,0 +1,565 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "bman_private.h" -+ -+/***************************/ -+/* Portal register assists */ -+/***************************/ -+ -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ -+/* Cache-inhibited register offsets */ -+#define BM_REG_RCR_PI_CINH 0x0000 -+#define BM_REG_RCR_CI_CINH 0x0004 -+#define BM_REG_RCR_ITR 0x0008 -+#define BM_REG_CFG 0x0100 -+#define BM_REG_SCN(n) (0x0200 + ((n) << 2)) -+#define BM_REG_ISR 0x0e00 -+#define BM_REG_IIR 0x0e0c -+ -+/* Cache-enabled register offsets */ -+#define BM_CL_CR 0x0000 -+#define BM_CL_RR0 0x0100 -+#define BM_CL_RR1 0x0140 -+#define BM_CL_RCR 0x1000 -+#define BM_CL_RCR_PI_CENA 0x3000 -+#define BM_CL_RCR_CI_CENA 0x3100 -+ -+#endif -+ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ -+/* Cache-inhibited register offsets */ -+#define BM_REG_RCR_PI_CINH 0x3000 -+#define BM_REG_RCR_CI_CINH 0x3100 -+#define BM_REG_RCR_ITR 0x3200 -+#define BM_REG_CFG 0x3300 -+#define BM_REG_SCN(n) (0x3400 + ((n) << 6)) -+#define BM_REG_ISR 0x3e00 -+#define BM_REG_IIR 0x3ec0 -+ -+/* Cache-enabled register offsets */ -+#define BM_CL_CR 0x0000 -+#define BM_CL_RR0 0x0100 -+#define BM_CL_RR1 0x0140 -+#define BM_CL_RCR 0x1000 -+#define BM_CL_RCR_PI_CENA 0x3000 -+#define BM_CL_RCR_CI_CENA 0x3100 -+ -+#endif -+ -+/* BTW, the drivers (and h/w programming model) already obtain the required -+ * synchronisation for portal accesses via lwsync(), hwsync(), and -+ * data-dependencies. Use of barrier()s or other order-preserving primitives -+ * simply degrade performance. Hence the use of the __raw_*() interfaces, which -+ * simply ensure that the compiler treats the portal registers as volatile (ie. -+ * non-coherent). */ -+ -+/* Cache-inhibited register access. */ -+#define __bm_in(bm, o) be32_to_cpu(__raw_readl((bm)->addr_ci + (o))) -+#define __bm_out(bm, o, val) __raw_writel(cpu_to_be32(val), \ -+ (bm)->addr_ci + (o)); -+#define bm_in(reg) __bm_in(&portal->addr, BM_REG_##reg) -+#define bm_out(reg, val) __bm_out(&portal->addr, BM_REG_##reg, val) -+ -+/* Cache-enabled (index) register access */ -+#define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->addr_ce + (o)) -+#define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->addr_ce + (o)) -+#define __bm_cl_in(bm, o) be32_to_cpu(__raw_readl((bm)->addr_ce + (o))) -+#define __bm_cl_out(bm, o, val) \ -+ do { \ -+ u32 *__tmpclout = (bm)->addr_ce + (o); \ -+ __raw_writel(cpu_to_be32(val), __tmpclout); \ -+ dcbf(__tmpclout); \ -+ } while (0) -+#define __bm_cl_invalidate(bm, o) dcbi((bm)->addr_ce + (o)) -+#define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA) -+#define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA) -+#define bm_cl_in(reg) __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA) -+#define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val) -+#define bm_cl_invalidate(reg)\ -+ __bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA) -+ -+/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf -+ * analysis, look at using the "extra" bit in the ring index registers to avoid -+ * cyclic issues. */ -+static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last) -+{ -+ /* 'first' is included, 'last' is excluded */ -+ if (first <= last) -+ return last - first; -+ return ringsize + last - first; -+} -+ -+/* Portal modes. -+ * Enum types; -+ * pmode == production mode -+ * cmode == consumption mode, -+ * Enum values use 3 letter codes. First letter matches the portal mode, -+ * remaining two letters indicate; -+ * ci == cache-inhibited portal register -+ * ce == cache-enabled portal register -+ * vb == in-band valid-bit (cache-enabled) -+ */ -+enum bm_rcr_pmode { /* matches BCSP_CFG::RPM */ -+ bm_rcr_pci = 0, /* PI index, cache-inhibited */ -+ bm_rcr_pce = 1, /* PI index, cache-enabled */ -+ bm_rcr_pvb = 2 /* valid-bit */ -+}; -+enum bm_rcr_cmode { /* s/w-only */ -+ bm_rcr_cci, /* CI index, cache-inhibited */ -+ bm_rcr_cce /* CI index, cache-enabled */ -+}; -+ -+ -+/* ------------------------- */ -+/* --- Portal structures --- */ -+ -+#define BM_RCR_SIZE 8 -+ -+struct bm_rcr { -+ struct bm_rcr_entry *ring, *cursor; -+ u8 ci, available, ithresh, vbit; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ u32 busy; -+ enum bm_rcr_pmode pmode; -+ enum bm_rcr_cmode cmode; -+#endif -+}; -+ -+struct bm_mc { -+ struct bm_mc_command *cr; -+ struct bm_mc_result *rr; -+ u8 rridx, vbit; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ enum { -+ /* Can only be _mc_start()ed */ -+ mc_idle, -+ /* Can only be _mc_commit()ed or _mc_abort()ed */ -+ mc_user, -+ /* Can only be _mc_retry()ed */ -+ mc_hw -+ } state; -+#endif -+}; -+ -+struct bm_addr { -+ void __iomem *addr_ce; /* cache-enabled */ -+ void __iomem *addr_ci; /* cache-inhibited */ -+}; -+ -+struct bm_portal { -+ struct bm_addr addr; -+ struct bm_rcr rcr; -+ struct bm_mc mc; -+ struct bm_portal_config config; -+} ____cacheline_aligned; -+ -+ -+/* --------------- */ -+/* --- RCR API --- */ -+ -+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ -+#define RCR_CARRYCLEAR(p) \ -+ (void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6))) -+ -+/* Bit-wise logic to convert a ring pointer to a ring index */ -+static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e) -+{ -+ return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1); -+} -+ -+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */ -+static inline void RCR_INC(struct bm_rcr *rcr) -+{ -+ /* NB: this is odd-looking, but experiments show that it generates -+ * fast code with essentially no branching overheads. We increment to -+ * the next RCR pointer and handle overflow and 'vbit'. */ -+ struct bm_rcr_entry *partial = rcr->cursor + 1; -+ rcr->cursor = RCR_CARRYCLEAR(partial); -+ if (partial != rcr->cursor) -+ rcr->vbit ^= BM_RCR_VERB_VBIT; -+} -+ -+static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode, -+ __maybe_unused enum bm_rcr_cmode cmode) -+{ -+ /* This use of 'register', as well as all other occurrences, is because -+ * it has been observed to generate much faster code with gcc than is -+ * otherwise the case. */ -+ register struct bm_rcr *rcr = &portal->rcr; -+ u32 cfg; -+ u8 pi; -+ -+ rcr->ring = portal->addr.addr_ce + BM_CL_RCR; -+ rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); -+ -+ pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1); -+ rcr->cursor = rcr->ring + pi; -+ rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ? BM_RCR_VERB_VBIT : 0; -+ rcr->available = BM_RCR_SIZE - 1 -+ - bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi); -+ rcr->ithresh = bm_in(RCR_ITR); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ rcr->busy = 0; -+ rcr->pmode = pmode; -+ rcr->cmode = cmode; -+#endif -+ cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */ -+ bm_out(CFG, cfg); -+ return 0; -+} -+ -+static inline void bm_rcr_finish(struct bm_portal *portal) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1); -+ u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); -+ DPA_ASSERT(!rcr->busy); -+ if (pi != RCR_PTR2IDX(rcr->cursor)) -+ pr_crit("losing uncommited RCR entries\n"); -+ if (ci != rcr->ci) -+ pr_crit("missing existing RCR completions\n"); -+ if (rcr->ci != RCR_PTR2IDX(rcr->cursor)) -+ pr_crit("RCR destroyed unquiesced\n"); -+} -+ -+static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ DPA_ASSERT(!rcr->busy); -+ if (!rcr->available) -+ return NULL; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ rcr->busy = 1; -+#endif -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ dcbz_64(rcr->cursor); -+#endif -+ return rcr->cursor; -+} -+ -+static inline void bm_rcr_abort(struct bm_portal *portal) -+{ -+ __maybe_unused register struct bm_rcr *rcr = &portal->rcr; -+ DPA_ASSERT(rcr->busy); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ rcr->busy = 0; -+#endif -+} -+ -+static inline struct bm_rcr_entry *bm_rcr_pend_and_next( -+ struct bm_portal *portal, u8 myverb) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ DPA_ASSERT(rcr->busy); -+ DPA_ASSERT(rcr->pmode != bm_rcr_pvb); -+ if (rcr->available == 1) -+ return NULL; -+ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; -+ dcbf_64(rcr->cursor); -+ RCR_INC(rcr); -+ rcr->available--; -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ dcbz_64(rcr->cursor); -+#endif -+ return rcr->cursor; -+} -+ -+static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ DPA_ASSERT(rcr->busy); -+ DPA_ASSERT(rcr->pmode == bm_rcr_pci); -+ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; -+ RCR_INC(rcr); -+ rcr->available--; -+ hwsync(); -+ bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor)); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ rcr->busy = 0; -+#endif -+} -+ -+static inline void bm_rcr_pce_prefetch(struct bm_portal *portal) -+{ -+ __maybe_unused register struct bm_rcr *rcr = &portal->rcr; -+ DPA_ASSERT(rcr->pmode == bm_rcr_pce); -+ bm_cl_invalidate(RCR_PI); -+ bm_cl_touch_rw(RCR_PI); -+} -+ -+static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ DPA_ASSERT(rcr->busy); -+ DPA_ASSERT(rcr->pmode == bm_rcr_pce); -+ rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit; -+ RCR_INC(rcr); -+ rcr->available--; -+ lwsync(); -+ bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor)); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ rcr->busy = 0; -+#endif -+} -+ -+static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ struct bm_rcr_entry *rcursor; -+ DPA_ASSERT(rcr->busy); -+ DPA_ASSERT(rcr->pmode == bm_rcr_pvb); -+ lwsync(); -+ rcursor = rcr->cursor; -+ rcursor->__dont_write_directly__verb = myverb | rcr->vbit; -+ dcbf_64(rcursor); -+ RCR_INC(rcr); -+ rcr->available--; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ rcr->busy = 0; -+#endif -+} -+ -+static inline u8 bm_rcr_cci_update(struct bm_portal *portal) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ u8 diff, old_ci = rcr->ci; -+ DPA_ASSERT(rcr->cmode == bm_rcr_cci); -+ rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1); -+ diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); -+ rcr->available += diff; -+ return diff; -+} -+ -+static inline void bm_rcr_cce_prefetch(struct bm_portal *portal) -+{ -+ __maybe_unused register struct bm_rcr *rcr = &portal->rcr; -+ DPA_ASSERT(rcr->cmode == bm_rcr_cce); -+ bm_cl_touch_ro(RCR_CI); -+} -+ -+static inline u8 bm_rcr_cce_update(struct bm_portal *portal) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ u8 diff, old_ci = rcr->ci; -+ DPA_ASSERT(rcr->cmode == bm_rcr_cce); -+ rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1); -+ bm_cl_invalidate(RCR_CI); -+ diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); -+ rcr->available += diff; -+ return diff; -+} -+ -+static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ return rcr->ithresh; -+} -+ -+static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ rcr->ithresh = ithresh; -+ bm_out(RCR_ITR, ithresh); -+} -+ -+static inline u8 bm_rcr_get_avail(struct bm_portal *portal) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ return rcr->available; -+} -+ -+static inline u8 bm_rcr_get_fill(struct bm_portal *portal) -+{ -+ register struct bm_rcr *rcr = &portal->rcr; -+ return BM_RCR_SIZE - 1 - rcr->available; -+} -+ -+ -+/* ------------------------------ */ -+/* --- Management command API --- */ -+ -+static inline int bm_mc_init(struct bm_portal *portal) -+{ -+ register struct bm_mc *mc = &portal->mc; -+ mc->cr = portal->addr.addr_ce + BM_CL_CR; -+ mc->rr = portal->addr.addr_ce + BM_CL_RR0; -+ mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) & -+ BM_MCC_VERB_VBIT) ? 0 : 1; -+ mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = mc_idle; -+#endif -+ return 0; -+} -+ -+static inline void bm_mc_finish(struct bm_portal *portal) -+{ -+ __maybe_unused register struct bm_mc *mc = &portal->mc; -+ DPA_ASSERT(mc->state == mc_idle); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (mc->state != mc_idle) -+ pr_crit("Losing incomplete MC command\n"); -+#endif -+} -+ -+static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal) -+{ -+ register struct bm_mc *mc = &portal->mc; -+ DPA_ASSERT(mc->state == mc_idle); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = mc_user; -+#endif -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ dcbz_64(mc->cr); -+#endif -+ return mc->cr; -+} -+ -+static inline void bm_mc_abort(struct bm_portal *portal) -+{ -+ __maybe_unused register struct bm_mc *mc = &portal->mc; -+ DPA_ASSERT(mc->state == mc_user); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = mc_idle; -+#endif -+} -+ -+static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb) -+{ -+ register struct bm_mc *mc = &portal->mc; -+ struct bm_mc_result *rr = mc->rr + mc->rridx; -+ DPA_ASSERT(mc->state == mc_user); -+ lwsync(); -+ mc->cr->__dont_write_directly__verb = myverb | mc->vbit; -+ dcbf(mc->cr); -+ dcbit_ro(rr); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = mc_hw; -+#endif -+} -+ -+static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal) -+{ -+ register struct bm_mc *mc = &portal->mc; -+ struct bm_mc_result *rr = mc->rr + mc->rridx; -+ DPA_ASSERT(mc->state == mc_hw); -+ /* The inactive response register's verb byte always returns zero until -+ * its command is submitted and completed. This includes the valid-bit, -+ * in case you were wondering... */ -+ if (!__raw_readb(&rr->verb)) { -+ dcbit_ro(rr); -+ return NULL; -+ } -+ mc->rridx ^= 1; -+ mc->vbit ^= BM_MCC_VERB_VBIT; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = mc_idle; -+#endif -+ return rr; -+} -+ -+ -+/* ------------------------------------- */ -+/* --- Portal interrupt register API --- */ -+ -+static inline int bm_isr_init(__always_unused struct bm_portal *portal) -+{ -+ return 0; -+} -+ -+static inline void bm_isr_finish(__always_unused struct bm_portal *portal) -+{ -+} -+ -+#define SCN_REG(bpid) BM_REG_SCN((bpid) / 32) -+#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31)) -+static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid, -+ int enable) -+{ -+ u32 val; -+ DPA_ASSERT(bpid < bman_pool_max); -+ /* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */ -+ val = __bm_in(&portal->addr, SCN_REG(bpid)); -+ if (enable) -+ val |= SCN_BIT(bpid); -+ else -+ val &= ~SCN_BIT(bpid); -+ __bm_out(&portal->addr, SCN_REG(bpid), val); -+} -+ -+static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n) -+{ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ return __bm_in(&portal->addr, BM_REG_ISR + (n << 6)); -+#else -+ return __bm_in(&portal->addr, BM_REG_ISR + (n << 2)); -+#endif -+} -+ -+static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n, -+ u32 val) -+{ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ __bm_out(&portal->addr, BM_REG_ISR + (n << 6), val); -+#else -+ __bm_out(&portal->addr, BM_REG_ISR + (n << 2), val); -+#endif -+} -+ -+/* Buffer Pool Cleanup */ -+static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid) -+{ -+ struct bm_mc_command *bm_cmd; -+ struct bm_mc_result *bm_res; -+ -+ int aq_count = 0; -+ bool stop = false; -+ while (!stop) { -+ /* Acquire buffers until empty */ -+ bm_cmd = bm_mc_start(p); -+ bm_cmd->acquire.bpid = bpid; -+ bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE | 1); -+ while (!(bm_res = bm_mc_result(p))) -+ cpu_relax(); -+ if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) { -+ /* Pool is empty */ -+ /* TBD : Should we do a few extra iterations in -+ case some other some blocks keep buffers 'on deck', -+ which may also be problematic */ -+ stop = true; -+ } else -+ ++aq_count; -+ } -+ return 0; -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_private.h -@@ -0,0 +1,166 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "dpa_sys.h" -+#include <linux/fsl_bman.h> -+ -+/* Revision info (for errata and feature handling) */ -+#define BMAN_REV10 0x0100 -+#define BMAN_REV20 0x0200 -+#define BMAN_REV21 0x0201 -+#define QBMAN_ANY_PORTAL_IDX 0xffffffff -+extern u16 bman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */ -+ -+/* -+ * Global variables of the max portal/pool number this bman version supported -+ */ -+extern u16 bman_pool_max; -+ -+/* used by CCSR and portal interrupt code */ -+enum bm_isr_reg { -+ bm_isr_status = 0, -+ bm_isr_enable = 1, -+ bm_isr_disable = 2, -+ bm_isr_inhibit = 3 -+}; -+ -+struct bm_portal_config { -+ /* Corenet portal addresses; -+ * [0]==cache-enabled, [1]==cache-inhibited. */ -+ __iomem void *addr_virt[2]; -+ struct resource addr_phys[2]; -+ /* Allow these to be joined in lists */ -+ struct list_head list; -+ /* User-visible portal configuration settings */ -+ struct bman_portal_config public_cfg; -+ /* power management saved data */ -+ u32 saved_isdr; -+}; -+ -+#ifdef CONFIG_FSL_BMAN_CONFIG -+/* Hooks from bman_driver.c to bman_config.c */ -+int bman_init_ccsr(struct device_node *node); -+#endif -+ -+/* Hooks from bman_driver.c in to bman_high.c */ -+struct bman_portal *bman_create_portal( -+ struct bman_portal *portal, -+ const struct bm_portal_config *config); -+struct bman_portal *bman_create_affine_portal( -+ const struct bm_portal_config *config); -+struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect, -+ int cpu); -+void bman_destroy_portal(struct bman_portal *bm); -+ -+const struct bm_portal_config *bman_destroy_affine_portal(void); -+ -+/* Hooks from fsl_usdpaa.c to bman_driver.c */ -+struct bm_portal_config *bm_get_unused_portal(void); -+struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx); -+void bm_put_unused_portal(struct bm_portal_config *pcfg); -+void bm_set_liodns(struct bm_portal_config *pcfg); -+ -+/* Pool logic in the portal driver, during initialisation, needs to know if -+ * there's access to CCSR or not (if not, it'll cripple the pool allocator). */ -+#ifdef CONFIG_FSL_BMAN_CONFIG -+int bman_have_ccsr(void); -+#else -+#define bman_have_ccsr() 0 -+#endif -+ -+/* Stockpile build constants. The _LOW value: when bman_acquire() is called and -+ * the stockpile fill-level is <= _LOW, an acquire is attempted from h/w but it -+ * might fail (if the buffer pool is depleted). So this value provides some -+ * "stagger" in that the bman_acquire() function will only fail if lots of bufs -+ * are requested at once or if h/w has been tested a couple of times without -+ * luck. The _HIGH value: when bman_release() is called and the stockpile -+ * fill-level is >= _HIGH, a release is attempted to h/w but it might fail (if -+ * the release ring is full). So this value provides some "stagger" so that -+ * ring-access is retried a couple of times prior to the API returning a -+ * failure. The following *must* be true; -+ * BMAN_STOCKPILE_HIGH-BMAN_STOCKPILE_LOW > 8 -+ * (to avoid thrashing) -+ * BMAN_STOCKPILE_SZ >= 16 -+ * (as the release logic expects to either send 8 buffers to hw prior to -+ * adding the given buffers to the stockpile or add the buffers to the -+ * stockpile before sending 8 to hw, as the API must be an all-or-nothing -+ * success/fail.) -+ */ -+#define BMAN_STOCKPILE_SZ 16u /* number of bufs in per-pool cache */ -+#define BMAN_STOCKPILE_LOW 2u /* when fill is <= this, acquire from hw */ -+#define BMAN_STOCKPILE_HIGH 14u /* when fill is >= this, release to hw */ -+ -+/*************************************************/ -+/* BMan s/w corenet portal, low-level i/face */ -+/*************************************************/ -+ -+/* Used by all portal interrupt registers except 'inhibit' -+ * This mask contains all the "irqsource" bits visible to API users -+ */ -+#define BM_PIRQ_VISIBLE (BM_PIRQ_RCRI | BM_PIRQ_BSCN) -+ -+/* These are bm_<reg>_<verb>(). So for example, bm_disable_write() means "write -+ * the disable register" rather than "disable the ability to write". */ -+#define bm_isr_status_read(bm) __bm_isr_read(bm, bm_isr_status) -+#define bm_isr_status_clear(bm, m) __bm_isr_write(bm, bm_isr_status, m) -+#define bm_isr_enable_read(bm) __bm_isr_read(bm, bm_isr_enable) -+#define bm_isr_enable_write(bm, v) __bm_isr_write(bm, bm_isr_enable, v) -+#define bm_isr_disable_read(bm) __bm_isr_read(bm, bm_isr_disable) -+#define bm_isr_disable_write(bm, v) __bm_isr_write(bm, bm_isr_disable, v) -+#define bm_isr_inhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 1) -+#define bm_isr_uninhibit(bm) __bm_isr_write(bm, bm_isr_inhibit, 0) -+ -+#ifdef CONFIG_FSL_BMAN_CONFIG -+/* Set depletion thresholds associated with a buffer pool. Requires that the -+ * operating system have access to Bman CCSR (ie. compiled in support and -+ * run-time access courtesy of the device-tree). */ -+int bm_pool_set(u32 bpid, const u32 *thresholds); -+#define BM_POOL_THRESH_SW_ENTER 0 -+#define BM_POOL_THRESH_SW_EXIT 1 -+#define BM_POOL_THRESH_HW_ENTER 2 -+#define BM_POOL_THRESH_HW_EXIT 3 -+ -+/* Read the free buffer count for a given buffer */ -+u32 bm_pool_free_buffers(u32 bpid); -+ -+__init int bman_init(void); -+__init int bman_resource_init(void); -+ -+const struct bm_portal_config *bman_get_bm_portal_config( -+ struct bman_portal *portal); -+ -+/* power management */ -+#ifdef CONFIG_SUSPEND -+void suspend_unused_bportal(void); -+void resume_unused_bportal(void); -+#endif -+ -+#endif /* CONFIG_FSL_BMAN_CONFIG */ ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_test.c -@@ -0,0 +1,56 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "bman_test.h" -+ -+MODULE_AUTHOR("Geoff Thorpe"); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("Bman testing"); -+ -+static int test_init(void) -+{ -+#ifdef CONFIG_FSL_BMAN_TEST_HIGH -+ int loop = 1; -+ while (loop--) -+ bman_test_high(); -+#endif -+#ifdef CONFIG_FSL_BMAN_TEST_THRESH -+ bman_test_thresh(); -+#endif -+ return 0; -+} -+ -+static void test_exit(void) -+{ -+} -+ -+module_init(test_init); -+module_exit(test_exit); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_test.h -@@ -0,0 +1,44 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <linux/kernel.h> -+#include <linux/errno.h> -+#include <linux/io.h> -+#include <linux/slab.h> -+#include <linux/module.h> -+#include <linux/interrupt.h> -+#include <linux/delay.h> -+#include <linux/kthread.h> -+ -+#include <linux/fsl_bman.h> -+ -+void bman_test_high(void); -+void bman_test_thresh(void); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_test_high.c -@@ -0,0 +1,183 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "bman_test.h" -+#include "bman_private.h" -+ -+/*************/ -+/* constants */ -+/*************/ -+ -+#define PORTAL_OPAQUE ((void *)0xf00dbeef) -+#define POOL_OPAQUE ((void *)0xdeadabba) -+#define NUM_BUFS 93 -+#define LOOPS 3 -+#define BMAN_TOKEN_MASK 0x00FFFFFFFFFFLLU -+ -+/***************/ -+/* global vars */ -+/***************/ -+ -+static struct bman_pool *pool; -+static int depleted; -+static struct bm_buffer bufs_in[NUM_BUFS] ____cacheline_aligned; -+static struct bm_buffer bufs_out[NUM_BUFS] ____cacheline_aligned; -+static int bufs_received; -+ -+/* Predeclare the callback so we can instantiate pool parameters */ -+static void depletion_cb(struct bman_portal *, struct bman_pool *, void *, int); -+ -+/**********************/ -+/* internal functions */ -+/**********************/ -+ -+static void bufs_init(void) -+{ -+ int i; -+ for (i = 0; i < NUM_BUFS; i++) -+ bm_buffer_set64(&bufs_in[i], 0xfedc01234567LLU * i); -+ bufs_received = 0; -+} -+ -+static inline int bufs_cmp(const struct bm_buffer *a, const struct bm_buffer *b) -+{ -+ if ((bman_ip_rev == BMAN_REV20) || (bman_ip_rev == BMAN_REV21)) { -+ -+ /* On SoCs with Bman revison 2.0, Bman only respects the 40 -+ * LS-bits of buffer addresses, masking off the upper 8-bits on -+ * release commands. The API provides for 48-bit addresses -+ * because some SoCs support all 48-bits. When generating -+ * garbage addresses for testing, we either need to zero the -+ * upper 8-bits when releasing to Bman (otherwise we'll be -+ * disappointed when the buffers we acquire back from Bman -+ * don't match), or we need to mask the upper 8-bits off when -+ * comparing. We do the latter. -+ */ -+ if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK) -+ < (bm_buffer_get64(b) & BMAN_TOKEN_MASK)) -+ return -1; -+ if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK) -+ > (bm_buffer_get64(b) & BMAN_TOKEN_MASK)) -+ return 1; -+ } else { -+ if (bm_buffer_get64(a) < bm_buffer_get64(b)) -+ return -1; -+ if (bm_buffer_get64(a) > bm_buffer_get64(b)) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static void bufs_confirm(void) -+{ -+ int i, j; -+ for (i = 0; i < NUM_BUFS; i++) { -+ int matches = 0; -+ for (j = 0; j < NUM_BUFS; j++) -+ if (!bufs_cmp(&bufs_in[i], &bufs_out[j])) -+ matches++; -+ BUG_ON(matches != 1); -+ } -+} -+ -+/********/ -+/* test */ -+/********/ -+ -+static void depletion_cb(struct bman_portal *__portal, struct bman_pool *__pool, -+ void *pool_ctx, int __depleted) -+{ -+ BUG_ON(__pool != pool); -+ BUG_ON(pool_ctx != POOL_OPAQUE); -+ depleted = __depleted; -+} -+ -+void bman_test_high(void) -+{ -+ struct bman_pool_params pparams = { -+ .flags = BMAN_POOL_FLAG_DEPLETION | BMAN_POOL_FLAG_DYNAMIC_BPID, -+ .cb = depletion_cb, -+ .cb_ctx = POOL_OPAQUE, -+ }; -+ int i, loops = LOOPS; -+ struct bm_buffer tmp_buf; -+ -+ bufs_init(); -+ -+ pr_info("BMAN: --- starting high-level test ---\n"); -+ -+ pool = bman_new_pool(&pparams); -+ BUG_ON(!pool); -+ -+ /*******************/ -+ /* Release buffers */ -+ /*******************/ -+do_loop: -+ i = 0; -+ while (i < NUM_BUFS) { -+ u32 flags = BMAN_RELEASE_FLAG_WAIT; -+ int num = 8; -+ if ((i + num) > NUM_BUFS) -+ num = NUM_BUFS - i; -+ if ((i + num) == NUM_BUFS) -+ flags |= BMAN_RELEASE_FLAG_WAIT_SYNC; -+ if (bman_release(pool, bufs_in + i, num, flags)) -+ panic("bman_release() failed\n"); -+ i += num; -+ } -+ -+ /*******************/ -+ /* Acquire buffers */ -+ /*******************/ -+ while (i > 0) { -+ int tmp, num = 8; -+ if (num > i) -+ num = i; -+ tmp = bman_acquire(pool, bufs_out + i - num, num, 0); -+ BUG_ON(tmp != num); -+ i -= num; -+ } -+ -+ i = bman_acquire(pool, &tmp_buf, 1, 0); -+ BUG_ON(i > 0); -+ -+ bufs_confirm(); -+ -+ if (--loops) -+ goto do_loop; -+ -+ /************/ -+ /* Clean up */ -+ /************/ -+ bman_free_pool(pool); -+ pr_info("BMAN: --- finished high-level test ---\n"); -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/bman_test_thresh.c -@@ -0,0 +1,196 @@ -+/* Copyright 2010-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "bman_test.h" -+ -+/* Test constants */ -+#define TEST_NUMBUFS 129728 -+#define TEST_EXIT 129536 -+#define TEST_ENTRY 129024 -+ -+struct affine_test_data { -+ struct task_struct *t; -+ int cpu; -+ int expect_affinity; -+ int drain; -+ int num_enter; -+ int num_exit; -+ struct list_head node; -+ struct completion wakethread; -+ struct completion wakeparent; -+}; -+ -+static void cb_depletion(struct bman_portal *portal, -+ struct bman_pool *pool, -+ void *opaque, -+ int depleted) -+{ -+ struct affine_test_data *data = opaque; -+ int c = smp_processor_id(); -+ pr_info("cb_depletion: bpid=%d, depleted=%d, cpu=%d, original=%d\n", -+ bman_get_params(pool)->bpid, !!depleted, c, data->cpu); -+ /* We should be executing on the CPU of the thread that owns the pool if -+ * and that CPU has an affine portal (ie. it isn't slaved). */ -+ BUG_ON((c != data->cpu) && data->expect_affinity); -+ BUG_ON((c == data->cpu) && !data->expect_affinity); -+ if (depleted) -+ data->num_enter++; -+ else -+ data->num_exit++; -+} -+ -+/* Params used to set up a pool, this also dynamically allocates a BPID */ -+static const struct bman_pool_params params_nocb = { -+ .flags = BMAN_POOL_FLAG_DYNAMIC_BPID | BMAN_POOL_FLAG_THRESH, -+ .thresholds = { TEST_ENTRY, TEST_EXIT, 0, 0 } -+}; -+ -+/* Params used to set up each cpu's pool with callbacks enabled */ -+static struct bman_pool_params params_cb = { -+ .bpid = 0, /* will be replaced to match pool_nocb */ -+ .flags = BMAN_POOL_FLAG_DEPLETION, -+ .cb = cb_depletion -+}; -+ -+static struct bman_pool *pool_nocb; -+static LIST_HEAD(threads); -+ -+static int affine_test(void *__data) -+{ -+ struct bman_pool *pool; -+ struct affine_test_data *data = __data; -+ struct bman_pool_params my_params = params_cb; -+ -+ pr_info("thread %d: starting\n", data->cpu); -+ /* create the pool */ -+ my_params.cb_ctx = data; -+ pool = bman_new_pool(&my_params); -+ BUG_ON(!pool); -+ complete(&data->wakeparent); -+ wait_for_completion(&data->wakethread); -+ init_completion(&data->wakethread); -+ -+ /* if we're the drainer, we get signalled for that */ -+ if (data->drain) { -+ struct bm_buffer buf; -+ int ret; -+ pr_info("thread %d: draining...\n", data->cpu); -+ do { -+ ret = bman_acquire(pool, &buf, 1, 0); -+ } while (ret > 0); -+ pr_info("thread %d: draining done.\n", data->cpu); -+ complete(&data->wakeparent); -+ wait_for_completion(&data->wakethread); -+ init_completion(&data->wakethread); -+ } -+ -+ /* cleanup */ -+ bman_free_pool(pool); -+ while (!kthread_should_stop()) -+ cpu_relax(); -+ pr_info("thread %d: exiting\n", data->cpu); -+ return 0; -+} -+ -+static struct affine_test_data *start_affine_test(int cpu, int drain) -+{ -+ struct affine_test_data *data = kmalloc(sizeof(*data), GFP_KERNEL); -+ -+ if (!data) -+ return NULL; -+ data->cpu = cpu; -+ data->expect_affinity = cpumask_test_cpu(cpu, bman_affine_cpus()); -+ data->drain = drain; -+ data->num_enter = 0; -+ data->num_exit = 0; -+ init_completion(&data->wakethread); -+ init_completion(&data->wakeparent); -+ list_add_tail(&data->node, &threads); -+ data->t = kthread_create(affine_test, data, "threshtest%d", cpu); -+ BUG_ON(IS_ERR(data->t)); -+ kthread_bind(data->t, cpu); -+ wake_up_process(data->t); -+ return data; -+} -+ -+void bman_test_thresh(void) -+{ -+ int loop = TEST_NUMBUFS; -+ int ret, num_cpus = 0; -+ struct affine_test_data *data, *drainer = NULL; -+ -+ pr_info("bman_test_thresh: start\n"); -+ -+ /* allocate a BPID and seed it */ -+ pool_nocb = bman_new_pool(¶ms_nocb); -+ BUG_ON(!pool_nocb); -+ while (loop--) { -+ struct bm_buffer buf; -+ bm_buffer_set64(&buf, 0x0badbeef + loop); -+ ret = bman_release(pool_nocb, &buf, 1, -+ BMAN_RELEASE_FLAG_WAIT); -+ BUG_ON(ret); -+ } -+ while (!bman_rcr_is_empty()) -+ cpu_relax(); -+ pr_info("bman_test_thresh: buffers are in\n"); -+ -+ /* create threads and wait for them to create pools */ -+ params_cb.bpid = bman_get_params(pool_nocb)->bpid; -+ for_each_cpu(loop, cpu_online_mask) { -+ data = start_affine_test(loop, drainer ? 0 : 1); -+ BUG_ON(!data); -+ if (!drainer) -+ drainer = data; -+ num_cpus++; -+ wait_for_completion(&data->wakeparent); -+ } -+ -+ /* signal the drainer to start draining */ -+ complete(&drainer->wakethread); -+ wait_for_completion(&drainer->wakeparent); -+ init_completion(&drainer->wakeparent); -+ -+ /* tear down */ -+ list_for_each_entry_safe(data, drainer, &threads, node) { -+ complete(&data->wakethread); -+ ret = kthread_stop(data->t); -+ BUG_ON(ret); -+ list_del(&data->node); -+ /* check that we get the expected callbacks (and no others) */ -+ BUG_ON(data->num_enter != 1); -+ BUG_ON(data->num_exit != 0); -+ kfree(data); -+ } -+ bman_free_pool(pool_nocb); -+ -+ pr_info("bman_test_thresh: done\n"); -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/dpa_alloc.c -@@ -0,0 +1,706 @@ -+/* Copyright 2009-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "dpa_sys.h" -+#include <linux/fsl_qman.h> -+#include <linux/fsl_bman.h> -+ -+/* Qman and Bman APIs are front-ends to the common code; */ -+ -+static DECLARE_DPA_ALLOC(bpalloc); /* BPID allocator */ -+static DECLARE_DPA_ALLOC(fqalloc); /* FQID allocator */ -+static DECLARE_DPA_ALLOC(qpalloc); /* pool-channel allocator */ -+static DECLARE_DPA_ALLOC(cgralloc); /* CGR ID allocator */ -+static DECLARE_DPA_ALLOC(ceetm0_challoc); /* CEETM Channel ID allocator */ -+static DECLARE_DPA_ALLOC(ceetm0_lfqidalloc); /* CEETM LFQID allocator */ -+static DECLARE_DPA_ALLOC(ceetm1_challoc); /* CEETM Channel ID allocator */ -+static DECLARE_DPA_ALLOC(ceetm1_lfqidalloc); /* CEETM LFQID allocator */ -+ -+/* This is a sort-of-conditional dpa_alloc_free() routine. Eg. when releasing -+ * FQIDs (probably from user-space), it can filter out those that aren't in the -+ * OOS state (better to leak a h/w resource than to crash). This function -+ * returns the number of invalid IDs that were not released. */ -+static u32 release_id_range(struct dpa_alloc *alloc, u32 id, u32 count, -+ int (*is_valid)(u32 id)) -+{ -+ int valid_mode = 0; -+ u32 loop = id, total_invalid = 0; -+ while (loop < (id + count)) { -+ int isvalid = is_valid ? is_valid(loop) : 1; -+ if (!valid_mode) { -+ /* We're looking for a valid ID to terminate an invalid -+ * range */ -+ if (isvalid) { -+ /* We finished a range of invalid IDs, a valid -+ * range is now underway */ -+ valid_mode = 1; -+ count -= (loop - id); -+ id = loop; -+ } else -+ total_invalid++; -+ } else { -+ /* We're looking for an invalid ID to terminate a -+ * valid range */ -+ if (!isvalid) { -+ /* Release the range of valid IDs, an unvalid -+ * range is now underway */ -+ if (loop > id) -+ dpa_alloc_free(alloc, id, loop - id); -+ valid_mode = 0; -+ } -+ } -+ loop++; -+ } -+ /* Release any unterminated range of valid IDs */ -+ if (valid_mode && count) -+ dpa_alloc_free(alloc, id, count); -+ return total_invalid; -+} -+ -+/* BPID allocator front-end */ -+ -+int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial) -+{ -+ return dpa_alloc_new(&bpalloc, result, count, align, partial); -+} -+EXPORT_SYMBOL(bman_alloc_bpid_range); -+ -+static int bp_cleanup(u32 bpid) -+{ -+ return bman_shutdown_pool(bpid) == 0; -+} -+void bman_release_bpid_range(u32 bpid, u32 count) -+{ -+ u32 total_invalid = release_id_range(&bpalloc, bpid, count, bp_cleanup); -+ if (total_invalid) -+ pr_err("BPID range [%d..%d] (%d) had %d leaks\n", -+ bpid, bpid + count - 1, count, total_invalid); -+} -+EXPORT_SYMBOL(bman_release_bpid_range); -+ -+void bman_seed_bpid_range(u32 bpid, u32 count) -+{ -+ dpa_alloc_seed(&bpalloc, bpid, count); -+} -+EXPORT_SYMBOL(bman_seed_bpid_range); -+ -+int bman_reserve_bpid_range(u32 bpid, u32 count) -+{ -+ return dpa_alloc_reserve(&bpalloc, bpid, count); -+} -+EXPORT_SYMBOL(bman_reserve_bpid_range); -+ -+ -+/* FQID allocator front-end */ -+ -+int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial) -+{ -+ return dpa_alloc_new(&fqalloc, result, count, align, partial); -+} -+EXPORT_SYMBOL(qman_alloc_fqid_range); -+ -+static int fq_cleanup(u32 fqid) -+{ -+ return qman_shutdown_fq(fqid) == 0; -+} -+void qman_release_fqid_range(u32 fqid, u32 count) -+{ -+ u32 total_invalid = release_id_range(&fqalloc, fqid, count, fq_cleanup); -+ if (total_invalid) -+ pr_err("FQID range [%d..%d] (%d) had %d leaks\n", -+ fqid, fqid + count - 1, count, total_invalid); -+} -+EXPORT_SYMBOL(qman_release_fqid_range); -+ -+int qman_reserve_fqid_range(u32 fqid, u32 count) -+{ -+ return dpa_alloc_reserve(&fqalloc, fqid, count); -+} -+EXPORT_SYMBOL(qman_reserve_fqid_range); -+ -+void qman_seed_fqid_range(u32 fqid, u32 count) -+{ -+ dpa_alloc_seed(&fqalloc, fqid, count); -+} -+EXPORT_SYMBOL(qman_seed_fqid_range); -+ -+/* Pool-channel allocator front-end */ -+ -+int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial) -+{ -+ return dpa_alloc_new(&qpalloc, result, count, align, partial); -+} -+EXPORT_SYMBOL(qman_alloc_pool_range); -+ -+static int qpool_cleanup(u32 qp) -+{ -+ /* We query all FQDs starting from -+ * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs -+ * whose destination channel is the pool-channel being released. -+ * When a non-OOS FQD is found we attempt to clean it up */ -+ struct qman_fq fq = { -+ .fqid = 1 -+ }; -+ int err; -+ do { -+ struct qm_mcr_queryfq_np np; -+ err = qman_query_fq_np(&fq, &np); -+ if (err) -+ /* FQID range exceeded, found no problems */ -+ return 1; -+ if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) { -+ struct qm_fqd fqd; -+ err = qman_query_fq(&fq, &fqd); -+ BUG_ON(err); -+ if (fqd.dest.channel == qp) { -+ /* The channel is the FQ's target, clean it */ -+ if (qman_shutdown_fq(fq.fqid) != 0) -+ /* Couldn't shut down the FQ -+ so the pool must be leaked */ -+ return 0; -+ } -+ } -+ /* Move to the next FQID */ -+ fq.fqid++; -+ } while (1); -+} -+void qman_release_pool_range(u32 qp, u32 count) -+{ -+ u32 total_invalid = release_id_range(&qpalloc, qp, -+ count, qpool_cleanup); -+ if (total_invalid) { -+ /* Pool channels are almost always used individually */ -+ if (count == 1) -+ pr_err("Pool channel 0x%x had %d leaks\n", -+ qp, total_invalid); -+ else -+ pr_err("Pool channels [%d..%d] (%d) had %d leaks\n", -+ qp, qp + count - 1, count, total_invalid); -+ } -+} -+EXPORT_SYMBOL(qman_release_pool_range); -+ -+ -+void qman_seed_pool_range(u32 poolid, u32 count) -+{ -+ dpa_alloc_seed(&qpalloc, poolid, count); -+ -+} -+EXPORT_SYMBOL(qman_seed_pool_range); -+ -+int qman_reserve_pool_range(u32 poolid, u32 count) -+{ -+ return dpa_alloc_reserve(&qpalloc, poolid, count); -+} -+EXPORT_SYMBOL(qman_reserve_pool_range); -+ -+ -+/* CGR ID allocator front-end */ -+ -+int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial) -+{ -+ return dpa_alloc_new(&cgralloc, result, count, align, partial); -+} -+EXPORT_SYMBOL(qman_alloc_cgrid_range); -+ -+static int cqr_cleanup(u32 cgrid) -+{ -+ /* We query all FQDs starting from -+ * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs -+ * whose CGR is the CGR being released. -+ */ -+ struct qman_fq fq = { -+ .fqid = 1 -+ }; -+ int err; -+ do { -+ struct qm_mcr_queryfq_np np; -+ err = qman_query_fq_np(&fq, &np); -+ if (err) -+ /* FQID range exceeded, found no problems */ -+ return 1; -+ if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) { -+ struct qm_fqd fqd; -+ err = qman_query_fq(&fq, &fqd); -+ BUG_ON(err); -+ if ((fqd.fq_ctrl & QM_FQCTRL_CGE) && -+ (fqd.cgid == cgrid)) { -+ pr_err("CRGID 0x%x is being used by FQID 0x%x," -+ " CGR will be leaked\n", -+ cgrid, fq.fqid); -+ return 1; -+ } -+ } -+ /* Move to the next FQID */ -+ fq.fqid++; -+ } while (1); -+} -+ -+void qman_release_cgrid_range(u32 cgrid, u32 count) -+{ -+ u32 total_invalid = release_id_range(&cgralloc, cgrid, -+ count, cqr_cleanup); -+ if (total_invalid) -+ pr_err("CGRID range [%d..%d] (%d) had %d leaks\n", -+ cgrid, cgrid + count - 1, count, total_invalid); -+} -+EXPORT_SYMBOL(qman_release_cgrid_range); -+ -+void qman_seed_cgrid_range(u32 cgrid, u32 count) -+{ -+ dpa_alloc_seed(&cgralloc, cgrid, count); -+ -+} -+EXPORT_SYMBOL(qman_seed_cgrid_range); -+ -+/* CEETM CHANNEL ID allocator front-end */ -+int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align, -+ int partial) -+{ -+ return dpa_alloc_new(&ceetm0_challoc, result, count, align, partial); -+} -+EXPORT_SYMBOL(qman_alloc_ceetm0_channel_range); -+ -+int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align, -+ int partial) -+{ -+ return dpa_alloc_new(&ceetm1_challoc, result, count, align, partial); -+} -+EXPORT_SYMBOL(qman_alloc_ceetm1_channel_range); -+ -+void qman_release_ceetm0_channel_range(u32 channelid, u32 count) -+{ -+ u32 total_invalid; -+ -+ total_invalid = release_id_range(&ceetm0_challoc, channelid, count, -+ NULL); -+ if (total_invalid) -+ pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n", -+ channelid, channelid + count - 1, count, total_invalid); -+} -+EXPORT_SYMBOL(qman_release_ceetm0_channel_range); -+ -+void qman_seed_ceetm0_channel_range(u32 channelid, u32 count) -+{ -+ dpa_alloc_seed(&ceetm0_challoc, channelid, count); -+ -+} -+EXPORT_SYMBOL(qman_seed_ceetm0_channel_range); -+ -+void qman_release_ceetm1_channel_range(u32 channelid, u32 count) -+{ -+ u32 total_invalid; -+ total_invalid = release_id_range(&ceetm1_challoc, channelid, count, -+ NULL); -+ if (total_invalid) -+ pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n", -+ channelid, channelid + count - 1, count, total_invalid); -+} -+EXPORT_SYMBOL(qman_release_ceetm1_channel_range); -+ -+void qman_seed_ceetm1_channel_range(u32 channelid, u32 count) -+{ -+ dpa_alloc_seed(&ceetm1_challoc, channelid, count); -+ -+} -+EXPORT_SYMBOL(qman_seed_ceetm1_channel_range); -+ -+/* CEETM LFQID allocator front-end */ -+int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align, -+ int partial) -+{ -+ return dpa_alloc_new(&ceetm0_lfqidalloc, result, count, align, partial); -+} -+EXPORT_SYMBOL(qman_alloc_ceetm0_lfqid_range); -+ -+int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align, -+ int partial) -+{ -+ return dpa_alloc_new(&ceetm1_lfqidalloc, result, count, align, partial); -+} -+EXPORT_SYMBOL(qman_alloc_ceetm1_lfqid_range); -+ -+void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count) -+{ -+ u32 total_invalid; -+ -+ total_invalid = release_id_range(&ceetm0_lfqidalloc, lfqid, count, -+ NULL); -+ if (total_invalid) -+ pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n", -+ lfqid, lfqid + count - 1, count, total_invalid); -+} -+EXPORT_SYMBOL(qman_release_ceetm0_lfqid_range); -+ -+void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count) -+{ -+ dpa_alloc_seed(&ceetm0_lfqidalloc, lfqid, count); -+ -+} -+EXPORT_SYMBOL(qman_seed_ceetm0_lfqid_range); -+ -+void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count) -+{ -+ u32 total_invalid; -+ -+ total_invalid = release_id_range(&ceetm1_lfqidalloc, lfqid, count, -+ NULL); -+ if (total_invalid) -+ pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n", -+ lfqid, lfqid + count - 1, count, total_invalid); -+} -+EXPORT_SYMBOL(qman_release_ceetm1_lfqid_range); -+ -+void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count) -+{ -+ dpa_alloc_seed(&ceetm1_lfqidalloc, lfqid, count); -+ -+} -+EXPORT_SYMBOL(qman_seed_ceetm1_lfqid_range); -+ -+ -+/* Everything else is the common backend to all the allocators */ -+ -+/* The allocator is a (possibly-empty) list of these; */ -+struct alloc_node { -+ struct list_head list; -+ u32 base; -+ u32 num; -+ /* refcount and is_alloced are only set -+ when the node is in the used list */ -+ unsigned int refcount; -+ int is_alloced; -+}; -+ -+/* #define DPA_ALLOC_DEBUG */ -+ -+#ifdef DPA_ALLOC_DEBUG -+#define DPRINT pr_info -+static void DUMP(struct dpa_alloc *alloc) -+{ -+ int off = 0; -+ char buf[256]; -+ struct alloc_node *p; -+ pr_info("Free Nodes\n"); -+ list_for_each_entry(p, &alloc->free, list) { -+ if (off < 255) -+ off += snprintf(buf + off, 255-off, "{%d,%d}", -+ p->base, p->base + p->num - 1); -+ } -+ pr_info("%s\n", buf); -+ -+ off = 0; -+ pr_info("Used Nodes\n"); -+ list_for_each_entry(p, &alloc->used, list) { -+ if (off < 255) -+ off += snprintf(buf + off, 255-off, "{%d,%d}", -+ p->base, p->base + p->num - 1); -+ } -+ pr_info("%s\n", buf); -+ -+ -+ -+} -+#else -+#define DPRINT(x...) -+#define DUMP(a) -+#endif -+ -+int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align, -+ int partial) -+{ -+ struct alloc_node *i = NULL, *next_best = NULL, *used_node = NULL; -+ u32 base, next_best_base = 0, num = 0, next_best_num = 0; -+ struct alloc_node *margin_left, *margin_right; -+ -+ *result = (u32)-1; -+ DPRINT("alloc_range(%d,%d,%d)\n", count, align, partial); -+ DUMP(alloc); -+ /* If 'align' is 0, it should behave as though it was 1 */ -+ if (!align) -+ align = 1; -+ margin_left = kmalloc(sizeof(*margin_left), GFP_KERNEL); -+ if (!margin_left) -+ goto err; -+ margin_right = kmalloc(sizeof(*margin_right), GFP_KERNEL); -+ if (!margin_right) { -+ kfree(margin_left); -+ goto err; -+ } -+ spin_lock_irq(&alloc->lock); -+ list_for_each_entry(i, &alloc->free, list) { -+ base = (i->base + align - 1) / align; -+ base *= align; -+ if ((base - i->base) >= i->num) -+ /* alignment is impossible, regardless of count */ -+ continue; -+ num = i->num - (base - i->base); -+ if (num >= count) { -+ /* this one will do nicely */ -+ num = count; -+ goto done; -+ } -+ if (num > next_best_num) { -+ next_best = i; -+ next_best_base = base; -+ next_best_num = num; -+ } -+ } -+ if (partial && next_best) { -+ i = next_best; -+ base = next_best_base; -+ num = next_best_num; -+ } else -+ i = NULL; -+done: -+ if (i) { -+ if (base != i->base) { -+ margin_left->base = i->base; -+ margin_left->num = base - i->base; -+ list_add_tail(&margin_left->list, &i->list); -+ } else -+ kfree(margin_left); -+ if ((base + num) < (i->base + i->num)) { -+ margin_right->base = base + num; -+ margin_right->num = (i->base + i->num) - -+ (base + num); -+ list_add(&margin_right->list, &i->list); -+ } else -+ kfree(margin_right); -+ list_del(&i->list); -+ kfree(i); -+ *result = base; -+ } else { -+ spin_unlock_irq(&alloc->lock); -+ kfree(margin_left); -+ kfree(margin_right); -+ } -+ -+err: -+ DPRINT("returning %d\n", i ? num : -ENOMEM); -+ DUMP(alloc); -+ if (!i) -+ return -ENOMEM; -+ -+ /* Add the allocation to the used list with a refcount of 1 */ -+ used_node = kmalloc(sizeof(*used_node), GFP_KERNEL); -+ if (!used_node) { -+ spin_unlock_irq(&alloc->lock); -+ return -ENOMEM; -+ } -+ used_node->base = *result; -+ used_node->num = num; -+ used_node->refcount = 1; -+ used_node->is_alloced = 1; -+ list_add_tail(&used_node->list, &alloc->used); -+ spin_unlock_irq(&alloc->lock); -+ return (int)num; -+} -+ -+/* Allocate the list node using GFP_ATOMIC, because we *really* want to avoid -+ * forcing error-handling on to users in the deallocation path. */ -+static void _dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count) -+{ -+ struct alloc_node *i, *node = kmalloc(sizeof(*node), GFP_ATOMIC); -+ BUG_ON(!node); -+ DPRINT("release_range(%d,%d)\n", base_id, count); -+ DUMP(alloc); -+ BUG_ON(!count); -+ spin_lock_irq(&alloc->lock); -+ -+ -+ node->base = base_id; -+ node->num = count; -+ list_for_each_entry(i, &alloc->free, list) { -+ if (i->base >= node->base) { -+ /* BUG_ON(any overlapping) */ -+ BUG_ON(i->base < (node->base + node->num)); -+ list_add_tail(&node->list, &i->list); -+ goto done; -+ } -+ } -+ list_add_tail(&node->list, &alloc->free); -+done: -+ /* Merge to the left */ -+ i = list_entry(node->list.prev, struct alloc_node, list); -+ if (node->list.prev != &alloc->free) { -+ BUG_ON((i->base + i->num) > node->base); -+ if ((i->base + i->num) == node->base) { -+ node->base = i->base; -+ node->num += i->num; -+ list_del(&i->list); -+ kfree(i); -+ } -+ } -+ /* Merge to the right */ -+ i = list_entry(node->list.next, struct alloc_node, list); -+ if (node->list.next != &alloc->free) { -+ BUG_ON((node->base + node->num) > i->base); -+ if ((node->base + node->num) == i->base) { -+ node->num += i->num; -+ list_del(&i->list); -+ kfree(i); -+ } -+ } -+ spin_unlock_irq(&alloc->lock); -+ DUMP(alloc); -+} -+ -+ -+void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count) -+{ -+ struct alloc_node *i = NULL; -+ spin_lock_irq(&alloc->lock); -+ -+ /* First find the node in the used list and decrement its ref count */ -+ list_for_each_entry(i, &alloc->used, list) { -+ if (i->base == base_id && i->num == count) { -+ --i->refcount; -+ if (i->refcount == 0) { -+ list_del(&i->list); -+ spin_unlock_irq(&alloc->lock); -+ if (i->is_alloced) -+ _dpa_alloc_free(alloc, base_id, count); -+ kfree(i); -+ return; -+ } -+ spin_unlock_irq(&alloc->lock); -+ return; -+ } -+ } -+ /* Couldn't find the allocation */ -+ pr_err("Attempt to free ID 0x%x COUNT %d that wasn't alloc'd or reserved\n", -+ base_id, count); -+ spin_unlock_irq(&alloc->lock); -+} -+ -+void dpa_alloc_seed(struct dpa_alloc *alloc, u32 base_id, u32 count) -+{ -+ /* Same as free but no previous allocation checking is needed */ -+ _dpa_alloc_free(alloc, base_id, count); -+} -+ -+ -+int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base, u32 num) -+{ -+ struct alloc_node *i = NULL, *used_node; -+ -+ DPRINT("alloc_reserve(%d,%d)\n", base, num); -+ DUMP(alloc); -+ -+ spin_lock_irq(&alloc->lock); -+ -+ /* Check for the node in the used list. -+ If found, increase it's refcount */ -+ list_for_each_entry(i, &alloc->used, list) { -+ if ((i->base == base) && (i->num == num)) { -+ ++i->refcount; -+ spin_unlock_irq(&alloc->lock); -+ return 0; -+ } -+ if ((base >= i->base) && (base < (i->base + i->num))) { -+ /* This is an attempt to reserve a region that was -+ already reserved or alloced with a different -+ base or num */ -+ pr_err("Cannot reserve %d - %d, it overlaps with" -+ " existing reservation from %d - %d\n", -+ base, base + num - 1, i->base, -+ i->base + i->num - 1); -+ spin_unlock_irq(&alloc->lock); -+ return -1; -+ } -+ } -+ /* Check to make sure this ID isn't in the free list */ -+ list_for_each_entry(i, &alloc->free, list) { -+ if ((base >= i->base) && (base < (i->base + i->num))) { -+ /* yep, the reservation is within this node */ -+ pr_err("Cannot reserve %d - %d, it overlaps with" -+ " free range %d - %d and must be alloced\n", -+ base, base + num - 1, -+ i->base, i->base + i->num - 1); -+ spin_unlock_irq(&alloc->lock); -+ return -1; -+ } -+ } -+ /* Add the allocation to the used list with a refcount of 1 */ -+ used_node = kmalloc(sizeof(*used_node), GFP_KERNEL); -+ if (!used_node) { -+ spin_unlock_irq(&alloc->lock); -+ return -ENOMEM; -+ -+ } -+ used_node->base = base; -+ used_node->num = num; -+ used_node->refcount = 1; -+ used_node->is_alloced = 0; -+ list_add_tail(&used_node->list, &alloc->used); -+ spin_unlock_irq(&alloc->lock); -+ return 0; -+} -+ -+ -+int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count) -+{ -+ struct alloc_node *i = NULL; -+ DPRINT("alloc_pop()\n"); -+ DUMP(alloc); -+ spin_lock_irq(&alloc->lock); -+ if (!list_empty(&alloc->free)) { -+ i = list_entry(alloc->free.next, struct alloc_node, list); -+ list_del(&i->list); -+ } -+ spin_unlock_irq(&alloc->lock); -+ DPRINT("returning %d\n", i ? 0 : -ENOMEM); -+ DUMP(alloc); -+ if (!i) -+ return -ENOMEM; -+ *result = i->base; -+ *count = i->num; -+ kfree(i); -+ return 0; -+} -+ -+int dpa_alloc_check(struct dpa_alloc *list_head, u32 item) -+{ -+ struct alloc_node *i = NULL; -+ int res = 0; -+ DPRINT("alloc_check()\n"); -+ spin_lock_irq(&list_head->lock); -+ -+ list_for_each_entry(i, &list_head->free, list) { -+ if ((item >= i->base) && (item < (i->base + i->num))) { -+ res = 1; -+ break; -+ } -+ } -+ spin_unlock_irq(&list_head->lock); -+ return res; -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/dpa_sys.h -@@ -0,0 +1,259 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef DPA_SYS_H -+#define DPA_SYS_H -+ -+#include <linux/kernel.h> -+#include <linux/errno.h> -+#include <linux/io.h> -+#include <linux/dma-mapping.h> -+#include <linux/bootmem.h> -+#include <linux/slab.h> -+#include <linux/module.h> -+#include <linux/init.h> -+#include <linux/interrupt.h> -+#include <linux/delay.h> -+#include <linux/of_platform.h> -+#include <linux/of_address.h> -+#include <linux/of_irq.h> -+#include <linux/kthread.h> -+#include <linux/memblock.h> -+#include <linux/completion.h> -+#include <linux/log2.h> -+#include <linux/types.h> -+#include <linux/ioctl.h> -+#include <linux/miscdevice.h> -+#include <linux/uaccess.h> -+#include <linux/debugfs.h> -+#include <linux/seq_file.h> -+#include <linux/device.h> -+#include <linux/uio_driver.h> -+#include <linux/smp.h> -+#include <linux/fsl_hypervisor.h> -+#include <linux/vmalloc.h> -+#include <linux/ctype.h> -+#include <linux/math64.h> -+#include <linux/bitops.h> -+ -+#include <linux/fsl_usdpaa.h> -+ -+/* When copying aligned words or shorts, try to avoid memcpy() */ -+#define CONFIG_TRY_BETTER_MEMCPY -+ -+/* For 2-element tables related to cache-inhibited and cache-enabled mappings */ -+#define DPA_PORTAL_CE 0 -+#define DPA_PORTAL_CI 1 -+ -+/***********************/ -+/* Misc inline assists */ -+/***********************/ -+ -+#if defined CONFIG_PPC32 -+#include "dpa_sys_ppc32.h" -+#elif defined CONFIG_PPC64 -+#include "dpa_sys_ppc64.h" -+#elif defined CONFIG_ARM -+#include "dpa_sys_arm.h" -+#elif defined CONFIG_ARM64 -+#include "dpa_sys_arm64.h" -+#endif -+ -+ -+#ifdef CONFIG_FSL_DPA_CHECKING -+#define DPA_ASSERT(x) \ -+ do { \ -+ if (!(x)) { \ -+ pr_crit("ASSERT: (%s:%d) %s\n", __FILE__, __LINE__, \ -+ __stringify_1(x)); \ -+ dump_stack(); \ -+ panic("assertion failure"); \ -+ } \ -+ } while (0) -+#else -+#define DPA_ASSERT(x) -+#endif -+ -+/* memcpy() stuff - when you know alignments in advance */ -+#ifdef CONFIG_TRY_BETTER_MEMCPY -+static inline void copy_words(void *dest, const void *src, size_t sz) -+{ -+ u32 *__dest = dest; -+ const u32 *__src = src; -+ size_t __sz = sz >> 2; -+ BUG_ON((unsigned long)dest & 0x3); -+ BUG_ON((unsigned long)src & 0x3); -+ BUG_ON(sz & 0x3); -+ while (__sz--) -+ *(__dest++) = *(__src++); -+} -+static inline void copy_shorts(void *dest, const void *src, size_t sz) -+{ -+ u16 *__dest = dest; -+ const u16 *__src = src; -+ size_t __sz = sz >> 1; -+ BUG_ON((unsigned long)dest & 0x1); -+ BUG_ON((unsigned long)src & 0x1); -+ BUG_ON(sz & 0x1); -+ while (__sz--) -+ *(__dest++) = *(__src++); -+} -+static inline void copy_bytes(void *dest, const void *src, size_t sz) -+{ -+ u8 *__dest = dest; -+ const u8 *__src = src; -+ while (sz--) -+ *(__dest++) = *(__src++); -+} -+#else -+#define copy_words memcpy -+#define copy_shorts memcpy -+#define copy_bytes memcpy -+#endif -+ -+/************/ -+/* RB-trees */ -+/************/ -+ -+/* We encapsulate RB-trees so that its easier to use non-linux forms in -+ * non-linux systems. This also encapsulates the extra plumbing that linux code -+ * usually provides when using RB-trees. This encapsulation assumes that the -+ * data type held by the tree is u32. */ -+ -+struct dpa_rbtree { -+ struct rb_root root; -+}; -+#define DPA_RBTREE { .root = RB_ROOT } -+ -+static inline void dpa_rbtree_init(struct dpa_rbtree *tree) -+{ -+ tree->root = RB_ROOT; -+} -+ -+#define IMPLEMENT_DPA_RBTREE(name, type, node_field, val_field) \ -+static inline int name##_push(struct dpa_rbtree *tree, type *obj) \ -+{ \ -+ struct rb_node *parent = NULL, **p = &tree->root.rb_node; \ -+ while (*p) { \ -+ u32 item; \ -+ parent = *p; \ -+ item = rb_entry(parent, type, node_field)->val_field; \ -+ if (obj->val_field < item) \ -+ p = &parent->rb_left; \ -+ else if (obj->val_field > item) \ -+ p = &parent->rb_right; \ -+ else \ -+ return -EBUSY; \ -+ } \ -+ rb_link_node(&obj->node_field, parent, p); \ -+ rb_insert_color(&obj->node_field, &tree->root); \ -+ return 0; \ -+} \ -+static inline void name##_del(struct dpa_rbtree *tree, type *obj) \ -+{ \ -+ rb_erase(&obj->node_field, &tree->root); \ -+} \ -+static inline type *name##_find(struct dpa_rbtree *tree, u32 val) \ -+{ \ -+ type *ret; \ -+ struct rb_node *p = tree->root.rb_node; \ -+ while (p) { \ -+ ret = rb_entry(p, type, node_field); \ -+ if (val < ret->val_field) \ -+ p = p->rb_left; \ -+ else if (val > ret->val_field) \ -+ p = p->rb_right; \ -+ else \ -+ return ret; \ -+ } \ -+ return NULL; \ -+} -+ -+/************/ -+/* Bootargs */ -+/************/ -+ -+/* Qman has "qportals=" and Bman has "bportals=", they use the same syntax -+ * though; a comma-separated list of items, each item being a cpu index and/or a -+ * range of cpu indices, and each item optionally be prefixed by "s" to indicate -+ * that the portal associated with that cpu should be shared. See bman_driver.c -+ * for more specifics. */ -+static int __parse_portals_cpu(const char **s, unsigned int *cpu) -+{ -+ *cpu = 0; -+ if (!isdigit(**s)) -+ return -EINVAL; -+ while (isdigit(**s)) -+ *cpu = *cpu * 10 + (*((*s)++) - '0'); -+ return 0; -+} -+static inline int parse_portals_bootarg(char *str, struct cpumask *want_shared, -+ struct cpumask *want_unshared, -+ const char *argname) -+{ -+ const char *s = str; -+ unsigned int shared, cpu1, cpu2, loop; -+ -+keep_going: -+ if (*s == 's') { -+ shared = 1; -+ s++; -+ } else -+ shared = 0; -+ if (__parse_portals_cpu(&s, &cpu1)) -+ goto err; -+ if (*s == '-') { -+ s++; -+ if (__parse_portals_cpu(&s, &cpu2)) -+ goto err; -+ if (cpu2 < cpu1) -+ goto err; -+ } else -+ cpu2 = cpu1; -+ for (loop = cpu1; loop <= cpu2; loop++) -+ cpumask_set_cpu(loop, shared ? want_shared : want_unshared); -+ if (*s == ',') { -+ s++; -+ goto keep_going; -+ } else if ((*s == '\0') || isspace(*s)) -+ return 0; -+err: -+ pr_crit("Malformed %s argument: %s, offset: %lu\n", argname, str, -+ (unsigned long)s - (unsigned long)str); -+ return -EINVAL; -+} -+#ifdef CONFIG_FSL_USDPAA -+/* Hooks from fsl_usdpaa_irq.c to fsl_usdpaa.c */ -+int usdpaa_get_portal_config(struct file *filp, void *cinh, -+ enum usdpaa_portal_type ptype, unsigned int *irq, -+ void **iir_reg); -+#endif -+#endif /* DPA_SYS_H */ ---- /dev/null -+++ b/drivers/staging/fsl_qbman/dpa_sys_arm.h -@@ -0,0 +1,95 @@ -+/* Copyright 2016 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef DPA_SYS_ARM_H -+#define DPA_SYS_ARM_H -+ -+#include <asm/cacheflush.h> -+#include <asm/barrier.h> -+ -+/* Implementation of ARM specific routines */ -+ -+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler -+ * barriers and that dcb*() won't fall victim to compiler or execution -+ * reordering with respect to other code/instructions that manipulate the same -+ * cacheline. */ -+#define hwsync() { asm volatile("dmb st" : : : "memory"); } -+#define lwsync() { asm volatile("dmb st" : : : "memory"); } -+#define dcbf(p) { asm volatile("mcr p15, 0, %0, c7, c10, 1" : : "r" (p) : "memory"); } -+#define dcbt_ro(p) { asm volatile("pld [%0, #64];": : "r" (p)); } -+#define dcbt_rw(p) { asm volatile("pldw [%0, #64];": : "r" (p)); } -+#define dcbi(p) { asm volatile("mcr p15, 0, %0, c7, c6, 1" : : "r" (p) : "memory"); } -+ -+#define dcbz_64(p) { memset(p, 0, sizeof(*p)); } -+ -+#define dcbf_64(p) \ -+ do { \ -+ dcbf((u32)p); \ -+ } while (0) -+/* Commonly used combo */ -+#define dcbit_ro(p) \ -+ do { \ -+ dcbi((u32)p); \ -+ dcbt_ro((u32)p); \ -+ } while (0) -+ -+static inline u64 mfatb(void) -+{ -+ return get_cycles(); -+} -+ -+static inline u32 in_be32(volatile void *addr) -+{ -+ return be32_to_cpu(*((volatile u32 *) addr)); -+} -+ -+static inline void out_be32(void *addr, u32 val) -+{ -+ *((u32 *) addr) = cpu_to_be32(val); -+} -+ -+ -+static inline void set_bits(unsigned long mask, volatile unsigned long *p) -+{ -+ *p |= mask; -+} -+static inline void clear_bits(unsigned long mask, volatile unsigned long *p) -+{ -+ *p &= ~mask; -+} -+ -+static inline void flush_dcache_range(unsigned long start, unsigned long stop) -+{ -+ __cpuc_flush_dcache_area((void *) start, stop - start); -+} -+ -+#define hard_smp_processor_id() raw_smp_processor_id() -+#endif ---- /dev/null -+++ b/drivers/staging/fsl_qbman/dpa_sys_arm64.h -@@ -0,0 +1,102 @@ -+/* Copyright 2014 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef DPA_SYS_ARM64_H -+#define DPA_SYS_ARM64_H -+ -+#include <asm/cacheflush.h> -+#include <asm/barrier.h> -+ -+/* Implementation of ARM 64 bit specific routines */ -+ -+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler -+ * barriers and that dcb*() won't fall victim to compiler or execution -+ * reordering with respect to other code/instructions that manipulate the same -+ * cacheline. */ -+#define hwsync() { asm volatile("dmb st" : : : "memory"); } -+#define lwsync() { asm volatile("dmb st" : : : "memory"); } -+#define dcbf(p) { asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); } -+#define dcbt_ro(p) { asm volatile("prfm pldl1keep, [%0, #0]" : : "r" (p)); } -+#define dcbt_rw(p) { asm volatile("prfm pstl1keep, [%0, #0]" : : "r" (p)); } -+#define dcbi(p) { asm volatile("dc ivac, %0" : : "r"(p) : "memory"); } -+#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); } -+ -+#define dcbz_64(p) \ -+ do { \ -+ dcbz(p); \ -+ } while (0) -+ -+#define dcbf_64(p) \ -+ do { \ -+ dcbf(p); \ -+ } while (0) -+/* Commonly used combo */ -+#define dcbit_ro(p) \ -+ do { \ -+ dcbi(p); \ -+ dcbt_ro(p); \ -+ } while (0) -+ -+static inline u64 mfatb(void) -+{ -+ return get_cycles(); -+} -+ -+static inline u32 in_be32(volatile void *addr) -+{ -+ return be32_to_cpu(*((volatile u32 *) addr)); -+} -+ -+static inline void out_be32(void *addr, u32 val) -+{ -+ *((u32 *) addr) = cpu_to_be32(val); -+} -+ -+ -+static inline void set_bits(unsigned long mask, volatile unsigned long *p) -+{ -+ *p |= mask; -+} -+static inline void clear_bits(unsigned long mask, volatile unsigned long *p) -+{ -+ *p &= ~mask; -+} -+ -+static inline void flush_dcache_range(unsigned long start, unsigned long stop) -+{ -+ __flush_dcache_area((void *) start, stop - start); -+} -+ -+#define hard_smp_processor_id() raw_smp_processor_id() -+ -+ -+ -+#endif ---- /dev/null -+++ b/drivers/staging/fsl_qbman/dpa_sys_ppc32.h -@@ -0,0 +1,70 @@ -+/* Copyright 2014 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef DPA_SYS_PPC32_H -+#define DPA_SYS_PPC32_H -+ -+/* Implementation of PowerPC 32 bit specific routines */ -+ -+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler -+ * barriers and that dcb*() won't fall victim to compiler or execution -+ * reordering with respect to other code/instructions that manipulate the same -+ * cacheline. */ -+#define hwsync() __asm__ __volatile__ ("sync" : : : "memory") -+#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory") -+#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory") -+#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p)) -+#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p)) -+#define dcbi(p) dcbf(p) -+ -+#define dcbzl(p) __asm__ __volatile__ ("dcbzl 0,%0" : : "r" (p)) -+#define dcbz_64(p) dcbzl(p) -+#define dcbf_64(p) dcbf(p) -+ -+/* Commonly used combo */ -+#define dcbit_ro(p) \ -+ do { \ -+ dcbi(p); \ -+ dcbt_ro(p); \ -+ } while (0) -+ -+static inline u64 mfatb(void) -+{ -+ u32 hi, lo, chk; -+ do { -+ hi = mfspr(SPRN_ATBU); -+ lo = mfspr(SPRN_ATBL); -+ chk = mfspr(SPRN_ATBU); -+ } while (unlikely(hi != chk)); -+ return ((u64)hi << 32) | (u64)lo; -+} -+ -+#endif ---- /dev/null -+++ b/drivers/staging/fsl_qbman/dpa_sys_ppc64.h -@@ -0,0 +1,79 @@ -+/* Copyright 2014 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef DPA_SYS_PPC64_H -+#define DPA_SYS_PPC64_H -+ -+/* Implementation of PowerPC 64 bit specific routines */ -+ -+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler -+ * barriers and that dcb*() won't fall victim to compiler or execution -+ * reordering with respect to other code/instructions that manipulate the same -+ * cacheline. */ -+#define hwsync() __asm__ __volatile__ ("sync" : : : "memory") -+#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory") -+#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory") -+#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p)) -+#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p)) -+#define dcbi(p) dcbf(p) -+ -+#define dcbz(p) __asm__ __volatile__ ("dcbz 0,%0" : : "r" (p)) -+#define dcbz_64(p) \ -+ do { \ -+ dcbz((void*)p + 32); \ -+ dcbz(p); \ -+ } while (0) -+#define dcbf_64(p) \ -+ do { \ -+ dcbf((void*)p + 32); \ -+ dcbf(p); \ -+ } while (0) -+/* Commonly used combo */ -+#define dcbit_ro(p) \ -+ do { \ -+ dcbi(p); \ -+ dcbi((void*)p + 32); \ -+ dcbt_ro(p); \ -+ dcbt_ro((void*)p + 32); \ -+ } while (0) -+ -+static inline u64 mfatb(void) -+{ -+ u32 hi, lo, chk; -+ do { -+ hi = mfspr(SPRN_ATBU); -+ lo = mfspr(SPRN_ATBL); -+ chk = mfspr(SPRN_ATBU); -+ } while (unlikely(hi != chk)); -+ return ((u64)hi << 32) | (u64)lo; -+} -+ -+#endif ---- /dev/null -+++ b/drivers/staging/fsl_qbman/fsl_usdpaa.c -@@ -0,0 +1,1984 @@ -+/* Copyright (C) 2008-2012 Freescale Semiconductor, Inc. -+ * Authors: Andy Fleming <afleming@freescale.com> -+ * Timur Tabi <timur@freescale.com> -+ * Geoff Thorpe <Geoff.Thorpe@freescale.com> -+ * -+ * This file is licensed under the terms of the GNU General Public License -+ * version 2. This program is licensed "as is" without any warranty of any -+ * kind, whether express or implied. -+ */ -+ -+ -+#include <linux/miscdevice.h> -+#include <linux/fs.h> -+#include <linux/cdev.h> -+#include <linux/mm.h> -+#include <linux/of.h> -+#include <linux/memblock.h> -+#include <linux/slab.h> -+#include <linux/mman.h> -+#include <linux/of_reserved_mem.h> -+ -+#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64)) -+#include <mm/mmu_decl.h> -+#endif -+ -+#include "dpa_sys.h" -+#include <linux/fsl_usdpaa.h> -+#include "bman_low.h" -+#include "qman_low.h" -+ -+/* Physical address range of the memory reservation, exported for mm/mem.c */ -+static u64 phys_start; -+static u64 phys_size; -+static u64 arg_phys_size; -+ -+/* PFN versions of the above */ -+static unsigned long pfn_start; -+static unsigned long pfn_size; -+ -+/* Memory reservations are manipulated under this spinlock (which is why 'refs' -+ * isn't atomic_t). */ -+static DEFINE_SPINLOCK(mem_lock); -+ -+/* The range of TLB1 indices */ -+static unsigned int first_tlb; -+static unsigned int num_tlb = 1; -+static unsigned int current_tlb; /* loops around for fault handling */ -+ -+/* Memory reservation is represented as a list of 'mem_fragment's, some of which -+ * may be mapped. Unmapped fragments are always merged where possible. */ -+static LIST_HEAD(mem_list); -+ -+struct mem_mapping; -+ -+/* Memory fragments are in 'mem_list'. */ -+struct mem_fragment { -+ u64 base; -+ u64 len; -+ unsigned long pfn_base; /* PFN version of 'base' */ -+ unsigned long pfn_len; /* PFN version of 'len' */ -+ unsigned int refs; /* zero if unmapped */ -+ u64 root_len; /* Size of the orignal fragment */ -+ unsigned long root_pfn; /* PFN of the orignal fragment */ -+ struct list_head list; -+ /* if mapped, flags+name captured at creation time */ -+ u32 flags; -+ char name[USDPAA_DMA_NAME_MAX]; -+ u64 map_len; -+ /* support multi-process locks per-memory-fragment. */ -+ int has_locking; -+ wait_queue_head_t wq; -+ struct mem_mapping *owner; -+}; -+ -+/* Mappings of memory fragments in 'struct ctx'. These are created from -+ * ioctl(USDPAA_IOCTL_DMA_MAP), though the actual mapping then happens via a -+ * mmap(). */ -+struct mem_mapping { -+ struct mem_fragment *root_frag; -+ u32 frag_count; -+ u64 total_size; -+ struct list_head list; -+ int refs; -+ void *virt_addr; -+}; -+ -+struct portal_mapping { -+ struct usdpaa_ioctl_portal_map user; -+ union { -+ struct qm_portal_config *qportal; -+ struct bm_portal_config *bportal; -+ }; -+ /* Declare space for the portals in case the process -+ exits unexpectedly and needs to be cleaned by the kernel */ -+ union { -+ struct qm_portal qman_portal_low; -+ struct bm_portal bman_portal_low; -+ }; -+ struct list_head list; -+ struct resource *phys; -+ struct iommu_domain *iommu_domain; -+}; -+ -+/* Track the DPAA resources the process is using */ -+struct active_resource { -+ struct list_head list; -+ u32 id; -+ u32 num; -+ unsigned int refcount; -+}; -+ -+/* Per-FD state (which should also be per-process but we don't enforce that) */ -+struct ctx { -+ /* Lock to protect the context */ -+ spinlock_t lock; -+ /* Allocated resources get put here for accounting */ -+ struct list_head resources[usdpaa_id_max]; -+ /* list of DMA maps */ -+ struct list_head maps; -+ /* list of portal maps */ -+ struct list_head portals; -+}; -+ -+/* Different resource classes */ -+static const struct alloc_backend { -+ enum usdpaa_id_type id_type; -+ int (*alloc)(u32 *, u32, u32, int); -+ void (*release)(u32 base, unsigned int count); -+ int (*reserve)(u32 base, unsigned int count); -+ const char *acronym; -+} alloc_backends[] = { -+ { -+ .id_type = usdpaa_id_fqid, -+ .alloc = qman_alloc_fqid_range, -+ .release = qman_release_fqid_range, -+ .reserve = qman_reserve_fqid_range, -+ .acronym = "FQID" -+ }, -+ { -+ .id_type = usdpaa_id_bpid, -+ .alloc = bman_alloc_bpid_range, -+ .release = bman_release_bpid_range, -+ .reserve = bman_reserve_bpid_range, -+ .acronym = "BPID" -+ }, -+ { -+ .id_type = usdpaa_id_qpool, -+ .alloc = qman_alloc_pool_range, -+ .release = qman_release_pool_range, -+ .reserve = qman_reserve_pool_range, -+ .acronym = "QPOOL" -+ }, -+ { -+ .id_type = usdpaa_id_cgrid, -+ .alloc = qman_alloc_cgrid_range, -+ .release = qman_release_cgrid_range, -+ .acronym = "CGRID" -+ }, -+ { -+ .id_type = usdpaa_id_ceetm0_lfqid, -+ .alloc = qman_alloc_ceetm0_lfqid_range, -+ .release = qman_release_ceetm0_lfqid_range, -+ .acronym = "CEETM0_LFQID" -+ }, -+ { -+ .id_type = usdpaa_id_ceetm0_channelid, -+ .alloc = qman_alloc_ceetm0_channel_range, -+ .release = qman_release_ceetm0_channel_range, -+ .acronym = "CEETM0_LFQID" -+ }, -+ { -+ .id_type = usdpaa_id_ceetm1_lfqid, -+ .alloc = qman_alloc_ceetm1_lfqid_range, -+ .release = qman_release_ceetm1_lfqid_range, -+ .acronym = "CEETM1_LFQID" -+ }, -+ { -+ .id_type = usdpaa_id_ceetm1_channelid, -+ .alloc = qman_alloc_ceetm1_channel_range, -+ .release = qman_release_ceetm1_channel_range, -+ .acronym = "CEETM1_LFQID" -+ }, -+ { -+ /* This terminates the array */ -+ .id_type = usdpaa_id_max -+ } -+}; -+ -+/* Determines the largest acceptable page size for a given size -+ The sizes are determined by what the TLB1 acceptable page sizes are */ -+static u32 largest_page_size(u32 size) -+{ -+ int shift = 30; /* Start at 1G size */ -+ if (size < 4096) -+ return 0; -+ do { -+ if (size >= (1<<shift)) -+ return 1<<shift; -+ shift -= 2; -+ } while (shift >= 12); /* Up to 4k */ -+ return 0; -+} -+ -+/* Determine if value is power of 4 */ -+static inline bool is_power_of_4(u64 x) -+{ -+ if (x == 0 || ((x & (x - 1)) != 0)) -+ return false; -+ return !!(x & 0x5555555555555555ull); -+} -+ -+/* Helper for ioctl_dma_map() when we have a larger fragment than we need. This -+ * splits the fragment into 4 and returns the upper-most. (The caller can loop -+ * until it has a suitable fragment size.) */ -+static struct mem_fragment *split_frag(struct mem_fragment *frag) -+{ -+ struct mem_fragment *x[3]; -+ -+ x[0] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC); -+ x[1] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC); -+ x[2] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC); -+ if (!x[0] || !x[1] || !x[2]) { -+ kfree(x[0]); -+ kfree(x[1]); -+ kfree(x[2]); -+ return NULL; -+ } -+ BUG_ON(frag->refs); -+ frag->len >>= 2; -+ frag->pfn_len >>= 2; -+ x[0]->base = frag->base + frag->len; -+ x[1]->base = x[0]->base + frag->len; -+ x[2]->base = x[1]->base + frag->len; -+ x[0]->len = x[1]->len = x[2]->len = frag->len; -+ x[0]->pfn_base = frag->pfn_base + frag->pfn_len; -+ x[1]->pfn_base = x[0]->pfn_base + frag->pfn_len; -+ x[2]->pfn_base = x[1]->pfn_base + frag->pfn_len; -+ x[0]->pfn_len = x[1]->pfn_len = x[2]->pfn_len = frag->pfn_len; -+ x[0]->refs = x[1]->refs = x[2]->refs = 0; -+ x[0]->root_len = x[1]->root_len = x[2]->root_len = frag->root_len; -+ x[0]->root_pfn = x[1]->root_pfn = x[2]->root_pfn = frag->root_pfn; -+ x[0]->name[0] = x[1]->name[0] = x[2]->name[0] = 0; -+ list_add_tail(&x[0]->list, &frag->list); -+ list_add_tail(&x[1]->list, &x[0]->list); -+ list_add_tail(&x[2]->list, &x[1]->list); -+ return x[2]; -+} -+ -+static __maybe_unused void dump_frags(void) -+{ -+ struct mem_fragment *frag; -+ int i = 0; -+ list_for_each_entry(frag, &mem_list, list) { -+ pr_info("FRAG %d: base 0x%llx pfn_base 0x%lx len 0x%llx root_len 0x%llx root_pfn 0x%lx refs %d name %s\n", -+ i, frag->base, frag->pfn_base, -+ frag->len, frag->root_len, frag->root_pfn, -+ frag->refs, frag->name); -+ ++i; -+ } -+} -+ -+/* Walk the list of fragments and adjoin neighbouring segments if possible */ -+static void compress_frags(void) -+{ -+ /* Walk the fragment list and combine fragments */ -+ struct mem_fragment *frag, *nxtfrag; -+ u64 len = 0; -+ -+ int i, numfrags; -+ -+ -+ frag = list_entry(mem_list.next, struct mem_fragment, list); -+ -+ while (&frag->list != &mem_list) { -+ /* Must combine consecutive fragemenst with -+ same root_pfn such that they are power of 4 */ -+ if (frag->refs != 0) { -+ frag = list_entry(frag->list.next, -+ struct mem_fragment, list); -+ continue; /* Not this window */ -+ } -+ len = frag->len; -+ numfrags = 0; -+ nxtfrag = list_entry(frag->list.next, -+ struct mem_fragment, list); -+ while (true) { -+ if (&nxtfrag->list == &mem_list) { -+ numfrags = 0; -+ break; /* End of list */ -+ } -+ if (nxtfrag->refs) { -+ numfrags = 0; -+ break; /* In use still */ -+ } -+ if (nxtfrag->root_pfn != frag->root_pfn) { -+ numfrags = 0; -+ break; /* Crosses root fragment boundary */ -+ } -+ len += nxtfrag->len; -+ numfrags++; -+ if (is_power_of_4(len)) { -+ /* These fragments can be combined */ -+ break; -+ } -+ nxtfrag = list_entry(nxtfrag->list.next, -+ struct mem_fragment, list); -+ } -+ if (numfrags == 0) { -+ frag = list_entry(frag->list.next, -+ struct mem_fragment, list); -+ continue; /* try the next window */ -+ } -+ for (i = 0; i < numfrags; i++) { -+ struct mem_fragment *todel = -+ list_entry(nxtfrag->list.prev, -+ struct mem_fragment, list); -+ nxtfrag->len += todel->len; -+ nxtfrag->pfn_len += todel->pfn_len; -+ list_del(&todel->list); -+ } -+ /* Re evaluate the list, things may merge now */ -+ frag = list_entry(mem_list.next, struct mem_fragment, list); -+ } -+} -+ -+/* Hook from arch/powerpc/mm/mem.c */ -+int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size) -+{ -+ struct mem_fragment *frag; -+ int idx = -1; -+ if ((pfn < pfn_start) || (pfn >= (pfn_start + pfn_size))) -+ return -1; -+ /* It's in-range, we need to find the fragment */ -+ spin_lock(&mem_lock); -+ list_for_each_entry(frag, &mem_list, list) { -+ if ((pfn >= frag->pfn_base) && (pfn < (frag->pfn_base + -+ frag->pfn_len))) { -+ *phys_addr = frag->base; -+ *size = frag->len; -+ idx = current_tlb++; -+ if (current_tlb >= (first_tlb + num_tlb)) -+ current_tlb = first_tlb; -+ break; -+ } -+ } -+ spin_unlock(&mem_lock); -+ return idx; -+} -+ -+static int usdpaa_open(struct inode *inode, struct file *filp) -+{ -+ const struct alloc_backend *backend = &alloc_backends[0]; -+ struct ctx *ctx = kmalloc(sizeof(struct ctx), GFP_KERNEL); -+ if (!ctx) -+ return -ENOMEM; -+ filp->private_data = ctx; -+ -+ while (backend->id_type != usdpaa_id_max) { -+ INIT_LIST_HEAD(&ctx->resources[backend->id_type]); -+ backend++; -+ } -+ -+ INIT_LIST_HEAD(&ctx->maps); -+ INIT_LIST_HEAD(&ctx->portals); -+ spin_lock_init(&ctx->lock); -+ -+ //filp->f_mapping->backing_dev_info = &directly_mappable_cdev_bdi; -+ -+ return 0; -+} -+ -+#define DQRR_MAXFILL 15 -+ -+/* Reset a QMan portal to its default state */ -+static int init_qm_portal(struct qm_portal_config *config, -+ struct qm_portal *portal) -+{ -+ const struct qm_dqrr_entry *dqrr = NULL; -+ int i; -+ -+ portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; -+ portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; -+ -+ /* Make sure interrupts are inhibited */ -+ qm_out(IIR, 1); -+ -+ /* Initialize the DQRR. This will stop any dequeue -+ commands that are in progress */ -+ if (qm_dqrr_init(portal, config, qm_dqrr_dpush, qm_dqrr_pvb, -+ qm_dqrr_cdc, DQRR_MAXFILL)) { -+ pr_err("qm_dqrr_init() failed when trying to" -+ " recover portal, portal will be leaked\n"); -+ return 1; -+ } -+ -+ /* Discard any entries on the DQRR */ -+ /* If we consume the ring twice something is wrong */ -+ for (i = 0; i < DQRR_MAXFILL * 2; i++) { -+ qm_dqrr_pvb_update(portal); -+ dqrr = qm_dqrr_current(portal); -+ if (!dqrr) -+ break; -+ qm_dqrr_cdc_consume_1ptr(portal, dqrr, 0); -+ qm_dqrr_pvb_update(portal); -+ qm_dqrr_next(portal); -+ } -+ /* Initialize the EQCR */ -+ if (qm_eqcr_init(portal, qm_eqcr_pvb, -+ qm_eqcr_get_ci_stashing(portal), 1)) { -+ pr_err("Qman EQCR initialisation failed\n"); -+ return 1; -+ } -+ /* initialize the MR */ -+ if (qm_mr_init(portal, qm_mr_pvb, qm_mr_cci)) { -+ pr_err("Qman MR initialisation failed\n"); -+ return 1; -+ } -+ qm_mr_pvb_update(portal); -+ while (qm_mr_current(portal)) { -+ qm_mr_next(portal); -+ qm_mr_cci_consume_to_current(portal); -+ qm_mr_pvb_update(portal); -+ } -+ -+ if (qm_mc_init(portal)) { -+ pr_err("Qman MC initialisation failed\n"); -+ return 1; -+ } -+ return 0; -+} -+ -+static int init_bm_portal(struct bm_portal_config *config, -+ struct bm_portal *portal) -+{ -+ portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; -+ portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; -+ -+ if (bm_rcr_init(portal, bm_rcr_pvb, bm_rcr_cce)) { -+ pr_err("Bman RCR initialisation failed\n"); -+ return 1; -+ } -+ if (bm_mc_init(portal)) { -+ pr_err("Bman MC initialisation failed\n"); -+ return 1; -+ } -+ return 0; -+} -+ -+/* Function that will scan all FQ's in the system. For each FQ that is not -+ OOS it will call the check_channel helper to determine if the FQ should -+ be torn down. If the check_channel helper returns true the FQ will be -+ transitioned to the OOS state */ -+static int qm_check_and_destroy_fqs(struct qm_portal *portal, void *ctx, -+ bool (*check_channel)(void*, u32)) -+{ -+ u32 fq_id = 0; -+ while (1) { -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ u8 state; -+ u32 channel; -+ -+ /* Determine the channel for the FQID */ -+ mcc = qm_mc_start(portal); -+ mcc->queryfq.fqid = fq_id; -+ qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ); -+ while (!(mcr = qm_mc_result(portal))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) -+ == QM_MCR_VERB_QUERYFQ); -+ if (mcr->result != QM_MCR_RESULT_OK) -+ break; /* End of valid FQIDs */ -+ -+ channel = mcr->queryfq.fqd.dest.channel; -+ /* Determine the state of the FQID */ -+ mcc = qm_mc_start(portal); -+ mcc->queryfq_np.fqid = fq_id; -+ qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ_NP); -+ while (!(mcr = qm_mc_result(portal))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) -+ == QM_MCR_VERB_QUERYFQ_NP); -+ state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; -+ if (state == QM_MCR_NP_STATE_OOS) -+ /* Already OOS, no need to do anymore checks */ -+ goto next; -+ -+ if (check_channel(ctx, channel)) -+ qm_shutdown_fq(&portal, 1, fq_id); -+ next: -+ ++fq_id; -+ } -+ return 0; -+} -+ -+static bool check_channel_device(void *_ctx, u32 channel) -+{ -+ struct ctx *ctx = _ctx; -+ struct portal_mapping *portal, *tmpportal; -+ struct active_resource *res; -+ -+ /* See if the FQ is destined for one of the portals we're cleaning up */ -+ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) { -+ if (portal->user.type == usdpaa_portal_qman) { -+ if (portal->qportal->public_cfg.channel == channel) { -+ /* This FQs destination is a portal -+ we're cleaning, send a retire */ -+ return true; -+ } -+ } -+ } -+ -+ /* Check the pool channels that will be released as well */ -+ list_for_each_entry(res, &ctx->resources[usdpaa_id_qpool], list) { -+ if ((res->id >= channel) && -+ ((res->id + res->num - 1) <= channel)) -+ return true; -+ } -+ return false; -+} -+ -+static bool check_portal_channel(void *ctx, u32 channel) -+{ -+ u32 portal_channel = *(u32 *)ctx; -+ if (portal_channel == channel) { -+ /* This FQs destination is a portal -+ we're cleaning, send a retire */ -+ return true; -+ } -+ return false; -+} -+ -+ -+ -+ -+static int usdpaa_release(struct inode *inode, struct file *filp) -+{ -+ struct ctx *ctx = filp->private_data; -+ struct mem_mapping *map, *tmpmap; -+ struct portal_mapping *portal, *tmpportal; -+ const struct alloc_backend *backend = &alloc_backends[0]; -+ struct active_resource *res; -+ struct qm_portal *qm_cleanup_portal = NULL; -+ struct bm_portal *bm_cleanup_portal = NULL; -+ struct qm_portal_config *qm_alloced_portal = NULL; -+ struct bm_portal_config *bm_alloced_portal = NULL; -+ -+ struct qm_portal *portal_array[qman_portal_max]; -+ int portal_count = 0; -+ -+ /* Ensure the release operation cannot be migrated to another -+ CPU as CPU specific variables may be needed during cleanup */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_disable(); -+#endif -+ /* The following logic is used to recover resources that were not -+ correctly released by the process that is closing the FD. -+ Step 1: syncronize the HW with the qm_portal/bm_portal structures -+ in the kernel -+ */ -+ -+ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) { -+ /* Try to recover any portals that weren't shut down */ -+ if (portal->user.type == usdpaa_portal_qman) { -+ portal_array[portal_count] = &portal->qman_portal_low; -+ ++portal_count; -+ init_qm_portal(portal->qportal, -+ &portal->qman_portal_low); -+ if (!qm_cleanup_portal) { -+ qm_cleanup_portal = &portal->qman_portal_low; -+ } else { -+ /* Clean FQs on the dedicated channel */ -+ u32 chan = portal->qportal->public_cfg.channel; -+ qm_check_and_destroy_fqs( -+ &portal->qman_portal_low, &chan, -+ check_portal_channel); -+ } -+ } else { -+ /* BMAN */ -+ init_bm_portal(portal->bportal, -+ &portal->bman_portal_low); -+ if (!bm_cleanup_portal) -+ bm_cleanup_portal = &portal->bman_portal_low; -+ } -+ } -+ /* If no portal was found, allocate one for cleanup */ -+ if (!qm_cleanup_portal) { -+ qm_alloced_portal = qm_get_unused_portal(); -+ if (!qm_alloced_portal) { -+ pr_crit("No QMan portal avalaible for cleanup\n"); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_enable(); -+#endif -+ return -1; -+ } -+ qm_cleanup_portal = kmalloc(sizeof(struct qm_portal), -+ GFP_KERNEL); -+ if (!qm_cleanup_portal) { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_enable(); -+#endif -+ return -ENOMEM; -+ } -+ init_qm_portal(qm_alloced_portal, qm_cleanup_portal); -+ portal_array[portal_count] = qm_cleanup_portal; -+ ++portal_count; -+ } -+ if (!bm_cleanup_portal) { -+ bm_alloced_portal = bm_get_unused_portal(); -+ if (!bm_alloced_portal) { -+ pr_crit("No BMan portal avalaible for cleanup\n"); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_enable(); -+#endif -+ return -1; -+ } -+ bm_cleanup_portal = kmalloc(sizeof(struct bm_portal), -+ GFP_KERNEL); -+ if (!bm_cleanup_portal) { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_enable(); -+#endif -+ return -ENOMEM; -+ } -+ init_bm_portal(bm_alloced_portal, bm_cleanup_portal); -+ } -+ -+ /* OOS the FQs associated with this process */ -+ qm_check_and_destroy_fqs(qm_cleanup_portal, ctx, check_channel_device); -+ -+ while (backend->id_type != usdpaa_id_max) { -+ int leaks = 0; -+ list_for_each_entry(res, &ctx->resources[backend->id_type], -+ list) { -+ if (backend->id_type == usdpaa_id_fqid) { -+ int i = 0; -+ for (; i < res->num; i++) { -+ /* Clean FQs with the cleanup portal */ -+ qm_shutdown_fq(portal_array, -+ portal_count, -+ res->id + i); -+ } -+ } -+ leaks += res->num; -+ backend->release(res->id, res->num); -+ } -+ if (leaks) -+ pr_crit("USDPAA process leaking %d %s%s\n", leaks, -+ backend->acronym, (leaks > 1) ? "s" : ""); -+ backend++; -+ } -+ /* Release any DMA regions */ -+ spin_lock(&mem_lock); -+ list_for_each_entry_safe(map, tmpmap, &ctx->maps, list) { -+ struct mem_fragment *current_frag = map->root_frag; -+ int i; -+ if (map->root_frag->has_locking && -+ (map->root_frag->owner == map)) { -+ map->root_frag->owner = NULL; -+ wake_up(&map->root_frag->wq); -+ } -+ /* Check each fragment and merge if the ref count is 0 */ -+ for (i = 0; i < map->frag_count; i++) { -+ --current_frag->refs; -+ current_frag = list_entry(current_frag->list.prev, -+ struct mem_fragment, list); -+ } -+ -+ compress_frags(); -+ list_del(&map->list); -+ kfree(map); -+ } -+ spin_unlock(&mem_lock); -+ -+ /* Return portals */ -+ list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) { -+ if (portal->user.type == usdpaa_portal_qman) { -+ /* Give the portal back to the allocator */ -+ init_qm_portal(portal->qportal, -+ &portal->qman_portal_low); -+ qm_put_unused_portal(portal->qportal); -+ } else { -+ init_bm_portal(portal->bportal, -+ &portal->bman_portal_low); -+ bm_put_unused_portal(portal->bportal); -+ } -+ list_del(&portal->list); -+ kfree(portal); -+ } -+ if (qm_alloced_portal) { -+ qm_put_unused_portal(qm_alloced_portal); -+ kfree(qm_cleanup_portal); -+ } -+ if (bm_alloced_portal) { -+ bm_put_unused_portal(bm_alloced_portal); -+ kfree(bm_cleanup_portal); -+ } -+ -+ kfree(ctx); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_enable(); -+#endif -+ return 0; -+} -+ -+static int check_mmap_dma(struct ctx *ctx, struct vm_area_struct *vma, -+ int *match, unsigned long *pfn) -+{ -+ struct mem_mapping *map; -+ -+ list_for_each_entry(map, &ctx->maps, list) { -+ int i; -+ struct mem_fragment *frag = map->root_frag; -+ -+ for (i = 0; i < map->frag_count; i++) { -+ if (frag->pfn_base == vma->vm_pgoff) { -+ *match = 1; -+ *pfn = frag->pfn_base; -+ return 0; -+ } -+ frag = list_entry(frag->list.next, struct mem_fragment, -+ list); -+ } -+ } -+ *match = 0; -+ return 0; -+} -+ -+static int check_mmap_resource(struct resource *res, struct vm_area_struct *vma, -+ int *match, unsigned long *pfn) -+{ -+ *pfn = res->start >> PAGE_SHIFT; -+ if (*pfn == vma->vm_pgoff) { -+ *match = 1; -+ if ((vma->vm_end - vma->vm_start) != resource_size(res)) -+ return -EINVAL; -+ } else -+ *match = 0; -+ return 0; -+} -+ -+static int check_mmap_portal(struct ctx *ctx, struct vm_area_struct *vma, -+ int *match, unsigned long *pfn) -+{ -+ struct portal_mapping *portal; -+ int ret; -+ -+ list_for_each_entry(portal, &ctx->portals, list) { -+ ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CE], vma, -+ match, pfn); -+ if (*match) { -+ vma->vm_page_prot = -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ pgprot_cached_ns(vma->vm_page_prot); -+#else -+ pgprot_cached_noncoherent(vma->vm_page_prot); -+#endif -+ return ret; -+ } -+ ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CI], vma, -+ match, pfn); -+ if (*match) { -+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); -+ return ret; -+ } -+ } -+ *match = 0; -+ return 0; -+} -+ -+static int usdpaa_mmap(struct file *filp, struct vm_area_struct *vma) -+{ -+ struct ctx *ctx = filp->private_data; -+ unsigned long pfn = 0; -+ int match, ret; -+ -+ spin_lock(&mem_lock); -+ ret = check_mmap_dma(ctx, vma, &match, &pfn); -+ if (!match) -+ ret = check_mmap_portal(ctx, vma, &match, &pfn); -+ spin_unlock(&mem_lock); -+ if (!match) -+ return -EINVAL; -+ if (!ret) -+ ret = remap_pfn_range(vma, vma->vm_start, pfn, -+ vma->vm_end - vma->vm_start, -+ vma->vm_page_prot); -+ return ret; -+} -+ -+/* Return the nearest rounded-up address >= 'addr' that is 'sz'-aligned. 'sz' -+ * must be a power of 2, but both 'addr' and 'sz' can be expressions. */ -+#define USDPAA_MEM_ROUNDUP(addr, sz) \ -+ ({ \ -+ unsigned long foo_align = (sz) - 1; \ -+ ((addr) + foo_align) & ~foo_align; \ -+ }) -+/* Searching for a size-aligned virtual address range starting from 'addr' */ -+static unsigned long usdpaa_get_unmapped_area(struct file *file, -+ unsigned long addr, -+ unsigned long len, -+ unsigned long pgoff, -+ unsigned long flags) -+{ -+ struct vm_area_struct *vma; -+ -+ if (len % PAGE_SIZE) -+ return -EINVAL; -+ if (!len) -+ return -EINVAL; -+ -+ /* Need to align the address to the largest pagesize of the mapping -+ * because the MMU requires the virtual address to have the same -+ * alignment as the physical address */ -+ addr = USDPAA_MEM_ROUNDUP(addr, largest_page_size(len)); -+ vma = find_vma(current->mm, addr); -+ /* Keep searching until we reach the end of currently-used virtual -+ * address-space or we find a big enough gap. */ -+ while (vma) { -+ if ((addr + len) < vma->vm_start) -+ return addr; -+ -+ addr = USDPAA_MEM_ROUNDUP(vma->vm_end, largest_page_size(len)); -+ vma = vma->vm_next; -+ } -+ if ((TASK_SIZE - len) < addr) -+ return -ENOMEM; -+ return addr; -+} -+ -+static long ioctl_id_alloc(struct ctx *ctx, void __user *arg) -+{ -+ struct usdpaa_ioctl_id_alloc i; -+ const struct alloc_backend *backend; -+ struct active_resource *res; -+ int ret = copy_from_user(&i, arg, sizeof(i)); -+ if (ret) -+ return ret; -+ if ((i.id_type >= usdpaa_id_max) || !i.num) -+ return -EINVAL; -+ backend = &alloc_backends[i.id_type]; -+ /* Allocate the required resource type */ -+ ret = backend->alloc(&i.base, i.num, i.align, i.partial); -+ if (ret < 0) -+ return ret; -+ i.num = ret; -+ /* Copy the result to user-space */ -+ ret = copy_to_user(arg, &i, sizeof(i)); -+ if (ret) { -+ backend->release(i.base, i.num); -+ return ret; -+ } -+ /* Assign the allocated range to the FD accounting */ -+ res = kmalloc(sizeof(*res), GFP_KERNEL); -+ if (!res) { -+ backend->release(i.base, i.num); -+ return -ENOMEM; -+ } -+ spin_lock(&ctx->lock); -+ res->id = i.base; -+ res->num = i.num; -+ res->refcount = 1; -+ list_add(&res->list, &ctx->resources[i.id_type]); -+ spin_unlock(&ctx->lock); -+ return 0; -+} -+ -+static long ioctl_id_release(struct ctx *ctx, void __user *arg) -+{ -+ struct usdpaa_ioctl_id_release i; -+ const struct alloc_backend *backend; -+ struct active_resource *tmp, *pos; -+ -+ int ret = copy_from_user(&i, arg, sizeof(i)); -+ if (ret) -+ return ret; -+ if ((i.id_type >= usdpaa_id_max) || !i.num) -+ return -EINVAL; -+ backend = &alloc_backends[i.id_type]; -+ /* Pull the range out of the FD accounting - the range is valid iff this -+ * succeeds. */ -+ spin_lock(&ctx->lock); -+ list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) { -+ if (pos->id == i.base && pos->num == i.num) { -+ pos->refcount--; -+ if (pos->refcount) { -+ spin_unlock(&ctx->lock); -+ return 0; /* Still being used */ -+ } -+ list_del(&pos->list); -+ kfree(pos); -+ spin_unlock(&ctx->lock); -+ goto found; -+ } -+ } -+ /* Failed to find the resource */ -+ spin_unlock(&ctx->lock); -+ pr_err("Couldn't find resource type %d base 0x%x num %d\n", -+ i.id_type, i.base, i.num); -+ return -EINVAL; -+found: -+ /* Release the resource to the backend */ -+ backend->release(i.base, i.num); -+ return 0; -+} -+ -+static long ioctl_id_reserve(struct ctx *ctx, void __user *arg) -+{ -+ struct usdpaa_ioctl_id_reserve i; -+ const struct alloc_backend *backend; -+ struct active_resource *tmp, *pos; -+ -+ int ret = copy_from_user(&i, arg, sizeof(i)); -+ if (ret) -+ return ret; -+ if ((i.id_type >= usdpaa_id_max) || !i.num) -+ return -EINVAL; -+ backend = &alloc_backends[i.id_type]; -+ if (!backend->reserve) -+ return -EINVAL; -+ /* Pull the range out of the FD accounting - the range is valid iff this -+ * succeeds. */ -+ spin_lock(&ctx->lock); -+ list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) { -+ if (pos->id == i.base && pos->num == i.num) { -+ pos->refcount++; -+ spin_unlock(&ctx->lock); -+ return 0; -+ } -+ } -+ -+ /* Failed to find the resource */ -+ spin_unlock(&ctx->lock); -+ -+ /* Reserve the resource in the backend */ -+ ret = backend->reserve(i.base, i.num); -+ if (ret) -+ return ret; -+ /* Assign the reserved range to the FD accounting */ -+ pos = kmalloc(sizeof(*pos), GFP_KERNEL); -+ if (!pos) { -+ backend->release(i.base, i.num); -+ return -ENOMEM; -+ } -+ spin_lock(&ctx->lock); -+ pos->id = i.base; -+ pos->num = i.num; -+ pos->refcount = 1; -+ list_add(&pos->list, &ctx->resources[i.id_type]); -+ spin_unlock(&ctx->lock); -+ return 0; -+} -+ -+static long ioctl_dma_map(struct file *fp, struct ctx *ctx, -+ struct usdpaa_ioctl_dma_map *i) -+{ -+ struct mem_fragment *frag, *start_frag, *next_frag; -+ struct mem_mapping *map, *tmp; -+ int ret = 0; -+ u32 largest_page, so_far = 0; -+ int frag_count = 0; -+ unsigned long next_addr = PAGE_SIZE, populate; -+ -+ /* error checking to ensure values copied from user space are valid */ -+ if (i->len % PAGE_SIZE) -+ return -EINVAL; -+ -+ map = kmalloc(sizeof(*map), GFP_KERNEL); -+ if (!map) -+ return -ENOMEM; -+ -+ spin_lock(&mem_lock); -+ if (i->flags & USDPAA_DMA_FLAG_SHARE) { -+ list_for_each_entry(frag, &mem_list, list) { -+ if (frag->refs && (frag->flags & -+ USDPAA_DMA_FLAG_SHARE) && -+ !strncmp(i->name, frag->name, -+ USDPAA_DMA_NAME_MAX)) { -+ /* Matching entry */ -+ if ((i->flags & USDPAA_DMA_FLAG_CREATE) && -+ !(i->flags & USDPAA_DMA_FLAG_LAZY)) { -+ ret = -EBUSY; -+ goto out; -+ } -+ -+ /* Check to ensure size matches record */ -+ if (i->len != frag->map_len && i->len) { -+ pr_err("ioctl_dma_map() Size requested does not match %s and is none zero\n", -+ frag->name); -+ return -EINVAL; -+ } -+ -+ /* Check if this has already been mapped -+ to this process */ -+ list_for_each_entry(tmp, &ctx->maps, list) -+ if (tmp->root_frag == frag) { -+ /* Already mapped, just need to -+ inc ref count */ -+ tmp->refs++; -+ kfree(map); -+ i->did_create = 0; -+ i->len = tmp->total_size; -+ i->phys_addr = frag->base; -+ i->ptr = tmp->virt_addr; -+ spin_unlock(&mem_lock); -+ return 0; -+ } -+ /* Matching entry - just need to map */ -+ i->has_locking = frag->has_locking; -+ i->did_create = 0; -+ i->len = frag->map_len; -+ start_frag = frag; -+ goto do_map; -+ } -+ } -+ /* No matching entry */ -+ if (!(i->flags & USDPAA_DMA_FLAG_CREATE)) { -+ pr_err("ioctl_dma_map() No matching entry\n"); -+ ret = -ENOMEM; -+ goto out; -+ } -+ } -+ /* New fragment required, size must be provided. */ -+ if (!i->len) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ /* Find one of more contiguous fragments that satisfy the total length -+ trying to minimize the number of fragments -+ compute the largest page size that the allocation could use */ -+ largest_page = largest_page_size(i->len); -+ start_frag = NULL; -+ while (largest_page && -+ largest_page <= largest_page_size(phys_size) && -+ start_frag == NULL) { -+ /* Search the list for a frag of that size */ -+ list_for_each_entry(frag, &mem_list, list) { -+ if (!frag->refs && (frag->len == largest_page)) { -+ /* See if the next x fragments are free -+ and can accomidate the size */ -+ u32 found_size = largest_page; -+ next_frag = list_entry(frag->list.prev, -+ struct mem_fragment, -+ list); -+ /* If the fragement is too small check -+ if the neighbours cab support it */ -+ while (found_size < i->len) { -+ if (&mem_list == &next_frag->list) -+ break; /* End of list */ -+ if (next_frag->refs != 0 || -+ next_frag->len == 0) -+ break; /* not enough space */ -+ found_size += next_frag->len; -+ next_frag = list_entry( -+ next_frag->list.prev, -+ struct mem_fragment, -+ list); -+ } -+ if (found_size >= i->len) { -+ /* Success! there is enough contigous -+ free space */ -+ start_frag = frag; -+ break; -+ } -+ } -+ } /* next frag loop */ -+ /* Couldn't statisfy the request with this -+ largest page size, try a smaller one */ -+ largest_page <<= 2; -+ } -+ if (start_frag == NULL) { -+ /* Couldn't find proper amount of space */ -+ ret = -ENOMEM; -+ goto out; -+ } -+ i->did_create = 1; -+do_map: -+ /* Verify there is sufficient space to do the mapping */ -+ down_write(¤t->mm->mmap_sem); -+ next_addr = usdpaa_get_unmapped_area(fp, next_addr, i->len, 0, 0); -+ up_write(¤t->mm->mmap_sem); -+ -+ if (next_addr & ~PAGE_MASK) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ /* We may need to divide the final fragment to accomidate the mapping */ -+ next_frag = start_frag; -+ while (so_far != i->len) { -+ BUG_ON(next_frag->len == 0); -+ while ((next_frag->len + so_far) > i->len) { -+ /* Split frag until they match */ -+ split_frag(next_frag); -+ } -+ so_far += next_frag->len; -+ next_frag->refs++; -+ ++frag_count; -+ next_frag = list_entry(next_frag->list.prev, -+ struct mem_fragment, list); -+ } -+ if (i->did_create) { -+ size_t name_len = 0; -+ start_frag->flags = i->flags; -+ strncpy(start_frag->name, i->name, USDPAA_DMA_NAME_MAX); -+ name_len = strnlen(start_frag->name, USDPAA_DMA_NAME_MAX); -+ if (name_len >= USDPAA_DMA_NAME_MAX) { -+ ret = -EFAULT; -+ goto out; -+ } -+ start_frag->map_len = i->len; -+ start_frag->has_locking = i->has_locking; -+ init_waitqueue_head(&start_frag->wq); -+ start_frag->owner = NULL; -+ } -+ -+ /* Setup the map entry */ -+ map->root_frag = start_frag; -+ map->total_size = i->len; -+ map->frag_count = frag_count; -+ map->refs = 1; -+ list_add(&map->list, &ctx->maps); -+ i->phys_addr = start_frag->base; -+out: -+ spin_unlock(&mem_lock); -+ -+ if (!ret) { -+ unsigned long longret; -+ down_write(¤t->mm->mmap_sem); -+ longret = do_mmap_pgoff(fp, next_addr, map->total_size, -+ PROT_READ | -+ (i->flags & -+ USDPAA_DMA_FLAG_RDONLY ? 0 -+ : PROT_WRITE), -+ MAP_SHARED, -+ start_frag->pfn_base, -+ &populate, -+ NULL); -+ up_write(¤t->mm->mmap_sem); -+ if (longret & ~PAGE_MASK) { -+ ret = (int)longret; -+ } else { -+ i->ptr = (void *)longret; -+ map->virt_addr = i->ptr; -+ } -+ } else -+ kfree(map); -+ return ret; -+} -+ -+static long ioctl_dma_unmap(struct ctx *ctx, void __user *arg) -+{ -+ struct mem_mapping *map; -+ struct vm_area_struct *vma; -+ int ret, i; -+ struct mem_fragment *current_frag; -+ size_t sz; -+ unsigned long base; -+ unsigned long vaddr; -+ -+ down_write(¤t->mm->mmap_sem); -+ vma = find_vma(current->mm, (unsigned long)arg); -+ if (!vma || (vma->vm_start > (unsigned long)arg)) { -+ up_write(¤t->mm->mmap_sem); -+ return -EFAULT; -+ } -+ spin_lock(&mem_lock); -+ list_for_each_entry(map, &ctx->maps, list) { -+ if (map->root_frag->pfn_base == vma->vm_pgoff) { -+ /* Drop the map lock if we hold it */ -+ if (map->root_frag->has_locking && -+ (map->root_frag->owner == map)) { -+ map->root_frag->owner = NULL; -+ wake_up(&map->root_frag->wq); -+ } -+ goto map_match; -+ } -+ } -+ /* Failed to find a matching mapping for this process */ -+ ret = -EFAULT; -+ spin_unlock(&mem_lock); -+ goto out; -+map_match: -+ map->refs--; -+ if (map->refs != 0) { -+ /* Another call the dma_map is referencing this */ -+ ret = 0; -+ spin_unlock(&mem_lock); -+ goto out; -+ } -+ -+ current_frag = map->root_frag; -+ vaddr = (unsigned long) map->virt_addr; -+ for (i = 0; i < map->frag_count; i++) { -+ DPA_ASSERT(current_frag->refs > 0); -+ --current_frag->refs; -+#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64)) -+ /* -+ * Make sure we invalidate the TLB entry for -+ * this fragment, otherwise a remap of a different -+ * page to this vaddr would give acces to an -+ * incorrect piece of memory -+ */ -+ cleartlbcam(vaddr, mfspr(SPRN_PID)); -+#endif -+ vaddr += current_frag->len; -+ current_frag = list_entry(current_frag->list.prev, -+ struct mem_fragment, list); -+ } -+ map->root_frag->name[0] = 0; -+ list_del(&map->list); -+ compress_frags(); -+ spin_unlock(&mem_lock); -+ -+ base = vma->vm_start; -+ sz = vma->vm_end - vma->vm_start; -+ do_munmap(current->mm, base, sz, NULL); -+ ret = 0; -+ out: -+ up_write(¤t->mm->mmap_sem); -+ return ret; -+} -+ -+static long ioctl_dma_stats(struct ctx *ctx, void __user *arg) -+{ -+ struct mem_fragment *frag; -+ struct usdpaa_ioctl_dma_used result; -+ -+ result.free_bytes = 0; -+ result.total_bytes = phys_size; -+ -+ list_for_each_entry(frag, &mem_list, list) { -+ if (frag->refs == 0) -+ result.free_bytes += frag->len; -+ } -+ -+ return copy_to_user(arg, &result, sizeof(result)); } -+ -+static int test_lock(struct mem_mapping *map) -+{ -+ int ret = 0; -+ spin_lock(&mem_lock); -+ if (!map->root_frag->owner) { -+ map->root_frag->owner = map; -+ ret = 1; -+ } -+ spin_unlock(&mem_lock); -+ return ret; -+} -+ -+static long ioctl_dma_lock(struct ctx *ctx, void __user *arg) -+{ -+ struct mem_mapping *map; -+ struct vm_area_struct *vma; -+ -+ down_read(¤t->mm->mmap_sem); -+ vma = find_vma(current->mm, (unsigned long)arg); -+ if (!vma || (vma->vm_start > (unsigned long)arg)) { -+ up_read(¤t->mm->mmap_sem); -+ return -EFAULT; -+ } -+ spin_lock(&mem_lock); -+ list_for_each_entry(map, &ctx->maps, list) { -+ if (map->root_frag->pfn_base == vma->vm_pgoff) -+ goto map_match; -+ } -+ map = NULL; -+map_match: -+ spin_unlock(&mem_lock); -+ up_read(¤t->mm->mmap_sem); -+ -+ if (!map) -+ return -EFAULT; -+ if (!map->root_frag->has_locking) -+ return -ENODEV; -+ return wait_event_interruptible(map->root_frag->wq, test_lock(map)); -+} -+ -+static long ioctl_dma_unlock(struct ctx *ctx, void __user *arg) -+{ -+ struct mem_mapping *map; -+ struct vm_area_struct *vma; -+ int ret; -+ -+ down_read(¤t->mm->mmap_sem); -+ vma = find_vma(current->mm, (unsigned long)arg); -+ if (!vma || (vma->vm_start > (unsigned long)arg)) -+ ret = -EFAULT; -+ else { -+ spin_lock(&mem_lock); -+ list_for_each_entry(map, &ctx->maps, list) { -+ if (map->root_frag->pfn_base == vma->vm_pgoff) { -+ if (!map->root_frag->has_locking) -+ ret = -ENODEV; -+ else if (map->root_frag->owner == map) { -+ map->root_frag->owner = NULL; -+ wake_up(&map->root_frag->wq); -+ ret = 0; -+ } else -+ ret = -EBUSY; -+ goto map_match; -+ } -+ } -+ ret = -EINVAL; -+map_match: -+ spin_unlock(&mem_lock); -+ } -+ up_read(¤t->mm->mmap_sem); -+ return ret; -+} -+ -+static int portal_mmap(struct file *fp, struct resource *res, void **ptr) -+{ -+ unsigned long longret = 0, populate; -+ resource_size_t len; -+ -+ down_write(¤t->mm->mmap_sem); -+ len = resource_size(res); -+ if (len != (unsigned long)len) -+ return -EINVAL; -+ longret = do_mmap_pgoff(fp, PAGE_SIZE, (unsigned long)len, -+ PROT_READ | PROT_WRITE, MAP_SHARED, -+ res->start >> PAGE_SHIFT, &populate, NULL); -+ up_write(¤t->mm->mmap_sem); -+ -+ if (longret & ~PAGE_MASK) -+ return (int)longret; -+ -+ *ptr = (void *) longret; -+ return 0; -+} -+ -+static void portal_munmap(struct resource *res, void *ptr) -+{ -+ down_write(¤t->mm->mmap_sem); -+ do_munmap(current->mm, (unsigned long)ptr, resource_size(res), NULL); -+ up_write(¤t->mm->mmap_sem); -+} -+ -+static long ioctl_portal_map(struct file *fp, struct ctx *ctx, -+ struct usdpaa_ioctl_portal_map *arg) -+{ -+ struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); -+ int ret; -+ -+ if (!mapping) -+ return -ENOMEM; -+ -+ mapping->user = *arg; -+ mapping->iommu_domain = NULL; -+ -+ if (mapping->user.type == usdpaa_portal_qman) { -+ mapping->qportal = -+ qm_get_unused_portal_idx(mapping->user.index); -+ if (!mapping->qportal) { -+ ret = -ENODEV; -+ goto err_get_portal; -+ } -+ mapping->phys = &mapping->qportal->addr_phys[0]; -+ mapping->user.channel = mapping->qportal->public_cfg.channel; -+ mapping->user.pools = mapping->qportal->public_cfg.pools; -+ mapping->user.index = mapping->qportal->public_cfg.index; -+ } else if (mapping->user.type == usdpaa_portal_bman) { -+ mapping->bportal = -+ bm_get_unused_portal_idx(mapping->user.index); -+ if (!mapping->bportal) { -+ ret = -ENODEV; -+ goto err_get_portal; -+ } -+ mapping->phys = &mapping->bportal->addr_phys[0]; -+ mapping->user.index = mapping->bportal->public_cfg.index; -+ } else { -+ ret = -EINVAL; -+ goto err_copy_from_user; -+ } -+ /* Need to put pcfg in ctx's list before the mmaps because the mmap -+ * handlers look it up. */ -+ spin_lock(&mem_lock); -+ list_add(&mapping->list, &ctx->portals); -+ spin_unlock(&mem_lock); -+ ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CE], -+ &mapping->user.addr.cena); -+ if (ret) -+ goto err_mmap_cena; -+ ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CI], -+ &mapping->user.addr.cinh); -+ if (ret) -+ goto err_mmap_cinh; -+ *arg = mapping->user; -+ return ret; -+ -+err_mmap_cinh: -+ portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena); -+err_mmap_cena: -+ if ((mapping->user.type == usdpaa_portal_qman) && mapping->qportal) -+ qm_put_unused_portal(mapping->qportal); -+ else if ((mapping->user.type == usdpaa_portal_bman) && mapping->bportal) -+ bm_put_unused_portal(mapping->bportal); -+ spin_lock(&mem_lock); -+ list_del(&mapping->list); -+ spin_unlock(&mem_lock); -+err_get_portal: -+err_copy_from_user: -+ kfree(mapping); -+ return ret; -+} -+ -+static long ioctl_portal_unmap(struct ctx *ctx, struct usdpaa_portal_map *i) -+{ -+ struct portal_mapping *mapping; -+ struct vm_area_struct *vma; -+ unsigned long pfn; -+ u32 channel; -+ -+ /* Get the PFN corresponding to one of the virt addresses */ -+ down_read(¤t->mm->mmap_sem); -+ vma = find_vma(current->mm, (unsigned long)i->cinh); -+ if (!vma || (vma->vm_start > (unsigned long)i->cinh)) { -+ up_read(¤t->mm->mmap_sem); -+ return -EFAULT; -+ } -+ pfn = vma->vm_pgoff; -+ up_read(¤t->mm->mmap_sem); -+ -+ /* Find the corresponding portal */ -+ spin_lock(&mem_lock); -+ list_for_each_entry(mapping, &ctx->portals, list) { -+ if (pfn == (mapping->phys[DPA_PORTAL_CI].start >> PAGE_SHIFT)) -+ goto found; -+ } -+ mapping = NULL; -+found: -+ if (mapping) -+ list_del(&mapping->list); -+ spin_unlock(&mem_lock); -+ if (!mapping) -+ return -ENODEV; -+ portal_munmap(&mapping->phys[DPA_PORTAL_CI], mapping->user.addr.cinh); -+ portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena); -+ if (mapping->user.type == usdpaa_portal_qman) { -+ init_qm_portal(mapping->qportal, -+ &mapping->qman_portal_low); -+ -+ /* Tear down any FQs this portal is referencing */ -+ channel = mapping->qportal->public_cfg.channel; -+ qm_check_and_destroy_fqs(&mapping->qman_portal_low, -+ &channel, -+ check_portal_channel); -+ qm_put_unused_portal(mapping->qportal); -+ } else if (mapping->user.type == usdpaa_portal_bman) { -+ init_bm_portal(mapping->bportal, -+ &mapping->bman_portal_low); -+ bm_put_unused_portal(mapping->bportal); -+ } -+ kfree(mapping); -+ return 0; -+} -+ -+static void portal_config_pamu(struct qm_portal_config *pcfg, uint8_t sdest, -+ uint32_t cpu, uint32_t cache, uint32_t window) -+{ -+#ifdef CONFIG_FSL_PAMU -+ int ret; -+ int window_count = 1; -+ struct iommu_domain_geometry geom_attr; -+ struct pamu_stash_attribute stash_attr; -+ -+ pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type); -+ if (!pcfg->iommu_domain) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed", -+ __func__); -+ goto _no_iommu; -+ } -+ geom_attr.aperture_start = 0; -+ geom_attr.aperture_end = -+ ((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1; -+ geom_attr.force_aperture = true; -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY, -+ &geom_attr); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS, -+ &window_count); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ stash_attr.cpu = cpu; -+ stash_attr.cache = cache; -+ /* set stash information for the window */ -+ stash_attr.window = 0; -+ -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, -+ DOMAIN_ATTR_FSL_PAMU_STASH, -+ &stash_attr); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36, -+ IOMMU_READ | IOMMU_WRITE); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, -+ DOMAIN_ATTR_FSL_PAMU_ENABLE, -+ &window_count); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_detach_device; -+ } -+_no_iommu: -+#endif -+ -+#ifdef CONFIG_FSL_QMAN_CONFIG -+ if (qman_set_sdest(pcfg->public_cfg.channel, sdest)) -+#endif -+ pr_warn("Failed to set QMan portal's stash request queue\n"); -+ -+ return; -+ -+#ifdef CONFIG_FSL_PAMU -+_iommu_detach_device: -+ iommu_detach_device(pcfg->iommu_domain, NULL); -+_iommu_domain_free: -+ iommu_domain_free(pcfg->iommu_domain); -+#endif -+} -+ -+static long ioctl_allocate_raw_portal(struct file *fp, struct ctx *ctx, -+ struct usdpaa_ioctl_raw_portal *arg) -+{ -+ struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); -+ int ret; -+ -+ if (!mapping) -+ return -ENOMEM; -+ -+ mapping->user.type = arg->type; -+ mapping->iommu_domain = NULL; -+ if (arg->type == usdpaa_portal_qman) { -+ mapping->qportal = qm_get_unused_portal_idx(arg->index); -+ if (!mapping->qportal) { -+ ret = -ENODEV; -+ goto err; -+ } -+ mapping->phys = &mapping->qportal->addr_phys[0]; -+ arg->index = mapping->qportal->public_cfg.index; -+ arg->cinh = mapping->qportal->addr_phys[DPA_PORTAL_CI].start; -+ arg->cena = mapping->qportal->addr_phys[DPA_PORTAL_CE].start; -+ if (arg->enable_stash) { -+ /* Setup the PAMU with the supplied parameters */ -+ portal_config_pamu(mapping->qportal, arg->sdest, -+ arg->cpu, arg->cache, arg->window); -+ } -+ } else if (mapping->user.type == usdpaa_portal_bman) { -+ mapping->bportal = -+ bm_get_unused_portal_idx(arg->index); -+ if (!mapping->bportal) { -+ ret = -ENODEV; -+ goto err; -+ } -+ mapping->phys = &mapping->bportal->addr_phys[0]; -+ arg->index = mapping->bportal->public_cfg.index; -+ arg->cinh = mapping->bportal->addr_phys[DPA_PORTAL_CI].start; -+ arg->cena = mapping->bportal->addr_phys[DPA_PORTAL_CE].start; -+ } else { -+ ret = -EINVAL; -+ goto err; -+ } -+ /* Need to put pcfg in ctx's list before the mmaps because the mmap -+ * handlers look it up. */ -+ spin_lock(&mem_lock); -+ list_add(&mapping->list, &ctx->portals); -+ spin_unlock(&mem_lock); -+ return 0; -+err: -+ kfree(mapping); -+ return ret; -+} -+ -+static long ioctl_free_raw_portal(struct file *fp, struct ctx *ctx, -+ struct usdpaa_ioctl_raw_portal *arg) -+{ -+ struct portal_mapping *mapping; -+ u32 channel; -+ -+ /* Find the corresponding portal */ -+ spin_lock(&mem_lock); -+ list_for_each_entry(mapping, &ctx->portals, list) { -+ if (mapping->phys[DPA_PORTAL_CI].start == arg->cinh) -+ goto found; -+ } -+ mapping = NULL; -+found: -+ if (mapping) -+ list_del(&mapping->list); -+ spin_unlock(&mem_lock); -+ if (!mapping) -+ return -ENODEV; -+ if (mapping->user.type == usdpaa_portal_qman) { -+ init_qm_portal(mapping->qportal, -+ &mapping->qman_portal_low); -+ -+ /* Tear down any FQs this portal is referencing */ -+ channel = mapping->qportal->public_cfg.channel; -+ qm_check_and_destroy_fqs(&mapping->qman_portal_low, -+ &channel, -+ check_portal_channel); -+ qm_put_unused_portal(mapping->qportal); -+ } else if (mapping->user.type == usdpaa_portal_bman) { -+ init_bm_portal(mapping->bportal, -+ &mapping->bman_portal_low); -+ bm_put_unused_portal(mapping->bportal); -+ } -+ kfree(mapping); -+ return 0; -+} -+ -+static long usdpaa_ioctl(struct file *fp, unsigned int cmd, unsigned long arg) -+{ -+ struct ctx *ctx = fp->private_data; -+ void __user *a = (void __user *)arg; -+ switch (cmd) { -+ case USDPAA_IOCTL_ID_ALLOC: -+ return ioctl_id_alloc(ctx, a); -+ case USDPAA_IOCTL_ID_RELEASE: -+ return ioctl_id_release(ctx, a); -+ case USDPAA_IOCTL_ID_RESERVE: -+ return ioctl_id_reserve(ctx, a); -+ case USDPAA_IOCTL_DMA_MAP: -+ { -+ struct usdpaa_ioctl_dma_map input; -+ int ret; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ ret = ioctl_dma_map(fp, ctx, &input); -+ if (copy_to_user(a, &input, sizeof(input))) -+ return -EFAULT; -+ return ret; -+ } -+ case USDPAA_IOCTL_DMA_UNMAP: -+ return ioctl_dma_unmap(ctx, a); -+ case USDPAA_IOCTL_DMA_LOCK: -+ return ioctl_dma_lock(ctx, a); -+ case USDPAA_IOCTL_DMA_UNLOCK: -+ return ioctl_dma_unlock(ctx, a); -+ case USDPAA_IOCTL_PORTAL_MAP: -+ { -+ struct usdpaa_ioctl_portal_map input; -+ int ret; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ ret = ioctl_portal_map(fp, ctx, &input); -+ if (copy_to_user(a, &input, sizeof(input))) -+ return -EFAULT; -+ return ret; -+ } -+ case USDPAA_IOCTL_PORTAL_UNMAP: -+ { -+ struct usdpaa_portal_map input; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ return ioctl_portal_unmap(ctx, &input); -+ } -+ case USDPAA_IOCTL_DMA_USED: -+ return ioctl_dma_stats(ctx, a); -+ case USDPAA_IOCTL_ALLOC_RAW_PORTAL: -+ { -+ struct usdpaa_ioctl_raw_portal input; -+ int ret; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ ret = ioctl_allocate_raw_portal(fp, ctx, &input); -+ if (copy_to_user(a, &input, sizeof(input))) -+ return -EFAULT; -+ return ret; -+ } -+ case USDPAA_IOCTL_FREE_RAW_PORTAL: -+ { -+ struct usdpaa_ioctl_raw_portal input; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ return ioctl_free_raw_portal(fp, ctx, &input); -+ } -+ } -+ return -EINVAL; -+} -+ -+static long usdpaa_ioctl_compat(struct file *fp, unsigned int cmd, -+ unsigned long arg) -+{ -+#ifdef CONFIG_COMPAT -+ struct ctx *ctx = fp->private_data; -+ void __user *a = (void __user *)arg; -+#endif -+ switch (cmd) { -+#ifdef CONFIG_COMPAT -+ case USDPAA_IOCTL_DMA_MAP_COMPAT: -+ { -+ int ret; -+ struct usdpaa_ioctl_dma_map_compat input; -+ struct usdpaa_ioctl_dma_map converted; -+ -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ -+ converted.ptr = compat_ptr(input.ptr); -+ converted.phys_addr = input.phys_addr; -+ converted.len = input.len; -+ converted.flags = input.flags; -+ strncpy(converted.name, input.name, USDPAA_DMA_NAME_MAX); -+ converted.has_locking = input.has_locking; -+ converted.did_create = input.did_create; -+ -+ ret = ioctl_dma_map(fp, ctx, &converted); -+ input.ptr = ptr_to_compat(converted.ptr); -+ input.phys_addr = converted.phys_addr; -+ input.len = converted.len; -+ input.flags = converted.flags; -+ strncpy(input.name, converted.name, USDPAA_DMA_NAME_MAX); -+ input.has_locking = converted.has_locking; -+ input.did_create = converted.did_create; -+ if (copy_to_user(a, &input, sizeof(input))) -+ return -EFAULT; -+ return ret; -+ } -+ case USDPAA_IOCTL_PORTAL_MAP_COMPAT: -+ { -+ int ret; -+ struct compat_usdpaa_ioctl_portal_map input; -+ struct usdpaa_ioctl_portal_map converted; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ converted.type = input.type; -+ converted.index = input.index; -+ ret = ioctl_portal_map(fp, ctx, &converted); -+ input.addr.cinh = ptr_to_compat(converted.addr.cinh); -+ input.addr.cena = ptr_to_compat(converted.addr.cena); -+ input.channel = converted.channel; -+ input.pools = converted.pools; -+ input.index = converted.index; -+ if (copy_to_user(a, &input, sizeof(input))) -+ return -EFAULT; -+ return ret; -+ } -+ case USDPAA_IOCTL_PORTAL_UNMAP_COMPAT: -+ { -+ struct usdpaa_portal_map_compat input; -+ struct usdpaa_portal_map converted; -+ -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ converted.cinh = compat_ptr(input.cinh); -+ converted.cena = compat_ptr(input.cena); -+ return ioctl_portal_unmap(ctx, &converted); -+ } -+ case USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT: -+ { -+ int ret; -+ struct usdpaa_ioctl_raw_portal converted; -+ struct compat_ioctl_raw_portal input; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ converted.type = input.type; -+ converted.index = input.index; -+ converted.enable_stash = input.enable_stash; -+ converted.cpu = input.cpu; -+ converted.cache = input.cache; -+ converted.window = input.window; -+ converted.sdest = input.sdest; -+ ret = ioctl_allocate_raw_portal(fp, ctx, &converted); -+ -+ input.cinh = converted.cinh; -+ input.cena = converted.cena; -+ input.index = converted.index; -+ -+ if (copy_to_user(a, &input, sizeof(input))) -+ return -EFAULT; -+ return ret; -+ } -+ case USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT: -+ { -+ struct usdpaa_ioctl_raw_portal converted; -+ struct compat_ioctl_raw_portal input; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ converted.type = input.type; -+ converted.index = input.index; -+ converted.cinh = input.cinh; -+ converted.cena = input.cena; -+ return ioctl_free_raw_portal(fp, ctx, &converted); -+ } -+#endif -+ default: -+ return usdpaa_ioctl(fp, cmd, arg); -+ } -+ return -EINVAL; -+} -+ -+int usdpaa_get_portal_config(struct file *filp, void *cinh, -+ enum usdpaa_portal_type ptype, unsigned int *irq, -+ void **iir_reg) -+{ -+ /* Walk the list of portals for filp and return the config -+ for the portal that matches the hint */ -+ struct ctx *context; -+ struct portal_mapping *portal; -+ -+ /* First sanitize the filp */ -+ if (filp->f_op->open != usdpaa_open) -+ return -ENODEV; -+ context = filp->private_data; -+ spin_lock(&context->lock); -+ list_for_each_entry(portal, &context->portals, list) { -+ if (portal->user.type == ptype && -+ portal->user.addr.cinh == cinh) { -+ if (ptype == usdpaa_portal_qman) { -+ *irq = portal->qportal->public_cfg.irq; -+ *iir_reg = portal->qportal->addr_virt[1] + -+ QM_REG_IIR; -+ } else { -+ *irq = portal->bportal->public_cfg.irq; -+ *iir_reg = portal->bportal->addr_virt[1] + -+ BM_REG_IIR; -+ } -+ spin_unlock(&context->lock); -+ return 0; -+ } -+ } -+ spin_unlock(&context->lock); -+ return -EINVAL; -+} -+ -+static const struct file_operations usdpaa_fops = { -+ .open = usdpaa_open, -+ .release = usdpaa_release, -+ .mmap = usdpaa_mmap, -+ .get_unmapped_area = usdpaa_get_unmapped_area, -+ .unlocked_ioctl = usdpaa_ioctl, -+ .compat_ioctl = usdpaa_ioctl_compat -+}; -+ -+static struct miscdevice usdpaa_miscdev = { -+ .name = "fsl-usdpaa", -+ .fops = &usdpaa_fops, -+ .minor = MISC_DYNAMIC_MINOR, -+}; -+ -+/* Early-boot memory allocation. The boot-arg "usdpaa_mem=<x>" is used to -+ * indicate how much memory (if any) to allocate during early boot. If the -+ * format "usdpaa_mem=<x>,<y>" is used, then <y> will be interpreted as the -+ * number of TLB1 entries to reserve (default is 1). If there are more mappings -+ * than there are TLB1 entries, fault-handling will occur. */ -+ -+static __init int usdpaa_mem(char *arg) -+{ -+ pr_warn("uspdaa_mem argument is depracated\n"); -+ arg_phys_size = memparse(arg, &arg); -+ num_tlb = 1; -+ if (*arg == ',') { -+ unsigned long ul; -+ int err = kstrtoul(arg + 1, 0, &ul); -+ if (err < 0) { -+ num_tlb = 1; -+ pr_warn("ERROR, usdpaa_mem arg is invalid\n"); -+ } else -+ num_tlb = (unsigned int)ul; -+ } -+ return 0; -+} -+early_param("usdpaa_mem", usdpaa_mem); -+ -+static int usdpaa_mem_init(struct reserved_mem *rmem) -+{ -+ phys_start = rmem->base; -+ phys_size = rmem->size; -+ -+ WARN_ON(!(phys_start && phys_size)); -+ -+ return 0; -+} -+RESERVEDMEM_OF_DECLARE(usdpaa_mem_init, "fsl,usdpaa-mem", usdpaa_mem_init); -+ -+__init int fsl_usdpaa_init_early(void) -+{ -+ if (!phys_size || !phys_start) { -+ pr_info("No USDPAA memory, no 'fsl,usdpaa-mem' in device-tree\n"); -+ return 0; -+ } -+ if (phys_size % PAGE_SIZE) { -+ pr_err("'fsl,usdpaa-mem' size must be a multiple of page size\n"); -+ phys_size = 0; -+ return 0; -+ } -+ if (arg_phys_size && phys_size != arg_phys_size) { -+ pr_err("'usdpaa_mem argument size (0x%llx) does not match device tree size (0x%llx)\n", -+ arg_phys_size, phys_size); -+ phys_size = 0; -+ return 0; -+ } -+ pfn_start = phys_start >> PAGE_SHIFT; -+ pfn_size = phys_size >> PAGE_SHIFT; -+#ifdef CONFIG_PPC -+ first_tlb = current_tlb = tlbcam_index; -+ tlbcam_index += num_tlb; -+#endif -+ pr_info("USDPAA region at %llx:%llx(%lx:%lx), %d TLB1 entries)\n", -+ phys_start, phys_size, pfn_start, pfn_size, num_tlb); -+ return 0; -+} -+subsys_initcall(fsl_usdpaa_init_early); -+ -+ -+static int __init usdpaa_init(void) -+{ -+ struct mem_fragment *frag; -+ int ret; -+ u64 tmp_size = phys_size; -+ u64 tmp_start = phys_start; -+ u64 tmp_pfn_size = pfn_size; -+ u64 tmp_pfn_start = pfn_start; -+ -+ pr_info("Freescale USDPAA process driver\n"); -+ if (!phys_start) { -+ pr_warn("fsl-usdpaa: no region found\n"); -+ return 0; -+ } -+ -+ while (tmp_size != 0) { -+ u32 frag_size = largest_page_size(tmp_size); -+ frag = kmalloc(sizeof(*frag), GFP_KERNEL); -+ if (!frag) { -+ pr_err("Failed to setup USDPAA memory accounting\n"); -+ return -ENOMEM; -+ } -+ frag->base = tmp_start; -+ frag->len = frag->root_len = frag_size; -+ frag->root_pfn = tmp_pfn_start; -+ frag->pfn_base = tmp_pfn_start; -+ frag->pfn_len = frag_size / PAGE_SIZE; -+ frag->refs = 0; -+ init_waitqueue_head(&frag->wq); -+ frag->owner = NULL; -+ list_add(&frag->list, &mem_list); -+ -+ /* Adjust for this frag */ -+ tmp_start += frag_size; -+ tmp_size -= frag_size; -+ tmp_pfn_start += frag_size / PAGE_SIZE; -+ tmp_pfn_size -= frag_size / PAGE_SIZE; -+ } -+ ret = misc_register(&usdpaa_miscdev); -+ if (ret) -+ pr_err("fsl-usdpaa: failed to register misc device\n"); -+ return ret; -+} -+ -+static void __exit usdpaa_exit(void) -+{ -+ misc_deregister(&usdpaa_miscdev); -+} -+ -+module_init(usdpaa_init); -+module_exit(usdpaa_exit); -+ -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("Freescale Semiconductor"); -+MODULE_DESCRIPTION("Freescale USDPAA process driver"); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/fsl_usdpaa_irq.c -@@ -0,0 +1,289 @@ -+/* Copyright (c) 2013 Freescale Semiconductor, Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+/* define a device that allows USPDAA processes to open a file -+ descriptor and specify which IRQ it wants to montior using an ioctl() -+ When an IRQ is received, the device becomes readable so that a process -+ can use read() or select() type calls to monitor for IRQs */ -+ -+#include <linux/miscdevice.h> -+#include <linux/fs.h> -+#include <linux/cdev.h> -+#include <linux/slab.h> -+#include <linux/interrupt.h> -+#include <linux/poll.h> -+#include <linux/uaccess.h> -+#include <linux/fsl_usdpaa.h> -+#include <linux/module.h> -+#include <linux/fdtable.h> -+#include <linux/file.h> -+ -+#include "qman_low.h" -+#include "bman_low.h" -+ -+struct usdpaa_irq_ctx { -+ int irq_set; /* Set to true once the irq is set via ioctl */ -+ unsigned int irq_num; -+ u32 last_irq_count; /* Last value returned from read */ -+ u32 irq_count; /* Number of irqs since last read */ -+ wait_queue_head_t wait_queue; /* Waiting processes */ -+ spinlock_t lock; -+ void *inhibit_addr; /* inhibit register address */ -+ struct file *usdpaa_filp; -+ char irq_name[128]; -+}; -+ -+static int usdpaa_irq_open(struct inode *inode, struct file *filp) -+{ -+ struct usdpaa_irq_ctx *ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); -+ if (!ctx) -+ return -ENOMEM; -+ ctx->irq_set = 0; -+ ctx->irq_count = 0; -+ ctx->last_irq_count = 0; -+ init_waitqueue_head(&ctx->wait_queue); -+ spin_lock_init(&ctx->lock); -+ filp->private_data = ctx; -+ return 0; -+} -+ -+static int usdpaa_irq_release(struct inode *inode, struct file *filp) -+{ -+ struct usdpaa_irq_ctx *ctx = filp->private_data; -+ if (ctx->irq_set) { -+ /* Inhibit the IRQ */ -+ out_be32(ctx->inhibit_addr, 0x1); -+ irq_set_affinity_hint(ctx->irq_num, NULL); -+ free_irq(ctx->irq_num, ctx); -+ ctx->irq_set = 0; -+ fput(ctx->usdpaa_filp); -+ } -+ kfree(filp->private_data); -+ return 0; -+} -+ -+static irqreturn_t usdpaa_irq_handler(int irq, void *_ctx) -+{ -+ unsigned long flags; -+ struct usdpaa_irq_ctx *ctx = _ctx; -+ spin_lock_irqsave(&ctx->lock, flags); -+ ++ctx->irq_count; -+ spin_unlock_irqrestore(&ctx->lock, flags); -+ wake_up_all(&ctx->wait_queue); -+ /* Set the inhibit register. This will be reenabled -+ once the USDPAA code handles the IRQ */ -+ out_be32(ctx->inhibit_addr, 0x1); -+ pr_info("Inhibit at %p count %d", ctx->inhibit_addr, ctx->irq_count); -+ return IRQ_HANDLED; -+} -+ -+static int map_irq(struct file *fp, struct usdpaa_ioctl_irq_map *irq_map) -+{ -+ struct usdpaa_irq_ctx *ctx = fp->private_data; -+ int ret; -+ -+ if (ctx->irq_set) { -+ pr_debug("Setting USDPAA IRQ when it was already set!\n"); -+ return -EBUSY; -+ } -+ -+ ctx->usdpaa_filp = fget(irq_map->fd); -+ if (!ctx->usdpaa_filp) { -+ pr_debug("USDPAA fget(%d) returned NULL\n", irq_map->fd); -+ return -EINVAL; -+ } -+ -+ ret = usdpaa_get_portal_config(ctx->usdpaa_filp, irq_map->portal_cinh, -+ irq_map->type, &ctx->irq_num, -+ &ctx->inhibit_addr); -+ if (ret) { -+ pr_debug("USDPAA IRQ couldn't identify portal\n"); -+ fput(ctx->usdpaa_filp); -+ return ret; -+ } -+ -+ ctx->irq_set = 1; -+ -+ snprintf(ctx->irq_name, sizeof(ctx->irq_name), -+ "usdpaa_irq %d", ctx->irq_num); -+ -+ ret = request_irq(ctx->irq_num, usdpaa_irq_handler, 0, -+ ctx->irq_name, ctx); -+ if (ret) { -+ pr_err("USDPAA request_irq(%d) failed, ret= %d\n", -+ ctx->irq_num, ret); -+ ctx->irq_set = 0; -+ fput(ctx->usdpaa_filp); -+ return ret; -+ } -+ ret = irq_set_affinity(ctx->irq_num, ¤t->cpus_allowed); -+ if (ret) -+ pr_err("USDPAA irq_set_affinity() failed, ret= %d\n", ret); -+ -+ ret = irq_set_affinity_hint(ctx->irq_num, ¤t->cpus_allowed); -+ if (ret) -+ pr_err("USDPAA irq_set_affinity_hint() failed, ret= %d\n", ret); -+ -+ return 0; -+} -+ -+static long usdpaa_irq_ioctl(struct file *fp, unsigned int cmd, -+ unsigned long arg) -+{ -+ int ret; -+ struct usdpaa_ioctl_irq_map irq_map; -+ -+ if (cmd != USDPAA_IOCTL_PORTAL_IRQ_MAP) { -+ pr_debug("USDPAA IRQ unknown command 0x%x\n", cmd); -+ return -EINVAL; -+ } -+ -+ ret = copy_from_user(&irq_map, (void __user *)arg, -+ sizeof(irq_map)); -+ if (ret) -+ return ret; -+ return map_irq(fp, &irq_map); -+} -+ -+static ssize_t usdpaa_irq_read(struct file *filp, char __user *buff, -+ size_t count, loff_t *offp) -+{ -+ struct usdpaa_irq_ctx *ctx = filp->private_data; -+ int ret; -+ -+ if (!ctx->irq_set) { -+ pr_debug("Reading USDPAA IRQ before it was set\n"); -+ return -EINVAL; -+ } -+ -+ if (count < sizeof(ctx->irq_count)) { -+ pr_debug("USDPAA IRQ Read too small\n"); -+ return -EINVAL; -+ } -+ if (ctx->irq_count == ctx->last_irq_count) { -+ if (filp->f_flags & O_NONBLOCK) -+ return -EAGAIN; -+ -+ ret = wait_event_interruptible(ctx->wait_queue, -+ ctx->irq_count != ctx->last_irq_count); -+ if (ret == -ERESTARTSYS) -+ return ret; -+ } -+ -+ ctx->last_irq_count = ctx->irq_count; -+ -+ if (copy_to_user(buff, &ctx->last_irq_count, -+ sizeof(ctx->last_irq_count))) -+ return -EFAULT; -+ return sizeof(ctx->irq_count); -+} -+ -+static unsigned int usdpaa_irq_poll(struct file *filp, poll_table *wait) -+{ -+ struct usdpaa_irq_ctx *ctx = filp->private_data; -+ unsigned int ret = 0; -+ unsigned long flags; -+ -+ if (!ctx->irq_set) -+ return POLLHUP; -+ -+ poll_wait(filp, &ctx->wait_queue, wait); -+ -+ spin_lock_irqsave(&ctx->lock, flags); -+ if (ctx->irq_count != ctx->last_irq_count) -+ ret |= POLLIN | POLLRDNORM; -+ spin_unlock_irqrestore(&ctx->lock, flags); -+ return ret; -+} -+ -+static long usdpaa_irq_ioctl_compat(struct file *fp, unsigned int cmd, -+ unsigned long arg) -+{ -+#ifdef CONFIG_COMPAT -+ void __user *a = (void __user *)arg; -+#endif -+ switch (cmd) { -+#ifdef CONFIG_COMPAT -+ case USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT: -+ { -+ struct compat_ioctl_irq_map input; -+ struct usdpaa_ioctl_irq_map converted; -+ if (copy_from_user(&input, a, sizeof(input))) -+ return -EFAULT; -+ converted.type = input.type; -+ converted.fd = input.fd; -+ converted.portal_cinh = compat_ptr(input.portal_cinh); -+ return map_irq(fp, &converted); -+ } -+#endif -+ default: -+ return usdpaa_irq_ioctl(fp, cmd, arg); -+ } -+} -+ -+static const struct file_operations usdpaa_irq_fops = { -+ .open = usdpaa_irq_open, -+ .release = usdpaa_irq_release, -+ .unlocked_ioctl = usdpaa_irq_ioctl, -+ .compat_ioctl = usdpaa_irq_ioctl_compat, -+ .read = usdpaa_irq_read, -+ .poll = usdpaa_irq_poll -+}; -+ -+static struct miscdevice usdpaa_miscdev = { -+ .name = "fsl-usdpaa-irq", -+ .fops = &usdpaa_irq_fops, -+ .minor = MISC_DYNAMIC_MINOR, -+}; -+ -+static int __init usdpaa_irq_init(void) -+{ -+ int ret; -+ -+ pr_info("Freescale USDPAA process IRQ driver\n"); -+ ret = misc_register(&usdpaa_miscdev); -+ if (ret) -+ pr_err("fsl-usdpaa-irq: failed to register misc device\n"); -+ return ret; -+} -+ -+static void __exit usdpaa_irq_exit(void) -+{ -+ misc_deregister(&usdpaa_miscdev); -+} -+ -+module_init(usdpaa_irq_init); -+module_exit(usdpaa_irq_exit); -+ -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("Freescale Semiconductor"); -+MODULE_DESCRIPTION("Freescale USDPAA process IRQ driver"); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qbman_driver.c -@@ -0,0 +1,88 @@ -+/* Copyright 2013 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <linux/time.h> -+#include "qman_private.h" -+#include "bman_private.h" -+__init void qman_init_early(void); -+__init void bman_init_early(void); -+ -+static __init int qbman_init(void) -+{ -+ struct device_node *dn; -+ u32 is_portal_available; -+ -+ bman_init(); -+ qman_init(); -+ -+ is_portal_available = 0; -+ for_each_compatible_node(dn, NULL, "fsl,qman-portal") { -+ if (!of_device_is_available(dn)) -+ continue; -+ else -+ is_portal_available = 1; -+ } -+ -+ if (!qman_have_ccsr() && is_portal_available) { -+ struct qman_fq fq = { -+ .fqid = 1 -+ }; -+ struct qm_mcr_queryfq_np np; -+ int err, retry = CONFIG_FSL_QMAN_INIT_TIMEOUT; -+ struct timespec nowts, diffts, startts = current_kernel_time(); -+ /* Loop while querying given fqid succeeds or time out */ -+ while (1) { -+ err = qman_query_fq_np(&fq, &np); -+ if (!err) { -+ /* success, control-plane has configured QMan */ -+ break; -+ } else if (err != -ERANGE) { -+ pr_err("QMan: I/O error, continuing anyway\n"); -+ break; -+ } -+ nowts = current_kernel_time(); -+ diffts = timespec_sub(nowts, startts); -+ if (diffts.tv_sec > 0) { -+ if (!retry--) { -+ pr_err("QMan: time out, control-plane" -+ " dead?\n"); -+ break; -+ } -+ pr_warn("QMan: polling for the control-plane" -+ " (%d)\n", retry); -+ } -+ } -+ } -+ bman_resource_init(); -+ qman_resource_init(); -+ return 0; -+} -+subsys_initcall(qbman_init); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_config.c -@@ -0,0 +1,1224 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <asm/cacheflush.h> -+#include "qman_private.h" -+#include <linux/highmem.h> -+#include <linux/of_reserved_mem.h> -+ -+/* Last updated for v00.800 of the BG */ -+ -+/* Register offsets */ -+#define REG_QCSP_LIO_CFG(n) (0x0000 + ((n) * 0x10)) -+#define REG_QCSP_IO_CFG(n) (0x0004 + ((n) * 0x10)) -+#define REG_QCSP_DD_CFG(n) (0x000c + ((n) * 0x10)) -+#define REG_DD_CFG 0x0200 -+#define REG_DCP_CFG(n) (0x0300 + ((n) * 0x10)) -+#define REG_DCP_DD_CFG(n) (0x0304 + ((n) * 0x10)) -+#define REG_DCP_DLM_AVG(n) (0x030c + ((n) * 0x10)) -+#define REG_PFDR_FPC 0x0400 -+#define REG_PFDR_FP_HEAD 0x0404 -+#define REG_PFDR_FP_TAIL 0x0408 -+#define REG_PFDR_FP_LWIT 0x0410 -+#define REG_PFDR_CFG 0x0414 -+#define REG_SFDR_CFG 0x0500 -+#define REG_SFDR_IN_USE 0x0504 -+#define REG_WQ_CS_CFG(n) (0x0600 + ((n) * 0x04)) -+#define REG_WQ_DEF_ENC_WQID 0x0630 -+#define REG_WQ_SC_DD_CFG(n) (0x640 + ((n) * 0x04)) -+#define REG_WQ_PC_DD_CFG(n) (0x680 + ((n) * 0x04)) -+#define REG_WQ_DC0_DD_CFG(n) (0x6c0 + ((n) * 0x04)) -+#define REG_WQ_DC1_DD_CFG(n) (0x700 + ((n) * 0x04)) -+#define REG_WQ_DCn_DD_CFG(n) (0x6c0 + ((n) * 0x40)) /* n=2,3 */ -+#define REG_CM_CFG 0x0800 -+#define REG_ECSR 0x0a00 -+#define REG_ECIR 0x0a04 -+#define REG_EADR 0x0a08 -+#define REG_ECIR2 0x0a0c -+#define REG_EDATA(n) (0x0a10 + ((n) * 0x04)) -+#define REG_SBEC(n) (0x0a80 + ((n) * 0x04)) -+#define REG_MCR 0x0b00 -+#define REG_MCP(n) (0x0b04 + ((n) * 0x04)) -+#define REG_MISC_CFG 0x0be0 -+#define REG_HID_CFG 0x0bf0 -+#define REG_IDLE_STAT 0x0bf4 -+#define REG_IP_REV_1 0x0bf8 -+#define REG_IP_REV_2 0x0bfc -+#define REG_FQD_BARE 0x0c00 -+#define REG_PFDR_BARE 0x0c20 -+#define REG_offset_BAR 0x0004 /* relative to REG_[FQD|PFDR]_BARE */ -+#define REG_offset_AR 0x0010 /* relative to REG_[FQD|PFDR]_BARE */ -+#define REG_QCSP_BARE 0x0c80 -+#define REG_QCSP_BAR 0x0c84 -+#define REG_CI_SCHED_CFG 0x0d00 -+#define REG_SRCIDR 0x0d04 -+#define REG_LIODNR 0x0d08 -+#define REG_CI_RLM_AVG 0x0d14 -+#define REG_ERR_ISR 0x0e00 /* + "enum qm_isr_reg" */ -+#define REG_REV3_QCSP_LIO_CFG(n) (0x1000 + ((n) * 0x10)) -+#define REG_REV3_QCSP_IO_CFG(n) (0x1004 + ((n) * 0x10)) -+#define REG_REV3_QCSP_DD_CFG(n) (0x100c + ((n) * 0x10)) -+#define REG_CEETM_CFG_IDX 0x900 -+#define REG_CEETM_CFG_PRES 0x904 -+#define REG_CEETM_XSFDR_IN_USE 0x908 -+ -+/* Assists for QMAN_MCR */ -+#define MCR_INIT_PFDR 0x01000000 -+#define MCR_get_rslt(v) (u8)((v) >> 24) -+#define MCR_rslt_idle(r) (!rslt || (rslt >= 0xf0)) -+#define MCR_rslt_ok(r) (rslt == 0xf0) -+#define MCR_rslt_eaccess(r) (rslt == 0xf8) -+#define MCR_rslt_inval(r) (rslt == 0xff) -+ -+struct qman; -+ -+/* Follows WQ_CS_CFG0-5 */ -+enum qm_wq_class { -+ qm_wq_portal = 0, -+ qm_wq_pool = 1, -+ qm_wq_fman0 = 2, -+ qm_wq_fman1 = 3, -+ qm_wq_caam = 4, -+ qm_wq_pme = 5, -+ qm_wq_first = qm_wq_portal, -+ qm_wq_last = qm_wq_pme -+}; -+ -+/* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */ -+enum qm_memory { -+ qm_memory_fqd, -+ qm_memory_pfdr -+}; -+ -+/* Used by all error interrupt registers except 'inhibit' */ -+#define QM_EIRQ_CIDE 0x20000000 /* Corenet Initiator Data Error */ -+#define QM_EIRQ_CTDE 0x10000000 /* Corenet Target Data Error */ -+#define QM_EIRQ_CITT 0x08000000 /* Corenet Invalid Target Transaction */ -+#define QM_EIRQ_PLWI 0x04000000 /* PFDR Low Watermark */ -+#define QM_EIRQ_MBEI 0x02000000 /* Multi-bit ECC Error */ -+#define QM_EIRQ_SBEI 0x01000000 /* Single-bit ECC Error */ -+#define QM_EIRQ_PEBI 0x00800000 /* PFDR Enqueues Blocked Interrupt */ -+#define QM_EIRQ_IFSI 0x00020000 /* Invalid FQ Flow Control State */ -+#define QM_EIRQ_ICVI 0x00010000 /* Invalid Command Verb */ -+#define QM_EIRQ_IDDI 0x00000800 /* Invalid Dequeue (Direct-connect) */ -+#define QM_EIRQ_IDFI 0x00000400 /* Invalid Dequeue FQ */ -+#define QM_EIRQ_IDSI 0x00000200 /* Invalid Dequeue Source */ -+#define QM_EIRQ_IDQI 0x00000100 /* Invalid Dequeue Queue */ -+#define QM_EIRQ_IECE 0x00000010 /* Invalid Enqueue Configuration */ -+#define QM_EIRQ_IEOI 0x00000008 /* Invalid Enqueue Overflow */ -+#define QM_EIRQ_IESI 0x00000004 /* Invalid Enqueue State */ -+#define QM_EIRQ_IECI 0x00000002 /* Invalid Enqueue Channel */ -+#define QM_EIRQ_IEQI 0x00000001 /* Invalid Enqueue Queue */ -+ -+/* QMAN_ECIR valid error bit */ -+#define PORTAL_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \ -+ QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \ -+ QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI) -+#define FQID_ECSR_ERR (QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \ -+ QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \ -+ QM_EIRQ_IFSI) -+ -+union qman_ecir { -+ u32 ecir_raw; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 __reserved:2; -+ u32 portal_type:1; -+ u32 portal_num:5; -+ u32 fqid:24; -+#else -+ u32 fqid:24; -+ u32 portal_num:5; -+ u32 portal_type:1; -+ u32 __reserved:2; -+#endif -+ } __packed info; -+}; -+ -+union qman_ecir2 { -+ u32 ecir2_raw; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 portal_type:1; -+ u32 __reserved:21; -+ u32 portal_num:10; -+#else -+ u32 portal_num:10; -+ u32 __reserved:21; -+ u32 portal_type:1; -+#endif -+ } __packed info; -+}; -+ -+union qman_eadr { -+ u32 eadr_raw; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 __reserved1:4; -+ u32 memid:4; -+ u32 __reserved2:12; -+ u32 eadr:12; -+#else -+ u32 eadr:12; -+ u32 __reserved2:12; -+ u32 memid:4; -+ u32 __reserved1:4; -+#endif -+ } __packed info; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 __reserved1:3; -+ u32 memid:5; -+ u32 __reserved:8; -+ u32 eadr:16; -+#else -+ u32 eadr:16; -+ u32 __reserved:8; -+ u32 memid:5; -+ u32 __reserved1:3; -+#endif -+ } __packed info_rev3; -+}; -+ -+struct qman_hwerr_txt { -+ u32 mask; -+ const char *txt; -+}; -+ -+#define QMAN_HWE_TXT(a, b) { .mask = QM_EIRQ_##a, .txt = b } -+ -+static const struct qman_hwerr_txt qman_hwerr_txts[] = { -+ QMAN_HWE_TXT(CIDE, "Corenet Initiator Data Error"), -+ QMAN_HWE_TXT(CTDE, "Corenet Target Data Error"), -+ QMAN_HWE_TXT(CITT, "Corenet Invalid Target Transaction"), -+ QMAN_HWE_TXT(PLWI, "PFDR Low Watermark"), -+ QMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"), -+ QMAN_HWE_TXT(SBEI, "Single-bit ECC Error"), -+ QMAN_HWE_TXT(PEBI, "PFDR Enqueues Blocked Interrupt"), -+ QMAN_HWE_TXT(ICVI, "Invalid Command Verb"), -+ QMAN_HWE_TXT(IFSI, "Invalid Flow Control State"), -+ QMAN_HWE_TXT(IDDI, "Invalid Dequeue (Direct-connect)"), -+ QMAN_HWE_TXT(IDFI, "Invalid Dequeue FQ"), -+ QMAN_HWE_TXT(IDSI, "Invalid Dequeue Source"), -+ QMAN_HWE_TXT(IDQI, "Invalid Dequeue Queue"), -+ QMAN_HWE_TXT(IECE, "Invalid Enqueue Configuration"), -+ QMAN_HWE_TXT(IEOI, "Invalid Enqueue Overflow"), -+ QMAN_HWE_TXT(IESI, "Invalid Enqueue State"), -+ QMAN_HWE_TXT(IECI, "Invalid Enqueue Channel"), -+ QMAN_HWE_TXT(IEQI, "Invalid Enqueue Queue") -+}; -+#define QMAN_HWE_COUNT (sizeof(qman_hwerr_txts)/sizeof(struct qman_hwerr_txt)) -+ -+struct qman_error_info_mdata { -+ u16 addr_mask; -+ u16 bits; -+ const char *txt; -+}; -+ -+#define QMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c} -+static const struct qman_error_info_mdata error_mdata[] = { -+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 0"), -+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 1"), -+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 2"), -+ QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 3"), -+ QMAN_ERR_MDATA(0x0FFF, 512, "FQD cache memory"), -+ QMAN_ERR_MDATA(0x07FF, 128, "SFDR memory"), -+ QMAN_ERR_MDATA(0x01FF, 72, "WQ context memory"), -+ QMAN_ERR_MDATA(0x00FF, 240, "CGR memory"), -+ QMAN_ERR_MDATA(0x00FF, 302, "Internal Order Restoration List memory"), -+ QMAN_ERR_MDATA(0x01FF, 256, "SW portal ring memory"), -+ QMAN_ERR_MDATA(0x07FF, 181, "CEETM class queue descriptor memory"), -+ QMAN_ERR_MDATA(0x0FFF, 140, "CEETM extended SFDR memory"), -+ QMAN_ERR_MDATA(0x0FFF, 25, "CEETM logical FQ mapping memory"), -+ QMAN_ERR_MDATA(0x0FFF, 96, "CEETM dequeue context memory"), -+ QMAN_ERR_MDATA(0x07FF, 396, "CEETM ccgr memory"), -+ QMAN_ERR_MDATA(0x00FF, 146, "CEETM CQ channel shaping memory"), -+ QMAN_ERR_MDATA(0x007F, 256, "CEETM CQ channel scheduling memory"), -+ QMAN_ERR_MDATA(0x01FF, 88, "CEETM dequeue statistics memory"), -+}; -+#define QMAN_ERR_MDATA_COUNT \ -+ (sizeof(error_mdata)/sizeof(struct qman_error_info_mdata)) -+ -+/* Add this in Kconfig */ -+#define QMAN_ERRS_TO_UNENABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI) -+ -+/** -+ * qm_err_isr_<reg>_<verb> - Manipulate global interrupt registers -+ * @v: for accessors that write values, this is the 32-bit value -+ * -+ * Manipulates QMAN_ERR_ISR, QMAN_ERR_IER, QMAN_ERR_ISDR, QMAN_ERR_IIR. All -+ * manipulations except qm_err_isr_[un]inhibit() use 32-bit masks composed of -+ * the QM_EIRQ_*** definitions. Note that "qm_err_isr_enable_write" means -+ * "write the enable register" rather than "enable the write register"! -+ */ -+#define qm_err_isr_status_read(qm) \ -+ __qm_err_isr_read(qm, qm_isr_status) -+#define qm_err_isr_status_clear(qm, m) \ -+ __qm_err_isr_write(qm, qm_isr_status, m) -+#define qm_err_isr_enable_read(qm) \ -+ __qm_err_isr_read(qm, qm_isr_enable) -+#define qm_err_isr_enable_write(qm, v) \ -+ __qm_err_isr_write(qm, qm_isr_enable, v) -+#define qm_err_isr_disable_read(qm) \ -+ __qm_err_isr_read(qm, qm_isr_disable) -+#define qm_err_isr_disable_write(qm, v) \ -+ __qm_err_isr_write(qm, qm_isr_disable, v) -+#define qm_err_isr_inhibit(qm) \ -+ __qm_err_isr_write(qm, qm_isr_inhibit, 1) -+#define qm_err_isr_uninhibit(qm) \ -+ __qm_err_isr_write(qm, qm_isr_inhibit, 0) -+ -+/* -+ * TODO: unimplemented registers -+ * -+ * Keeping a list here of Qman registers I have not yet covered; -+ * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR, -+ * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG, -+ * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12 -+ */ -+ -+/* Encapsulate "struct qman *" as a cast of the register space address. */ -+ -+static struct qman *qm_create(void *regs) -+{ -+ return (struct qman *)regs; -+} -+ -+static inline u32 __qm_in(struct qman *qm, u32 offset) -+{ -+ return in_be32((void *)qm + offset); -+} -+static inline void __qm_out(struct qman *qm, u32 offset, u32 val) -+{ -+ out_be32((void *)qm + offset, val); -+} -+#define qm_in(reg) __qm_in(qm, REG_##reg) -+#define qm_out(reg, val) __qm_out(qm, REG_##reg, val) -+ -+static u32 __qm_err_isr_read(struct qman *qm, enum qm_isr_reg n) -+{ -+ return __qm_in(qm, REG_ERR_ISR + (n << 2)); -+} -+ -+static void __qm_err_isr_write(struct qman *qm, enum qm_isr_reg n, u32 val) -+{ -+ __qm_out(qm, REG_ERR_ISR + (n << 2), val); -+} -+ -+static void qm_set_dc(struct qman *qm, enum qm_dc_portal portal, -+ int ed, u8 sernd) -+{ -+ DPA_ASSERT(!ed || (portal == qm_dc_portal_fman0) || -+ (portal == qm_dc_portal_fman1)); -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) -+ qm_out(DCP_CFG(portal), (ed ? 0x1000 : 0) | (sernd & 0x3ff)); -+ else -+ qm_out(DCP_CFG(portal), (ed ? 0x100 : 0) | (sernd & 0x1f)); -+} -+ -+static void qm_set_wq_scheduling(struct qman *qm, enum qm_wq_class wq_class, -+ u8 cs_elev, u8 csw2, u8 csw3, u8 csw4, u8 csw5, -+ u8 csw6, u8 csw7) -+{ -+ qm_out(WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) | -+ ((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) | -+ ((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) | -+ ((csw6 & 0x7) << 4) | (csw7 & 0x7)); -+} -+ -+static void qm_set_hid(struct qman *qm) -+{ -+ qm_out(HID_CFG, 0); -+} -+ -+static void qm_set_corenet_initiator(struct qman *qm) -+{ -+ qm_out(CI_SCHED_CFG, -+ 0x80000000 | /* write srcciv enable */ -+ (CONFIG_FSL_QMAN_CI_SCHED_CFG_SRCCIV << 24) | -+ (CONFIG_FSL_QMAN_CI_SCHED_CFG_SRQ_W << 8) | -+ (CONFIG_FSL_QMAN_CI_SCHED_CFG_RW_W << 4) | -+ CONFIG_FSL_QMAN_CI_SCHED_CFG_BMAN_W); -+} -+ -+static void qm_get_version(struct qman *qm, u16 *id, u8 *major, u8 *minor, -+ u8 *cfg) -+{ -+ u32 v = qm_in(IP_REV_1); -+ u32 v2 = qm_in(IP_REV_2); -+ *id = (v >> 16); -+ *major = (v >> 8) & 0xff; -+ *minor = v & 0xff; -+ *cfg = v2 & 0xff; -+} -+ -+static void qm_set_memory(struct qman *qm, enum qm_memory memory, u64 ba, -+ int enable, int prio, int stash, u32 size) -+{ -+ u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE; -+ u32 exp = ilog2(size); -+ /* choke if size isn't within range */ -+ DPA_ASSERT((size >= 4096) && (size <= 1073741824) && -+ is_power_of_2(size)); -+ /* choke if 'ba' has lower-alignment than 'size' */ -+ DPA_ASSERT(!(ba & (size - 1))); -+ __qm_out(qm, offset, upper_32_bits(ba)); -+ __qm_out(qm, offset + REG_offset_BAR, lower_32_bits(ba)); -+ __qm_out(qm, offset + REG_offset_AR, -+ (enable ? 0x80000000 : 0) | -+ (prio ? 0x40000000 : 0) | -+ (stash ? 0x20000000 : 0) | -+ (exp - 1)); -+} -+ -+static void qm_set_pfdr_threshold(struct qman *qm, u32 th, u8 k) -+{ -+ qm_out(PFDR_FP_LWIT, th & 0xffffff); -+ qm_out(PFDR_CFG, k); -+} -+ -+static void qm_set_sfdr_threshold(struct qman *qm, u16 th) -+{ -+ qm_out(SFDR_CFG, th & 0x3ff); -+} -+ -+static int qm_init_pfdr(struct qman *qm, u32 pfdr_start, u32 num) -+{ -+ u8 rslt = MCR_get_rslt(qm_in(MCR)); -+ -+ DPA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num); -+ /* Make sure the command interface is 'idle' */ -+ if (!MCR_rslt_idle(rslt)) -+ panic("QMAN_MCR isn't idle"); -+ -+ /* Write the MCR command params then the verb */ -+ qm_out(MCP(0), pfdr_start); -+ /* TODO: remove this - it's a workaround for a model bug that is -+ * corrected in more recent versions. We use the workaround until -+ * everyone has upgraded. */ -+ qm_out(MCP(1), (pfdr_start + num - 16)); -+ lwsync(); -+ qm_out(MCR, MCR_INIT_PFDR); -+ /* Poll for the result */ -+ do { -+ rslt = MCR_get_rslt(qm_in(MCR)); -+ } while (!MCR_rslt_idle(rslt)); -+ if (MCR_rslt_ok(rslt)) -+ return 0; -+ if (MCR_rslt_eaccess(rslt)) -+ return -EACCES; -+ if (MCR_rslt_inval(rslt)) -+ return -EINVAL; -+ pr_crit("Unexpected result from MCR_INIT_PFDR: %02x\n", rslt); -+ return -ENOSYS; -+} -+ -+/*****************/ -+/* Config driver */ -+/*****************/ -+ -+#define DEFAULT_FQD_SZ (PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ) -+#define DEFAULT_PFDR_SZ (PAGE_SIZE << CONFIG_FSL_QMAN_PFDR_SZ) -+ -+/* We support only one of these */ -+static struct qman *qm; -+static struct device_node *qm_node; -+ -+/* And this state belongs to 'qm'. It is set during fsl_qman_init(), but used -+ * during qman_init_ccsr(). */ -+static dma_addr_t fqd_a, pfdr_a; -+static size_t fqd_sz = DEFAULT_FQD_SZ, pfdr_sz = DEFAULT_PFDR_SZ; -+ -+static int qman_fqd(struct reserved_mem *rmem) -+{ -+ fqd_a = rmem->base; -+ fqd_sz = rmem->size; -+ -+ WARN_ON(!(fqd_a && fqd_sz)); -+ -+ return 0; -+} -+RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd); -+ -+static int qman_pfdr(struct reserved_mem *rmem) -+{ -+ pfdr_a = rmem->base; -+ pfdr_sz = rmem->size; -+ -+ WARN_ON(!(pfdr_a && pfdr_sz)); -+ -+ return 0; -+} -+RESERVEDMEM_OF_DECLARE(qman_fbpr, "fsl,qman-pfdr", qman_pfdr); -+ -+size_t get_qman_fqd_size() -+{ -+ return fqd_sz; -+} -+ -+/* Parse the <name> property to extract the memory location and size and -+ * memblock_reserve() it. If it isn't supplied, memblock_alloc() the default -+ * size. Also flush this memory range from data cache so that QMAN originated -+ * transactions for this memory region could be marked non-coherent. -+ */ -+static __init int parse_mem_property(struct device_node *node, const char *name, -+ dma_addr_t *addr, size_t *sz, int zero) -+{ -+ int ret; -+ -+ /* If using a "zero-pma", don't try to zero it, even if you asked */ -+ if (zero && of_find_property(node, "zero-pma", &ret)) { -+ pr_info(" it's a 'zero-pma', not zeroing from s/w\n"); -+ zero = 0; -+ } -+ -+ if (zero) { -+ /* map as cacheable, non-guarded */ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ void __iomem *tmpp = ioremap_cache(*addr, *sz); -+#else -+ void __iomem *tmpp = ioremap(*addr, *sz); -+#endif -+ -+ if (!tmpp) -+ return -ENOMEM; -+ memset_io(tmpp, 0, *sz); -+ flush_dcache_range((unsigned long)tmpp, -+ (unsigned long)tmpp + *sz); -+ iounmap(tmpp); -+ } -+ -+ return 0; -+} -+ -+/* TODO: -+ * - there is obviously no handling of errors, -+ * - the calls to qm_set_memory() hard-code the priority and CPC-stashing for -+ * both memory resources to zero. -+ */ -+static int __init fsl_qman_init(struct device_node *node) -+{ -+ struct resource res; -+ resource_size_t len; -+ u32 __iomem *regs; -+ const char *s; -+ int ret, standby = 0; -+ u16 id; -+ u8 major, minor, cfg; -+ ret = of_address_to_resource(node, 0, &res); -+ if (ret) { -+ pr_err("Can't get %s property '%s'\n", node->full_name, "reg"); -+ return ret; -+ } -+ s = of_get_property(node, "fsl,hv-claimable", &ret); -+ if (s && !strcmp(s, "standby")) -+ standby = 1; -+ if (!standby) { -+ ret = parse_mem_property(node, "fsl,qman-fqd", -+ &fqd_a, &fqd_sz, 1); -+ pr_info("qman-fqd addr %pad size 0x%zx\n", &fqd_a, fqd_sz); -+ BUG_ON(ret); -+ ret = parse_mem_property(node, "fsl,qman-pfdr", -+ &pfdr_a, &pfdr_sz, 0); -+ pr_info("qman-pfdr addr %pad size 0x%zx\n", &pfdr_a, pfdr_sz); -+ BUG_ON(ret); -+ } -+ /* Global configuration */ -+ len = resource_size(&res); -+ if (len != (unsigned long)len) -+ return -EINVAL; -+ regs = ioremap(res.start, (unsigned long)len); -+ qm = qm_create(regs); -+ qm_node = node; -+ qm_get_version(qm, &id, &major, &minor, &cfg); -+ pr_info("Qman ver:%04x,%02x,%02x,%02x\n", id, major, minor, cfg); -+ if (!qman_ip_rev) { -+ if ((major == 1) && (minor == 0)) { -+ pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n"); -+ iounmap(regs); -+ return -ENODEV; -+ } else if ((major == 1) && (minor == 1)) -+ qman_ip_rev = QMAN_REV11; -+ else if ((major == 1) && (minor == 2)) -+ qman_ip_rev = QMAN_REV12; -+ else if ((major == 2) && (minor == 0)) -+ qman_ip_rev = QMAN_REV20; -+ else if ((major == 3) && (minor == 0)) -+ qman_ip_rev = QMAN_REV30; -+ else if ((major == 3) && (minor == 1)) -+ qman_ip_rev = QMAN_REV31; -+ else if ((major == 3) && (minor == 2)) -+ qman_ip_rev = QMAN_REV32; -+ else { -+ pr_warn("unknown Qman version, default to rev1.1\n"); -+ qman_ip_rev = QMAN_REV11; -+ } -+ qman_ip_cfg = cfg; -+ } -+ -+ if (standby) { -+ pr_info(" -> in standby mode\n"); -+ return 0; -+ } -+ return 0; -+} -+ -+int qman_have_ccsr(void) -+{ -+ return qm ? 1 : 0; -+} -+ -+__init int qman_init_early(void) -+{ -+ struct device_node *dn; -+ int ret; -+ -+ for_each_compatible_node(dn, NULL, "fsl,qman") { -+ if (qm) -+ pr_err("%s: only one 'fsl,qman' allowed\n", -+ dn->full_name); -+ else { -+ if (!of_device_is_available(dn)) -+ continue; -+ -+ ret = fsl_qman_init(dn); -+ BUG_ON(ret); -+ } -+ } -+ return 0; -+} -+postcore_initcall_sync(qman_init_early); -+ -+static void log_edata_bits(u32 bit_count) -+{ -+ u32 i, j, mask = 0xffffffff; -+ -+ pr_warn("Qman ErrInt, EDATA:\n"); -+ i = bit_count/32; -+ if (bit_count%32) { -+ i++; -+ mask = ~(mask << bit_count%32); -+ } -+ j = 16-i; -+ pr_warn(" 0x%08x\n", qm_in(EDATA(j)) & mask); -+ j++; -+ for (; j < 16; j++) -+ pr_warn(" 0x%08x\n", qm_in(EDATA(j))); -+} -+ -+static void log_additional_error_info(u32 isr_val, u32 ecsr_val) -+{ -+ union qman_ecir ecir_val; -+ union qman_eadr eadr_val; -+ -+ ecir_val.ecir_raw = qm_in(ECIR); -+ /* Is portal info valid */ -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { -+ union qman_ecir2 ecir2_val; -+ ecir2_val.ecir2_raw = qm_in(ECIR2); -+ if (ecsr_val & PORTAL_ECSR_ERR) { -+ pr_warn("Qman ErrInt: %s id %d\n", -+ (ecir2_val.info.portal_type) ? -+ "DCP" : "SWP", ecir2_val.info.portal_num); -+ } -+ if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE)) { -+ pr_warn("Qman ErrInt: ecir.fqid 0x%x\n", -+ ecir_val.info.fqid); -+ } -+ if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) { -+ eadr_val.eadr_raw = qm_in(EADR); -+ pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n", -+ error_mdata[eadr_val.info_rev3.memid].txt, -+ error_mdata[eadr_val.info_rev3.memid].addr_mask -+ & eadr_val.info_rev3.eadr); -+ log_edata_bits( -+ error_mdata[eadr_val.info_rev3.memid].bits); -+ } -+ } else { -+ if (ecsr_val & PORTAL_ECSR_ERR) { -+ pr_warn("Qman ErrInt: %s id %d\n", -+ (ecir_val.info.portal_type) ? -+ "DCP" : "SWP", ecir_val.info.portal_num); -+ } -+ if (ecsr_val & FQID_ECSR_ERR) { -+ pr_warn("Qman ErrInt: ecir.fqid 0x%x\n", -+ ecir_val.info.fqid); -+ } -+ if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) { -+ eadr_val.eadr_raw = qm_in(EADR); -+ pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n", -+ error_mdata[eadr_val.info.memid].txt, -+ error_mdata[eadr_val.info.memid].addr_mask -+ & eadr_val.info.eadr); -+ log_edata_bits(error_mdata[eadr_val.info.memid].bits); -+ } -+ } -+} -+ -+/* Qman interrupt handler */ -+static irqreturn_t qman_isr(int irq, void *ptr) -+{ -+ u32 isr_val, ier_val, ecsr_val, isr_mask, i; -+ -+ ier_val = qm_err_isr_enable_read(qm); -+ isr_val = qm_err_isr_status_read(qm); -+ ecsr_val = qm_in(ECSR); -+ isr_mask = isr_val & ier_val; -+ -+ if (!isr_mask) -+ return IRQ_NONE; -+ for (i = 0; i < QMAN_HWE_COUNT; i++) { -+ if (qman_hwerr_txts[i].mask & isr_mask) { -+ pr_warn("Qman ErrInt: %s\n", qman_hwerr_txts[i].txt); -+ if (qman_hwerr_txts[i].mask & ecsr_val) { -+ log_additional_error_info(isr_mask, ecsr_val); -+ /* Re-arm error capture registers */ -+ qm_out(ECSR, ecsr_val); -+ } -+ if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_UNENABLE) { -+ pr_devel("Qman un-enabling error 0x%x\n", -+ qman_hwerr_txts[i].mask); -+ ier_val &= ~qman_hwerr_txts[i].mask; -+ qm_err_isr_enable_write(qm, ier_val); -+ } -+ } -+ } -+ qm_err_isr_status_clear(qm, isr_val); -+ return IRQ_HANDLED; -+} -+ -+static int __bind_irq(void) -+{ -+ int ret, err_irq; -+ -+ err_irq = of_irq_to_resource(qm_node, 0, NULL); -+ if (err_irq == 0) { -+ pr_info("Can't get %s property '%s'\n", qm_node->full_name, -+ "interrupts"); -+ return -ENODEV; -+ } -+ ret = request_irq(err_irq, qman_isr, IRQF_SHARED, "qman-err", qm_node); -+ if (ret) { -+ pr_err("request_irq() failed %d for '%s'\n", ret, -+ qm_node->full_name); -+ return -ENODEV; -+ } -+ /* Write-to-clear any stale bits, (eg. starvation being asserted prior -+ * to resource allocation during driver init). */ -+ qm_err_isr_status_clear(qm, 0xffffffff); -+ /* Enable Error Interrupts */ -+ qm_err_isr_enable_write(qm, 0xffffffff); -+ return 0; -+} -+ -+int qman_init_ccsr(struct device_node *node) -+{ -+ int ret; -+ if (!qman_have_ccsr()) -+ return 0; -+ if (node != qm_node) -+ return -EINVAL; -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ /* TEMP for LS1043 : should be done in uboot */ -+ qm_out(QCSP_BARE, 0x5); -+ qm_out(QCSP_BAR, 0x0); -+#endif -+ /* FQD memory */ -+ qm_set_memory(qm, qm_memory_fqd, fqd_a, 1, 0, 0, fqd_sz); -+ /* PFDR memory */ -+ qm_set_memory(qm, qm_memory_pfdr, pfdr_a, 1, 0, 0, pfdr_sz); -+ qm_init_pfdr(qm, 8, pfdr_sz / 64 - 8); -+ /* thresholds */ -+ qm_set_pfdr_threshold(qm, 512, 64); -+ qm_set_sfdr_threshold(qm, 128); -+ /* clear stale PEBI bit from interrupt status register */ -+ qm_err_isr_status_clear(qm, QM_EIRQ_PEBI); -+ /* corenet initiator settings */ -+ qm_set_corenet_initiator(qm); -+ /* HID settings */ -+ qm_set_hid(qm); -+ /* Set scheduling weights to defaults */ -+ for (ret = qm_wq_first; ret <= qm_wq_last; ret++) -+ qm_set_wq_scheduling(qm, ret, 0, 0, 0, 0, 0, 0, 0); -+ /* We are not prepared to accept ERNs for hardware enqueues */ -+ qm_set_dc(qm, qm_dc_portal_fman0, 1, 0); -+ qm_set_dc(qm, qm_dc_portal_fman1, 1, 0); -+ /* Initialise Error Interrupt Handler */ -+ ret = __bind_irq(); -+ if (ret) -+ return ret; -+ return 0; -+} -+ -+#define LIO_CFG_LIODN_MASK 0x0fff0000 -+void qman_liodn_fixup(u16 channel) -+{ -+ static int done; -+ static u32 liodn_offset; -+ u32 before, after; -+ int idx = channel - QM_CHANNEL_SWPORTAL0; -+ -+ if (!qman_have_ccsr()) -+ return; -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) -+ before = qm_in(REV3_QCSP_LIO_CFG(idx)); -+ else -+ before = qm_in(QCSP_LIO_CFG(idx)); -+ if (!done) { -+ liodn_offset = before & LIO_CFG_LIODN_MASK; -+ done = 1; -+ return; -+ } -+ after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset; -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) -+ qm_out(REV3_QCSP_LIO_CFG(idx), after); -+ else -+ qm_out(QCSP_LIO_CFG(idx), after); -+} -+ -+#define IO_CFG_SDEST_MASK 0x00ff0000 -+int qman_set_sdest(u16 channel, unsigned int cpu_idx) -+{ -+ int idx = channel - QM_CHANNEL_SWPORTAL0; -+ u32 before, after; -+ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ if ((qman_ip_rev & 0xFF00) == QMAN_REV31) { -+ /* LS1043A - only one L2 cache */ -+ cpu_idx = 0; -+ } -+ -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { -+ before = qm_in(REV3_QCSP_IO_CFG(idx)); -+ /* Each pair of vcpu share the same SRQ(SDEST) */ -+ cpu_idx /= 2; -+ after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16); -+ qm_out(REV3_QCSP_IO_CFG(idx), after); -+ } else { -+ before = qm_in(QCSP_IO_CFG(idx)); -+ after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16); -+ qm_out(QCSP_IO_CFG(idx), after); -+ } -+ return 0; -+} -+ -+#define MISC_CFG_WPM_MASK 0x00000002 -+int qm_set_wpm(int wpm) -+{ -+ u32 before; -+ u32 after; -+ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ -+ before = qm_in(MISC_CFG); -+ after = (before & (~MISC_CFG_WPM_MASK)) | (wpm << 1); -+ qm_out(MISC_CFG, after); -+ return 0; -+} -+ -+int qm_get_wpm(int *wpm) -+{ -+ u32 before; -+ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ -+ before = qm_in(MISC_CFG); -+ *wpm = (before & MISC_CFG_WPM_MASK) >> 1; -+ return 0; -+} -+ -+/* CEETM_CFG_PRES register has PRES field which is calculated by: -+ * PRES = (2^22 / credit update reference period) * QMan clock period -+ * = (2^22 * 10^9)/ CONFIG_QMAN_CEETM_UPDATE_PERIOD) / qman_clk -+ */ -+ -+int qman_ceetm_set_prescaler(enum qm_dc_portal portal) -+{ -+ u64 temp; -+ u16 pres; -+ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ -+ temp = 0x400000 * 100; -+ do_div(temp, CONFIG_QMAN_CEETM_UPDATE_PERIOD); -+ temp *= 10000000; -+ do_div(temp, qman_clk); -+ pres = (u16) temp; -+ qm_out(CEETM_CFG_IDX, portal); -+ qm_out(CEETM_CFG_PRES, pres); -+ return 0; -+} -+ -+int qman_ceetm_get_prescaler(u16 *pres) -+{ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ *pres = (u16)qm_in(CEETM_CFG_PRES); -+ return 0; -+} -+ -+#define DCP_CFG_CEETME_MASK 0xFFFF0000 -+#define QM_SP_ENABLE_CEETM(n) (0x80000000 >> (n)) -+int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal) -+{ -+ u32 dcp_cfg; -+ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ -+ dcp_cfg = qm_in(DCP_CFG(portal)); -+ dcp_cfg |= QM_SP_ENABLE_CEETM(sub_portal); -+ qm_out(DCP_CFG(portal), dcp_cfg); -+ return 0; -+} -+ -+int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal) -+{ -+ u32 dcp_cfg; -+ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ dcp_cfg = qm_in(DCP_CFG(portal)); -+ dcp_cfg &= ~(QM_SP_ENABLE_CEETM(sub_portal)); -+ qm_out(DCP_CFG(portal), dcp_cfg); -+ return 0; -+} -+ -+int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num) -+{ -+ if (!qman_have_ccsr()) -+ return -ENODEV; -+ *num = qm_in(CEETM_XSFDR_IN_USE); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_get_xsfdr); -+ -+#ifdef CONFIG_SYSFS -+ -+#define DRV_NAME "fsl-qman" -+#define DCP_MAX_ID 3 -+#define DCP_MIN_ID 0 -+ -+static ssize_t show_pfdr_fpc(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_FPC)); -+}; -+ -+static ssize_t show_dlm_avg(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ u32 data; -+ int i; -+ -+ if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i)) -+ return -EINVAL; -+ if (i < DCP_MIN_ID || i > DCP_MAX_ID) -+ return -EINVAL; -+ data = qm_in(DCP_DLM_AVG(i)); -+ return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8, -+ (data & 0x000000ff)*390625); -+}; -+ -+static ssize_t set_dlm_avg(struct device *dev, -+ struct device_attribute *dev_attr, const char *buf, size_t count) -+{ -+ unsigned long val; -+ int i; -+ -+ if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i)) -+ return -EINVAL; -+ if (i < DCP_MIN_ID || i > DCP_MAX_ID) -+ return -EINVAL; -+ if (kstrtoul(buf, 0, &val)) { -+ dev_dbg(dev, "invalid input %s\n", buf); -+ return -EINVAL; -+ } -+ qm_out(DCP_DLM_AVG(i), val); -+ return count; -+}; -+ -+static ssize_t show_pfdr_cfg(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_CFG)); -+}; -+ -+static ssize_t set_pfdr_cfg(struct device *dev, -+ struct device_attribute *dev_attr, const char *buf, size_t count) -+{ -+ unsigned long val; -+ -+ if (kstrtoul(buf, 0, &val)) { -+ dev_dbg(dev, "invalid input %s\n", buf); -+ return -EINVAL; -+ } -+ qm_out(PFDR_CFG, val); -+ return count; -+}; -+ -+static ssize_t show_sfdr_in_use(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SFDR_IN_USE)); -+}; -+ -+static ssize_t show_idle_stat(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(IDLE_STAT)); -+}; -+ -+static ssize_t show_ci_rlm_avg(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ u32 data = qm_in(CI_RLM_AVG); -+ return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8, -+ (data & 0x000000ff)*390625); -+}; -+ -+static ssize_t set_ci_rlm_avg(struct device *dev, -+ struct device_attribute *dev_attr, const char *buf, size_t count) -+{ -+ unsigned long val; -+ -+ if (kstrtoul(buf, 0, &val)) { -+ dev_dbg(dev, "invalid input %s\n", buf); -+ return -EINVAL; -+ } -+ qm_out(CI_RLM_AVG, val); -+ return count; -+}; -+ -+static ssize_t show_err_isr(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ return snprintf(buf, PAGE_SIZE, "0x%08x\n", qm_in(ERR_ISR)); -+}; -+ -+#define SBEC_MAX_ID 14 -+#define SBEC_MIN_ID 0 -+ -+static ssize_t show_sbec(struct device *dev, -+ struct device_attribute *dev_attr, char *buf) -+{ -+ int i; -+ -+ if (!sscanf(dev_attr->attr.name, "sbec_%d", &i)) -+ return -EINVAL; -+ if (i < SBEC_MIN_ID || i > SBEC_MAX_ID) -+ return -EINVAL; -+ return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SBEC(i))); -+}; -+ -+static DEVICE_ATTR(pfdr_fpc, S_IRUSR, show_pfdr_fpc, NULL); -+static DEVICE_ATTR(pfdr_cfg, S_IRUSR, show_pfdr_cfg, set_pfdr_cfg); -+static DEVICE_ATTR(idle_stat, S_IRUSR, show_idle_stat, NULL); -+static DEVICE_ATTR(ci_rlm_avg, (S_IRUSR|S_IWUSR), -+ show_ci_rlm_avg, set_ci_rlm_avg); -+static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL); -+static DEVICE_ATTR(sfdr_in_use, S_IRUSR, show_sfdr_in_use, NULL); -+ -+static DEVICE_ATTR(dcp0_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); -+static DEVICE_ATTR(dcp1_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); -+static DEVICE_ATTR(dcp2_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); -+static DEVICE_ATTR(dcp3_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg); -+ -+static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_2, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_3, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_4, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_5, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_6, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_7, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_8, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_9, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_10, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_11, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_12, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_13, S_IRUSR, show_sbec, NULL); -+static DEVICE_ATTR(sbec_14, S_IRUSR, show_sbec, NULL); -+ -+static struct attribute *qman_dev_attributes[] = { -+ &dev_attr_pfdr_fpc.attr, -+ &dev_attr_pfdr_cfg.attr, -+ &dev_attr_idle_stat.attr, -+ &dev_attr_ci_rlm_avg.attr, -+ &dev_attr_err_isr.attr, -+ &dev_attr_dcp0_dlm_avg.attr, -+ &dev_attr_dcp1_dlm_avg.attr, -+ &dev_attr_dcp2_dlm_avg.attr, -+ &dev_attr_dcp3_dlm_avg.attr, -+ /* sfdr_in_use will be added if necessary */ -+ NULL -+}; -+ -+static struct attribute *qman_dev_ecr_attributes[] = { -+ &dev_attr_sbec_0.attr, -+ &dev_attr_sbec_1.attr, -+ &dev_attr_sbec_2.attr, -+ &dev_attr_sbec_3.attr, -+ &dev_attr_sbec_4.attr, -+ &dev_attr_sbec_5.attr, -+ &dev_attr_sbec_6.attr, -+ &dev_attr_sbec_7.attr, -+ &dev_attr_sbec_8.attr, -+ &dev_attr_sbec_9.attr, -+ &dev_attr_sbec_10.attr, -+ &dev_attr_sbec_11.attr, -+ &dev_attr_sbec_12.attr, -+ &dev_attr_sbec_13.attr, -+ &dev_attr_sbec_14.attr, -+ NULL -+}; -+ -+/* root level */ -+static const struct attribute_group qman_dev_attr_grp = { -+ .name = NULL, -+ .attrs = qman_dev_attributes -+}; -+static const struct attribute_group qman_dev_ecr_grp = { -+ .name = "error_capture", -+ .attrs = qman_dev_ecr_attributes -+}; -+ -+static int of_fsl_qman_remove(struct platform_device *ofdev) -+{ -+ sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp); -+ return 0; -+}; -+ -+static int of_fsl_qman_probe(struct platform_device *ofdev) -+{ -+ int ret; -+ -+ ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_attr_grp); -+ if (ret) -+ goto done; -+ ret = sysfs_add_file_to_group(&ofdev->dev.kobj, -+ &dev_attr_sfdr_in_use.attr, qman_dev_attr_grp.name); -+ if (ret) -+ goto del_group_0; -+ ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_ecr_grp); -+ if (ret) -+ goto del_group_0; -+ -+ goto done; -+ -+del_group_0: -+ sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp); -+done: -+ if (ret) -+ dev_err(&ofdev->dev, -+ "Cannot create dev attributes ret=%d\n", ret); -+ return ret; -+}; -+ -+static struct of_device_id of_fsl_qman_ids[] = { -+ { -+ .compatible = "fsl,qman", -+ }, -+ {} -+}; -+MODULE_DEVICE_TABLE(of, of_fsl_qman_ids); -+ -+#ifdef CONFIG_SUSPEND -+ -+static u32 saved_isdr; -+static int qman_pm_suspend_noirq(struct device *dev) -+{ -+ uint32_t idle_state; -+ -+ suspend_unused_qportal(); -+ /* save isdr, disable all, clear isr */ -+ saved_isdr = qm_err_isr_disable_read(qm); -+ qm_err_isr_disable_write(qm, 0xffffffff); -+ qm_err_isr_status_clear(qm, 0xffffffff); -+ idle_state = qm_in(IDLE_STAT); -+ if (!(idle_state & 0x1)) { -+ pr_err("Qman not idle 0x%x aborting\n", idle_state); -+ qm_err_isr_disable_write(qm, saved_isdr); -+ resume_unused_qportal(); -+ return -EBUSY; -+ } -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Qman suspend code, IDLE_STAT = 0x%x\n", idle_state); -+#endif -+ return 0; -+} -+ -+static int qman_pm_resume_noirq(struct device *dev) -+{ -+ /* restore isdr */ -+ qm_err_isr_disable_write(qm, saved_isdr); -+ resume_unused_qportal(); -+ return 0; -+} -+#else -+#define qman_pm_suspend_noirq NULL -+#define qman_pm_resume_noirq NULL -+#endif -+ -+static const struct dev_pm_ops qman_pm_ops = { -+ .suspend_noirq = qman_pm_suspend_noirq, -+ .resume_noirq = qman_pm_resume_noirq, -+}; -+ -+static struct platform_driver of_fsl_qman_driver = { -+ .driver = { -+ .owner = THIS_MODULE, -+ .name = DRV_NAME, -+ .of_match_table = of_fsl_qman_ids, -+ .pm = &qman_pm_ops, -+ }, -+ .probe = of_fsl_qman_probe, -+ .remove = of_fsl_qman_remove, -+}; -+ -+static int qman_ctrl_init(void) -+{ -+ return platform_driver_register(&of_fsl_qman_driver); -+} -+ -+static void qman_ctrl_exit(void) -+{ -+ platform_driver_unregister(&of_fsl_qman_driver); -+} -+ -+module_init(qman_ctrl_init); -+module_exit(qman_ctrl_exit); -+ -+#endif /* CONFIG_SYSFS */ ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_debugfs.c -@@ -0,0 +1,1594 @@ -+/* Copyright 2010-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+#include "qman_private.h" -+ -+#define MAX_FQID (0x00ffffff) -+#define QM_FQD_BLOCK_SIZE 64 -+#define QM_FQD_AR (0xC10) -+ -+static u32 fqid_max; -+static u64 qman_ccsr_start; -+static u64 qman_ccsr_size; -+ -+static const char * const state_txt[] = { -+ "Out of Service", -+ "Retired", -+ "Tentatively Scheduled", -+ "Truly Scheduled", -+ "Parked", -+ "Active, Active Held or Held Suspended", -+ "Unknown State 6", -+ "Unknown State 7", -+ NULL, -+}; -+ -+static const u8 fqd_states[] = { -+ QM_MCR_NP_STATE_OOS, QM_MCR_NP_STATE_RETIRED, QM_MCR_NP_STATE_TEN_SCHED, -+ QM_MCR_NP_STATE_TRU_SCHED, QM_MCR_NP_STATE_PARKED, -+ QM_MCR_NP_STATE_ACTIVE}; -+ -+struct mask_to_text { -+ u16 mask; -+ const char *txt; -+}; -+ -+struct mask_filter_s { -+ u16 mask; -+ u8 filter; -+}; -+ -+static const struct mask_filter_s mask_filter[] = { -+ {QM_FQCTRL_PREFERINCACHE, 0}, -+ {QM_FQCTRL_PREFERINCACHE, 1}, -+ {QM_FQCTRL_HOLDACTIVE, 0}, -+ {QM_FQCTRL_HOLDACTIVE, 1}, -+ {QM_FQCTRL_AVOIDBLOCK, 0}, -+ {QM_FQCTRL_AVOIDBLOCK, 1}, -+ {QM_FQCTRL_FORCESFDR, 0}, -+ {QM_FQCTRL_FORCESFDR, 1}, -+ {QM_FQCTRL_CPCSTASH, 0}, -+ {QM_FQCTRL_CPCSTASH, 1}, -+ {QM_FQCTRL_CTXASTASHING, 0}, -+ {QM_FQCTRL_CTXASTASHING, 1}, -+ {QM_FQCTRL_ORP, 0}, -+ {QM_FQCTRL_ORP, 1}, -+ {QM_FQCTRL_TDE, 0}, -+ {QM_FQCTRL_TDE, 1}, -+ {QM_FQCTRL_CGE, 0}, -+ {QM_FQCTRL_CGE, 1} -+}; -+ -+static const struct mask_to_text fq_ctrl_text_list[] = { -+ { -+ .mask = QM_FQCTRL_PREFERINCACHE, -+ .txt = "Prefer in cache", -+ }, -+ { -+ .mask = QM_FQCTRL_HOLDACTIVE, -+ .txt = "Hold active in portal", -+ }, -+ { -+ .mask = QM_FQCTRL_AVOIDBLOCK, -+ .txt = "Avoid Blocking", -+ }, -+ { -+ .mask = QM_FQCTRL_FORCESFDR, -+ .txt = "High-priority SFDRs", -+ }, -+ { -+ .mask = QM_FQCTRL_CPCSTASH, -+ .txt = "CPC Stash Enable", -+ }, -+ { -+ .mask = QM_FQCTRL_CTXASTASHING, -+ .txt = "Context-A stashing", -+ }, -+ { -+ .mask = QM_FQCTRL_ORP, -+ .txt = "ORP Enable", -+ }, -+ { -+ .mask = QM_FQCTRL_TDE, -+ .txt = "Tail-Drop Enable", -+ }, -+ { -+ .mask = QM_FQCTRL_CGE, -+ .txt = "Congestion Group Enable", -+ }, -+ { -+ .mask = 0, -+ .txt = NULL, -+ } -+}; -+ -+static const char *get_fqd_ctrl_text(u16 mask) -+{ -+ int i = 0; -+ -+ while (fq_ctrl_text_list[i].txt != NULL) { -+ if (fq_ctrl_text_list[i].mask == mask) -+ return fq_ctrl_text_list[i].txt; -+ i++; -+ } -+ return NULL; -+} -+ -+static const struct mask_to_text stashing_text_list[] = { -+ { -+ .mask = QM_STASHING_EXCL_CTX, -+ .txt = "FQ Ctx Stash" -+ }, -+ { -+ .mask = QM_STASHING_EXCL_DATA, -+ .txt = "Frame Data Stash", -+ }, -+ { -+ .mask = QM_STASHING_EXCL_ANNOTATION, -+ .txt = "Frame Annotation Stash", -+ }, -+ { -+ .mask = 0, -+ .txt = NULL, -+ }, -+}; -+ -+static int user_input_convert(const char __user *user_buf, size_t count, -+ unsigned long *val) -+{ -+ char buf[12]; -+ -+ if (count > sizeof(buf) - 1) -+ return -EINVAL; -+ if (copy_from_user(buf, user_buf, count)) -+ return -EFAULT; -+ buf[count] = '\0'; -+ if (kstrtoul(buf, 0, val)) -+ return -EINVAL; -+ return 0; -+} -+ -+struct line_buffer_fq { -+ u32 buf[8]; -+ u32 buf_cnt; -+ int line_cnt; -+}; -+ -+static void add_to_line_buffer(struct line_buffer_fq *line_buf, u32 fqid, -+ struct seq_file *file) -+{ -+ line_buf->buf[line_buf->buf_cnt] = fqid; -+ line_buf->buf_cnt++; -+ if (line_buf->buf_cnt == 8) { -+ /* Buffer is full, flush it */ -+ if (line_buf->line_cnt != 0) -+ seq_puts(file, ",\n"); -+ seq_printf(file, "0x%06x,0x%06x,0x%06x,0x%06x,0x%06x," -+ "0x%06x,0x%06x,0x%06x", -+ line_buf->buf[0], line_buf->buf[1], line_buf->buf[2], -+ line_buf->buf[3], line_buf->buf[4], line_buf->buf[5], -+ line_buf->buf[6], line_buf->buf[7]); -+ line_buf->buf_cnt = 0; -+ line_buf->line_cnt++; -+ } -+} -+ -+static void flush_line_buffer(struct line_buffer_fq *line_buf, -+ struct seq_file *file) -+{ -+ if (line_buf->buf_cnt) { -+ int y = 0; -+ if (line_buf->line_cnt != 0) -+ seq_puts(file, ",\n"); -+ while (y != line_buf->buf_cnt) { -+ if (y+1 == line_buf->buf_cnt) -+ seq_printf(file, "0x%06x", line_buf->buf[y]); -+ else -+ seq_printf(file, "0x%06x,", line_buf->buf[y]); -+ y++; -+ } -+ line_buf->line_cnt++; -+ } -+ if (line_buf->line_cnt) -+ seq_putc(file, '\n'); -+} -+ -+static struct dentry *dfs_root; /* debugfs root directory */ -+ -+/******************************************************************************* -+ * Query Frame Queue Non Programmable Fields -+ ******************************************************************************/ -+struct query_fq_np_fields_data_s { -+ u32 fqid; -+}; -+static struct query_fq_np_fields_data_s query_fq_np_fields_data = { -+ .fqid = 1, -+}; -+ -+static int query_fq_np_fields_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_mcr_queryfq_np np; -+ struct qman_fq fq; -+ -+ fq.fqid = query_fq_np_fields_data.fqid; -+ ret = qman_query_fq_np(&fq, &np); -+ if (ret) -+ return ret; -+ /* Print state */ -+ seq_printf(file, "Query FQ Non Programmable Fields Result fqid 0x%x\n", -+ fq.fqid); -+ seq_printf(file, " force eligible pending: %s\n", -+ (np.state & QM_MCR_NP_STATE_FE) ? "yes" : "no"); -+ seq_printf(file, " retirement pending: %s\n", -+ (np.state & QM_MCR_NP_STATE_R) ? "yes" : "no"); -+ seq_printf(file, " state: %s\n", -+ state_txt[np.state & QM_MCR_NP_STATE_MASK]); -+ seq_printf(file, " fq_link: 0x%x\n", np.fqd_link); -+ seq_printf(file, " odp_seq: %u\n", np.odp_seq); -+ seq_printf(file, " orp_nesn: %u\n", np.orp_nesn); -+ seq_printf(file, " orp_ea_hseq: %u\n", np.orp_ea_hseq); -+ seq_printf(file, " orp_ea_tseq: %u\n", np.orp_ea_tseq); -+ seq_printf(file, " orp_ea_hptr: 0x%x\n", np.orp_ea_hptr); -+ seq_printf(file, " orp_ea_tptr: 0x%x\n", np.orp_ea_tptr); -+ seq_printf(file, " pfdr_hptr: 0x%x\n", np.pfdr_hptr); -+ seq_printf(file, " pfdr_tptr: 0x%x\n", np.pfdr_tptr); -+ seq_printf(file, " is: ics_surp contains a %s\n", -+ (np.is) ? "deficit" : "surplus"); -+ seq_printf(file, " ics_surp: %u\n", np.ics_surp); -+ seq_printf(file, " byte_cnt: %u\n", np.byte_cnt); -+ seq_printf(file, " frm_cnt: %u\n", np.frm_cnt); -+ seq_printf(file, " ra1_sfdr: 0x%x\n", np.ra1_sfdr); -+ seq_printf(file, " ra2_sfdr: 0x%x\n", np.ra2_sfdr); -+ seq_printf(file, " od1_sfdr: 0x%x\n", np.od1_sfdr); -+ seq_printf(file, " od2_sfdr: 0x%x\n", np.od2_sfdr); -+ seq_printf(file, " od3_sfdr: 0x%x\n", np.od3_sfdr); -+ return 0; -+} -+ -+static int query_fq_np_fields_open(struct inode *inode, -+ struct file *file) -+{ -+ return single_open(file, query_fq_np_fields_show, NULL); -+} -+ -+static ssize_t query_fq_np_fields_write(struct file *f, -+ const char __user *buf, size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ if (val > MAX_FQID) -+ return -EINVAL; -+ query_fq_np_fields_data.fqid = (u32)val; -+ return count; -+} -+ -+static const struct file_operations query_fq_np_fields_fops = { -+ .owner = THIS_MODULE, -+ .open = query_fq_np_fields_open, -+ .read = seq_read, -+ .write = query_fq_np_fields_write, -+ .release = single_release, -+}; -+ -+/******************************************************************************* -+ * Frame Queue Programmable Fields -+ ******************************************************************************/ -+struct query_fq_fields_data_s { -+ u32 fqid; -+}; -+ -+static struct query_fq_fields_data_s query_fq_fields_data = { -+ .fqid = 1, -+}; -+ -+static int query_fq_fields_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_fqd fqd; -+ struct qman_fq fq; -+ int i = 0; -+ -+ memset(&fqd, 0, sizeof(struct qm_fqd)); -+ fq.fqid = query_fq_fields_data.fqid; -+ ret = qman_query_fq(&fq, &fqd); -+ if (ret) -+ return ret; -+ seq_printf(file, "Query FQ Programmable Fields Result fqid 0x%x\n", -+ fq.fqid); -+ seq_printf(file, " orprws: %u\n", fqd.orprws); -+ seq_printf(file, " oa: %u\n", fqd.oa); -+ seq_printf(file, " olws: %u\n", fqd.olws); -+ -+ seq_printf(file, " cgid: %u\n", fqd.cgid); -+ -+ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) == 0) -+ seq_puts(file, " fq_ctrl: None\n"); -+ else { -+ i = 0; -+ seq_puts(file, " fq_ctrl:\n"); -+ while (fq_ctrl_text_list[i].txt != NULL) { -+ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) & -+ fq_ctrl_text_list[i].mask) -+ seq_printf(file, " %s\n", -+ fq_ctrl_text_list[i].txt); -+ i++; -+ } -+ } -+ seq_printf(file, " dest_channel: %u\n", fqd.dest.channel); -+ seq_printf(file, " dest_wq: %u\n", fqd.dest.wq); -+ seq_printf(file, " ics_cred: %u\n", fqd.ics_cred); -+ seq_printf(file, " td_mant: %u\n", fqd.td.mant); -+ seq_printf(file, " td_exp: %u\n", fqd.td.exp); -+ -+ seq_printf(file, " ctx_b: 0x%x\n", fqd.context_b); -+ -+ seq_printf(file, " ctx_a: 0x%llx\n", qm_fqd_stashing_get64(&fqd)); -+ /* Any stashing configured */ -+ if ((fqd.context_a.stashing.exclusive & 0x7) == 0) -+ seq_puts(file, " ctx_a_stash_exclusive: None\n"); -+ else { -+ seq_puts(file, " ctx_a_stash_exclusive:\n"); -+ i = 0; -+ while (stashing_text_list[i].txt != NULL) { -+ if ((fqd.fq_ctrl & 0x7) & stashing_text_list[i].mask) -+ seq_printf(file, " %s\n", -+ stashing_text_list[i].txt); -+ i++; -+ } -+ } -+ seq_printf(file, " ctx_a_stash_annotation_cl: %u\n", -+ fqd.context_a.stashing.annotation_cl); -+ seq_printf(file, " ctx_a_stash_data_cl: %u\n", -+ fqd.context_a.stashing.data_cl); -+ seq_printf(file, " ctx_a_stash_context_cl: %u\n", -+ fqd.context_a.stashing.context_cl); -+ return 0; -+} -+ -+static int query_fq_fields_open(struct inode *inode, -+ struct file *file) -+{ -+ return single_open(file, query_fq_fields_show, NULL); -+} -+ -+static ssize_t query_fq_fields_write(struct file *f, -+ const char __user *buf, size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ if (val > MAX_FQID) -+ return -EINVAL; -+ query_fq_fields_data.fqid = (u32)val; -+ return count; -+} -+ -+static const struct file_operations query_fq_fields_fops = { -+ .owner = THIS_MODULE, -+ .open = query_fq_fields_open, -+ .read = seq_read, -+ .write = query_fq_fields_write, -+ .release = single_release, -+}; -+ -+/******************************************************************************* -+ * Query WQ lengths -+ ******************************************************************************/ -+struct query_wq_lengths_data_s { -+ union { -+ u16 channel_wq; /* ignores wq (3 lsbits) */ -+ struct { -+ u16 id:13; /* qm_channel */ -+ u16 __reserved:3; -+ } __packed channel; -+ }; -+}; -+static struct query_wq_lengths_data_s query_wq_lengths_data; -+static int query_wq_lengths_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_mcr_querywq wq; -+ int i; -+ -+ memset(&wq, 0, sizeof(struct qm_mcr_querywq)); -+ wq.channel.id = query_wq_lengths_data.channel.id; -+ ret = qman_query_wq(0, &wq); -+ if (ret) -+ return ret; -+ seq_printf(file, "Query Result For Channel: 0x%x\n", wq.channel.id); -+ for (i = 0; i < 8; i++) -+ /* mask out upper 4 bits since they are not part of length */ -+ seq_printf(file, " wq%d_len : %u\n", i, wq.wq_len[i] & 0x0fff); -+ return 0; -+} -+ -+static int query_wq_lengths_open(struct inode *inode, -+ struct file *file) -+{ -+ return single_open(file, query_wq_lengths_show, NULL); -+} -+ -+static ssize_t query_wq_lengths_write(struct file *f, -+ const char __user *buf, size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ if (val > 0xfff8) -+ return -EINVAL; -+ query_wq_lengths_data.channel.id = (u16)val; -+ return count; -+} -+ -+static const struct file_operations query_wq_lengths_fops = { -+ .owner = THIS_MODULE, -+ .open = query_wq_lengths_open, -+ .read = seq_read, -+ .write = query_wq_lengths_write, -+ .release = single_release, -+}; -+ -+/******************************************************************************* -+ * Query CGR -+ ******************************************************************************/ -+struct query_cgr_s { -+ u8 cgid; -+}; -+static struct query_cgr_s query_cgr_data; -+ -+static int query_cgr_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_mcr_querycgr cgrd; -+ struct qman_cgr cgr; -+ int i, j; -+ u32 mask; -+ -+ memset(&cgr, 0, sizeof(cgr)); -+ memset(&cgrd, 0, sizeof(cgrd)); -+ cgr.cgrid = query_cgr_data.cgid; -+ ret = qman_query_cgr(&cgr, &cgrd); -+ if (ret) -+ return ret; -+ seq_printf(file, "Query CGR id 0x%x\n", cgr.cgrid); -+ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ cgrd.cgr.wr_parm_g.MA, cgrd.cgr.wr_parm_g.Mn, -+ cgrd.cgr.wr_parm_g.SA, cgrd.cgr.wr_parm_g.Sn, -+ cgrd.cgr.wr_parm_g.Pn); -+ -+ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ cgrd.cgr.wr_parm_y.MA, cgrd.cgr.wr_parm_y.Mn, -+ cgrd.cgr.wr_parm_y.SA, cgrd.cgr.wr_parm_y.Sn, -+ cgrd.cgr.wr_parm_y.Pn); -+ -+ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ cgrd.cgr.wr_parm_r.MA, cgrd.cgr.wr_parm_r.Mn, -+ cgrd.cgr.wr_parm_r.SA, cgrd.cgr.wr_parm_r.Sn, -+ cgrd.cgr.wr_parm_r.Pn); -+ -+ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n", -+ cgrd.cgr.wr_en_g, cgrd.cgr.wr_en_y, cgrd.cgr.wr_en_r); -+ -+ seq_printf(file, " cscn_en: %u\n", cgrd.cgr.cscn_en); -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) { -+ seq_puts(file, " cscn_targ_dcp:\n"); -+ mask = 0x80000000; -+ for (i = 0; i < 32; i++) { -+ if (cgrd.cgr.cscn_targ & mask) -+ seq_printf(file, " send CSCN to dcp %u\n", -+ (31 - i)); -+ mask >>= 1; -+ } -+ -+ seq_puts(file, " cscn_targ_swp:\n"); -+ for (i = 0; i < 4; i++) { -+ mask = 0x80000000; -+ for (j = 0; j < 32; j++) { -+ if (cgrd.cscn_targ_swp[i] & mask) -+ seq_printf(file, " send CSCN to swp" -+ " %u\n", (127 - (i * 32) - j)); -+ mask >>= 1; -+ } -+ } -+ } else { -+ seq_printf(file, " cscn_targ: %u\n", cgrd.cgr.cscn_targ); -+ } -+ seq_printf(file, " cstd_en: %u\n", cgrd.cgr.cstd_en); -+ seq_printf(file, " cs: %u\n", cgrd.cgr.cs); -+ -+ seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n", -+ cgrd.cgr.cs_thres.TA, cgrd.cgr.cs_thres.Tn); -+ -+ seq_printf(file, " mode: %s\n", -+ (cgrd.cgr.mode & QMAN_CGR_MODE_FRAME) ? -+ "frame count" : "byte count"); -+ seq_printf(file, " i_bcnt: %llu\n", qm_mcr_querycgr_i_get64(&cgrd)); -+ seq_printf(file, " a_bcnt: %llu\n", qm_mcr_querycgr_a_get64(&cgrd)); -+ -+ return 0; -+} -+ -+static int query_cgr_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, query_cgr_show, NULL); -+} -+ -+static ssize_t query_cgr_write(struct file *f, const char __user *buf, -+ size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ if (val > 0xff) -+ return -EINVAL; -+ query_cgr_data.cgid = (u8)val; -+ return count; -+} -+ -+static const struct file_operations query_cgr_fops = { -+ .owner = THIS_MODULE, -+ .open = query_cgr_open, -+ .read = seq_read, -+ .write = query_cgr_write, -+ .release = single_release, -+}; -+ -+/******************************************************************************* -+ * Test Write CGR -+ ******************************************************************************/ -+struct test_write_cgr_s { -+ u64 i_bcnt; -+ u8 cgid; -+}; -+static struct test_write_cgr_s test_write_cgr_data; -+ -+static int testwrite_cgr_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_mcr_cgrtestwrite result; -+ struct qman_cgr cgr; -+ u64 i_bcnt; -+ -+ memset(&cgr, 0, sizeof(struct qman_cgr)); -+ memset(&result, 0, sizeof(struct qm_mcr_cgrtestwrite)); -+ cgr.cgrid = test_write_cgr_data.cgid; -+ i_bcnt = test_write_cgr_data.i_bcnt; -+ ret = qman_testwrite_cgr(&cgr, i_bcnt, &result); -+ if (ret) -+ return ret; -+ seq_printf(file, "CGR Test Write CGR id 0x%x\n", cgr.cgrid); -+ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ result.cgr.wr_parm_g.MA, result.cgr.wr_parm_g.Mn, -+ result.cgr.wr_parm_g.SA, result.cgr.wr_parm_g.Sn, -+ result.cgr.wr_parm_g.Pn); -+ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ result.cgr.wr_parm_y.MA, result.cgr.wr_parm_y.Mn, -+ result.cgr.wr_parm_y.SA, result.cgr.wr_parm_y.Sn, -+ result.cgr.wr_parm_y.Pn); -+ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ result.cgr.wr_parm_r.MA, result.cgr.wr_parm_r.Mn, -+ result.cgr.wr_parm_r.SA, result.cgr.wr_parm_r.Sn, -+ result.cgr.wr_parm_r.Pn); -+ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n", -+ result.cgr.wr_en_g, result.cgr.wr_en_y, result.cgr.wr_en_r); -+ seq_printf(file, " cscn_en: %u\n", result.cgr.cscn_en); -+ seq_printf(file, " cscn_targ: %u\n", result.cgr.cscn_targ); -+ seq_printf(file, " cstd_en: %u\n", result.cgr.cstd_en); -+ seq_printf(file, " cs: %u\n", result.cgr.cs); -+ seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n", -+ result.cgr.cs_thres.TA, result.cgr.cs_thres.Tn); -+ -+ /* Add Mode for Si 2 */ -+ seq_printf(file, " mode: %s\n", -+ (result.cgr.mode & QMAN_CGR_MODE_FRAME) ? -+ "frame count" : "byte count"); -+ -+ seq_printf(file, " i_bcnt: %llu\n", -+ qm_mcr_cgrtestwrite_i_get64(&result)); -+ seq_printf(file, " a_bcnt: %llu\n", -+ qm_mcr_cgrtestwrite_a_get64(&result)); -+ seq_printf(file, " wr_prob_g: %u\n", result.wr_prob_g); -+ seq_printf(file, " wr_prob_y: %u\n", result.wr_prob_y); -+ seq_printf(file, " wr_prob_r: %u\n", result.wr_prob_r); -+ return 0; -+} -+ -+static int testwrite_cgr_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, testwrite_cgr_show, NULL); -+} -+ -+static const struct file_operations testwrite_cgr_fops = { -+ .owner = THIS_MODULE, -+ .open = testwrite_cgr_open, -+ .read = seq_read, -+ .release = single_release, -+}; -+ -+ -+static int testwrite_cgr_ibcnt_show(struct seq_file *file, void *offset) -+{ -+ seq_printf(file, "i_bcnt: %llu\n", test_write_cgr_data.i_bcnt); -+ return 0; -+} -+static int testwrite_cgr_ibcnt_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, testwrite_cgr_ibcnt_show, NULL); -+} -+ -+static ssize_t testwrite_cgr_ibcnt_write(struct file *f, const char __user *buf, -+ size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ test_write_cgr_data.i_bcnt = val; -+ return count; -+} -+ -+static const struct file_operations teswrite_cgr_ibcnt_fops = { -+ .owner = THIS_MODULE, -+ .open = testwrite_cgr_ibcnt_open, -+ .read = seq_read, -+ .write = testwrite_cgr_ibcnt_write, -+ .release = single_release, -+}; -+ -+static int testwrite_cgr_cgrid_show(struct seq_file *file, void *offset) -+{ -+ seq_printf(file, "cgrid: %u\n", (u32)test_write_cgr_data.cgid); -+ return 0; -+} -+static int testwrite_cgr_cgrid_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, testwrite_cgr_cgrid_show, NULL); -+} -+ -+static ssize_t testwrite_cgr_cgrid_write(struct file *f, const char __user *buf, -+ size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ if (val > 0xff) -+ return -EINVAL; -+ test_write_cgr_data.cgid = (u8)val; -+ return count; -+} -+ -+static const struct file_operations teswrite_cgr_cgrid_fops = { -+ .owner = THIS_MODULE, -+ .open = testwrite_cgr_cgrid_open, -+ .read = seq_read, -+ .write = testwrite_cgr_cgrid_write, -+ .release = single_release, -+}; -+ -+/******************************************************************************* -+ * Query Congestion State -+ ******************************************************************************/ -+static int query_congestion_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_mcr_querycongestion cs; -+ int i, j, in_cong = 0; -+ u32 mask; -+ -+ memset(&cs, 0, sizeof(struct qm_mcr_querycongestion)); -+ ret = qman_query_congestion(&cs); -+ if (ret) -+ return ret; -+ seq_puts(file, "Query Congestion Result\n"); -+ for (i = 0; i < 8; i++) { -+ mask = 0x80000000; -+ for (j = 0; j < 32; j++) { -+ if (cs.state.__state[i] & mask) { -+ in_cong = 1; -+ seq_printf(file, " cg %u: %s\n", (i*32)+j, -+ "in congestion"); -+ } -+ mask >>= 1; -+ } -+ } -+ if (!in_cong) -+ seq_puts(file, " All congestion groups not congested.\n"); -+ return 0; -+} -+ -+static int query_congestion_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, query_congestion_show, NULL); -+} -+ -+static const struct file_operations query_congestion_fops = { -+ .owner = THIS_MODULE, -+ .open = query_congestion_open, -+ .read = seq_read, -+ .release = single_release, -+}; -+ -+/******************************************************************************* -+ * Query CCGR -+ ******************************************************************************/ -+struct query_ccgr_s { -+ u32 ccgid; -+}; -+static struct query_ccgr_s query_ccgr_data; -+ -+static int query_ccgr_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_mcr_ceetm_ccgr_query ccgr_query; -+ struct qm_mcc_ceetm_ccgr_query query_opts; -+ int i, j; -+ u32 mask; -+ -+ memset(&ccgr_query, 0, sizeof(struct qm_mcr_ceetm_ccgr_query)); -+ memset(&query_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_query)); -+ -+ if ((qman_ip_rev & 0xFF00) < QMAN_REV30) -+ return -EINVAL; -+ -+ seq_printf(file, "Query CCGID %x\n", query_ccgr_data.ccgid); -+ query_opts.dcpid = ((query_ccgr_data.ccgid & 0xFF000000) >> 24); -+ query_opts.ccgrid = query_ccgr_data.ccgid & 0x000001FF; -+ ret = qman_ceetm_query_ccgr(&query_opts, &ccgr_query); -+ if (ret) -+ return ret; -+ seq_printf(file, "Query CCGR id %x in DCP %d\n", query_opts.ccgrid, -+ query_opts.dcpid); -+ seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ ccgr_query.cm_query.wr_parm_g.MA, -+ ccgr_query.cm_query.wr_parm_g.Mn, -+ ccgr_query.cm_query.wr_parm_g.SA, -+ ccgr_query.cm_query.wr_parm_g.Sn, -+ ccgr_query.cm_query.wr_parm_g.Pn); -+ -+ seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ ccgr_query.cm_query.wr_parm_y.MA, -+ ccgr_query.cm_query.wr_parm_y.Mn, -+ ccgr_query.cm_query.wr_parm_y.SA, -+ ccgr_query.cm_query.wr_parm_y.Sn, -+ ccgr_query.cm_query.wr_parm_y.Pn); -+ -+ seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n", -+ ccgr_query.cm_query.wr_parm_r.MA, -+ ccgr_query.cm_query.wr_parm_r.Mn, -+ ccgr_query.cm_query.wr_parm_r.SA, -+ ccgr_query.cm_query.wr_parm_r.Sn, -+ ccgr_query.cm_query.wr_parm_r.Pn); -+ -+ seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n", -+ ccgr_query.cm_query.ctl_wr_en_g, -+ ccgr_query.cm_query.ctl_wr_en_y, -+ ccgr_query.cm_query.ctl_wr_en_r); -+ -+ seq_printf(file, " cscn_en: %u\n", ccgr_query.cm_query.ctl_cscn_en); -+ seq_puts(file, " cscn_targ_dcp:\n"); -+ mask = 0x80000000; -+ for (i = 0; i < 32; i++) { -+ if (ccgr_query.cm_query.cscn_targ_dcp & mask) -+ seq_printf(file, " send CSCN to dcp %u\n", (31 - i)); -+ mask >>= 1; -+ } -+ -+ seq_puts(file, " cscn_targ_swp:\n"); -+ for (i = 0; i < 4; i++) { -+ mask = 0x80000000; -+ for (j = 0; j < 32; j++) { -+ if (ccgr_query.cm_query.cscn_targ_swp[i] & mask) -+ seq_printf(file, " send CSCN to swp" -+ "%u\n", (127 - (i * 32) - j)); -+ mask >>= 1; -+ } -+ } -+ -+ seq_printf(file, " td_en: %u\n", ccgr_query.cm_query.ctl_td_en); -+ -+ seq_printf(file, " cs_thresh_in_TA: %u, cs_thresh_in_Tn: %u\n", -+ ccgr_query.cm_query.cs_thres.TA, -+ ccgr_query.cm_query.cs_thres.Tn); -+ -+ seq_printf(file, " cs_thresh_out_TA: %u, cs_thresh_out_Tn: %u\n", -+ ccgr_query.cm_query.cs_thres_x.TA, -+ ccgr_query.cm_query.cs_thres_x.Tn); -+ -+ seq_printf(file, " td_thresh_TA: %u, td_thresh_Tn: %u\n", -+ ccgr_query.cm_query.td_thres.TA, -+ ccgr_query.cm_query.td_thres.Tn); -+ -+ seq_printf(file, " mode: %s\n", -+ (ccgr_query.cm_query.ctl_mode & -+ QMAN_CGR_MODE_FRAME) ? -+ "frame count" : "byte count"); -+ seq_printf(file, " i_cnt: %llu\n", (u64)ccgr_query.cm_query.i_cnt); -+ seq_printf(file, " a_cnt: %llu\n", (u64)ccgr_query.cm_query.a_cnt); -+ -+ return 0; -+} -+ -+static int query_ccgr_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, query_ccgr_show, NULL); -+} -+ -+static ssize_t query_ccgr_write(struct file *f, const char __user *buf, -+ size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ query_ccgr_data.ccgid = val; -+ return count; -+} -+ -+static const struct file_operations query_ccgr_fops = { -+ .owner = THIS_MODULE, -+ .open = query_ccgr_open, -+ .read = seq_read, -+ .write = query_ccgr_write, -+ .release = single_release, -+}; -+/******************************************************************************* -+ * QMan register -+ ******************************************************************************/ -+struct qman_register_s { -+ u32 val; -+}; -+static struct qman_register_s qman_register_data; -+ -+static void init_ccsrmempeek(void) -+{ -+ struct device_node *dn; -+ const u32 *regaddr_p; -+ -+ dn = of_find_compatible_node(NULL, NULL, "fsl,qman"); -+ if (!dn) { -+ pr_info("No fsl,qman node\n"); -+ return; -+ } -+ regaddr_p = of_get_address(dn, 0, &qman_ccsr_size, NULL); -+ if (!regaddr_p) { -+ of_node_put(dn); -+ return; -+ } -+ qman_ccsr_start = of_translate_address(dn, regaddr_p); -+ of_node_put(dn); -+} -+/* This function provides access to QMan ccsr memory map */ -+static int qman_ccsrmempeek(u32 *val, u32 offset) -+{ -+ void __iomem *addr; -+ u64 phys_addr; -+ -+ if (!qman_ccsr_start) -+ return -EINVAL; -+ -+ if (offset > (qman_ccsr_size - sizeof(u32))) -+ return -EINVAL; -+ -+ phys_addr = qman_ccsr_start + offset; -+ addr = ioremap(phys_addr, sizeof(u32)); -+ if (!addr) { -+ pr_err("ccsrmempeek, ioremap failed\n"); -+ return -EINVAL; -+ } -+ *val = in_be32(addr); -+ iounmap(addr); -+ return 0; -+} -+ -+static int qman_ccsrmempeek_show(struct seq_file *file, void *offset) -+{ -+ u32 b; -+ -+ qman_ccsrmempeek(&b, qman_register_data.val); -+ seq_printf(file, "QMan register offset = 0x%x\n", -+ qman_register_data.val); -+ seq_printf(file, "value = 0x%08x\n", b); -+ -+ return 0; -+} -+ -+static int qman_ccsrmempeek_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, qman_ccsrmempeek_show, NULL); -+} -+ -+static ssize_t qman_ccsrmempeek_write(struct file *f, const char __user *buf, -+ size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ /* multiple of 4 */ -+ if (val > (qman_ccsr_size - sizeof(u32))) { -+ pr_info("Input 0x%lx > 0x%llx\n", -+ val, (qman_ccsr_size - sizeof(u32))); -+ return -EINVAL; -+ } -+ if (val & 0x3) { -+ pr_info("Input 0x%lx not multiple of 4\n", val); -+ return -EINVAL; -+ } -+ qman_register_data.val = val; -+ return count; -+} -+ -+static const struct file_operations qman_ccsrmempeek_fops = { -+ .owner = THIS_MODULE, -+ .open = qman_ccsrmempeek_open, -+ .read = seq_read, -+ .write = qman_ccsrmempeek_write, -+}; -+ -+/******************************************************************************* -+ * QMan state -+ ******************************************************************************/ -+static int qman_fqd_state_show(struct seq_file *file, void *offset) -+{ -+ struct qm_mcr_queryfq_np np; -+ struct qman_fq fq; -+ struct line_buffer_fq line_buf; -+ int ret, i; -+ u8 *state = file->private; -+ u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)]; -+ -+ memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt)); -+ memset(&line_buf, 0, sizeof(line_buf)); -+ -+ seq_printf(file, "List of fq ids in state: %s\n", state_txt[*state]); -+ -+ for (i = 1; i < fqid_max; i++) { -+ fq.fqid = i; -+ ret = qman_query_fq_np(&fq, &np); -+ if (ret) -+ return ret; -+ if (*state == (np.state & QM_MCR_NP_STATE_MASK)) -+ add_to_line_buffer(&line_buf, fq.fqid, file); -+ /* Keep a summary count of all states */ -+ if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states)) -+ qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++; -+ } -+ flush_line_buffer(&line_buf, file); -+ -+ for (i = 0; i < ARRAY_SIZE(fqd_states); i++) { -+ seq_printf(file, "%s count = %u\n", state_txt[i], -+ qm_fq_state_cnt[i]); -+ } -+ return 0; -+} -+ -+static int qman_fqd_state_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, qman_fqd_state_show, inode->i_private); -+} -+ -+static const struct file_operations qman_fqd_state_fops = { -+ .owner = THIS_MODULE, -+ .open = qman_fqd_state_open, -+ .read = seq_read, -+}; -+ -+static int qman_fqd_ctrl_show(struct seq_file *file, void *offset) -+{ -+ struct qm_fqd fqd; -+ struct qman_fq fq; -+ u32 fq_en_cnt = 0, fq_di_cnt = 0; -+ int ret, i; -+ struct mask_filter_s *data = file->private; -+ const char *ctrl_txt = get_fqd_ctrl_text(data->mask); -+ struct line_buffer_fq line_buf; -+ -+ memset(&line_buf, 0, sizeof(line_buf)); -+ seq_printf(file, "List of fq ids with: %s :%s\n", -+ ctrl_txt, (data->filter) ? "enabled" : "disabled"); -+ for (i = 1; i < fqid_max; i++) { -+ fq.fqid = i; -+ memset(&fqd, 0, sizeof(struct qm_fqd)); -+ ret = qman_query_fq(&fq, &fqd); -+ if (ret) -+ return ret; -+ if (data->filter) { -+ if (fqd.fq_ctrl & data->mask) -+ add_to_line_buffer(&line_buf, fq.fqid, file); -+ } else { -+ if (!(fqd.fq_ctrl & data->mask)) -+ add_to_line_buffer(&line_buf, fq.fqid, file); -+ } -+ if (fqd.fq_ctrl & data->mask) -+ fq_en_cnt++; -+ else -+ fq_di_cnt++; -+ } -+ flush_line_buffer(&line_buf, file); -+ -+ seq_printf(file, "Total FQD with: %s : enabled = %u\n", -+ ctrl_txt, fq_en_cnt); -+ seq_printf(file, "Total FQD with: %s : disabled = %u\n", -+ ctrl_txt, fq_di_cnt); -+ return 0; -+} -+ -+/******************************************************************************* -+ * QMan ctrl CGE, TDE, ORP, CTX, CPC, SFDR, BLOCK, HOLD, CACHE -+ ******************************************************************************/ -+static int qman_fqd_ctrl_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, qman_fqd_ctrl_show, inode->i_private); -+} -+ -+static const struct file_operations qman_fqd_ctrl_fops = { -+ .owner = THIS_MODULE, -+ .open = qman_fqd_ctrl_open, -+ .read = seq_read, -+}; -+ -+/******************************************************************************* -+ * QMan ctrl summary -+ ******************************************************************************/ -+/******************************************************************************* -+ * QMan summary state -+ ******************************************************************************/ -+static int qman_fqd_non_prog_summary_show(struct seq_file *file, void *offset) -+{ -+ struct qm_mcr_queryfq_np np; -+ struct qman_fq fq; -+ int ret, i; -+ u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)]; -+ -+ memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt)); -+ -+ for (i = 1; i < fqid_max; i++) { -+ fq.fqid = i; -+ ret = qman_query_fq_np(&fq, &np); -+ if (ret) -+ return ret; -+ /* Keep a summary count of all states */ -+ if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states)) -+ qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++; -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(fqd_states); i++) { -+ seq_printf(file, "%s count = %u\n", state_txt[i], -+ qm_fq_state_cnt[i]); -+ } -+ return 0; -+} -+ -+static int qman_fqd_prog_summary_show(struct seq_file *file, void *offset) -+{ -+ struct qm_fqd fqd; -+ struct qman_fq fq; -+ int ret, i , j; -+ u32 qm_prog_cnt[ARRAY_SIZE(mask_filter)/2]; -+ -+ memset(qm_prog_cnt, 0, sizeof(qm_prog_cnt)); -+ -+ for (i = 1; i < fqid_max; i++) { -+ memset(&fqd, 0, sizeof(struct qm_fqd)); -+ fq.fqid = i; -+ ret = qman_query_fq(&fq, &fqd); -+ if (ret) -+ return ret; -+ /* Keep a summary count of all states */ -+ for (j = 0; j < ARRAY_SIZE(mask_filter); j += 2) -+ if ((fqd.fq_ctrl & QM_FQCTRL_MASK) & -+ mask_filter[j].mask) -+ qm_prog_cnt[j/2]++; -+ } -+ for (i = 0; i < ARRAY_SIZE(mask_filter) / 2; i++) { -+ seq_printf(file, "%s count = %u\n", -+ get_fqd_ctrl_text(mask_filter[i*2].mask), -+ qm_prog_cnt[i]); -+ } -+ return 0; -+} -+ -+static int qman_fqd_summary_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ -+ /* Display summary of non programmable fields */ -+ ret = qman_fqd_non_prog_summary_show(file, offset); -+ if (ret) -+ return ret; -+ seq_puts(file, "-----------------------------------------\n"); -+ /* Display programmable fields */ -+ ret = qman_fqd_prog_summary_show(file, offset); -+ if (ret) -+ return ret; -+ return 0; -+} -+ -+static int qman_fqd_summary_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, qman_fqd_summary_show, NULL); -+} -+ -+static const struct file_operations qman_fqd_summary_fops = { -+ .owner = THIS_MODULE, -+ .open = qman_fqd_summary_open, -+ .read = seq_read, -+}; -+ -+/******************************************************************************* -+ * QMan destination work queue -+ ******************************************************************************/ -+struct qman_dest_wq_s { -+ u16 wq_id; -+}; -+static struct qman_dest_wq_s qman_dest_wq_data = { -+ .wq_id = 0, -+}; -+ -+static int qman_fqd_dest_wq_show(struct seq_file *file, void *offset) -+{ -+ struct qm_fqd fqd; -+ struct qman_fq fq; -+ int ret, i; -+ u16 *wq, wq_id = qman_dest_wq_data.wq_id; -+ struct line_buffer_fq line_buf; -+ -+ memset(&line_buf, 0, sizeof(line_buf)); -+ /* use vmalloc : need to allocate large memory region and don't -+ * require the memory to be physically contiguous. */ -+ wq = vzalloc(sizeof(u16) * (0xFFFF+1)); -+ if (!wq) -+ return -ENOMEM; -+ -+ seq_printf(file, "List of fq ids with destination work queue id" -+ " = 0x%x\n", wq_id); -+ -+ for (i = 1; i < fqid_max; i++) { -+ fq.fqid = i; -+ memset(&fqd, 0, sizeof(struct qm_fqd)); -+ ret = qman_query_fq(&fq, &fqd); -+ if (ret) { -+ vfree(wq); -+ return ret; -+ } -+ if (wq_id == fqd.dest_wq) -+ add_to_line_buffer(&line_buf, fq.fqid, file); -+ wq[fqd.dest_wq]++; -+ } -+ flush_line_buffer(&line_buf, file); -+ -+ seq_puts(file, "Summary of all FQD destination work queue values\n"); -+ for (i = 0; i < 0xFFFF; i++) { -+ if (wq[i]) -+ seq_printf(file, "Channel: 0x%x WQ: 0x%x WQ_ID: 0x%x, " -+ "count = %u\n", i >> 3, i & 0x3, i, wq[i]); -+ } -+ vfree(wq); -+ return 0; -+} -+ -+static ssize_t qman_fqd_dest_wq_write(struct file *f, const char __user *buf, -+ size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ if (val > 0xFFFF) -+ return -EINVAL; -+ qman_dest_wq_data.wq_id = val; -+ return count; -+} -+ -+static int qman_fqd_dest_wq_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, qman_fqd_dest_wq_show, NULL); -+} -+ -+static const struct file_operations qman_fqd_dest_wq_fops = { -+ .owner = THIS_MODULE, -+ .open = qman_fqd_dest_wq_open, -+ .read = seq_read, -+ .write = qman_fqd_dest_wq_write, -+}; -+ -+/******************************************************************************* -+ * QMan Intra-Class Scheduling Credit -+ ******************************************************************************/ -+static int qman_fqd_cred_show(struct seq_file *file, void *offset) -+{ -+ struct qm_fqd fqd; -+ struct qman_fq fq; -+ int ret, i; -+ u32 fq_cnt = 0; -+ struct line_buffer_fq line_buf; -+ -+ memset(&line_buf, 0, sizeof(line_buf)); -+ seq_puts(file, "List of fq ids with Intra-Class Scheduling Credit > 0" -+ "\n"); -+ -+ for (i = 1; i < fqid_max; i++) { -+ fq.fqid = i; -+ memset(&fqd, 0, sizeof(struct qm_fqd)); -+ ret = qman_query_fq(&fq, &fqd); -+ if (ret) -+ return ret; -+ if (fqd.ics_cred > 0) { -+ add_to_line_buffer(&line_buf, fq.fqid, file); -+ fq_cnt++; -+ } -+ } -+ flush_line_buffer(&line_buf, file); -+ -+ seq_printf(file, "Total FQD with ics_cred > 0 = %d\n", fq_cnt); -+ return 0; -+} -+ -+static int qman_fqd_cred_open(struct inode *inode, struct file *file) -+{ -+ return single_open(file, qman_fqd_cred_show, NULL); -+} -+ -+static const struct file_operations qman_fqd_cred_fops = { -+ .owner = THIS_MODULE, -+ .open = qman_fqd_cred_open, -+ .read = seq_read, -+}; -+ -+/******************************************************************************* -+ * Class Queue Fields -+ ******************************************************************************/ -+struct query_cq_fields_data_s { -+ u32 cqid; -+}; -+ -+static struct query_cq_fields_data_s query_cq_fields_data = { -+ .cqid = 1, -+}; -+ -+static int query_cq_fields_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ struct qm_mcr_ceetm_cq_query query_result; -+ unsigned int cqid; -+ unsigned int portal; -+ -+ if ((qman_ip_rev & 0xFF00) < QMAN_REV30) -+ return -EINVAL; -+ -+ cqid = query_cq_fields_data.cqid & 0x00FFFFFF; -+ portal = query_cq_fields_data.cqid >> 24; -+ if (portal > qm_dc_portal_fman1) -+ return -EINVAL; -+ -+ ret = qman_ceetm_query_cq(cqid, portal, &query_result); -+ if (ret) -+ return ret; -+ seq_printf(file, "Query CQ Fields Result cqid 0x%x on DCP %d\n", -+ cqid, portal); -+ seq_printf(file, " ccgid: %u\n", query_result.ccgid); -+ seq_printf(file, " state: %u\n", query_result.state); -+ seq_printf(file, " pfdr_hptr: %u\n", query_result.pfdr_hptr); -+ seq_printf(file, " pfdr_tptr: %u\n", query_result.pfdr_tptr); -+ seq_printf(file, " od1_xsfdr: %u\n", query_result.od1_xsfdr); -+ seq_printf(file, " od2_xsfdr: %u\n", query_result.od2_xsfdr); -+ seq_printf(file, " od3_xsfdr: %u\n", query_result.od3_xsfdr); -+ seq_printf(file, " od4_xsfdr: %u\n", query_result.od4_xsfdr); -+ seq_printf(file, " od5_xsfdr: %u\n", query_result.od5_xsfdr); -+ seq_printf(file, " od6_xsfdr: %u\n", query_result.od6_xsfdr); -+ seq_printf(file, " ra1_xsfdr: %u\n", query_result.ra1_xsfdr); -+ seq_printf(file, " ra2_xsfdr: %u\n", query_result.ra2_xsfdr); -+ seq_printf(file, " frame_count: %u\n", query_result.frm_cnt); -+ -+ return 0; -+} -+ -+static int query_cq_fields_open(struct inode *inode, -+ struct file *file) -+{ -+ return single_open(file, query_cq_fields_show, NULL); -+} -+ -+static ssize_t query_cq_fields_write(struct file *f, -+ const char __user *buf, size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ query_cq_fields_data.cqid = (u32)val; -+ return count; -+} -+ -+static const struct file_operations query_cq_fields_fops = { -+ .owner = THIS_MODULE, -+ .open = query_cq_fields_open, -+ .read = seq_read, -+ .write = query_cq_fields_write, -+ .release = single_release, -+}; -+ -+/******************************************************************************* -+ * READ CEETM_XSFDR_IN_USE -+ ******************************************************************************/ -+struct query_ceetm_xsfdr_data_s { -+ enum qm_dc_portal dcp_portal; -+}; -+ -+static struct query_ceetm_xsfdr_data_s query_ceetm_xsfdr_data; -+ -+static int query_ceetm_xsfdr_show(struct seq_file *file, void *offset) -+{ -+ int ret; -+ unsigned int xsfdr_in_use; -+ enum qm_dc_portal portal; -+ -+ -+ if (qman_ip_rev < QMAN_REV31) -+ return -EINVAL; -+ -+ portal = query_ceetm_xsfdr_data.dcp_portal; -+ ret = qman_ceetm_get_xsfdr(portal, &xsfdr_in_use); -+ if (ret) { -+ seq_printf(file, "Read CEETM_XSFDR_IN_USE on DCP %d failed\n", -+ portal); -+ return ret; -+ } -+ -+ seq_printf(file, "DCP%d: CEETM_XSFDR_IN_USE number is %u\n", portal, -+ (xsfdr_in_use & 0x1FFF)); -+ return 0; -+} -+ -+static int query_ceetm_xsfdr_open(struct inode *inode, -+ struct file *file) -+{ -+ return single_open(file, query_ceetm_xsfdr_show, NULL); -+} -+ -+static ssize_t query_ceetm_xsfdr_write(struct file *f, -+ const char __user *buf, size_t count, loff_t *off) -+{ -+ int ret; -+ unsigned long val; -+ -+ ret = user_input_convert(buf, count, &val); -+ if (ret) -+ return ret; -+ if (val > qm_dc_portal_fman1) -+ return -EINVAL; -+ query_ceetm_xsfdr_data.dcp_portal = (u32)val; -+ return count; -+} -+ -+static const struct file_operations query_ceetm_xsfdr_fops = { -+ .owner = THIS_MODULE, -+ .open = query_ceetm_xsfdr_open, -+ .read = seq_read, -+ .write = query_ceetm_xsfdr_write, -+ .release = single_release, -+}; -+ -+/* helper macros used in qman_debugfs_module_init */ -+#define QMAN_DBGFS_ENTRY(name, mode, parent, data, fops) \ -+ do { \ -+ d = debugfs_create_file(name, \ -+ mode, parent, \ -+ data, \ -+ fops); \ -+ if (d == NULL) { \ -+ ret = -ENOMEM; \ -+ goto _return; \ -+ } \ -+ } while (0) -+ -+/* dfs_root as parent */ -+#define QMAN_DBGFS_ENTRY_ROOT(name, mode, data, fops) \ -+ QMAN_DBGFS_ENTRY(name, mode, dfs_root, data, fops) -+ -+/* fqd_root as parent */ -+#define QMAN_DBGFS_ENTRY_FQDROOT(name, mode, data, fops) \ -+ QMAN_DBGFS_ENTRY(name, mode, fqd_root, data, fops) -+ -+/* fqd state */ -+#define QMAN_DBGFS_ENTRY_FQDSTATE(name, index) \ -+ QMAN_DBGFS_ENTRY_FQDROOT(name, S_IRUGO, \ -+ (void *)&mask_filter[index], &qman_fqd_ctrl_fops) -+ -+static int __init qman_debugfs_module_init(void) -+{ -+ int ret = 0; -+ struct dentry *d, *fqd_root; -+ u32 reg; -+ -+ fqid_max = 0; -+ init_ccsrmempeek(); -+ if (qman_ccsr_start) { -+ if (!qman_ccsrmempeek(®, QM_FQD_AR)) { -+ /* extract the size of the FQD window */ -+ reg = reg & 0x3f; -+ /* calculate valid frame queue descriptor range */ -+ fqid_max = (1 << (reg + 1)) / QM_FQD_BLOCK_SIZE; -+ } -+ } -+ dfs_root = debugfs_create_dir("qman", NULL); -+ fqd_root = debugfs_create_dir("fqd", dfs_root); -+ if (dfs_root == NULL || fqd_root == NULL) { -+ ret = -ENOMEM; -+ pr_err("Cannot create qman/fqd debugfs dir\n"); -+ goto _return; -+ } -+ if (fqid_max) { -+ QMAN_DBGFS_ENTRY_ROOT("ccsrmempeek", S_IRUGO | S_IWUGO, -+ NULL, &qman_ccsrmempeek_fops); -+ } -+ QMAN_DBGFS_ENTRY_ROOT("query_fq_np_fields", S_IRUGO | S_IWUGO, -+ &query_fq_np_fields_data, &query_fq_np_fields_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("query_fq_fields", S_IRUGO | S_IWUGO, -+ &query_fq_fields_data, &query_fq_fields_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("query_wq_lengths", S_IRUGO | S_IWUGO, -+ &query_wq_lengths_data, &query_wq_lengths_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("query_cgr", S_IRUGO | S_IWUGO, -+ &query_cgr_data, &query_cgr_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("query_congestion", S_IRUGO, -+ NULL, &query_congestion_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr", S_IRUGO, -+ NULL, &testwrite_cgr_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_cgrid", S_IRUGO | S_IWUGO, -+ NULL, &teswrite_cgr_cgrid_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_ibcnt", S_IRUGO | S_IWUGO, -+ NULL, &teswrite_cgr_ibcnt_fops); -+ -+ QMAN_DBGFS_ENTRY_ROOT("query_ceetm_ccgr", S_IRUGO | S_IWUGO, -+ &query_ccgr_data, &query_ccgr_fops); -+ /* Create files with fqd_root as parent */ -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("stateoos", S_IRUGO, -+ (void *)&fqd_states[QM_MCR_NP_STATE_OOS], &qman_fqd_state_fops); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("state_retired", S_IRUGO, -+ (void *)&fqd_states[QM_MCR_NP_STATE_RETIRED], -+ &qman_fqd_state_fops); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("state_tentatively_sched", S_IRUGO, -+ (void *)&fqd_states[QM_MCR_NP_STATE_TEN_SCHED], -+ &qman_fqd_state_fops); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("state_truly_sched", S_IRUGO, -+ (void *)&fqd_states[QM_MCR_NP_STATE_TRU_SCHED], -+ &qman_fqd_state_fops); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("state_parked", S_IRUGO, -+ (void *)&fqd_states[QM_MCR_NP_STATE_PARKED], -+ &qman_fqd_state_fops); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("state_active", S_IRUGO, -+ (void *)&fqd_states[QM_MCR_NP_STATE_ACTIVE], -+ &qman_fqd_state_fops); -+ QMAN_DBGFS_ENTRY_ROOT("query_cq_fields", S_IRUGO | S_IWUGO, -+ &query_cq_fields_data, &query_cq_fields_fops); -+ QMAN_DBGFS_ENTRY_ROOT("query_ceetm_xsfdr_in_use", S_IRUGO | S_IWUGO, -+ &query_ceetm_xsfdr_data, &query_ceetm_xsfdr_fops); -+ -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("cge_enable", 17); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("cge_disable", 16); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("tde_enable", 15); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("tde_disable", 14); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("orp_enable", 13); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("orp_disable", 12); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_enable", 11); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_disable", 10); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("cpc_enable", 9); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("cpc_disable", 8); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_enable", 7); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_disable", 6); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_enable", 5); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_disable", 4); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_enable", 3); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_disable", 2); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_enable", 1); -+ -+ QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_disable", 0); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("summary", S_IRUGO, -+ NULL, &qman_fqd_summary_fops); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("wq", S_IRUGO | S_IWUGO, -+ NULL, &qman_fqd_dest_wq_fops); -+ -+ QMAN_DBGFS_ENTRY_FQDROOT("cred", S_IRUGO, -+ NULL, &qman_fqd_cred_fops); -+ -+ return 0; -+ -+_return: -+ debugfs_remove_recursive(dfs_root); -+ return ret; -+} -+ -+static void __exit qman_debugfs_module_exit(void) -+{ -+ debugfs_remove_recursive(dfs_root); -+} -+ -+module_init(qman_debugfs_module_init); -+module_exit(qman_debugfs_module_exit); -+MODULE_LICENSE("Dual BSD/GPL"); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_driver.c -@@ -0,0 +1,961 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "qman_private.h" -+ -+#include <asm/smp.h> /* hard_smp_processor_id() if !CONFIG_SMP */ -+#ifdef CONFIG_HOTPLUG_CPU -+#include <linux/cpu.h> -+#endif -+ -+/* Global variable containing revision id (even on non-control plane systems -+ * where CCSR isn't available) */ -+u16 qman_ip_rev; -+EXPORT_SYMBOL(qman_ip_rev); -+u8 qman_ip_cfg; -+EXPORT_SYMBOL(qman_ip_cfg); -+u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1; -+EXPORT_SYMBOL(qm_channel_pool1); -+u16 qm_channel_caam = QMAN_CHANNEL_CAAM; -+EXPORT_SYMBOL(qm_channel_caam); -+u16 qm_channel_pme = QMAN_CHANNEL_PME; -+EXPORT_SYMBOL(qm_channel_pme); -+u16 qm_channel_dce = QMAN_CHANNEL_DCE; -+EXPORT_SYMBOL(qm_channel_dce); -+u16 qman_portal_max; -+EXPORT_SYMBOL(qman_portal_max); -+ -+u32 qman_clk; -+struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX]; -+/* the qman ceetm instances on the given SoC */ -+u8 num_ceetms; -+ -+/* For these variables, and the portal-initialisation logic, the -+ * comments in bman_driver.c apply here so won't be repeated. */ -+static struct qman_portal *shared_portals[NR_CPUS]; -+static int num_shared_portals; -+static int shared_portals_idx; -+static LIST_HEAD(unused_pcfgs); -+static DEFINE_SPINLOCK(unused_pcfgs_lock); -+ -+/* A SDQCR mask comprising all the available/visible pool channels */ -+static u32 pools_sdqcr; -+ -+#define STR_ERR_NOPROP "No '%s' property in node %s\n" -+#define STR_ERR_CELL "'%s' is not a %d-cell range in node %s\n" -+#define STR_FQID_RANGE "fsl,fqid-range" -+#define STR_POOL_CHAN_RANGE "fsl,pool-channel-range" -+#define STR_CGRID_RANGE "fsl,cgrid-range" -+ -+/* A "fsl,fqid-range" node; release the given range to the allocator */ -+static __init int fsl_fqid_range_init(struct device_node *node) -+{ -+ int ret; -+ const u32 *range = of_get_property(node, STR_FQID_RANGE, &ret); -+ if (!range) { -+ pr_err(STR_ERR_NOPROP, STR_FQID_RANGE, node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err(STR_ERR_CELL, STR_FQID_RANGE, 2, node->full_name); -+ return -EINVAL; -+ } -+ qman_seed_fqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ pr_info("Qman: FQID allocator includes range %d:%d\n", -+ be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ return 0; -+} -+ -+/* A "fsl,pool-channel-range" node; add to the SDQCR mask only */ -+static __init int fsl_pool_channel_range_sdqcr(struct device_node *node) -+{ -+ int ret; -+ const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret); -+ if (!chanid) { -+ pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name); -+ return -EINVAL; -+ } -+ for (ret = 0; ret < be32_to_cpu(chanid[1]); ret++) -+ pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(be32_to_cpu(chanid[0]) + ret); -+ return 0; -+} -+ -+/* A "fsl,pool-channel-range" node; release the given range to the allocator */ -+static __init int fsl_pool_channel_range_init(struct device_node *node) -+{ -+ int ret; -+ const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret); -+ if (!chanid) { -+ pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name); -+ return -EINVAL; -+ } -+ qman_seed_pool_range(be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1])); -+ pr_info("Qman: pool channel allocator includes range %d:%d\n", -+ be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1])); -+ return 0; -+} -+ -+/* A "fsl,cgrid-range" node; release the given range to the allocator */ -+static __init int fsl_cgrid_range_init(struct device_node *node) -+{ -+ struct qman_cgr cgr; -+ int ret, errors = 0; -+ const u32 *range = of_get_property(node, STR_CGRID_RANGE, &ret); -+ if (!range) { -+ pr_err(STR_ERR_NOPROP, STR_CGRID_RANGE, node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err(STR_ERR_CELL, STR_CGRID_RANGE, 2, node->full_name); -+ return -EINVAL; -+ } -+ qman_seed_cgrid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ pr_info("Qman: CGRID allocator includes range %d:%d\n", -+ be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ for (cgr.cgrid = 0; cgr.cgrid < __CGR_NUM; cgr.cgrid++) { -+ ret = qman_modify_cgr(&cgr, QMAN_CGR_FLAG_USE_INIT, NULL); -+ if (ret) -+ errors++; -+ } -+ if (errors) -+ pr_err("Warning: %d error%s while initialising CGRs %d:%d\n", -+ errors, (errors > 1) ? "s" : "", range[0], range[1]); -+ return 0; -+} -+ -+static __init int fsl_ceetm_init(struct device_node *node) -+{ -+ enum qm_dc_portal dcp_portal; -+ struct qm_ceetm_sp *sp; -+ struct qm_ceetm_lni *lni; -+ int ret, i; -+ const u32 *range; -+ -+ /* Find LFQID range */ -+ range = of_get_property(node, "fsl,ceetm-lfqid-range", &ret); -+ if (!range) { -+ pr_err("No fsl,ceetm-lfqid-range in node %s\n", -+ node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err("fsl,ceetm-lfqid-range is not a 2-cell range in node" -+ " %s\n", node->full_name); -+ return -EINVAL; -+ } -+ -+ dcp_portal = (be32_to_cpu(range[0]) & 0x0F0000) >> 16; -+ if (dcp_portal > qm_dc_portal_fman1) { -+ pr_err("The DCP portal %d doesn't support CEETM\n", dcp_portal); -+ return -EINVAL; -+ } -+ -+ if (dcp_portal == qm_dc_portal_fman0) -+ qman_seed_ceetm0_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ if (dcp_portal == qm_dc_portal_fman1) -+ qman_seed_ceetm1_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ pr_debug("Qman: The lfqid allocator of CEETM %d includes range" -+ " 0x%x:0x%x\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ -+ qman_ceetms[dcp_portal].idx = dcp_portal; -+ INIT_LIST_HEAD(&qman_ceetms[dcp_portal].sub_portals); -+ INIT_LIST_HEAD(&qman_ceetms[dcp_portal].lnis); -+ -+ /* Find Sub-portal range */ -+ range = of_get_property(node, "fsl,ceetm-sp-range", &ret); -+ if (!range) { -+ pr_err("No fsl,ceetm-sp-range in node %s\n", node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err("fsl,ceetm-sp-range is not a 2-cell range in node %s\n", -+ node->full_name); -+ return -EINVAL; -+ } -+ -+ for (i = 0; i < be32_to_cpu(range[1]); i++) { -+ sp = kzalloc(sizeof(*sp), GFP_KERNEL); -+ if (!sp) { -+ pr_err("Can't alloc memory for sub-portal %d\n", -+ range[0] + i); -+ return -ENOMEM; -+ } -+ sp->idx = be32_to_cpu(range[0]) + i; -+ sp->dcp_idx = dcp_portal; -+ sp->is_claimed = 0; -+ list_add_tail(&sp->node, &qman_ceetms[dcp_portal].sub_portals); -+ sp++; -+ } -+ pr_debug("Qman: Reserve sub-portal %d:%d for CEETM %d\n", -+ be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal); -+ qman_ceetms[dcp_portal].sp_range[0] = be32_to_cpu(range[0]); -+ qman_ceetms[dcp_portal].sp_range[1] = be32_to_cpu(range[1]); -+ -+ /* Find LNI range */ -+ range = of_get_property(node, "fsl,ceetm-lni-range", &ret); -+ if (!range) { -+ pr_err("No fsl,ceetm-lni-range in node %s\n", node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err("fsl,ceetm-lni-range is not a 2-cell range in node %s\n", -+ node->full_name); -+ return -EINVAL; -+ } -+ -+ for (i = 0; i < be32_to_cpu(range[1]); i++) { -+ lni = kzalloc(sizeof(*lni), GFP_KERNEL); -+ if (!lni) { -+ pr_err("Can't alloc memory for LNI %d\n", -+ range[0] + i); -+ return -ENOMEM; -+ } -+ lni->idx = be32_to_cpu(range[0]) + i; -+ lni->dcp_idx = dcp_portal; -+ lni->is_claimed = 0; -+ INIT_LIST_HEAD(&lni->channels); -+ list_add_tail(&lni->node, &qman_ceetms[dcp_portal].lnis); -+ lni++; -+ } -+ pr_debug("Qman: Reserve LNI %d:%d for CEETM %d\n", -+ be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal); -+ qman_ceetms[dcp_portal].lni_range[0] = be32_to_cpu(range[0]); -+ qman_ceetms[dcp_portal].lni_range[1] = be32_to_cpu(range[1]); -+ -+ /* Find CEETM channel range */ -+ range = of_get_property(node, "fsl,ceetm-channel-range", &ret); -+ if (!range) { -+ pr_err("No fsl,ceetm-channel-range in node %s\n", -+ node->full_name); -+ return -EINVAL; -+ } -+ if (ret != 8) { -+ pr_err("fsl,ceetm-channel-range is not a 2-cell range in node" -+ "%s\n", node->full_name); -+ return -EINVAL; -+ } -+ -+ if (dcp_portal == qm_dc_portal_fman0) -+ qman_seed_ceetm0_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ if (dcp_portal == qm_dc_portal_fman1) -+ qman_seed_ceetm1_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ pr_debug("Qman: The channel allocator of CEETM %d includes" -+ " range %d:%d\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1])); -+ -+ /* Set CEETM PRES register */ -+ ret = qman_ceetm_set_prescaler(dcp_portal); -+ if (ret) -+ return ret; -+ return 0; -+} -+ -+static void qman_get_ip_revision(struct device_node *dn) -+{ -+ u16 ip_rev = 0; -+ u8 ip_cfg = QMAN_REV_CFG_0; -+ for_each_compatible_node(dn, NULL, "fsl,qman-portal") { -+ if (!of_device_is_available(dn)) -+ continue; -+ if (of_device_is_compatible(dn, "fsl,qman-portal-1.0") || -+ of_device_is_compatible(dn, "fsl,qman-portal-1.0.0")) { -+ pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n"); -+ BUG_ON(1); -+ } else if (of_device_is_compatible(dn, "fsl,qman-portal-1.1") || -+ of_device_is_compatible(dn, "fsl,qman-portal-1.1.0")) { -+ ip_rev = QMAN_REV11; -+ qman_portal_max = 10; -+ } else if (of_device_is_compatible(dn, "fsl,qman-portal-1.2") || -+ of_device_is_compatible(dn, "fsl,qman-portal-1.2.0")) { -+ ip_rev = QMAN_REV12; -+ qman_portal_max = 10; -+ } else if (of_device_is_compatible(dn, "fsl,qman-portal-2.0") || -+ of_device_is_compatible(dn, "fsl,qman-portal-2.0.0")) { -+ ip_rev = QMAN_REV20; -+ qman_portal_max = 3; -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.0.0")) { -+ ip_rev = QMAN_REV30; -+ qman_portal_max = 50; -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.0.1")) { -+ ip_rev = QMAN_REV30; -+ qman_portal_max = 25; -+ ip_cfg = QMAN_REV_CFG_1; -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.1.0")) { -+ ip_rev = QMAN_REV31; -+ qman_portal_max = 50; -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.1.1")) { -+ ip_rev = QMAN_REV31; -+ qman_portal_max = 25; -+ ip_cfg = QMAN_REV_CFG_1; -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.1.2")) { -+ ip_rev = QMAN_REV31; -+ qman_portal_max = 18; -+ ip_cfg = QMAN_REV_CFG_2; -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.1.3")) { -+ ip_rev = QMAN_REV31; -+ qman_portal_max = 10; -+ ip_cfg = QMAN_REV_CFG_3; -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.2.0")) { -+ ip_rev = QMAN_REV32; -+ qman_portal_max = 10; -+ ip_cfg = QMAN_REV_CFG_3; // TODO: Verify for ls1043 -+ } else if (of_device_is_compatible(dn, -+ "fsl,qman-portal-3.2.1")) { -+ ip_rev = QMAN_REV32; -+ qman_portal_max = 10; -+ ip_cfg = QMAN_REV_CFG_3; -+ } else { -+ pr_warn("unknown QMan version in portal node," -+ "default to rev1.1\n"); -+ ip_rev = QMAN_REV11; -+ qman_portal_max = 10; -+ } -+ -+ if (!qman_ip_rev) { -+ if (ip_rev) { -+ qman_ip_rev = ip_rev; -+ qman_ip_cfg = ip_cfg; -+ } else { -+ pr_warn("unknown Qman version," -+ " default to rev1.1\n"); -+ qman_ip_rev = QMAN_REV11; -+ qman_ip_cfg = QMAN_REV_CFG_0; -+ } -+ } else if (ip_rev && (qman_ip_rev != ip_rev)) -+ pr_warn("Revision=0x%04x, but portal '%s' has" -+ " 0x%04x\n", -+ qman_ip_rev, dn->full_name, ip_rev); -+ if (qman_ip_rev == ip_rev) -+ break; -+ } -+} -+ -+/* Parse a portal node, perform generic mapping duties and return the config. It -+ * is not known at this stage for what purpose (or even if) the portal will be -+ * used. */ -+static struct qm_portal_config * __init parse_pcfg(struct device_node *node) -+{ -+ struct qm_portal_config *pcfg; -+ const u32 *index_p; -+ u32 index, channel; -+ int irq, ret; -+ resource_size_t len; -+ -+ pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL); -+ if (!pcfg) { -+ pr_err("can't allocate portal config"); -+ return NULL; -+ } -+ -+ /* -+ * This is a *horrible hack*, but the IOMMU/PAMU driver needs a -+ * 'struct device' in order to get the PAMU stashing setup and the QMan -+ * portal [driver] won't function at all without ring stashing -+ * -+ * Making the QMan portal driver nice and proper is part of the -+ * upstreaming effort -+ */ -+ pcfg->dev.bus = &platform_bus_type; -+ pcfg->dev.of_node = node; -+#ifdef CONFIG_FSL_PAMU -+ pcfg->dev.archdata.iommu_domain = NULL; -+#endif -+ -+ ret = of_address_to_resource(node, DPA_PORTAL_CE, -+ &pcfg->addr_phys[DPA_PORTAL_CE]); -+ if (ret) { -+ pr_err("Can't get %s property '%s'\n", node->full_name, -+ "reg::CE"); -+ goto err; -+ } -+ ret = of_address_to_resource(node, DPA_PORTAL_CI, -+ &pcfg->addr_phys[DPA_PORTAL_CI]); -+ if (ret) { -+ pr_err("Can't get %s property '%s'\n", node->full_name, -+ "reg::CI"); -+ goto err; -+ } -+ index_p = of_get_property(node, "cell-index", &ret); -+ if (!index_p || (ret != 4)) { -+ pr_err("Can't get %s property '%s'\n", node->full_name, -+ "cell-index"); -+ goto err; -+ } -+ index = be32_to_cpu(*index_p); -+ if (index >= qman_portal_max) { -+ pr_err("QMan portal index %d is beyond max (%d)\n", -+ index, qman_portal_max); -+ goto err; -+ } -+ -+ channel = index + QM_CHANNEL_SWPORTAL0; -+ pcfg->public_cfg.channel = channel; -+ pcfg->public_cfg.cpu = -1; -+ irq = irq_of_parse_and_map(node, 0); -+ if (irq == 0) { -+ pr_err("Can't get %s property '%s'\n", node->full_name, -+ "interrupts"); -+ goto err; -+ } -+ pcfg->public_cfg.irq = irq; -+ pcfg->public_cfg.index = index; -+#ifdef CONFIG_FSL_QMAN_CONFIG -+ /* We need the same LIODN offset for all portals */ -+ qman_liodn_fixup(pcfg->public_cfg.channel); -+#endif -+ -+ len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]); -+ if (len != (unsigned long)len) -+ goto err; -+ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns( -+ pcfg->addr_phys[DPA_PORTAL_CE].start, -+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CE])); -+ -+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap( -+ pcfg->addr_phys[DPA_PORTAL_CI].start, -+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI])); -+#else -+ pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot( -+ pcfg->addr_phys[DPA_PORTAL_CE].start, -+ (unsigned long)len, -+ 0); -+ pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot( -+ pcfg->addr_phys[DPA_PORTAL_CI].start, -+ resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]), -+ _PAGE_GUARDED | _PAGE_NO_CACHE); -+#endif -+ return pcfg; -+err: -+ kfree(pcfg); -+ return NULL; -+} -+ -+static struct qm_portal_config *get_pcfg(struct list_head *list) -+{ -+ struct qm_portal_config *pcfg; -+ if (list_empty(list)) -+ return NULL; -+ pcfg = list_entry(list->prev, struct qm_portal_config, list); -+ list_del(&pcfg->list); -+ return pcfg; -+} -+ -+static struct qm_portal_config *get_pcfg_idx(struct list_head *list, u32 idx) -+{ -+ struct qm_portal_config *pcfg; -+ if (list_empty(list)) -+ return NULL; -+ list_for_each_entry(pcfg, list, list) { -+ if (pcfg->public_cfg.index == idx) { -+ list_del(&pcfg->list); -+ return pcfg; -+ } -+ } -+ return NULL; -+} -+ -+static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu) -+{ -+#ifdef CONFIG_FSL_PAMU -+ int ret; -+ int window_count = 1; -+ struct iommu_domain_geometry geom_attr; -+ struct pamu_stash_attribute stash_attr; -+ -+ pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type); -+ if (!pcfg->iommu_domain) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed", -+ __func__); -+ goto _no_iommu; -+ } -+ geom_attr.aperture_start = 0; -+ geom_attr.aperture_end = -+ ((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1; -+ geom_attr.force_aperture = true; -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY, -+ &geom_attr); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS, -+ &window_count); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ stash_attr.cpu = cpu; -+ stash_attr.cache = PAMU_ATTR_CACHE_L1; -+ /* set stash information for the window */ -+ stash_attr.window = 0; -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, -+ DOMAIN_ATTR_FSL_PAMU_STASH, -+ &stash_attr); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36, -+ IOMMU_READ | IOMMU_WRITE); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d", -+ __func__, ret); -+ goto _iommu_domain_free; -+ } -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, -+ DOMAIN_ATTR_FSL_PAMU_ENABLE, -+ &window_count); -+ if (ret < 0) { -+ pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d", -+ __func__, ret); -+ goto _iommu_detach_device; -+ } -+ -+_no_iommu: -+#endif -+#ifdef CONFIG_FSL_QMAN_CONFIG -+ if (qman_set_sdest(pcfg->public_cfg.channel, cpu)) -+#endif -+ pr_warn("Failed to set QMan portal's stash request queue\n"); -+ -+ return; -+ -+#ifdef CONFIG_FSL_PAMU -+_iommu_detach_device: -+ iommu_detach_device(pcfg->iommu_domain, NULL); -+_iommu_domain_free: -+ iommu_domain_free(pcfg->iommu_domain); -+#endif -+} -+ -+struct qm_portal_config *qm_get_unused_portal_idx(u32 idx) -+{ -+ struct qm_portal_config *ret; -+ spin_lock(&unused_pcfgs_lock); -+ if (idx == QBMAN_ANY_PORTAL_IDX) -+ ret = get_pcfg(&unused_pcfgs); -+ else -+ ret = get_pcfg_idx(&unused_pcfgs, idx); -+ spin_unlock(&unused_pcfgs_lock); -+ /* Bind stashing LIODNs to the CPU we are currently executing on, and -+ * set the portal to use the stashing request queue corresonding to the -+ * cpu as well. The user-space driver assumption is that the pthread has -+ * to already be affine to one cpu only before opening a portal. If that -+ * check is circumvented, the only risk is a performance degradation - -+ * stashing will go to whatever cpu they happened to be running on when -+ * opening the device file, and if that isn't the cpu they subsequently -+ * bind to and do their polling on, tough. */ -+ if (ret) -+ portal_set_cpu(ret, hard_smp_processor_id()); -+ return ret; -+} -+ -+struct qm_portal_config *qm_get_unused_portal(void) -+{ -+ return qm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX); -+} -+ -+void qm_put_unused_portal(struct qm_portal_config *pcfg) -+{ -+ spin_lock(&unused_pcfgs_lock); -+ list_add(&pcfg->list, &unused_pcfgs); -+ spin_unlock(&unused_pcfgs_lock); -+} -+ -+static struct qman_portal *init_pcfg(struct qm_portal_config *pcfg) -+{ -+ struct qman_portal *p; -+ -+ pcfg->iommu_domain = NULL; -+ portal_set_cpu(pcfg, pcfg->public_cfg.cpu); -+ p = qman_create_affine_portal(pcfg, NULL); -+ if (p) { -+ u32 irq_sources = 0; -+ /* Determine what should be interrupt-vs-poll driven */ -+#ifdef CONFIG_FSL_DPA_PIRQ_SLOW -+ irq_sources |= QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI | -+ QM_PIRQ_CSCI | QM_PIRQ_CCSCI; -+#endif -+#ifdef CONFIG_FSL_DPA_PIRQ_FAST -+ irq_sources |= QM_PIRQ_DQRI; -+#endif -+ qman_p_irqsource_add(p, irq_sources); -+ pr_info("Qman portal %sinitialised, cpu %d\n", -+ pcfg->public_cfg.is_shared ? "(shared) " : "", -+ pcfg->public_cfg.cpu); -+ } else -+ pr_crit("Qman portal failure on cpu %d\n", -+ pcfg->public_cfg.cpu); -+ return p; -+} -+ -+static void init_slave(int cpu) -+{ -+ struct qman_portal *p; -+ struct cpumask oldmask = current->cpus_allowed; -+ set_cpus_allowed_ptr(current, get_cpu_mask(cpu)); -+ p = qman_create_affine_slave(shared_portals[shared_portals_idx++], cpu); -+ if (!p) -+ pr_err("Qman slave portal failure on cpu %d\n", cpu); -+ else -+ pr_info("Qman portal %sinitialised, cpu %d\n", "(slave) ", cpu); -+ set_cpus_allowed_ptr(current, &oldmask); -+ if (shared_portals_idx >= num_shared_portals) -+ shared_portals_idx = 0; -+} -+ -+static struct cpumask want_unshared __initdata; -+static struct cpumask want_shared __initdata; -+ -+static int __init parse_qportals(char *str) -+{ -+ return parse_portals_bootarg(str, &want_shared, &want_unshared, -+ "qportals"); -+} -+__setup("qportals=", parse_qportals); -+ -+static void qman_portal_update_sdest(const struct qm_portal_config *pcfg, -+ unsigned int cpu) -+{ -+#ifdef CONFIG_FSL_PAMU -+ struct pamu_stash_attribute stash_attr; -+ int ret; -+ -+ if (pcfg->iommu_domain) { -+ stash_attr.cpu = cpu; -+ stash_attr.cache = PAMU_ATTR_CACHE_L1; -+ /* set stash information for the window */ -+ stash_attr.window = 0; -+ ret = iommu_domain_set_attr(pcfg->iommu_domain, -+ DOMAIN_ATTR_FSL_PAMU_STASH, &stash_attr); -+ if (ret < 0) { -+ pr_err("Failed to update pamu stash setting\n"); -+ return; -+ } -+ } -+#endif -+#ifdef CONFIG_FSL_QMAN_CONFIG -+ if (qman_set_sdest(pcfg->public_cfg.channel, cpu)) -+ pr_warn("Failed to update portal's stash request queue\n"); -+#endif -+} -+ -+static int qman_offline_cpu(unsigned int cpu) -+{ -+ struct qman_portal *p; -+ const struct qm_portal_config *pcfg; -+ p = (struct qman_portal *)affine_portals[cpu]; -+ if (p) { -+ pcfg = qman_get_qm_portal_config(p); -+ if (pcfg) { -+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0)); -+ qman_portal_update_sdest(pcfg, 0); -+ } -+ } -+ return 0; -+} -+ -+#ifdef CONFIG_HOTPLUG_CPU -+static int qman_online_cpu(unsigned int cpu) -+{ -+ struct qman_portal *p; -+ const struct qm_portal_config *pcfg; -+ p = (struct qman_portal *)affine_portals[cpu]; -+ if (p) { -+ pcfg = qman_get_qm_portal_config(p); -+ if (pcfg) { -+ irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu)); -+ qman_portal_update_sdest(pcfg, cpu); -+ } -+ } -+ return 0; -+} -+ -+#endif /* CONFIG_HOTPLUG_CPU */ -+ -+__init int qman_init(void) -+{ -+ struct cpumask slave_cpus; -+ struct cpumask unshared_cpus = *cpu_none_mask; -+ struct cpumask shared_cpus = *cpu_none_mask; -+ LIST_HEAD(unshared_pcfgs); -+ LIST_HEAD(shared_pcfgs); -+ struct device_node *dn; -+ struct qm_portal_config *pcfg; -+ struct qman_portal *p; -+ int cpu, ret; -+ const u32 *clk; -+ struct cpumask offline_cpus; -+ -+ /* Initialise the Qman (CCSR) device */ -+ for_each_compatible_node(dn, NULL, "fsl,qman") { -+ if (!qman_init_ccsr(dn)) -+ pr_info("Qman err interrupt handler present\n"); -+ else -+ pr_err("Qman CCSR setup failed\n"); -+ -+ clk = of_get_property(dn, "clock-frequency", NULL); -+ if (!clk) -+ pr_warn("Can't find Qman clock frequency\n"); -+ else -+ qman_clk = be32_to_cpu(*clk); -+ } -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ /* Setup lookup table for FQ demux */ -+ ret = qman_setup_fq_lookup_table(get_qman_fqd_size()/64); -+ if (ret) -+ return ret; -+#endif -+ -+ /* Get qman ip revision */ -+ qman_get_ip_revision(dn); -+ if ((qman_ip_rev & 0xff00) >= QMAN_REV30) { -+ qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3; -+ qm_channel_caam = QMAN_CHANNEL_CAAM_REV3; -+ qm_channel_pme = QMAN_CHANNEL_PME_REV3; -+ } -+ -+ if ((qman_ip_rev == QMAN_REV31) && (qman_ip_cfg == QMAN_REV_CFG_2)) -+ qm_channel_dce = QMAN_CHANNEL_DCE_QMANREV312; -+ -+ /* -+ * Parse the ceetm node to get how many ceetm instances are supported -+ * on the current silicon. num_ceetms must be confirmed before portals -+ * are intiailized. -+ */ -+ num_ceetms = 0; -+ for_each_compatible_node(dn, NULL, "fsl,qman-ceetm") -+ num_ceetms++; -+ -+ /* Parse pool channels into the SDQCR mask. (Must happen before portals -+ * are initialised.) */ -+ for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") { -+ ret = fsl_pool_channel_range_sdqcr(dn); -+ if (ret) -+ return ret; -+ } -+ -+ memset(affine_portals, 0, sizeof(void *) * num_possible_cpus()); -+ /* Initialise portals. See bman_driver.c for comments */ -+ for_each_compatible_node(dn, NULL, "fsl,qman-portal") { -+ if (!of_device_is_available(dn)) -+ continue; -+ pcfg = parse_pcfg(dn); -+ if (pcfg) { -+ pcfg->public_cfg.pools = pools_sdqcr; -+ list_add_tail(&pcfg->list, &unused_pcfgs); -+ } -+ } -+ for_each_possible_cpu(cpu) { -+ if (cpumask_test_cpu(cpu, &want_shared)) { -+ pcfg = get_pcfg(&unused_pcfgs); -+ if (!pcfg) -+ break; -+ pcfg->public_cfg.cpu = cpu; -+ list_add_tail(&pcfg->list, &shared_pcfgs); -+ cpumask_set_cpu(cpu, &shared_cpus); -+ } -+ if (cpumask_test_cpu(cpu, &want_unshared)) { -+ if (cpumask_test_cpu(cpu, &shared_cpus)) -+ continue; -+ pcfg = get_pcfg(&unused_pcfgs); -+ if (!pcfg) -+ break; -+ pcfg->public_cfg.cpu = cpu; -+ list_add_tail(&pcfg->list, &unshared_pcfgs); -+ cpumask_set_cpu(cpu, &unshared_cpus); -+ } -+ } -+ if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) { -+ for_each_online_cpu(cpu) { -+ pcfg = get_pcfg(&unused_pcfgs); -+ if (!pcfg) -+ break; -+ pcfg->public_cfg.cpu = cpu; -+ list_add_tail(&pcfg->list, &unshared_pcfgs); -+ cpumask_set_cpu(cpu, &unshared_cpus); -+ } -+ } -+ cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus); -+ cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus); -+ if (cpumask_empty(&slave_cpus)) { -+ if (!list_empty(&shared_pcfgs)) { -+ cpumask_or(&unshared_cpus, &unshared_cpus, -+ &shared_cpus); -+ cpumask_clear(&shared_cpus); -+ list_splice_tail(&shared_pcfgs, &unshared_pcfgs); -+ INIT_LIST_HEAD(&shared_pcfgs); -+ } -+ } else { -+ if (list_empty(&shared_pcfgs)) { -+ pcfg = get_pcfg(&unshared_pcfgs); -+ if (!pcfg) { -+ pr_crit("No QMan portals available!\n"); -+ return 0; -+ } -+ cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus); -+ cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus); -+ list_add_tail(&pcfg->list, &shared_pcfgs); -+ } -+ } -+ list_for_each_entry(pcfg, &unshared_pcfgs, list) { -+ pcfg->public_cfg.is_shared = 0; -+ p = init_pcfg(pcfg); -+ if (!p) { -+ pr_crit("Unable to configure portals\n"); -+ return 0; -+ } -+ } -+ list_for_each_entry(pcfg, &shared_pcfgs, list) { -+ pcfg->public_cfg.is_shared = 1; -+ p = init_pcfg(pcfg); -+ if (p) -+ shared_portals[num_shared_portals++] = p; -+ } -+ if (!cpumask_empty(&slave_cpus)) -+ for_each_cpu(cpu, &slave_cpus) -+ init_slave(cpu); -+ pr_info("Qman portals initialised\n"); -+ cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask); -+ for_each_cpu(cpu, &offline_cpus) -+ qman_offline_cpu(cpu); -+#ifdef CONFIG_HOTPLUG_CPU -+ ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, -+ "soc/qman_portal:online", -+ qman_online_cpu, qman_offline_cpu); -+ if (ret < 0) { -+ pr_err("qman: failed to register hotplug callbacks.\n"); -+ return ret; -+ } -+#endif -+ return 0; -+} -+ -+__init int qman_resource_init(void) -+{ -+ struct device_node *dn; -+ int ret; -+ -+ /* Initialise FQID allocation ranges */ -+ for_each_compatible_node(dn, NULL, "fsl,fqid-range") { -+ ret = fsl_fqid_range_init(dn); -+ if (ret) -+ return ret; -+ } -+ /* Initialise CGRID allocation ranges */ -+ for_each_compatible_node(dn, NULL, "fsl,cgrid-range") { -+ ret = fsl_cgrid_range_init(dn); -+ if (ret) -+ return ret; -+ } -+ /* Parse pool channels into the allocator. (Must happen after portals -+ * are initialised.) */ -+ for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") { -+ ret = fsl_pool_channel_range_init(dn); -+ if (ret) -+ return ret; -+ } -+ -+ /* Parse CEETM */ -+ for_each_compatible_node(dn, NULL, "fsl,qman-ceetm") { -+ ret = fsl_ceetm_init(dn); -+ if (ret) -+ return ret; -+ } -+ return 0; -+} -+ -+#ifdef CONFIG_SUSPEND -+void suspend_unused_qportal(void) -+{ -+ struct qm_portal_config *pcfg; -+ -+ if (list_empty(&unused_pcfgs)) -+ return; -+ -+ list_for_each_entry(pcfg, &unused_pcfgs, list) { -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Need to save qportal %d\n", pcfg->public_cfg.index); -+#endif -+ /* save isdr, disable all via isdr, clear isr */ -+ pcfg->saved_isdr = -+ __raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); -+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + -+ 0xe08); -+ __raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] + -+ 0xe00); -+ } -+ return; -+} -+ -+void resume_unused_qportal(void) -+{ -+ struct qm_portal_config *pcfg; -+ -+ if (list_empty(&unused_pcfgs)) -+ return; -+ -+ list_for_each_entry(pcfg, &unused_pcfgs, list) { -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Need to resume qportal %d\n", pcfg->public_cfg.index); -+#endif -+ /* restore isdr */ -+ __raw_writel(pcfg->saved_isdr, -+ pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08); -+ } -+ return; -+} -+#endif ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_high.c -@@ -0,0 +1,5669 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "qman_low.h" -+ -+/* Compilation constants */ -+#define DQRR_MAXFILL 15 -+#define EQCR_ITHRESH 4 /* if EQCR congests, interrupt threshold */ -+#define IRQNAME "QMan portal %d" -+#define MAX_IRQNAME 16 /* big enough for "QMan portal %d" */ -+ -+/* Divide 'n' by 'd', rounding down if 'r' is negative, rounding up if it's -+ * positive, and rounding to the closest value if it's zero. NB, this macro -+ * implicitly upgrades parameters to unsigned 64-bit, so feed it with types -+ * that are compatible with this. NB, these arguments should not be expressions -+ * unless it is safe for them to be evaluated multiple times. Eg. do not pass -+ * in "some_value++" as a parameter to the macro! */ -+#define ROUNDING(n, d, r) \ -+ (((r) < 0) ? div64_u64((n), (d)) : \ -+ (((r) > 0) ? div64_u64(((n) + (d) - 1), (d)) : \ -+ div64_u64(((n) + ((d) / 2)), (d)))) -+ -+/* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about -+ * inter-processor locking only. Note, FQLOCK() is always called either under a -+ * local_irq_save() or from interrupt context - hence there's no need for irq -+ * protection (and indeed, attempting to nest irq-protection doesn't work, as -+ * the "irq en/disable" machinery isn't recursive...). */ -+#define FQLOCK(fq) \ -+ do { \ -+ struct qman_fq *__fq478 = (fq); \ -+ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ -+ spin_lock(&__fq478->fqlock); \ -+ } while (0) -+#define FQUNLOCK(fq) \ -+ do { \ -+ struct qman_fq *__fq478 = (fq); \ -+ if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \ -+ spin_unlock(&__fq478->fqlock); \ -+ } while (0) -+ -+static inline void fq_set(struct qman_fq *fq, u32 mask) -+{ -+ set_bits(mask, &fq->flags); -+} -+static inline void fq_clear(struct qman_fq *fq, u32 mask) -+{ -+ clear_bits(mask, &fq->flags); -+} -+static inline int fq_isset(struct qman_fq *fq, u32 mask) -+{ -+ return fq->flags & mask; -+} -+static inline int fq_isclear(struct qman_fq *fq, u32 mask) -+{ -+ return !(fq->flags & mask); -+} -+ -+struct qman_portal { -+ struct qm_portal p; -+ unsigned long bits; /* PORTAL_BITS_*** - dynamic, strictly internal */ -+ unsigned long irq_sources; -+ u32 use_eqcr_ci_stashing; -+ u32 slowpoll; /* only used when interrupts are off */ -+ struct qman_fq *vdqcr_owned; /* only 1 volatile dequeue at a time */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ struct qman_fq *eqci_owned; /* only 1 enqueue WAIT_SYNC at a time */ -+#endif -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ raw_spinlock_t sharing_lock; /* only used if is_shared */ -+ int is_shared; -+ struct qman_portal *sharing_redirect; -+#endif -+ u32 sdqcr; -+ int dqrr_disable_ref; -+ /* A portal-specific handler for DCP ERNs. If this is NULL, the global -+ * handler is called instead. */ -+ qman_cb_dc_ern cb_dc_ern; -+ /* When the cpu-affine portal is activated, this is non-NULL */ -+ const struct qm_portal_config *config; -+ /* This is needed for providing a non-NULL device to dma_map_***() */ -+ struct platform_device *pdev; -+ struct dpa_rbtree retire_table; -+ char irqname[MAX_IRQNAME]; -+ /* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */ -+ struct qman_cgrs *cgrs; -+ /* linked-list of CSCN handlers. */ -+ struct list_head cgr_cbs; -+ /* list lock */ -+ spinlock_t cgr_lock; -+ /* 2-element array. ccgrs[0] is mask, ccgrs[1] is snapshot. */ -+ struct qman_ccgrs *ccgrs[QMAN_CEETM_MAX]; -+ /* 256-element array, each is a linked-list of CCSCN handlers. */ -+ struct list_head ccgr_cbs[QMAN_CEETM_MAX]; -+ /* list lock */ -+ spinlock_t ccgr_lock; -+ /* track if memory was allocated by the driver */ -+ u8 alloced; -+ /* power management data */ -+ u32 save_isdr; -+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -+ /* Keep a shadow copy of the DQRR on LE systems as the SW needs to -+ * do byte swaps of DQRR read only memory. First entry must be aligned -+ * to 2 ** 10 to ensure DQRR index calculations based shadow copy -+ * address (6 bits for address shift + 4 bits for the DQRR size). -+ */ -+ struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE] __aligned(1024); -+#endif -+}; -+ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+#define PORTAL_IRQ_LOCK(p, irqflags) \ -+ do { \ -+ if ((p)->is_shared) \ -+ raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \ -+ else \ -+ local_irq_save(irqflags); \ -+ } while (0) -+#define PORTAL_IRQ_UNLOCK(p, irqflags) \ -+ do { \ -+ if ((p)->is_shared) \ -+ raw_spin_unlock_irqrestore(&(p)->sharing_lock, \ -+ irqflags); \ -+ else \ -+ local_irq_restore(irqflags); \ -+ } while (0) -+#else -+#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags) -+#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags) -+#endif -+ -+/* Global handler for DCP ERNs. Used when the portal receiving the message does -+ * not have a portal-specific handler. */ -+static qman_cb_dc_ern cb_dc_ern; -+ -+static cpumask_t affine_mask; -+static DEFINE_SPINLOCK(affine_mask_lock); -+static u16 affine_channels[NR_CPUS]; -+static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal); -+void *affine_portals[NR_CPUS]; -+ -+/* "raw" gets the cpu-local struct whether it's a redirect or not. */ -+static inline struct qman_portal *get_raw_affine_portal(void) -+{ -+ return &get_cpu_var(qman_affine_portal); -+} -+/* For ops that can redirect, this obtains the portal to use */ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+static inline struct qman_portal *get_affine_portal(void) -+{ -+ struct qman_portal *p = get_raw_affine_portal(); -+ if (p->sharing_redirect) -+ return p->sharing_redirect; -+ return p; -+} -+#else -+#define get_affine_portal() get_raw_affine_portal() -+#endif -+/* For every "get", there must be a "put" */ -+static inline void put_affine_portal(void) -+{ -+ put_cpu_var(qman_affine_portal); -+} -+/* Exception: poll functions assume the caller is cpu-affine and in no risk of -+ * re-entrance, which are the two reasons we usually use the get/put_cpu_var() -+ * semantic - ie. to disable pre-emption. Some use-cases expect the execution -+ * context to remain as non-atomic during poll-triggered callbacks as it was -+ * when the poll API was first called (eg. NAPI), so we go out of our way in -+ * this case to not disable pre-emption. */ -+static inline struct qman_portal *get_poll_portal(void) -+{ -+ return &get_cpu_var(qman_affine_portal); -+} -+#define put_poll_portal() -+ -+/* This gives a FQID->FQ lookup to cover the fact that we can't directly demux -+ * retirement notifications (the fact they are sometimes h/w-consumed means that -+ * contextB isn't always a s/w demux - and as we can't know which case it is -+ * when looking at the notification, we have to use the slow lookup for all of -+ * them). NB, it's possible to have multiple FQ objects refer to the same FQID -+ * (though at most one of them should be the consumer), so this table isn't for -+ * all FQs - FQs are added when retirement commands are issued, and removed when -+ * they complete, which also massively reduces the size of this table. */ -+IMPLEMENT_DPA_RBTREE(fqtree, struct qman_fq, node, fqid); -+ -+/* This is what everything can wait on, even if it migrates to a different cpu -+ * to the one whose affine portal it is waiting on. */ -+static DECLARE_WAIT_QUEUE_HEAD(affine_queue); -+ -+static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq) -+{ -+ int ret = fqtree_push(&p->retire_table, fq); -+ if (ret) -+ pr_err("ERROR: double FQ-retirement %d\n", fq->fqid); -+ return ret; -+} -+ -+static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq) -+{ -+ fqtree_del(&p->retire_table, fq); -+} -+ -+static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid) -+{ -+ return fqtree_find(&p->retire_table, fqid); -+} -+ -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+static void **qman_fq_lookup_table; -+static size_t qman_fq_lookup_table_size; -+ -+int qman_setup_fq_lookup_table(size_t num_entries) -+{ -+ num_entries++; -+ /* Allocate 1 more entry since the first entry is not used */ -+ qman_fq_lookup_table = vzalloc((num_entries * sizeof(void *))); -+ if (!qman_fq_lookup_table) { -+ pr_err("QMan: Could not allocate fq lookup table\n"); -+ return -ENOMEM; -+ } -+ qman_fq_lookup_table_size = num_entries; -+ pr_info("QMan: Allocated lookup table at %p, entry count %lu\n", -+ qman_fq_lookup_table, -+ (unsigned long)qman_fq_lookup_table_size); -+ return 0; -+} -+ -+/* global structure that maintains fq object mapping */ -+static DEFINE_SPINLOCK(fq_hash_table_lock); -+ -+static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq) -+{ -+ u32 i; -+ -+ spin_lock(&fq_hash_table_lock); -+ /* Can't use index zero because this has special meaning -+ * in context_b field. */ -+ for (i = 1; i < qman_fq_lookup_table_size; i++) { -+ if (qman_fq_lookup_table[i] == NULL) { -+ *entry = i; -+ qman_fq_lookup_table[i] = fq; -+ spin_unlock(&fq_hash_table_lock); -+ return 0; -+ } -+ } -+ spin_unlock(&fq_hash_table_lock); -+ return -ENOMEM; -+} -+ -+static void clear_fq_table_entry(u32 entry) -+{ -+ spin_lock(&fq_hash_table_lock); -+ BUG_ON(entry >= qman_fq_lookup_table_size); -+ qman_fq_lookup_table[entry] = NULL; -+ spin_unlock(&fq_hash_table_lock); -+} -+ -+static inline struct qman_fq *get_fq_table_entry(u32 entry) -+{ -+ BUG_ON(entry >= qman_fq_lookup_table_size); -+ return qman_fq_lookup_table[entry]; -+} -+#endif -+ -+static inline void cpu_to_hw_fqd(struct qm_fqd *fqd) -+{ -+ /* Byteswap the FQD to HW format */ -+ fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl); -+ fqd->dest_wq = cpu_to_be16(fqd->dest_wq); -+ fqd->ics_cred = cpu_to_be16(fqd->ics_cred); -+ fqd->context_b = cpu_to_be32(fqd->context_b); -+ fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque); -+} -+ -+static inline void hw_fqd_to_cpu(struct qm_fqd *fqd) -+{ -+ /* Byteswap the FQD to CPU format */ -+ fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl); -+ fqd->dest_wq = be16_to_cpu(fqd->dest_wq); -+ fqd->ics_cred = be16_to_cpu(fqd->ics_cred); -+ fqd->context_b = be32_to_cpu(fqd->context_b); -+ fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque); -+} -+ -+/* Swap a 40 bit address */ -+static inline u64 cpu_to_be40(u64 in) -+{ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ return in; -+#else -+ u64 out = 0; -+ u8 *p = (u8 *) &out; -+ p[0] = in >> 32; -+ p[1] = in >> 24; -+ p[2] = in >> 16; -+ p[3] = in >> 8; -+ p[4] = in >> 0; -+ return out; -+#endif -+} -+static inline u64 be40_to_cpu(u64 in) -+{ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ return in; -+#else -+ u64 out = 0; -+ u8 *pout = (u8 *) &out; -+ u8 *pin = (u8 *) ∈ -+ pout[0] = pin[4]; -+ pout[1] = pin[3]; -+ pout[2] = pin[2]; -+ pout[3] = pin[1]; -+ pout[4] = pin[0]; -+ return out; -+#endif -+} -+ -+/* Swap a 24 bit value */ -+static inline u32 cpu_to_be24(u32 in) -+{ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ return in; -+#else -+ u32 out = 0; -+ u8 *p = (u8 *) &out; -+ p[0] = in >> 16; -+ p[1] = in >> 8; -+ p[2] = in >> 0; -+ return out; -+#endif -+} -+ -+static inline u32 be24_to_cpu(u32 in) -+{ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ return in; -+#else -+ u32 out = 0; -+ u8 *pout = (u8 *) &out; -+ u8 *pin = (u8 *) ∈ -+ pout[0] = pin[2]; -+ pout[1] = pin[1]; -+ pout[2] = pin[0]; -+ return out; -+#endif -+} -+ -+static inline u64 be48_to_cpu(u64 in) -+{ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ return in; -+#else -+ u64 out = 0; -+ u8 *pout = (u8 *) &out; -+ u8 *pin = (u8 *) ∈ -+ -+ pout[0] = pin[5]; -+ pout[1] = pin[4]; -+ pout[2] = pin[3]; -+ pout[3] = pin[2]; -+ pout[4] = pin[1]; -+ pout[5] = pin[0]; -+ return out; -+#endif -+} -+static inline void cpu_to_hw_fd(struct qm_fd *fd) -+{ -+ fd->opaque_addr = cpu_to_be64(fd->opaque_addr); -+ fd->status = cpu_to_be32(fd->status); -+ fd->opaque = cpu_to_be32(fd->opaque); -+} -+ -+static inline void hw_fd_to_cpu(struct qm_fd *fd) -+{ -+ fd->opaque_addr = be64_to_cpu(fd->opaque_addr); -+ fd->status = be32_to_cpu(fd->status); -+ fd->opaque = be32_to_cpu(fd->opaque); -+} -+ -+static inline void hw_cq_query_to_cpu(struct qm_mcr_ceetm_cq_query *cq_query) -+{ -+ cq_query->ccgid = be16_to_cpu(cq_query->ccgid); -+ cq_query->state = be16_to_cpu(cq_query->state); -+ cq_query->pfdr_hptr = be24_to_cpu(cq_query->pfdr_hptr); -+ cq_query->pfdr_tptr = be24_to_cpu(cq_query->pfdr_tptr); -+ cq_query->od1_xsfdr = be16_to_cpu(cq_query->od1_xsfdr); -+ cq_query->od2_xsfdr = be16_to_cpu(cq_query->od2_xsfdr); -+ cq_query->od3_xsfdr = be16_to_cpu(cq_query->od3_xsfdr); -+ cq_query->od4_xsfdr = be16_to_cpu(cq_query->od4_xsfdr); -+ cq_query->od5_xsfdr = be16_to_cpu(cq_query->od5_xsfdr); -+ cq_query->od6_xsfdr = be16_to_cpu(cq_query->od6_xsfdr); -+ cq_query->ra1_xsfdr = be16_to_cpu(cq_query->ra1_xsfdr); -+ cq_query->ra2_xsfdr = be16_to_cpu(cq_query->ra2_xsfdr); -+ cq_query->frm_cnt = be24_to_cpu(cq_query->frm_cnt); -+} -+ -+static inline void hw_ccgr_query_to_cpu(struct qm_mcr_ceetm_ccgr_query *ccgr_q) -+{ -+ int i; -+ -+ ccgr_q->cm_query.cs_thres.hword = -+ be16_to_cpu(ccgr_q->cm_query.cs_thres.hword); -+ ccgr_q->cm_query.cs_thres_x.hword = -+ be16_to_cpu(ccgr_q->cm_query.cs_thres_x.hword); -+ ccgr_q->cm_query.td_thres.hword = -+ be16_to_cpu(ccgr_q->cm_query.td_thres.hword); -+ ccgr_q->cm_query.wr_parm_g.word = -+ be32_to_cpu(ccgr_q->cm_query.wr_parm_g.word); -+ ccgr_q->cm_query.wr_parm_y.word = -+ be32_to_cpu(ccgr_q->cm_query.wr_parm_y.word); -+ ccgr_q->cm_query.wr_parm_r.word = -+ be32_to_cpu(ccgr_q->cm_query.wr_parm_r.word); -+ ccgr_q->cm_query.cscn_targ_dcp = -+ be16_to_cpu(ccgr_q->cm_query.cscn_targ_dcp); -+ ccgr_q->cm_query.i_cnt = be40_to_cpu(ccgr_q->cm_query.i_cnt); -+ ccgr_q->cm_query.a_cnt = be40_to_cpu(ccgr_q->cm_query.a_cnt); -+ for (i = 0; i < ARRAY_SIZE(ccgr_q->cm_query.cscn_targ_swp); i++) -+ ccgr_q->cm_query.cscn_targ_swp[i] = -+ be32_to_cpu(ccgr_q->cm_query.cscn_targ_swp[i]); -+} -+ -+/* In the case that slow- and fast-path handling are both done by qman_poll() -+ * (ie. because there is no interrupt handling), we ought to balance how often -+ * we do the fast-path poll versus the slow-path poll. We'll use two decrementer -+ * sources, so we call the fast poll 'n' times before calling the slow poll -+ * once. The idle decrementer constant is used when the last slow-poll detected -+ * no work to do, and the busy decrementer constant when the last slow-poll had -+ * work to do. */ -+#define SLOW_POLL_IDLE 1000 -+#define SLOW_POLL_BUSY 10 -+static u32 __poll_portal_slow(struct qman_portal *p, u32 is); -+static inline unsigned int __poll_portal_fast(struct qman_portal *p, -+ unsigned int poll_limit); -+ -+/* Portal interrupt handler */ -+static irqreturn_t portal_isr(__always_unused int irq, void *ptr) -+{ -+ struct qman_portal *p = ptr; -+ /* -+ * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because -+ * it could race against a Query Congestion State command also given -+ * as part of the handling of this interrupt source. We mustn't -+ * clear it a second time in this top-level function. -+ */ -+ u32 clear = QM_DQAVAIL_MASK | (p->irq_sources & -+ ~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI)); -+ u32 is = qm_isr_status_read(&p->p) & p->irq_sources; -+ /* DQRR-handling if it's interrupt-driven */ -+ if (is & QM_PIRQ_DQRI) -+ __poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT); -+ /* Handling of anything else that's interrupt-driven */ -+ clear |= __poll_portal_slow(p, is); -+ qm_isr_status_clear(&p->p, clear); -+ return IRQ_HANDLED; -+} -+ -+/* This inner version is used privately by qman_create_affine_portal(), as well -+ * as by the exported qman_stop_dequeues(). */ -+static inline void qman_stop_dequeues_ex(struct qman_portal *p) -+{ -+ unsigned long irqflags __maybe_unused; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ if (!(p->dqrr_disable_ref++)) -+ qm_dqrr_set_maxfill(&p->p, 0); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+} -+ -+static int drain_mr_fqrni(struct qm_portal *p) -+{ -+ const struct qm_mr_entry *msg; -+loop: -+ msg = qm_mr_current(p); -+ if (!msg) { -+ /* if MR was full and h/w had other FQRNI entries to produce, we -+ * need to allow it time to produce those entries once the -+ * existing entries are consumed. A worst-case situation -+ * (fully-loaded system) means h/w sequencers may have to do 3-4 -+ * other things before servicing the portal's MR pump, each of -+ * which (if slow) may take ~50 qman cycles (which is ~200 -+ * processor cycles). So rounding up and then multiplying this -+ * worst-case estimate by a factor of 10, just to be -+ * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume -+ * one entry at a time, so h/w has an opportunity to produce new -+ * entries well before the ring has been fully consumed, so -+ * we're being *really* paranoid here. */ -+ u64 now, then = mfatb(); -+ do { -+ now = mfatb(); -+ } while ((then + 10000) > now); -+ msg = qm_mr_current(p); -+ if (!msg) -+ return 0; -+ } -+ if ((msg->verb & QM_MR_VERB_TYPE_MASK) != QM_MR_VERB_FQRNI) { -+ /* We aren't draining anything but FQRNIs */ -+ pr_err("QMan found verb 0x%x in MR\n", msg->verb); -+ return -1; -+ } -+ qm_mr_next(p); -+ qm_mr_cci_consume(p, 1); -+ goto loop; -+} -+ -+#ifdef CONFIG_SUSPEND -+static int _qman_portal_suspend_noirq(struct device *dev) -+{ -+ struct qman_portal *p = (struct qman_portal *)dev->platform_data; -+#ifdef CONFIG_PM_DEBUG -+ struct platform_device *pdev = to_platform_device(dev); -+#endif -+ -+ p->save_isdr = qm_isr_disable_read(&p->p); -+ qm_isr_disable_write(&p->p, 0xffffffff); -+ qm_isr_status_clear(&p->p, 0xffffffff); -+#ifdef CONFIG_PM_DEBUG -+ pr_info("Suspend for %s\n", pdev->name); -+#endif -+ return 0; -+} -+ -+static int _qman_portal_resume_noirq(struct device *dev) -+{ -+ struct qman_portal *p = (struct qman_portal *)dev->platform_data; -+ -+ /* restore isdr */ -+ qm_isr_disable_write(&p->p, p->save_isdr); -+ return 0; -+} -+#else -+#define _qman_portal_suspend_noirq NULL -+#define _qman_portal_resume_noirq NULL -+#endif -+ -+struct dev_pm_domain qman_portal_device_pm_domain = { -+ .ops = { -+ USE_PLATFORM_PM_SLEEP_OPS -+ .suspend_noirq = _qman_portal_suspend_noirq, -+ .resume_noirq = _qman_portal_resume_noirq, -+ } -+}; -+ -+struct qman_portal *qman_create_portal( -+ struct qman_portal *portal, -+ const struct qm_portal_config *config, -+ const struct qman_cgrs *cgrs) -+{ -+ struct qm_portal *__p; -+ char buf[16]; -+ int ret; -+ u32 isdr; -+ -+ if (!portal) { -+ portal = kmalloc(sizeof(*portal), GFP_KERNEL); -+ if (!portal) -+ return portal; -+ portal->alloced = 1; -+ } else -+ portal->alloced = 0; -+ -+ __p = &portal->p; -+ -+#if (defined CONFIG_PPC || defined CONFIG_PPC64) && defined CONFIG_FSL_PAMU -+ /* PAMU is required for stashing */ -+ portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ? -+ 1 : 0); -+#elif defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ portal->use_eqcr_ci_stashing = 1; -+#else -+ portal->use_eqcr_ci_stashing = 0; -+#endif -+ -+ /* prep the low-level portal struct with the mapped addresses from the -+ * config, everything that follows depends on it and "config" is more -+ * for (de)reference... */ -+ __p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE]; -+ __p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI]; -+ /* -+ * If CI-stashing is used, the current defaults use a threshold of 3, -+ * and stash with high-than-DQRR priority. -+ */ -+ if (qm_eqcr_init(__p, qm_eqcr_pvb, -+ portal->use_eqcr_ci_stashing ? 3 : 0, 1)) { -+ pr_err("Qman EQCR initialisation failed\n"); -+ goto fail_eqcr; -+ } -+ if (qm_dqrr_init(__p, config, qm_dqrr_dpush, qm_dqrr_pvb, -+ qm_dqrr_cdc, DQRR_MAXFILL)) { -+ pr_err("Qman DQRR initialisation failed\n"); -+ goto fail_dqrr; -+ } -+ if (qm_mr_init(__p, qm_mr_pvb, qm_mr_cci)) { -+ pr_err("Qman MR initialisation failed\n"); -+ goto fail_mr; -+ } -+ if (qm_mc_init(__p)) { -+ pr_err("Qman MC initialisation failed\n"); -+ goto fail_mc; -+ } -+ if (qm_isr_init(__p)) { -+ pr_err("Qman ISR initialisation failed\n"); -+ goto fail_isr; -+ } -+ /* static interrupt-gating controls */ -+ qm_dqrr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_DQRR_ITHRESH); -+ qm_mr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_MR_ITHRESH); -+ qm_isr_set_iperiod(__p, CONFIG_FSL_QMAN_PIRQ_IPERIOD); -+ portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL); -+ if (!portal->cgrs) -+ goto fail_cgrs; -+ /* initial snapshot is no-depletion */ -+ qman_cgrs_init(&portal->cgrs[1]); -+ if (cgrs) -+ portal->cgrs[0] = *cgrs; -+ else -+ /* if the given mask is NULL, assume all CGRs can be seen */ -+ qman_cgrs_fill(&portal->cgrs[0]); -+ INIT_LIST_HEAD(&portal->cgr_cbs); -+ spin_lock_init(&portal->cgr_lock); -+ if (num_ceetms) { -+ for (ret = 0; ret < num_ceetms; ret++) { -+ portal->ccgrs[ret] = kmalloc(2 * -+ sizeof(struct qman_ccgrs), GFP_KERNEL); -+ if (!portal->ccgrs[ret]) -+ goto fail_ccgrs; -+ qman_ccgrs_init(&portal->ccgrs[ret][1]); -+ qman_ccgrs_fill(&portal->ccgrs[ret][0]); -+ INIT_LIST_HEAD(&portal->ccgr_cbs[ret]); -+ } -+ } -+ spin_lock_init(&portal->ccgr_lock); -+ portal->bits = 0; -+ portal->slowpoll = 0; -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ portal->eqci_owned = NULL; -+#endif -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ raw_spin_lock_init(&portal->sharing_lock); -+ portal->is_shared = config->public_cfg.is_shared; -+ portal->sharing_redirect = NULL; -+#endif -+ portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 | -+ QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS | -+ QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED; -+ portal->dqrr_disable_ref = 0; -+ portal->cb_dc_ern = NULL; -+ sprintf(buf, "qportal-%d", config->public_cfg.channel); -+ portal->pdev = platform_device_alloc(buf, -1); -+ if (!portal->pdev) { -+ pr_err("qman_portal - platform_device_alloc() failed\n"); -+ goto fail_devalloc; -+ } -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ portal->pdev->dev.coherent_dma_mask = DMA_BIT_MASK(40); -+ portal->pdev->dev.dma_mask = &portal->pdev->dev.coherent_dma_mask; -+#else -+ if (dma_set_mask(&portal->pdev->dev, DMA_BIT_MASK(40))) { -+ pr_err("qman_portal - dma_set_mask() failed\n"); -+ goto fail_devadd; -+ } -+#endif -+ portal->pdev->dev.pm_domain = &qman_portal_device_pm_domain; -+ portal->pdev->dev.platform_data = portal; -+ ret = platform_device_add(portal->pdev); -+ if (ret) { -+ pr_err("qman_portal - platform_device_add() failed\n"); -+ goto fail_devadd; -+ } -+ dpa_rbtree_init(&portal->retire_table); -+ isdr = 0xffffffff; -+ qm_isr_disable_write(__p, isdr); -+ portal->irq_sources = 0; -+ qm_isr_enable_write(__p, portal->irq_sources); -+ qm_isr_status_clear(__p, 0xffffffff); -+ snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu); -+ if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname, -+ portal)) { -+ pr_err("request_irq() failed\n"); -+ goto fail_irq; -+ } -+ if ((config->public_cfg.cpu != -1) && -+ irq_can_set_affinity(config->public_cfg.irq) && -+ irq_set_affinity(config->public_cfg.irq, -+ cpumask_of(config->public_cfg.cpu))) { -+ pr_err("irq_set_affinity() failed\n"); -+ goto fail_affinity; -+ } -+ -+ /* Need EQCR to be empty before continuing */ -+ isdr ^= QM_PIRQ_EQCI; -+ qm_isr_disable_write(__p, isdr); -+ ret = qm_eqcr_get_fill(__p); -+ if (ret) { -+ pr_err("Qman EQCR unclean\n"); -+ goto fail_eqcr_empty; -+ } -+ isdr ^= (QM_PIRQ_DQRI | QM_PIRQ_MRI); -+ qm_isr_disable_write(__p, isdr); -+ if (qm_dqrr_current(__p) != NULL) { -+ pr_err("Qman DQRR unclean\n"); -+ qm_dqrr_cdc_consume_n(__p, 0xffff); -+ } -+ if (qm_mr_current(__p) != NULL) { -+ /* special handling, drain just in case it's a few FQRNIs */ -+ if (drain_mr_fqrni(__p)) { -+ const struct qm_mr_entry *e = qm_mr_current(__p); -+ /* -+ * Message ring cannot be empty no need to check -+ * qm_mr_current returned successfully -+ */ -+ pr_err("Qman MR unclean, MR VERB 0x%x, rc 0x%x\n, addr 0x%x", -+ e->verb, e->ern.rc, e->ern.fd.addr_lo); -+ goto fail_dqrr_mr_empty; -+ } -+ } -+ /* Success */ -+ portal->config = config; -+ qm_isr_disable_write(__p, 0); -+ qm_isr_uninhibit(__p); -+ /* Write a sane SDQCR */ -+ qm_dqrr_sdqcr_set(__p, portal->sdqcr); -+ return portal; -+fail_dqrr_mr_empty: -+fail_eqcr_empty: -+fail_affinity: -+ free_irq(config->public_cfg.irq, portal); -+fail_irq: -+ platform_device_del(portal->pdev); -+fail_devadd: -+ platform_device_put(portal->pdev); -+fail_devalloc: -+ if (num_ceetms) -+ for (ret = 0; ret < num_ceetms; ret++) -+ kfree(portal->ccgrs[ret]); -+fail_ccgrs: -+ kfree(portal->cgrs); -+fail_cgrs: -+ qm_isr_finish(__p); -+fail_isr: -+ qm_mc_finish(__p); -+fail_mc: -+ qm_mr_finish(__p); -+fail_mr: -+ qm_dqrr_finish(__p); -+fail_dqrr: -+ qm_eqcr_finish(__p); -+fail_eqcr: -+ if (portal->alloced) -+ kfree(portal); -+ return NULL; -+} -+ -+struct qman_portal *qman_create_affine_portal( -+ const struct qm_portal_config *config, -+ const struct qman_cgrs *cgrs) -+{ -+ struct qman_portal *res; -+ struct qman_portal *portal; -+ -+ portal = &per_cpu(qman_affine_portal, config->public_cfg.cpu); -+ res = qman_create_portal(portal, config, cgrs); -+ if (res) { -+ spin_lock(&affine_mask_lock); -+ cpumask_set_cpu(config->public_cfg.cpu, &affine_mask); -+ affine_channels[config->public_cfg.cpu] = -+ config->public_cfg.channel; -+ affine_portals[config->public_cfg.cpu] = portal; -+ spin_unlock(&affine_mask_lock); -+ } -+ return res; -+} -+ -+/* These checks are BUG_ON()s because the driver is already supposed to avoid -+ * these cases. */ -+struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect, -+ int cpu) -+{ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ struct qman_portal *p; -+ p = &per_cpu(qman_affine_portal, cpu); -+ /* Check that we don't already have our own portal */ -+ BUG_ON(p->config); -+ /* Check that we aren't already slaving to another portal */ -+ BUG_ON(p->is_shared); -+ /* Check that 'redirect' is prepared to have us */ -+ BUG_ON(!redirect->config->public_cfg.is_shared); -+ /* These are the only elements to initialise when redirecting */ -+ p->irq_sources = 0; -+ p->sharing_redirect = redirect; -+ affine_portals[cpu] = p; -+ return p; -+#else -+ BUG(); -+ return NULL; -+#endif -+} -+ -+void qman_destroy_portal(struct qman_portal *qm) -+{ -+ const struct qm_portal_config *pcfg; -+ int i; -+ -+ /* Stop dequeues on the portal */ -+ qm_dqrr_sdqcr_set(&qm->p, 0); -+ -+ /* NB we do this to "quiesce" EQCR. If we add enqueue-completions or -+ * something related to QM_PIRQ_EQCI, this may need fixing. -+ * Also, due to the prefetching model used for CI updates in the enqueue -+ * path, this update will only invalidate the CI cacheline *after* -+ * working on it, so we need to call this twice to ensure a full update -+ * irrespective of where the enqueue processing was at when the teardown -+ * began. */ -+ qm_eqcr_cce_update(&qm->p); -+ qm_eqcr_cce_update(&qm->p); -+ pcfg = qm->config; -+ -+ free_irq(pcfg->public_cfg.irq, qm); -+ -+ kfree(qm->cgrs); -+ if (num_ceetms) -+ for (i = 0; i < num_ceetms; i++) -+ kfree(qm->ccgrs[i]); -+ qm_isr_finish(&qm->p); -+ qm_mc_finish(&qm->p); -+ qm_mr_finish(&qm->p); -+ qm_dqrr_finish(&qm->p); -+ qm_eqcr_finish(&qm->p); -+ -+ platform_device_del(qm->pdev); -+ platform_device_put(qm->pdev); -+ -+ qm->config = NULL; -+ if (qm->alloced) -+ kfree(qm); -+} -+ -+const struct qm_portal_config *qman_destroy_affine_portal(void) -+{ -+ /* We don't want to redirect if we're a slave, use "raw" */ -+ struct qman_portal *qm = get_raw_affine_portal(); -+ const struct qm_portal_config *pcfg; -+ int cpu; -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (qm->sharing_redirect) { -+ qm->sharing_redirect = NULL; -+ put_affine_portal(); -+ return NULL; -+ } -+ qm->is_shared = 0; -+#endif -+ pcfg = qm->config; -+ cpu = pcfg->public_cfg.cpu; -+ -+ qman_destroy_portal(qm); -+ -+ spin_lock(&affine_mask_lock); -+ cpumask_clear_cpu(cpu, &affine_mask); -+ spin_unlock(&affine_mask_lock); -+ put_affine_portal(); -+ return pcfg; -+} -+ -+const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal *p) -+{ -+ return &p->config->public_cfg; -+} -+EXPORT_SYMBOL(qman_p_get_portal_config); -+ -+const struct qman_portal_config *qman_get_portal_config(void) -+{ -+ struct qman_portal *p = get_affine_portal(); -+ const struct qman_portal_config *ret = qman_p_get_portal_config(p); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_get_portal_config); -+ -+/* Inline helper to reduce nesting in __poll_portal_slow() */ -+static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_mr_entry *msg, u8 verb) -+{ -+ FQLOCK(fq); -+ switch (verb) { -+ case QM_MR_VERB_FQRL: -+ DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL)); -+ fq_clear(fq, QMAN_FQ_STATE_ORL); -+ table_del_fq(p, fq); -+ break; -+ case QM_MR_VERB_FQRN: -+ DPA_ASSERT((fq->state == qman_fq_state_parked) || -+ (fq->state == qman_fq_state_sched)); -+ DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING)); -+ fq_clear(fq, QMAN_FQ_STATE_CHANGING); -+ if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY) -+ fq_set(fq, QMAN_FQ_STATE_NE); -+ if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT) -+ fq_set(fq, QMAN_FQ_STATE_ORL); -+ else -+ table_del_fq(p, fq); -+ fq->state = qman_fq_state_retired; -+ break; -+ case QM_MR_VERB_FQPN: -+ DPA_ASSERT(fq->state == qman_fq_state_sched); -+ DPA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING)); -+ fq->state = qman_fq_state_parked; -+ } -+ FQUNLOCK(fq); -+} -+ -+static u32 __poll_portal_slow(struct qman_portal *p, u32 is) -+{ -+ const struct qm_mr_entry *msg; -+ struct qm_mr_entry swapped_msg; -+ int k; -+ -+ if (is & QM_PIRQ_CSCI) { -+ struct qman_cgrs rr, c; -+ struct qm_mc_result *mcr; -+ struct qman_cgr *cgr; -+ unsigned long irqflags __maybe_unused; -+ -+ spin_lock_irqsave(&p->cgr_lock, irqflags); -+ /* -+ * The CSCI bit must be cleared _before_ issuing the -+ * Query Congestion State command, to ensure that a long -+ * CGR State Change callback cannot miss an intervening -+ * state change. -+ */ -+ qm_isr_status_clear(&p->p, QM_PIRQ_CSCI); -+ qm_mc_start(&p->p); -+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ for (k = 0; k < 8; k++) -+ mcr->querycongestion.state.__state[k] = be32_to_cpu( -+ mcr->querycongestion.state.__state[k]); -+ /* mask out the ones I'm not interested in */ -+ qman_cgrs_and(&rr, (const struct qman_cgrs *) -+ &mcr->querycongestion.state, &p->cgrs[0]); -+ /* check previous snapshot for delta, enter/exit congestion */ -+ qman_cgrs_xor(&c, &rr, &p->cgrs[1]); -+ /* update snapshot */ -+ qman_cgrs_cp(&p->cgrs[1], &rr); -+ /* Invoke callback */ -+ list_for_each_entry(cgr, &p->cgr_cbs, node) -+ if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid)) -+ cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid)); -+ spin_unlock_irqrestore(&p->cgr_lock, irqflags); -+ } -+ if (is & QM_PIRQ_CCSCI) { -+ struct qman_ccgrs rr, c, congestion_result; -+ struct qm_mc_result *mcr; -+ struct qm_mc_command *mcc; -+ struct qm_ceetm_ccg *ccg; -+ unsigned long irqflags __maybe_unused; -+ int i, j; -+ -+ spin_lock_irqsave(&p->ccgr_lock, irqflags); -+ /* -+ * The CCSCI bit must be cleared _before_ issuing the -+ * Query Congestion State command, to ensure that a long -+ * CCGR State Change callback cannot miss an intervening -+ * state change. -+ */ -+ qm_isr_status_clear(&p->p, QM_PIRQ_CCSCI); -+ -+ for (i = 0; i < num_ceetms; i++) { -+ for (j = 0; j < 2; j++) { -+ mcc = qm_mc_start(&p->p); -+ mcc->ccgr_query.ccgrid = cpu_to_be16( -+ CEETM_QUERY_CONGESTION_STATE | j); -+ mcc->ccgr_query.dcpid = i; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ for (k = 0; k < 8; k++) -+ mcr->ccgr_query.congestion_state.state. -+ __state[k] = be32_to_cpu( -+ mcr->ccgr_query. -+ congestion_state.state. -+ __state[k]); -+ congestion_result.q[j] = -+ mcr->ccgr_query.congestion_state.state; -+ } -+ /* mask out the ones I'm not interested in */ -+ qman_ccgrs_and(&rr, &congestion_result, -+ &p->ccgrs[i][0]); -+ /* -+ * check previous snapshot for delta, enter/exit -+ * congestion. -+ */ -+ qman_ccgrs_xor(&c, &rr, &p->ccgrs[i][1]); -+ /* update snapshot */ -+ qman_ccgrs_cp(&p->ccgrs[i][1], &rr); -+ /* Invoke callback */ -+ list_for_each_entry(ccg, &p->ccgr_cbs[i], cb_node) -+ if (ccg->cb && qman_ccgrs_get(&c, -+ (ccg->parent->idx << 4) | ccg->idx)) -+ ccg->cb(ccg, ccg->cb_ctx, -+ qman_ccgrs_get(&rr, -+ (ccg->parent->idx << 4) -+ | ccg->idx)); -+ } -+ spin_unlock_irqrestore(&p->ccgr_lock, irqflags); -+ } -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (is & QM_PIRQ_EQCI) { -+ unsigned long irqflags; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ p->eqci_owned = NULL; -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ wake_up(&affine_queue); -+ } -+#endif -+ -+ if (is & QM_PIRQ_EQRI) { -+ unsigned long irqflags __maybe_unused; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ qm_eqcr_cce_update(&p->p); -+ qm_eqcr_set_ithresh(&p->p, 0); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ wake_up(&affine_queue); -+ } -+ -+ if (is & QM_PIRQ_MRI) { -+ struct qman_fq *fq; -+ u8 verb, num = 0; -+mr_loop: -+ qm_mr_pvb_update(&p->p); -+ msg = qm_mr_current(&p->p); -+ if (!msg) -+ goto mr_done; -+ swapped_msg = *msg; -+ hw_fd_to_cpu(&swapped_msg.ern.fd); -+ verb = msg->verb & QM_MR_VERB_TYPE_MASK; -+ /* The message is a software ERN iff the 0x20 bit is set */ -+ if (verb & 0x20) { -+ switch (verb) { -+ case QM_MR_VERB_FQRNI: -+ /* nada, we drop FQRNIs on the floor */ -+ break; -+ case QM_MR_VERB_FQRN: -+ case QM_MR_VERB_FQRL: -+ /* Lookup in the retirement table */ -+ fq = table_find_fq(p, be32_to_cpu(msg->fq.fqid)); -+ BUG_ON(!fq); -+ fq_state_change(p, fq, &swapped_msg, verb); -+ if (fq->cb.fqs) -+ fq->cb.fqs(p, fq, &swapped_msg); -+ break; -+ case QM_MR_VERB_FQPN: -+ /* Parked */ -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ fq = get_fq_table_entry( -+ be32_to_cpu(msg->fq.contextB)); -+#else -+ fq = (void *)(uintptr_t) -+ be32_to_cpu(msg->fq.contextB); -+#endif -+ fq_state_change(p, fq, msg, verb); -+ if (fq->cb.fqs) -+ fq->cb.fqs(p, fq, &swapped_msg); -+ break; -+ case QM_MR_VERB_DC_ERN: -+ /* DCP ERN */ -+ if (p->cb_dc_ern) -+ p->cb_dc_ern(p, msg); -+ else if (cb_dc_ern) -+ cb_dc_ern(p, msg); -+ else { -+ static int warn_once; -+ if (!warn_once) { -+ pr_crit("Leaking DCP ERNs!\n"); -+ warn_once = 1; -+ } -+ } -+ break; -+ default: -+ pr_crit("Invalid MR verb 0x%02x\n", verb); -+ } -+ } else { -+ /* Its a software ERN */ -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag)); -+#else -+ fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag); -+#endif -+ fq->cb.ern(p, fq, &swapped_msg); -+ } -+ num++; -+ qm_mr_next(&p->p); -+ goto mr_loop; -+mr_done: -+ qm_mr_cci_consume(&p->p, num); -+ } -+ /* -+ * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific -+ * processing. If that interrupt source has meanwhile been re-asserted, -+ * we mustn't clear it here (or in the top-level interrupt handler). -+ */ -+ return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI); -+} -+ -+/* remove some slowish-path stuff from the "fast path" and make sure it isn't -+ * inlined. */ -+static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq) -+{ -+ p->vdqcr_owned = NULL; -+ FQLOCK(fq); -+ fq_clear(fq, QMAN_FQ_STATE_VDQCR); -+ FQUNLOCK(fq); -+ wake_up(&affine_queue); -+} -+ -+/* Copy a DQRR entry ensuring reads reach QBMan in order */ -+static inline void safe_copy_dqrr(struct qm_dqrr_entry *dst, -+ const struct qm_dqrr_entry *src) -+{ -+ int i = 0; -+ const u64 *s64 = (u64*)src; -+ u64 *d64 = (u64*)dst; -+ -+ /* DQRR only has 32 bytes of valid data so only need to -+ * copy 4 - 64 bit values */ -+ *d64 = *s64; -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ { -+ u32 res, zero = 0; -+ /* Create a dependancy after copying first bytes ensures no wrap -+ transaction generated to QBMan */ -+ /* Logical AND the value pointed to by s64 with 0x0 and -+ store the result in res */ -+ asm volatile("and %[result], %[in1], %[in2]" -+ : [result] "=r" (res) -+ : [in1] "r" (zero), [in2] "r" (*s64) -+ : "memory"); -+ /* Add res to s64 - this creates a dependancy on the result of -+ reading the value of s64 before the next read. The side -+ effect of this is that the core must stall until the first -+ aligned read is complete therefore preventing a WRAP -+ transaction to be seen by the QBMan */ -+ asm volatile("add %[result], %[in1], %[in2]" -+ : [result] "=r" (s64) -+ : [in1] "r" (res), [in2] "r" (s64) -+ : "memory"); -+ } -+#endif -+ /* Copy the last 3 64 bit parts */ -+ d64++; s64++; -+ for (;i<3; i++) -+ *d64++ = *s64++; -+} -+ -+/* Look: no locks, no irq_save()s, no preempt_disable()s! :-) The only states -+ * that would conflict with other things if they ran at the same time on the -+ * same cpu are; -+ * -+ * (i) setting/clearing vdqcr_owned, and -+ * (ii) clearing the NE (Not Empty) flag. -+ * -+ * Both are safe. Because; -+ * -+ * (i) this clearing can only occur after qman_volatile_dequeue() has set the -+ * vdqcr_owned field (which it does before setting VDQCR), and -+ * qman_volatile_dequeue() blocks interrupts and preemption while this is -+ * done so that we can't interfere. -+ * (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as -+ * with (i) that API prevents us from interfering until it's safe. -+ * -+ * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far -+ * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett -+ * advantage comes from this function not having to "lock" anything at all. -+ * -+ * Note also that the callbacks are invoked at points which are safe against the -+ * above potential conflicts, but that this function itself is not re-entrant -+ * (this is because the function tracks one end of each FIFO in the portal and -+ * we do *not* want to lock that). So the consequence is that it is safe for -+ * user callbacks to call into any Qman API *except* qman_poll() (as that's the -+ * sole API that could be invoking the callback through this function). -+ */ -+static inline unsigned int __poll_portal_fast(struct qman_portal *p, -+ unsigned int poll_limit) -+{ -+ const struct qm_dqrr_entry *dq; -+ struct qman_fq *fq; -+ enum qman_cb_dqrr_result res; -+ unsigned int limit = 0; -+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -+ struct qm_dqrr_entry *shadow; -+ const struct qm_dqrr_entry *orig_dq; -+#endif -+loop: -+ qm_dqrr_pvb_update(&p->p); -+ dq = qm_dqrr_current(&p->p); -+ if (!dq) -+ goto done; -+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -+ /* If running on an LE system the fields of the -+ dequeue entry must be swapped. Because the -+ QMan HW will ignore writes the DQRR entry is -+ copied and the index stored within the copy */ -+ shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)]; -+ /* Use safe copy here to avoid WRAP transaction */ -+ safe_copy_dqrr(shadow, dq); -+ orig_dq = dq; -+ dq = shadow; -+ shadow->fqid = be32_to_cpu(shadow->fqid); -+ shadow->contextB = be32_to_cpu(shadow->contextB); -+ shadow->seqnum = be16_to_cpu(shadow->seqnum); -+ hw_fd_to_cpu(&shadow->fd); -+#endif -+ if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) { -+ /* VDQCR: don't trust contextB as the FQ may have been -+ * configured for h/w consumption and we're draining it -+ * post-retirement. */ -+ fq = p->vdqcr_owned; -+ /* We only set QMAN_FQ_STATE_NE when retiring, so we only need -+ * to check for clearing it when doing volatile dequeues. It's -+ * one less thing to check in the critical path (SDQCR). */ -+ if (dq->stat & QM_DQRR_STAT_FQ_EMPTY) -+ fq_clear(fq, QMAN_FQ_STATE_NE); -+ /* this is duplicated from the SDQCR code, but we have stuff to -+ * do before *and* after this callback, and we don't want -+ * multiple if()s in the critical path (SDQCR). */ -+ res = fq->cb.dqrr(p, fq, dq); -+ if (res == qman_cb_dqrr_stop) -+ goto done; -+ /* Check for VDQCR completion */ -+ if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED) -+ clear_vdqcr(p, fq); -+ } else { -+ /* SDQCR: contextB points to the FQ */ -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ fq = get_fq_table_entry(dq->contextB); -+#else -+ fq = (void *)(uintptr_t)dq->contextB; -+#endif -+ /* Now let the callback do its stuff */ -+ res = fq->cb.dqrr(p, fq, dq); -+ -+ /* The callback can request that we exit without consuming this -+ * entry nor advancing; */ -+ if (res == qman_cb_dqrr_stop) -+ goto done; -+ } -+ /* Interpret 'dq' from a driver perspective. */ -+ /* Parking isn't possible unless HELDACTIVE was set. NB, -+ * FORCEELIGIBLE implies HELDACTIVE, so we only need to -+ * check for HELDACTIVE to cover both. */ -+ DPA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) || -+ (res != qman_cb_dqrr_park)); -+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -+ if (res != qman_cb_dqrr_defer) -+ qm_dqrr_cdc_consume_1ptr(&p->p, orig_dq, -+ (res == qman_cb_dqrr_park)); -+#else -+ /* Defer just means "skip it, I'll consume it myself later on" */ -+ if (res != qman_cb_dqrr_defer) -+ qm_dqrr_cdc_consume_1ptr(&p->p, dq, (res == qman_cb_dqrr_park)); -+#endif -+ /* Move forward */ -+ qm_dqrr_next(&p->p); -+ /* Entry processed and consumed, increment our counter. The callback can -+ * request that we exit after consuming the entry, and we also exit if -+ * we reach our processing limit, so loop back only if neither of these -+ * conditions is met. */ -+ if ((++limit < poll_limit) && (res != qman_cb_dqrr_consume_stop)) -+ goto loop; -+done: -+ return limit; -+} -+ -+u32 qman_irqsource_get(void) -+{ -+ /* "irqsource" and "poll" APIs mustn't redirect when sharing, they -+ * should shut the user out if they are not the primary CPU hosting the -+ * portal. That's why we use the "raw" interface. */ -+ struct qman_portal *p = get_raw_affine_portal(); -+ u32 ret = p->irq_sources & QM_PIRQ_VISIBLE; -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_irqsource_get); -+ -+int qman_p_irqsource_add(struct qman_portal *p, u32 bits __maybe_unused) -+{ -+ __maybe_unused unsigned long irqflags; -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (p->sharing_redirect) -+ return -EINVAL; -+ else -+#endif -+ { -+ bits = bits & QM_PIRQ_VISIBLE; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ /* Clear any previously remaining interrupt conditions in -+ * QCSP_ISR. This prevents raising a false interrupt when -+ * interrupt conditions are enabled in QCSP_IER. -+ */ -+ qm_isr_status_clear(&p->p, bits); -+ set_bits(bits, &p->irq_sources); -+ qm_isr_enable_write(&p->p, p->irq_sources); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_p_irqsource_add); -+ -+int qman_irqsource_add(u32 bits __maybe_unused) -+{ -+ struct qman_portal *p = get_raw_affine_portal(); -+ int ret; -+ ret = qman_p_irqsource_add(p, bits); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_irqsource_add); -+ -+int qman_p_irqsource_remove(struct qman_portal *p, u32 bits) -+{ -+ __maybe_unused unsigned long irqflags; -+ u32 ier; -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (p->sharing_redirect) { -+ put_affine_portal(); -+ return -EINVAL; -+ } -+#endif -+ /* Our interrupt handler only processes+clears status register bits that -+ * are in p->irq_sources. As we're trimming that mask, if one of them -+ * were to assert in the status register just before we remove it from -+ * the enable register, there would be an interrupt-storm when we -+ * release the IRQ lock. So we wait for the enable register update to -+ * take effect in h/w (by reading it back) and then clear all other bits -+ * in the status register. Ie. we clear them from ISR once it's certain -+ * IER won't allow them to reassert. */ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ bits &= QM_PIRQ_VISIBLE; -+ clear_bits(bits, &p->irq_sources); -+ qm_isr_enable_write(&p->p, p->irq_sources); -+ -+ ier = qm_isr_enable_read(&p->p); -+ /* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a -+ * data-dependency, ie. to protect against re-ordering. */ -+ qm_isr_status_clear(&p->p, ~ier); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ return 0; -+} -+EXPORT_SYMBOL(qman_p_irqsource_remove); -+ -+int qman_irqsource_remove(u32 bits) -+{ -+ struct qman_portal *p = get_raw_affine_portal(); -+ int ret; -+ ret = qman_p_irqsource_remove(p, bits); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_irqsource_remove); -+ -+const cpumask_t *qman_affine_cpus(void) -+{ -+ return &affine_mask; -+} -+EXPORT_SYMBOL(qman_affine_cpus); -+ -+u16 qman_affine_channel(int cpu) -+{ -+ if (cpu < 0) { -+ struct qman_portal *portal = get_raw_affine_portal(); -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ BUG_ON(portal->sharing_redirect); -+#endif -+ cpu = portal->config->public_cfg.cpu; -+ put_affine_portal(); -+ } -+ BUG_ON(!cpumask_test_cpu(cpu, &affine_mask)); -+ return affine_channels[cpu]; -+} -+EXPORT_SYMBOL(qman_affine_channel); -+ -+void *qman_get_affine_portal(int cpu) -+{ -+ return affine_portals[cpu]; -+} -+EXPORT_SYMBOL(qman_get_affine_portal); -+ -+int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit) -+{ -+ int ret; -+ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (unlikely(p->sharing_redirect)) -+ ret = -EINVAL; -+ else -+#endif -+ { -+ BUG_ON(p->irq_sources & QM_PIRQ_DQRI); -+ ret = __poll_portal_fast(p, limit); -+ } -+ return ret; -+} -+EXPORT_SYMBOL(qman_p_poll_dqrr); -+ -+int qman_poll_dqrr(unsigned int limit) -+{ -+ struct qman_portal *p = get_poll_portal(); -+ int ret; -+ ret = qman_p_poll_dqrr(p, limit); -+ put_poll_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_poll_dqrr); -+ -+u32 qman_p_poll_slow(struct qman_portal *p) -+{ -+ u32 ret; -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (unlikely(p->sharing_redirect)) -+ ret = (u32)-1; -+ else -+#endif -+ { -+ u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; -+ ret = __poll_portal_slow(p, is); -+ qm_isr_status_clear(&p->p, ret); -+ } -+ return ret; -+} -+EXPORT_SYMBOL(qman_p_poll_slow); -+ -+u32 qman_poll_slow(void) -+{ -+ struct qman_portal *p = get_poll_portal(); -+ u32 ret; -+ ret = qman_p_poll_slow(p); -+ put_poll_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_poll_slow); -+ -+/* Legacy wrapper */ -+void qman_p_poll(struct qman_portal *p) -+{ -+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE -+ if (unlikely(p->sharing_redirect)) -+ return; -+#endif -+ if ((~p->irq_sources) & QM_PIRQ_SLOW) { -+ if (!(p->slowpoll--)) { -+ u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources; -+ u32 active = __poll_portal_slow(p, is); -+ if (active) { -+ qm_isr_status_clear(&p->p, active); -+ p->slowpoll = SLOW_POLL_BUSY; -+ } else -+ p->slowpoll = SLOW_POLL_IDLE; -+ } -+ } -+ if ((~p->irq_sources) & QM_PIRQ_DQRI) -+ __poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT); -+} -+EXPORT_SYMBOL(qman_p_poll); -+ -+void qman_poll(void) -+{ -+ struct qman_portal *p = get_poll_portal(); -+ qman_p_poll(p); -+ put_poll_portal(); -+} -+EXPORT_SYMBOL(qman_poll); -+ -+void qman_p_stop_dequeues(struct qman_portal *p) -+{ -+ qman_stop_dequeues_ex(p); -+} -+EXPORT_SYMBOL(qman_p_stop_dequeues); -+ -+void qman_stop_dequeues(void) -+{ -+ struct qman_portal *p = get_affine_portal(); -+ qman_p_stop_dequeues(p); -+ put_affine_portal(); -+} -+EXPORT_SYMBOL(qman_stop_dequeues); -+ -+void qman_p_start_dequeues(struct qman_portal *p) -+{ -+ unsigned long irqflags __maybe_unused; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ DPA_ASSERT(p->dqrr_disable_ref > 0); -+ if (!(--p->dqrr_disable_ref)) -+ qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+} -+EXPORT_SYMBOL(qman_p_start_dequeues); -+ -+void qman_start_dequeues(void) -+{ -+ struct qman_portal *p = get_affine_portal(); -+ qman_p_start_dequeues(p); -+ put_affine_portal(); -+} -+EXPORT_SYMBOL(qman_start_dequeues); -+ -+void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools) -+{ -+ unsigned long irqflags __maybe_unused; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ pools &= p->config->public_cfg.pools; -+ p->sdqcr |= pools; -+ qm_dqrr_sdqcr_set(&p->p, p->sdqcr); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+} -+EXPORT_SYMBOL(qman_p_static_dequeue_add); -+ -+void qman_static_dequeue_add(u32 pools) -+{ -+ struct qman_portal *p = get_affine_portal(); -+ qman_p_static_dequeue_add(p, pools); -+ put_affine_portal(); -+} -+EXPORT_SYMBOL(qman_static_dequeue_add); -+ -+void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools) -+{ -+ unsigned long irqflags __maybe_unused; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ pools &= p->config->public_cfg.pools; -+ p->sdqcr &= ~pools; -+ qm_dqrr_sdqcr_set(&p->p, p->sdqcr); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+} -+EXPORT_SYMBOL(qman_p_static_dequeue_del); -+ -+void qman_static_dequeue_del(u32 pools) -+{ -+ struct qman_portal *p = get_affine_portal(); -+ qman_p_static_dequeue_del(p, pools); -+ put_affine_portal(); -+} -+EXPORT_SYMBOL(qman_static_dequeue_del); -+ -+u32 qman_p_static_dequeue_get(struct qman_portal *p) -+{ -+ return p->sdqcr; -+} -+EXPORT_SYMBOL(qman_p_static_dequeue_get); -+ -+u32 qman_static_dequeue_get(void) -+{ -+ struct qman_portal *p = get_affine_portal(); -+ u32 ret = qman_p_static_dequeue_get(p); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_static_dequeue_get); -+ -+void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq, -+ int park_request) -+{ -+ qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request); -+} -+EXPORT_SYMBOL(qman_p_dca); -+ -+void qman_dca(struct qm_dqrr_entry *dq, int park_request) -+{ -+ struct qman_portal *p = get_affine_portal(); -+ qman_p_dca(p, dq, park_request); -+ put_affine_portal(); -+} -+EXPORT_SYMBOL(qman_dca); -+ -+/*******************/ -+/* Frame queue API */ -+/*******************/ -+ -+static const char *mcr_result_str(u8 result) -+{ -+ switch (result) { -+ case QM_MCR_RESULT_NULL: -+ return "QM_MCR_RESULT_NULL"; -+ case QM_MCR_RESULT_OK: -+ return "QM_MCR_RESULT_OK"; -+ case QM_MCR_RESULT_ERR_FQID: -+ return "QM_MCR_RESULT_ERR_FQID"; -+ case QM_MCR_RESULT_ERR_FQSTATE: -+ return "QM_MCR_RESULT_ERR_FQSTATE"; -+ case QM_MCR_RESULT_ERR_NOTEMPTY: -+ return "QM_MCR_RESULT_ERR_NOTEMPTY"; -+ case QM_MCR_RESULT_PENDING: -+ return "QM_MCR_RESULT_PENDING"; -+ case QM_MCR_RESULT_ERR_BADCOMMAND: -+ return "QM_MCR_RESULT_ERR_BADCOMMAND"; -+ } -+ return "<unknown MCR result>"; -+} -+ -+int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq) -+{ -+ struct qm_fqd fqd; -+ struct qm_mcr_queryfq_np np; -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ -+ if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) { -+ int ret = qman_alloc_fqid(&fqid); -+ if (ret) -+ return ret; -+ } -+ spin_lock_init(&fq->fqlock); -+ fq->fqid = fqid; -+ fq->flags = flags; -+ fq->state = qman_fq_state_oos; -+ fq->cgr_groupid = 0; -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) -+ return -ENOMEM; -+#endif -+ if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY)) -+ return 0; -+ /* Everything else is AS_IS support */ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ mcc = qm_mc_start(&p->p); -+ mcc->queryfq.fqid = cpu_to_be32(fqid); -+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ); -+ if (mcr->result != QM_MCR_RESULT_OK) { -+ pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result)); -+ goto err; -+ } -+ fqd = mcr->queryfq.fqd; -+ hw_fqd_to_cpu(&fqd); -+ mcc = qm_mc_start(&p->p); -+ mcc->queryfq_np.fqid = cpu_to_be32(fqid); -+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP); -+ if (mcr->result != QM_MCR_RESULT_OK) { -+ pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result)); -+ goto err; -+ } -+ np = mcr->queryfq_np; -+ /* Phew, have queryfq and queryfq_np results, stitch together -+ * the FQ object from those. */ -+ fq->cgr_groupid = fqd.cgid; -+ switch (np.state & QM_MCR_NP_STATE_MASK) { -+ case QM_MCR_NP_STATE_OOS: -+ break; -+ case QM_MCR_NP_STATE_RETIRED: -+ fq->state = qman_fq_state_retired; -+ if (np.frm_cnt) -+ fq_set(fq, QMAN_FQ_STATE_NE); -+ break; -+ case QM_MCR_NP_STATE_TEN_SCHED: -+ case QM_MCR_NP_STATE_TRU_SCHED: -+ case QM_MCR_NP_STATE_ACTIVE: -+ fq->state = qman_fq_state_sched; -+ if (np.state & QM_MCR_NP_STATE_R) -+ fq_set(fq, QMAN_FQ_STATE_CHANGING); -+ break; -+ case QM_MCR_NP_STATE_PARKED: -+ fq->state = qman_fq_state_parked; -+ break; -+ default: -+ DPA_ASSERT(NULL == "invalid FQ state"); -+ } -+ if (fqd.fq_ctrl & QM_FQCTRL_CGE) -+ fq->state |= QMAN_FQ_STATE_CGR_EN; -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return 0; -+err: -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) -+ qman_release_fqid(fqid); -+ return -EIO; -+} -+EXPORT_SYMBOL(qman_create_fq); -+ -+void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused) -+{ -+ -+ /* We don't need to lock the FQ as it is a pre-condition that the FQ be -+ * quiesced. Instead, run some checks. */ -+ switch (fq->state) { -+ case qman_fq_state_parked: -+ DPA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED); -+ case qman_fq_state_oos: -+ if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID)) -+ qman_release_fqid(fq->fqid); -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ clear_fq_table_entry(fq->key); -+#endif -+ return; -+ default: -+ break; -+ } -+ DPA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!"); -+} -+EXPORT_SYMBOL(qman_destroy_fq); -+ -+u32 qman_fq_fqid(struct qman_fq *fq) -+{ -+ return fq->fqid; -+} -+EXPORT_SYMBOL(qman_fq_fqid); -+ -+void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags) -+{ -+ if (state) -+ *state = fq->state; -+ if (flags) -+ *flags = fq->flags; -+} -+EXPORT_SYMBOL(qman_fq_state); -+ -+int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ? -+ QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED; -+ -+ if ((fq->state != qman_fq_state_oos) && -+ (fq->state != qman_fq_state_parked)) -+ return -EINVAL; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) -+ return -EINVAL; -+#endif -+ if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) { -+ /* And can't be set at the same time as TDTHRESH */ -+ if (opts->we_mask & QM_INITFQ_WE_TDTHRESH) -+ return -EINVAL; -+ } -+ /* Issue an INITFQ_[PARKED|SCHED] management command */ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ FQLOCK(fq); -+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || -+ ((fq->state != qman_fq_state_oos) && -+ (fq->state != qman_fq_state_parked)))) { -+ FQUNLOCK(fq); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return -EBUSY; -+ } -+ mcc = qm_mc_start(&p->p); -+ if (opts) -+ mcc->initfq = *opts; -+ mcc->initfq.fqid = cpu_to_be32(fq->fqid); -+ mcc->initfq.count = 0; -+ -+ /* If the FQ does *not* have the TO_DCPORTAL flag, contextB is set as a -+ * demux pointer. Otherwise, the caller-provided value is allowed to -+ * stand, don't overwrite it. */ -+ if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) { -+ dma_addr_t phys_fq; -+ mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB; -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ mcc->initfq.fqd.context_b = fq->key; -+#else -+ mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq; -+#endif -+ /* and the physical address - NB, if the user wasn't trying to -+ * set CONTEXTA, clear the stashing settings. */ -+ if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) { -+ mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA; -+ memset(&mcc->initfq.fqd.context_a, 0, -+ sizeof(mcc->initfq.fqd.context_a)); -+ } else { -+ phys_fq = dma_map_single(&p->pdev->dev, fq, sizeof(*fq), -+ DMA_TO_DEVICE); -+ qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq); -+ } -+ } -+ if (flags & QMAN_INITFQ_FLAG_LOCAL) { -+ mcc->initfq.fqd.dest.channel = p->config->public_cfg.channel; -+ if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) { -+ mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ; -+ mcc->initfq.fqd.dest.wq = 4; -+ } -+ } -+ mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask); -+ cpu_to_hw_fqd(&mcc->initfq.fqd); -+ qm_mc_commit(&p->p, myverb); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ FQUNLOCK(fq); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return -EIO; -+ } -+ if (opts) { -+ if (opts->we_mask & QM_INITFQ_WE_FQCTRL) { -+ if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE) -+ fq_set(fq, QMAN_FQ_STATE_CGR_EN); -+ else -+ fq_clear(fq, QMAN_FQ_STATE_CGR_EN); -+ } -+ if (opts->we_mask & QM_INITFQ_WE_CGID) -+ fq->cgr_groupid = opts->fqd.cgid; -+ } -+ fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ? -+ qman_fq_state_sched : qman_fq_state_parked; -+ FQUNLOCK(fq); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return 0; -+} -+EXPORT_SYMBOL(qman_init_fq); -+ -+int qman_schedule_fq(struct qman_fq *fq) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ int ret = 0; -+ u8 res; -+ -+ if (fq->state != qman_fq_state_parked) -+ return -EINVAL; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) -+ return -EINVAL; -+#endif -+ /* Issue a ALTERFQ_SCHED management command */ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ FQLOCK(fq); -+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || -+ (fq->state != qman_fq_state_parked))) { -+ ret = -EBUSY; -+ goto out; -+ } -+ mcc = qm_mc_start(&p->p); -+ mcc->alterfq.fqid = cpu_to_be32(fq->fqid); -+ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED); -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ ret = -EIO; -+ goto out; -+ } -+ fq->state = qman_fq_state_sched; -+out: -+ FQUNLOCK(fq); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_schedule_fq); -+ -+int qman_retire_fq(struct qman_fq *fq, u32 *flags) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ int rval; -+ u8 res; -+ -+ if ((fq->state != qman_fq_state_parked) && -+ (fq->state != qman_fq_state_sched)) -+ return -EINVAL; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) -+ return -EINVAL; -+#endif -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ FQLOCK(fq); -+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || -+ (fq->state == qman_fq_state_retired) || -+ (fq->state == qman_fq_state_oos))) { -+ rval = -EBUSY; -+ goto out; -+ } -+ rval = table_push_fq(p, fq); -+ if (rval) -+ goto out; -+ mcc = qm_mc_start(&p->p); -+ mcc->alterfq.fqid = cpu_to_be32(fq->fqid); -+ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE); -+ res = mcr->result; -+ /* "Elegant" would be to treat OK/PENDING the same way; set CHANGING, -+ * and defer the flags until FQRNI or FQRN (respectively) show up. But -+ * "Friendly" is to process OK immediately, and not set CHANGING. We do -+ * friendly, otherwise the caller doesn't necessarily have a fully -+ * "retired" FQ on return even if the retirement was immediate. However -+ * this does mean some code duplication between here and -+ * fq_state_change(). */ -+ if (likely(res == QM_MCR_RESULT_OK)) { -+ rval = 0; -+ /* Process 'fq' right away, we'll ignore FQRNI */ -+ if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) -+ fq_set(fq, QMAN_FQ_STATE_NE); -+ if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT) -+ fq_set(fq, QMAN_FQ_STATE_ORL); -+ else -+ table_del_fq(p, fq); -+ if (flags) -+ *flags = fq->flags; -+ fq->state = qman_fq_state_retired; -+ if (fq->cb.fqs) { -+ /* Another issue with supporting "immediate" retirement -+ * is that we're forced to drop FQRNIs, because by the -+ * time they're seen it may already be "too late" (the -+ * fq may have been OOS'd and free()'d already). But if -+ * the upper layer wants a callback whether it's -+ * immediate or not, we have to fake a "MR" entry to -+ * look like an FQRNI... */ -+ struct qm_mr_entry msg; -+ msg.verb = QM_MR_VERB_FQRNI; -+ msg.fq.fqs = mcr->alterfq.fqs; -+ msg.fq.fqid = fq->fqid; -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ msg.fq.contextB = fq->key; -+#else -+ msg.fq.contextB = (u32)(uintptr_t)fq; -+#endif -+ fq->cb.fqs(p, fq, &msg); -+ } -+ } else if (res == QM_MCR_RESULT_PENDING) { -+ rval = 1; -+ fq_set(fq, QMAN_FQ_STATE_CHANGING); -+ } else { -+ rval = -EIO; -+ table_del_fq(p, fq); -+ } -+out: -+ FQUNLOCK(fq); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return rval; -+} -+EXPORT_SYMBOL(qman_retire_fq); -+ -+int qman_oos_fq(struct qman_fq *fq) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ int ret = 0; -+ u8 res; -+ -+ if (fq->state != qman_fq_state_retired) -+ return -EINVAL; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) -+ return -EINVAL; -+#endif -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ FQLOCK(fq); -+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) || -+ (fq->state != qman_fq_state_retired))) { -+ ret = -EBUSY; -+ goto out; -+ } -+ mcc = qm_mc_start(&p->p); -+ mcc->alterfq.fqid = cpu_to_be32(fq->fqid); -+ qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS); -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ ret = -EIO; -+ goto out; -+ } -+ fq->state = qman_fq_state_oos; -+out: -+ FQUNLOCK(fq); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_oos_fq); -+ -+int qman_fq_flow_control(struct qman_fq *fq, int xon) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ int ret = 0; -+ u8 res; -+ u8 myverb; -+ -+ if ((fq->state == qman_fq_state_oos) || -+ (fq->state == qman_fq_state_retired) || -+ (fq->state == qman_fq_state_parked)) -+ return -EINVAL; -+ -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY))) -+ return -EINVAL; -+#endif -+ /* Issue a ALTER_FQXON or ALTER_FQXOFF management command */ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ FQLOCK(fq); -+ if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) || -+ (fq->state == qman_fq_state_parked) || -+ (fq->state == qman_fq_state_oos) || -+ (fq->state == qman_fq_state_retired))) { -+ ret = -EBUSY; -+ goto out; -+ } -+ mcc = qm_mc_start(&p->p); -+ mcc->alterfq.fqid = fq->fqid; -+ mcc->alterfq.count = 0; -+ myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF; -+ -+ qm_mc_commit(&p->p, myverb); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); -+ -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ ret = -EIO; -+ goto out; -+ } -+out: -+ FQUNLOCK(fq); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_fq_flow_control); -+ -+int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ mcc = qm_mc_start(&p->p); -+ mcc->queryfq.fqid = cpu_to_be32(fq->fqid); -+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) -+ *fqd = mcr->queryfq.fqd; -+ hw_fqd_to_cpu(fqd); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res != QM_MCR_RESULT_OK) -+ return -EIO; -+ return 0; -+} -+EXPORT_SYMBOL(qman_query_fq); -+ -+int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ mcc = qm_mc_start(&p->p); -+ mcc->queryfq.fqid = cpu_to_be32(fq->fqid); -+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) { -+ *np = mcr->queryfq_np; -+ np->fqd_link = be24_to_cpu(np->fqd_link); -+ np->odp_seq = be16_to_cpu(np->odp_seq); -+ np->orp_nesn = be16_to_cpu(np->orp_nesn); -+ np->orp_ea_hseq = be16_to_cpu(np->orp_ea_hseq); -+ np->orp_ea_tseq = be16_to_cpu(np->orp_ea_tseq); -+ np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr); -+ np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr); -+ np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr); -+ np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr); -+ np->ics_surp = be16_to_cpu(np->ics_surp); -+ np->byte_cnt = be32_to_cpu(np->byte_cnt); -+ np->frm_cnt = be24_to_cpu(np->frm_cnt); -+ np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr); -+ np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr); -+ np->od1_sfdr = be16_to_cpu(np->od1_sfdr); -+ np->od2_sfdr = be16_to_cpu(np->od2_sfdr); -+ np->od3_sfdr = be16_to_cpu(np->od3_sfdr); -+ } -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res == QM_MCR_RESULT_ERR_FQID) -+ return -ERANGE; -+ else if (res != QM_MCR_RESULT_OK) -+ return -EIO; -+ return 0; -+} -+EXPORT_SYMBOL(qman_query_fq_np); -+ -+int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res, myverb; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED : -+ QM_MCR_VERB_QUERYWQ; -+ mcc = qm_mc_start(&p->p); -+ mcc->querywq.channel.id = cpu_to_be16(wq->channel.id); -+ qm_mc_commit(&p->p, myverb); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) { -+ int i, array_len; -+ wq->channel.id = be16_to_cpu(mcr->querywq.channel.id); -+ array_len = ARRAY_SIZE(mcr->querywq.wq_len); -+ for (i = 0; i < array_len; i++) -+ wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]); -+ } -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("QUERYWQ failed: %s\n", mcr_result_str(res)); -+ return -EIO; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_query_wq); -+ -+int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, -+ struct qm_mcr_cgrtestwrite *result) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ mcc = qm_mc_start(&p->p); -+ mcc->cgrtestwrite.cgid = cgr->cgrid; -+ mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32); -+ mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt; -+ qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) -+ *result = mcr->cgrtestwrite; -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res)); -+ return -EIO; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_testwrite_cgr); -+ -+int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ int i; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ mcc = qm_mc_start(&p->p); -+ mcc->querycgr.cgid = cgr->cgrid; -+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) -+ *cgrd = mcr->querycgr; -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res)); -+ return -EIO; -+ } -+ cgrd->cgr.wr_parm_g.word = -+ be32_to_cpu(cgrd->cgr.wr_parm_g.word); -+ cgrd->cgr.wr_parm_y.word = -+ be32_to_cpu(cgrd->cgr.wr_parm_y.word); -+ cgrd->cgr.wr_parm_r.word = -+ be32_to_cpu(cgrd->cgr.wr_parm_r.word); -+ cgrd->cgr.cscn_targ = be32_to_cpu(cgrd->cgr.cscn_targ); -+ cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres); -+ for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++) -+ be32_to_cpus(&cgrd->cscn_targ_swp[i]); -+ return 0; -+} -+EXPORT_SYMBOL(qman_query_cgr); -+ -+int qman_query_congestion(struct qm_mcr_querycongestion *congestion) -+{ -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ int i; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ qm_mc_start(&p->p); -+ qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_MCC_VERB_QUERYCONGESTION); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) -+ memcpy_fromio(congestion, &mcr->querycongestion, -+ sizeof(*congestion)); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res)); -+ return -EIO; -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(congestion->state.__state); i++) -+ be32_to_cpus(&congestion->state.__state[i]); -+ return 0; -+} -+EXPORT_SYMBOL(qman_query_congestion); -+ -+/* internal function used as a wait_event() expression */ -+static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr) -+{ -+ unsigned long irqflags __maybe_unused; -+ int ret = -EBUSY; -+ PORTAL_IRQ_LOCK(p, irqflags); -+ if (!p->vdqcr_owned) { -+ FQLOCK(fq); -+ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) -+ goto escape; -+ fq_set(fq, QMAN_FQ_STATE_VDQCR); -+ FQUNLOCK(fq); -+ p->vdqcr_owned = fq; -+ ret = 0; -+ } -+escape: -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ if (!ret) -+ qm_dqrr_vdqcr_set(&p->p, vdqcr); -+ return ret; -+} -+ -+static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr) -+{ -+ int ret; -+ *p = get_affine_portal(); -+ ret = set_p_vdqcr(*p, fq, vdqcr); -+ put_affine_portal(); -+ return ret; -+} -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+static int wait_p_vdqcr_start(struct qman_portal *p, struct qman_fq *fq, -+ u32 vdqcr, u32 flags) -+{ -+ int ret = 0; -+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) -+ ret = wait_event_interruptible(affine_queue, -+ !(ret = set_p_vdqcr(p, fq, vdqcr))); -+ else -+ wait_event(affine_queue, !(ret = set_p_vdqcr(p, fq, vdqcr))); -+ return ret; -+} -+ -+static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq, -+ u32 vdqcr, u32 flags) -+{ -+ int ret = 0; -+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) -+ ret = wait_event_interruptible(affine_queue, -+ !(ret = set_vdqcr(p, fq, vdqcr))); -+ else -+ wait_event(affine_queue, !(ret = set_vdqcr(p, fq, vdqcr))); -+ return ret; -+} -+#endif -+ -+int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq, -+ u32 flags __maybe_unused, u32 vdqcr) -+{ -+ int ret; -+ -+ if ((fq->state != qman_fq_state_parked) && -+ (fq->state != qman_fq_state_retired)) -+ return -EINVAL; -+ if (vdqcr & QM_VDQCR_FQID_MASK) -+ return -EINVAL; -+ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) -+ return -EBUSY; -+ vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_VOLATILE_FLAG_WAIT) -+ ret = wait_p_vdqcr_start(p, fq, vdqcr, flags); -+ else -+#endif -+ ret = set_p_vdqcr(p, fq, vdqcr); -+ if (ret) -+ return ret; -+ /* VDQCR is set */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_VOLATILE_FLAG_FINISH) { -+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) -+ /* NB: don't propagate any error - the caller wouldn't -+ * know whether the VDQCR was issued or not. A signal -+ * could arrive after returning anyway, so the caller -+ * can check signal_pending() if that's an issue. */ -+ wait_event_interruptible(affine_queue, -+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); -+ else -+ wait_event(affine_queue, -+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_p_volatile_dequeue); -+ -+int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused, -+ u32 vdqcr) -+{ -+ struct qman_portal *p; -+ int ret; -+ -+ if ((fq->state != qman_fq_state_parked) && -+ (fq->state != qman_fq_state_retired)) -+ return -EINVAL; -+ if (vdqcr & QM_VDQCR_FQID_MASK) -+ return -EINVAL; -+ if (fq_isset(fq, QMAN_FQ_STATE_VDQCR)) -+ return -EBUSY; -+ vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid; -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_VOLATILE_FLAG_WAIT) -+ ret = wait_vdqcr_start(&p, fq, vdqcr, flags); -+ else -+#endif -+ ret = set_vdqcr(&p, fq, vdqcr); -+ if (ret) -+ return ret; -+ /* VDQCR is set */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_VOLATILE_FLAG_FINISH) { -+ if (flags & QMAN_VOLATILE_FLAG_WAIT_INT) -+ /* NB: don't propagate any error - the caller wouldn't -+ * know whether the VDQCR was issued or not. A signal -+ * could arrive after returning anyway, so the caller -+ * can check signal_pending() if that's an issue. */ -+ wait_event_interruptible(affine_queue, -+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); -+ else -+ wait_event(affine_queue, -+ !fq_isset(fq, QMAN_FQ_STATE_VDQCR)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_volatile_dequeue); -+ -+static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail) -+{ -+ if (avail) -+ qm_eqcr_cce_prefetch(&p->p); -+ else -+ qm_eqcr_cce_update(&p->p); -+} -+ -+int qman_eqcr_is_empty(void) -+{ -+ unsigned long irqflags __maybe_unused; -+ struct qman_portal *p = get_affine_portal(); -+ u8 avail; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ update_eqcr_ci(p, 0); -+ avail = qm_eqcr_get_fill(&p->p); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return avail == 0; -+} -+EXPORT_SYMBOL(qman_eqcr_is_empty); -+ -+void qman_set_dc_ern(qman_cb_dc_ern handler, int affine) -+{ -+ if (affine) { -+ unsigned long irqflags __maybe_unused; -+ struct qman_portal *p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ p->cb_dc_ern = handler; -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ } else -+ cb_dc_ern = handler; -+} -+EXPORT_SYMBOL(qman_set_dc_ern); -+ -+static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p, -+ unsigned long *irqflags __maybe_unused, -+ struct qman_fq *fq, -+ const struct qm_fd *fd, -+ u32 flags) -+{ -+ struct qm_eqcr_entry *eq; -+ u8 avail; -+ PORTAL_IRQ_LOCK(p, (*irqflags)); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { -+ if (p->eqci_owned) { -+ PORTAL_IRQ_UNLOCK(p, (*irqflags)); -+ return NULL; -+ } -+ p->eqci_owned = fq; -+ } -+#endif -+ if (p->use_eqcr_ci_stashing) { -+ /* -+ * The stashing case is easy, only update if we need to in -+ * order to try and liberate ring entries. -+ */ -+ eq = qm_eqcr_start_stash(&p->p); -+ } else { -+ /* -+ * The non-stashing case is harder, need to prefetch ahead of -+ * time. -+ */ -+ avail = qm_eqcr_get_avail(&p->p); -+ if (avail < 2) -+ update_eqcr_ci(p, avail); -+ eq = qm_eqcr_start_no_stash(&p->p); -+ } -+ -+ if (unlikely(!eq)) { -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) -+ p->eqci_owned = NULL; -+#endif -+ PORTAL_IRQ_UNLOCK(p, (*irqflags)); -+ return NULL; -+ } -+ if (flags & QMAN_ENQUEUE_FLAG_DCA) -+ eq->dca = QM_EQCR_DCA_ENABLE | -+ ((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ? -+ QM_EQCR_DCA_PARK : 0) | -+ ((flags >> 8) & QM_EQCR_DCA_IDXMASK); -+ eq->fqid = cpu_to_be32(fq->fqid); -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ eq->tag = cpu_to_be32(fq->key); -+#else -+ eq->tag = cpu_to_be32((u32)(uintptr_t)fq); -+#endif -+ eq->fd = *fd; -+ cpu_to_hw_fd(&eq->fd); -+ return eq; -+} -+ -+static inline struct qm_eqcr_entry *try_eq_start(struct qman_portal **p, -+ unsigned long *irqflags __maybe_unused, -+ struct qman_fq *fq, -+ const struct qm_fd *fd, -+ u32 flags) -+{ -+ struct qm_eqcr_entry *eq; -+ *p = get_affine_portal(); -+ eq = try_p_eq_start(*p, irqflags, fq, fd, flags); -+ if (!eq) -+ put_affine_portal(); -+ return eq; -+} -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+static noinline struct qm_eqcr_entry *__wait_eq_start(struct qman_portal **p, -+ unsigned long *irqflags __maybe_unused, -+ struct qman_fq *fq, -+ const struct qm_fd *fd, -+ u32 flags) -+{ -+ struct qm_eqcr_entry *eq = try_eq_start(p, irqflags, fq, fd, flags); -+ if (!eq) -+ qm_eqcr_set_ithresh(&(*p)->p, EQCR_ITHRESH); -+ return eq; -+} -+static noinline struct qm_eqcr_entry *wait_eq_start(struct qman_portal **p, -+ unsigned long *irqflags __maybe_unused, -+ struct qman_fq *fq, -+ const struct qm_fd *fd, -+ u32 flags) -+{ -+ struct qm_eqcr_entry *eq; -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return NULL if signal occurs before completion. Signal -+ * can occur during return. Caller must check for signal */ -+ wait_event_interruptible(affine_queue, -+ (eq = __wait_eq_start(p, irqflags, fq, fd, flags))); -+ else -+ wait_event(affine_queue, -+ (eq = __wait_eq_start(p, irqflags, fq, fd, flags))); -+ return eq; -+} -+static noinline struct qm_eqcr_entry *__wait_p_eq_start(struct qman_portal *p, -+ unsigned long *irqflags __maybe_unused, -+ struct qman_fq *fq, -+ const struct qm_fd *fd, -+ u32 flags) -+{ -+ struct qm_eqcr_entry *eq = try_p_eq_start(p, irqflags, fq, fd, flags); -+ if (!eq) -+ qm_eqcr_set_ithresh(&p->p, EQCR_ITHRESH); -+ return eq; -+} -+static noinline struct qm_eqcr_entry *wait_p_eq_start(struct qman_portal *p, -+ unsigned long *irqflags __maybe_unused, -+ struct qman_fq *fq, -+ const struct qm_fd *fd, -+ u32 flags) -+{ -+ struct qm_eqcr_entry *eq; -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return NULL if signal occurs before completion. Signal -+ * can occur during return. Caller must check for signal */ -+ wait_event_interruptible(affine_queue, -+ (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags))); -+ else -+ wait_event(affine_queue, -+ (eq = __wait_p_eq_start(p, irqflags, fq, fd, flags))); -+ return eq; -+} -+#endif -+ -+int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_fd *fd, u32 flags) -+{ -+ struct qm_eqcr_entry *eq; -+ unsigned long irqflags __maybe_unused; -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT) -+ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); -+ else -+#endif -+ eq = try_p_eq_start(p, &irqflags, fq, fd, flags); -+ if (!eq) -+ return -EBUSY; -+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ -+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | -+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); -+ /* Factor the below out, it's used from qman_enqueue_orp() too */ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return success even if signal occurs before -+ * condition is true. pvb_commit guarantees success */ -+ wait_event_interruptible(affine_queue, -+ (p->eqci_owned != fq)); -+ else -+ wait_event(affine_queue, (p->eqci_owned != fq)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_p_enqueue); -+ -+int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags) -+{ -+ struct qman_portal *p; -+ struct qm_eqcr_entry *eq; -+ unsigned long irqflags __maybe_unused; -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT) -+ eq = wait_eq_start(&p, &irqflags, fq, fd, flags); -+ else -+#endif -+ eq = try_eq_start(&p, &irqflags, fq, fd, flags); -+ if (!eq) -+ return -EBUSY; -+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ -+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | -+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); -+ /* Factor the below out, it's used from qman_enqueue_orp() too */ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return success even if signal occurs before -+ * condition is true. pvb_commit guarantees success */ -+ wait_event_interruptible(affine_queue, -+ (p->eqci_owned != fq)); -+ else -+ wait_event(affine_queue, (p->eqci_owned != fq)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_enqueue); -+ -+int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_fd *fd, u32 flags, -+ struct qman_fq *orp, u16 orp_seqnum) -+{ -+ struct qm_eqcr_entry *eq; -+ unsigned long irqflags __maybe_unused; -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT) -+ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); -+ else -+#endif -+ eq = try_p_eq_start(p, &irqflags, fq, fd, flags); -+ if (!eq) -+ return -EBUSY; -+ /* Process ORP-specifics here */ -+ if (flags & QMAN_ENQUEUE_FLAG_NLIS) -+ orp_seqnum |= QM_EQCR_SEQNUM_NLIS; -+ else { -+ orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; -+ if (flags & QMAN_ENQUEUE_FLAG_NESN) -+ orp_seqnum |= QM_EQCR_SEQNUM_NESN; -+ else -+ /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ -+ orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; -+ } -+ eq->seqnum = cpu_to_be16(orp_seqnum); -+ eq->orp = cpu_to_be32(orp->fqid); -+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ -+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | -+ ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? -+ 0 : QM_EQCR_VERB_CMD_ENQUEUE) | -+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return success even if signal occurs before -+ * condition is true. pvb_commit guarantees success */ -+ wait_event_interruptible(affine_queue, -+ (p->eqci_owned != fq)); -+ else -+ wait_event(affine_queue, (p->eqci_owned != fq)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_p_enqueue_orp); -+ -+int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, -+ struct qman_fq *orp, u16 orp_seqnum) -+{ -+ struct qman_portal *p; -+ struct qm_eqcr_entry *eq; -+ unsigned long irqflags __maybe_unused; -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT) -+ eq = wait_eq_start(&p, &irqflags, fq, fd, flags); -+ else -+#endif -+ eq = try_eq_start(&p, &irqflags, fq, fd, flags); -+ if (!eq) -+ return -EBUSY; -+ /* Process ORP-specifics here */ -+ if (flags & QMAN_ENQUEUE_FLAG_NLIS) -+ orp_seqnum |= QM_EQCR_SEQNUM_NLIS; -+ else { -+ orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS; -+ if (flags & QMAN_ENQUEUE_FLAG_NESN) -+ orp_seqnum |= QM_EQCR_SEQNUM_NESN; -+ else -+ /* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */ -+ orp_seqnum &= ~QM_EQCR_SEQNUM_NESN; -+ } -+ eq->seqnum = cpu_to_be16(orp_seqnum); -+ eq->orp = cpu_to_be32(orp->fqid); -+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ -+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP | -+ ((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ? -+ 0 : QM_EQCR_VERB_CMD_ENQUEUE) | -+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return success even if signal occurs before -+ * condition is true. pvb_commit guarantees success */ -+ wait_event_interruptible(affine_queue, -+ (p->eqci_owned != fq)); -+ else -+ wait_event(affine_queue, (p->eqci_owned != fq)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_enqueue_orp); -+ -+int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_fd *fd, u32 flags, -+ qman_cb_precommit cb, void *cb_arg) -+{ -+ struct qm_eqcr_entry *eq; -+ unsigned long irqflags __maybe_unused; -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT) -+ eq = wait_p_eq_start(p, &irqflags, fq, fd, flags); -+ else -+#endif -+ eq = try_p_eq_start(p, &irqflags, fq, fd, flags); -+ if (!eq) -+ return -EBUSY; -+ /* invoke user supplied callback function before writing commit verb */ -+ if (cb(cb_arg)) { -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ return -EINVAL; -+ } -+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ -+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | -+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); -+ /* Factor the below out, it's used from qman_enqueue_orp() too */ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return success even if signal occurs before -+ * condition is true. pvb_commit guarantees success */ -+ wait_event_interruptible(affine_queue, -+ (p->eqci_owned != fq)); -+ else -+ wait_event(affine_queue, (p->eqci_owned != fq)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_p_enqueue_precommit); -+ -+int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd, -+ u32 flags, qman_cb_precommit cb, void *cb_arg) -+{ -+ struct qman_portal *p; -+ struct qm_eqcr_entry *eq; -+ unsigned long irqflags __maybe_unused; -+ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT) -+ eq = wait_eq_start(&p, &irqflags, fq, fd, flags); -+ else -+#endif -+ eq = try_eq_start(&p, &irqflags, fq, fd, flags); -+ if (!eq) -+ return -EBUSY; -+ /* invoke user supplied callback function before writing commit verb */ -+ if (cb(cb_arg)) { -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return -EINVAL; -+ } -+ /* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */ -+ qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE | -+ (flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT))); -+ /* Factor the below out, it's used from qman_enqueue_orp() too */ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+ if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) && -+ (flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) { -+ if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT) -+ /* NB: return success even if signal occurs before -+ * condition is true. pvb_commit guarantees success */ -+ wait_event_interruptible(affine_queue, -+ (p->eqci_owned != fq)); -+ else -+ wait_event(affine_queue, (p->eqci_owned != fq)); -+ } -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_enqueue_precommit); -+ -+int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, -+ struct qm_mcc_initcgr *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ u8 verb = QM_MCC_VERB_MODIFYCGR; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ mcc = qm_mc_start(&p->p); -+ if (opts) -+ mcc->initcgr = *opts; -+ mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask); -+ mcc->initcgr.cgr.wr_parm_g.word = -+ cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word); -+ mcc->initcgr.cgr.wr_parm_y.word = -+ cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word); -+ mcc->initcgr.cgr.wr_parm_r.word = -+ cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word); -+ mcc->initcgr.cgr.cscn_targ = cpu_to_be32(mcc->initcgr.cgr.cscn_targ); -+ mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres); -+ -+ mcc->initcgr.cgid = cgr->cgrid; -+ if (flags & QMAN_CGR_FLAG_USE_INIT) -+ verb = QM_MCC_VERB_INITCGR; -+ qm_mc_commit(&p->p, verb); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb); -+ res = mcr->result; -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return (res == QM_MCR_RESULT_OK) ? 0 : -EIO; -+} -+EXPORT_SYMBOL(qman_modify_cgr); -+ -+#define TARG_MASK(n) (0x80000000 >> (n->config->public_cfg.channel - \ -+ QM_CHANNEL_SWPORTAL0)) -+#define TARG_DCP_MASK(n) (0x80000000 >> (10 + n)) -+#define PORTAL_IDX(n) (n->config->public_cfg.channel - QM_CHANNEL_SWPORTAL0) -+ -+static u8 qman_cgr_cpus[__CGR_NUM]; -+ -+int qman_create_cgr(struct qman_cgr *cgr, u32 flags, -+ struct qm_mcc_initcgr *opts) -+{ -+ unsigned long irqflags __maybe_unused; -+ struct qm_mcr_querycgr cgr_state; -+ struct qm_mcc_initcgr local_opts; -+ int ret; -+ struct qman_portal *p; -+ -+ /* We have to check that the provided CGRID is within the limits of the -+ * data-structures, for obvious reasons. However we'll let h/w take -+ * care of determining whether it's within the limits of what exists on -+ * the SoC. */ -+ if (cgr->cgrid >= __CGR_NUM) -+ return -EINVAL; -+ -+ preempt_disable(); -+ p = get_affine_portal(); -+ qman_cgr_cpus[cgr->cgrid] = smp_processor_id(); -+ preempt_enable(); -+ -+ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); -+ cgr->chan = p->config->public_cfg.channel; -+ spin_lock_irqsave(&p->cgr_lock, irqflags); -+ -+ /* if no opts specified, just add it to the list */ -+ if (!opts) -+ goto add_list; -+ -+ ret = qman_query_cgr(cgr, &cgr_state); -+ if (ret) -+ goto release_lock; -+ if (opts) -+ local_opts = *opts; -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) -+ local_opts.cgr.cscn_targ_upd_ctrl = -+ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p); -+ else -+ /* Overwrite TARG */ -+ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | -+ TARG_MASK(p); -+ local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; -+ -+ /* send init if flags indicate so */ -+ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) -+ ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts); -+ else -+ ret = qman_modify_cgr(cgr, 0, &local_opts); -+ if (ret) -+ goto release_lock; -+add_list: -+ list_add(&cgr->node, &p->cgr_cbs); -+ -+ /* Determine if newly added object requires its callback to be called */ -+ ret = qman_query_cgr(cgr, &cgr_state); -+ if (ret) { -+ /* we can't go back, so proceed and return success, but screen -+ * and wail to the log file */ -+ pr_crit("CGR HW state partially modified\n"); -+ ret = 0; -+ goto release_lock; -+ } -+ if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1], -+ cgr->cgrid)) -+ cgr->cb(p, cgr, 1); -+release_lock: -+ spin_unlock_irqrestore(&p->cgr_lock, irqflags); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_create_cgr); -+ -+int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, -+ struct qm_mcc_initcgr *opts) -+{ -+ unsigned long irqflags __maybe_unused; -+ struct qm_mcc_initcgr local_opts; -+ struct qm_mcr_querycgr cgr_state; -+ int ret; -+ -+ if ((qman_ip_rev & 0xFF00) < QMAN_REV30) { -+ pr_warn("This QMan version doesn't support to send CSCN to DCP portal\n"); -+ return -EINVAL; -+ } -+ /* We have to check that the provided CGRID is within the limits of the -+ * data-structures, for obvious reasons. However we'll let h/w take -+ * care of determining whether it's within the limits of what exists on -+ * the SoC. -+ */ -+ if (cgr->cgrid >= __CGR_NUM) -+ return -EINVAL; -+ -+ ret = qman_query_cgr(cgr, &cgr_state); -+ if (ret) -+ return ret; -+ -+ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); -+ if (opts) -+ local_opts = *opts; -+ -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) -+ local_opts.cgr.cscn_targ_upd_ctrl = -+ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | -+ QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal; -+ else -+ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ | -+ TARG_DCP_MASK(dcp_portal); -+ local_opts.we_mask |= QM_CGR_WE_CSCN_TARG; -+ -+ /* send init if flags indicate so */ -+ if (opts && (flags & QMAN_CGR_FLAG_USE_INIT)) -+ ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, -+ &local_opts); -+ else -+ ret = qman_modify_cgr(cgr, 0, &local_opts); -+ -+ return ret; -+} -+EXPORT_SYMBOL(qman_create_cgr_to_dcp); -+ -+int qman_delete_cgr(struct qman_cgr *cgr) -+{ -+ unsigned long irqflags __maybe_unused; -+ struct qm_mcr_querycgr cgr_state; -+ struct qm_mcc_initcgr local_opts; -+ int ret = 0; -+ struct qman_cgr *i; -+ struct qman_portal *p = get_affine_portal(); -+ -+ if (cgr->chan != p->config->public_cfg.channel) { -+ pr_crit("Attempting to delete cgr from different portal " -+ "than it was create: create 0x%x, delete 0x%x\n", -+ cgr->chan, p->config->public_cfg.channel); -+ ret = -EINVAL; -+ goto put_portal; -+ } -+ memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr)); -+ spin_lock_irqsave(&p->cgr_lock, irqflags); -+ list_del(&cgr->node); -+ /* -+ * If there are no other CGR objects for this CGRID in the list, update -+ * CSCN_TARG accordingly -+ */ -+ list_for_each_entry(i, &p->cgr_cbs, node) -+ if ((i->cgrid == cgr->cgrid) && i->cb) -+ goto release_lock; -+ ret = qman_query_cgr(cgr, &cgr_state); -+ if (ret) { -+ /* add back to the list */ -+ list_add(&cgr->node, &p->cgr_cbs); -+ goto release_lock; -+ } -+ /* Overwrite TARG */ -+ local_opts.we_mask = QM_CGR_WE_CSCN_TARG; -+ if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) -+ local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p); -+ else -+ local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ & -+ ~(TARG_MASK(p)); -+ ret = qman_modify_cgr(cgr, 0, &local_opts); -+ if (ret) -+ /* add back to the list */ -+ list_add(&cgr->node, &p->cgr_cbs); -+release_lock: -+ spin_unlock_irqrestore(&p->cgr_lock, irqflags); -+put_portal: -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_delete_cgr); -+ -+struct cgr_comp { -+ struct qman_cgr *cgr; -+ struct completion completion; -+}; -+ -+static int qman_delete_cgr_thread(void *p) -+{ -+ struct cgr_comp *cgr_comp = (struct cgr_comp *)p; -+ int res; -+ -+ res = qman_delete_cgr((struct qman_cgr *)cgr_comp->cgr); -+ complete(&cgr_comp->completion); -+ -+ return res; -+} -+ -+void qman_delete_cgr_safe(struct qman_cgr *cgr) -+{ -+ struct task_struct *thread; -+ struct cgr_comp cgr_comp; -+ -+ preempt_disable(); -+ if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) { -+ init_completion(&cgr_comp.completion); -+ cgr_comp.cgr = cgr; -+ thread = kthread_create(qman_delete_cgr_thread, &cgr_comp, -+ "cgr_del"); -+ -+ if (likely(!IS_ERR(thread))) { -+ kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]); -+ wake_up_process(thread); -+ wait_for_completion(&cgr_comp.completion); -+ preempt_enable(); -+ return; -+ } -+ } -+ qman_delete_cgr(cgr); -+ preempt_enable(); -+} -+EXPORT_SYMBOL(qman_delete_cgr_safe); -+ -+int qm_get_clock(u64 *clock_hz) -+{ -+ if (!qman_clk) { -+ pr_warn("Qman clock speed is unknown\n"); -+ return -EINVAL; -+ } -+ *clock_hz = (u64)qman_clk; -+ return 0; -+} -+EXPORT_SYMBOL(qm_get_clock); -+ -+int qm_set_clock(u64 clock_hz) -+{ -+ if (qman_clk) -+ return -1; -+ qman_clk = (u32)clock_hz; -+ return 0; -+} -+EXPORT_SYMBOL(qm_set_clock); -+ -+/* CEETM management command */ -+static int qman_ceetm_configure_lfqmt(struct qm_mcc_ceetm_lfqmt_config *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->lfqmt_config = *opts; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_CONFIG); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_LFQMT_CONFIG); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: CONFIGURE LFQMT failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+ -+int qman_ceetm_query_lfqmt(int lfqid, -+ struct qm_mcr_ceetm_lfqmt_query *lfqmt_query) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->lfqmt_query.lfqid = lfqid; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_QUERY); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_LFQMT_QUERY); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) -+ *lfqmt_query = mcr->lfqmt_query; -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: QUERY LFQMT failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_query_lfqmt); -+ -+static int qman_ceetm_configure_cq(struct qm_mcc_ceetm_cq_config *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->cq_config = *opts; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_CONFIG); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ res = mcr->result; -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_CONFIG); -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: CONFIGURE CQ failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+ -+int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid, -+ struct qm_mcr_ceetm_cq_query *cq_query) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->cq_query.cqid = cpu_to_be16(cqid); -+ mcc->cq_query.dcpid = dcpid; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_QUERY); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_QUERY); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) { -+ *cq_query = mcr->cq_query; -+ hw_cq_query_to_cpu(cq_query); -+ } -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: QUERY CQ failed\n"); -+ return -EIO; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_query_cq); -+ -+static int qman_ceetm_configure_dct(struct qm_mcc_ceetm_dct_config *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->dct_config = *opts; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_CONFIG); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_CONFIG); -+ res = mcr->result; -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: CONFIGURE DCT failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+ -+static int qman_ceetm_query_dct(struct qm_mcc_ceetm_dct_query *opts, -+ struct qm_mcr_ceetm_dct_query *dct_query) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p = get_affine_portal(); -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->dct_query = *opts; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_QUERY); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_QUERY); -+ res = mcr->result; -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: QUERY DCT failed\n"); -+ return -EIO; -+ } -+ -+ *dct_query = mcr->dct_query; -+ return 0; -+} -+ -+static int qman_ceetm_configure_class_scheduler( -+ struct qm_mcc_ceetm_class_scheduler_config *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->csch_config = *opts; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG); -+ res = mcr->result; -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: CONFIGURE CLASS SCHEDULER failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+ -+static int qman_ceetm_query_class_scheduler(struct qm_ceetm_channel *channel, -+ struct qm_mcr_ceetm_class_scheduler_query *query) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->csch_query.cqcid = cpu_to_be16(channel->idx); -+ mcc->csch_query.dcpid = channel->dcp_idx; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_QUERY); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_CLASS_SCHEDULER_QUERY); -+ res = mcr->result; -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: QUERY CLASS SCHEDULER failed\n"); -+ return -EIO; -+ } -+ *query = mcr->csch_query; -+ return 0; -+} -+ -+static int qman_ceetm_configure_mapping_shaper_tcfc( -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->mst_config = *opts; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG); -+ res = mcr->result; -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: CONFIGURE CHANNEL MAPPING failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+ -+static int qman_ceetm_query_mapping_shaper_tcfc( -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query *opts, -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query *response) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->mst_query = *opts; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY); -+ res = mcr->result; -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: QUERY CHANNEL MAPPING failed\n"); -+ return -EIO; -+ } -+ -+ *response = mcr->mst_query; -+ return 0; -+} -+ -+static int qman_ceetm_configure_ccgr(struct qm_mcc_ceetm_ccgr_config *opts) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->ccgr_config = *opts; -+ -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_CONFIG); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_CONFIG); -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: CONFIGURE CCGR failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+ -+int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query, -+ struct qm_mcr_ceetm_ccgr_query *response) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->ccgr_query.ccgrid = cpu_to_be16(ccgr_query->ccgrid); -+ mcc->ccgr_query.dcpid = ccgr_query->dcpid; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY); -+ -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_QUERY); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) { -+ *response = mcr->ccgr_query; -+ hw_ccgr_query_to_cpu(response); -+ } -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: QUERY CCGR failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_query_ccgr); -+ -+static int qman_ceetm_cq_peek_pop_xsfdrread(struct qm_ceetm_cq *cq, -+ u8 command_type, u16 xsfdr, -+ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread *cq_ppxr) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ switch (command_type) { -+ case 0: -+ case 1: -+ mcc->cq_ppxr.cqid = (cq->parent->idx << 4) | cq->idx; -+ break; -+ case 2: -+ mcc->cq_ppxr.xsfdr = xsfdr; -+ break; -+ default: -+ break; -+ } -+ mcc->cq_ppxr.ct = command_type; -+ mcc->cq_ppxr.dcpid = cq->parent->dcp_idx; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD); -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD); -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: CQ PEEK/POP/XSFDR READ failed\n"); -+ return -EIO; -+ } -+ *cq_ppxr = mcr->cq_ppxr; -+ return 0; -+} -+ -+static int qman_ceetm_query_statistics(u16 cid, -+ enum qm_dc_portal dcp_idx, -+ u16 command_type, -+ struct qm_mcr_ceetm_statistics_query *query_result) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->stats_query_write.cid = cid; -+ mcc->stats_query_write.dcpid = dcp_idx; -+ mcc->stats_query_write.ct = command_type; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE); -+ -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_STATISTICS_QUERY_WRITE); -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: STATISTICS QUERY failed\n"); -+ return -EIO; -+ } -+ *query_result = mcr->stats_query; -+ return 0; -+} -+ -+int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx, -+ u16 command_type, u64 frame_count, -+ u64 byte_count) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ mcc->stats_query_write.cid = cid; -+ mcc->stats_query_write.dcpid = dcp_idx; -+ mcc->stats_query_write.ct = command_type; -+ mcc->stats_query_write.frm_cnt = frame_count; -+ mcc->stats_query_write.byte_cnt = byte_count; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE); -+ -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_STATISTICS_QUERY_WRITE); -+ -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ -+ res = mcr->result; -+ if (res != QM_MCR_RESULT_OK) { -+ pr_err("CEETM: STATISTICS WRITE failed\n"); -+ return -EIO; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_query_write_statistics); -+ -+int qman_ceetm_bps2tokenrate(u64 bps, struct qm_ceetm_rate *token_rate, -+ int rounding) -+{ -+ u16 pres; -+ u64 temp; -+ u64 qman_freq; -+ int ret; -+ -+ /* Read PRES from CEET_CFG_PRES register */ -+ ret = qman_ceetm_get_prescaler(&pres); -+ if (ret) -+ return -EINVAL; -+ -+ ret = qm_get_clock(&qman_freq); -+ if (ret) -+ return -EINVAL; -+ -+ /* token-rate = bytes-per-second * update-reference-period -+ * -+ * Where token-rate is N/8192 for a integer N, and -+ * update-reference-period is (2^22)/(PRES*QHz), where PRES -+ * is the prescalar value and QHz is the QMan clock frequency. -+ * So: -+ * -+ * token-rate = (byte-per-second*2^22)/PRES*QHZ) -+ * -+ * Converting to bits-per-second gives; -+ * -+ * token-rate = (bps*2^19) / (PRES*QHZ) -+ * N = (bps*2^32) / (PRES*QHz) -+ * -+ * And to avoid 64-bit overflow if 'bps' is larger than 4Gbps -+ * (yet minimise rounding error if 'bps' is small), we reorganise -+ * the formula to use two 16-bit shifts rather than 1 32-bit shift. -+ * N = (((bps*2^16)/PRES)*2^16)/QHz -+ */ -+ temp = ROUNDING((bps << 16), pres, rounding); -+ temp = ROUNDING((temp << 16), qman_freq, rounding); -+ token_rate->whole = temp >> 13; -+ token_rate->fraction = temp & (((u64)1 << 13) - 1); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_bps2tokenrate); -+ -+int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, u64 *bps, -+ int rounding) -+{ -+ u16 pres; -+ u64 temp; -+ u64 qman_freq; -+ int ret; -+ -+ /* Read PRES from CEET_CFG_PRES register */ -+ ret = qman_ceetm_get_prescaler(&pres); -+ if (ret) -+ return -EINVAL; -+ -+ ret = qm_get_clock(&qman_freq); -+ if (ret) -+ return -EINVAL; -+ -+ /* bytes-per-second = token-rate / update-reference-period -+ * -+ * where "token-rate" is N/8192 for an integer N, and -+ * "update-reference-period" is (2^22)/(PRES*QHz), where PRES is -+ * the prescalar value and QHz is the QMan clock frequency. So; -+ * -+ * bytes-per-second = (N/8192) / (4194304/PRES*QHz) -+ * = N*PRES*QHz / (4194304*8192) -+ * = N*PRES*QHz / (2^35) -+ * -+ * Converting to bits-per-second gives; -+ * -+ * bps = N*PRES*QHZ / (2^32) -+ * -+ * Note, the numerator has a maximum width of 72 bits! So to -+ * avoid 64-bit overflow errors, we calculate PRES*QHZ (maximum -+ * width 48 bits) divided by 2^9 (reducing to maximum 39 bits), before -+ * multiplying by N (goes to maximum of 63 bits). -+ * -+ * temp = PRES*QHZ / (2^16) -+ * kbps = temp*N / (2^16) -+ */ -+ temp = ROUNDING(qman_freq * pres, (u64)1 << 16 , rounding); -+ temp *= ((token_rate->whole << 13) + token_rate->fraction); -+ *bps = ROUNDING(temp, (u64)(1) << 16, rounding); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_tokenrate2bps); -+ -+int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, enum qm_dc_portal dcp_idx, -+ unsigned int sp_idx) -+{ -+ struct qm_ceetm_sp *p; -+ -+ DPA_ASSERT((dcp_idx == qm_dc_portal_fman0) || -+ (dcp_idx == qm_dc_portal_fman1)); -+ -+ if ((sp_idx < qman_ceetms[dcp_idx].sp_range[0]) || -+ (sp_idx >= (qman_ceetms[dcp_idx].sp_range[0] + -+ qman_ceetms[dcp_idx].sp_range[1]))) { -+ pr_err("Sub-portal index doesn't exist\n"); -+ return -EINVAL; -+ } -+ -+ list_for_each_entry(p, &qman_ceetms[dcp_idx].sub_portals, node) { -+ if ((p->idx == sp_idx) && (p->is_claimed == 0)) { -+ p->is_claimed = 1; -+ *sp = p; -+ return 0; -+ } -+ } -+ pr_err("The sub-portal#%d is not available!\n", sp_idx); -+ return -ENODEV; -+} -+EXPORT_SYMBOL(qman_ceetm_sp_claim); -+ -+int qman_ceetm_sp_release(struct qm_ceetm_sp *sp) -+{ -+ struct qm_ceetm_sp *p; -+ -+ if (sp->lni && sp->lni->is_claimed == 1) { -+ pr_err("The dependency of sub-portal has not been released!\n"); -+ return -EBUSY; -+ } -+ -+ list_for_each_entry(p, &qman_ceetms[sp->dcp_idx].sub_portals, node) { -+ if (p->idx == sp->idx) { -+ p->is_claimed = 0; -+ p->lni = NULL; -+ } -+ } -+ /* Disable CEETM mode of this sub-portal */ -+ qman_sp_disable_ceetm_mode(sp->dcp_idx, sp->idx); -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_sp_release); -+ -+int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, enum qm_dc_portal dcp_idx, -+ unsigned int lni_idx) -+{ -+ struct qm_ceetm_lni *p; -+ -+ if ((lni_idx < qman_ceetms[dcp_idx].lni_range[0]) || -+ (lni_idx >= (qman_ceetms[dcp_idx].lni_range[0] + -+ qman_ceetms[dcp_idx].lni_range[1]))) { -+ pr_err("The lni index is out of range\n"); -+ return -EINVAL; -+ } -+ -+ list_for_each_entry(p, &qman_ceetms[dcp_idx].lnis, node) { -+ if ((p->idx == lni_idx) && (p->is_claimed == 0)) { -+ *lni = p; -+ p->is_claimed = 1; -+ return 0; -+ } -+ } -+ -+ pr_err("The LNI#%d is not available!\n", lni_idx); -+ return -EINVAL; -+} -+EXPORT_SYMBOL(qman_ceetm_lni_claim); -+ -+int qman_ceetm_lni_release(struct qm_ceetm_lni *lni) -+{ -+ struct qm_ceetm_lni *p; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ if (!list_empty(&lni->channels)) { -+ pr_err("The LNI dependencies are not released!\n"); -+ return -EBUSY; -+ } -+ -+ list_for_each_entry(p, &qman_ceetms[lni->dcp_idx].lnis, node) { -+ if (p->idx == lni->idx) { -+ p->shaper_enable = 0; -+ p->shaper_couple = 0; -+ p->cr_token_rate.whole = 0; -+ p->cr_token_rate.fraction = 0; -+ p->er_token_rate.whole = 0; -+ p->er_token_rate.fraction = 0; -+ p->cr_token_bucket_limit = 0; -+ p->er_token_bucket_limit = 0; -+ p->is_claimed = 0; -+ } -+ } -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ config_opts.dcpid = lni->dcp_idx; -+ memset(&config_opts.shaper_config, 0, -+ sizeof(config_opts.shaper_config)); -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_release); -+ -+int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, struct qm_ceetm_lni *lni) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx); -+ config_opts.dcpid = sp->dcp_idx; -+ config_opts.sp_mapping.map_lni_id = lni->idx; -+ sp->lni = lni; -+ -+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) -+ return -EINVAL; -+ -+ /* Enable CEETM mode for this sub-portal */ -+ return qman_sp_enable_ceetm_mode(sp->dcp_idx, sp->idx); -+} -+EXPORT_SYMBOL(qman_ceetm_sp_set_lni); -+ -+int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, unsigned int *lni_idx) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx); -+ query_opts.dcpid = sp->dcp_idx; -+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { -+ pr_err("Can't get SP <-> LNI mapping\n"); -+ return -EINVAL; -+ } -+ *lni_idx = query_result.sp_mapping_query.map_lni_id; -+ sp->lni->idx = query_result.sp_mapping_query.map_lni_id; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_sp_get_lni); -+ -+int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled, -+ int oal) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ if (lni->shaper_enable) { -+ pr_err("The shaper has already been enabled\n"); -+ return -EINVAL; -+ } -+ lni->shaper_enable = 1; -+ lni->shaper_couple = coupled; -+ lni->oal = oal; -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ config_opts.dcpid = lni->dcp_idx; -+ config_opts.shaper_config.cpl = coupled; -+ config_opts.shaper_config.oal = oal; -+ config_opts.shaper_config.crtcr = cpu_to_be24((lni->cr_token_rate.whole -+ << 13) | lni->cr_token_rate.fraction); -+ config_opts.shaper_config.ertcr = cpu_to_be24((lni->er_token_rate.whole -+ << 13) | lni->er_token_rate.fraction); -+ config_opts.shaper_config.crtbl = -+ cpu_to_be16(lni->cr_token_bucket_limit); -+ config_opts.shaper_config.ertbl = -+ cpu_to_be16(lni->er_token_bucket_limit); -+ config_opts.shaper_config.mps = 60; -+ -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_enable_shaper); -+ -+int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ if (!lni->shaper_enable) { -+ pr_err("The shaper has been disabled\n"); -+ return -EINVAL; -+ } -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ config_opts.dcpid = lni->dcp_idx; -+ config_opts.shaper_config.cpl = lni->shaper_couple; -+ config_opts.shaper_config.oal = lni->oal; -+ config_opts.shaper_config.crtbl = -+ cpu_to_be16(lni->cr_token_bucket_limit); -+ config_opts.shaper_config.ertbl = -+ cpu_to_be16(lni->er_token_bucket_limit); -+ /* Set CR/ER rate with all 1's to configure an infinite rate, thus -+ * disable the shaping. -+ */ -+ config_opts.shaper_config.crtcr = 0xFFFFFF; -+ config_opts.shaper_config.ertcr = 0xFFFFFF; -+ config_opts.shaper_config.mps = 60; -+ lni->shaper_enable = 0; -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_disable_shaper); -+ -+int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni) -+{ -+ return lni->shaper_enable; -+} -+EXPORT_SYMBOL(qman_ceetm_lni_is_shaper_enabled); -+ -+int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ lni->cr_token_rate.whole = token_rate->whole; -+ lni->cr_token_rate.fraction = token_rate->fraction; -+ lni->cr_token_bucket_limit = token_limit; -+ if (!lni->shaper_enable) -+ return 0; -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ query_opts.dcpid = lni->dcp_idx; -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, -+ &query_result); -+ if (ret) { -+ pr_err("Fail to get current LNI shaper setting\n"); -+ return -EINVAL; -+ } -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ config_opts.dcpid = lni->dcp_idx; -+ config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole << 13) -+ | (token_rate->fraction)); -+ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit); -+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; -+ config_opts.shaper_config.oal = query_result.shaper_query.oal; -+ config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr; -+ config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl; -+ config_opts.shaper_config.mps = query_result.shaper_query.mps; -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate); -+ -+int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni, -+ u64 bps, -+ u16 token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); -+ if (ret) { -+ pr_err("Can not convert bps to token rate\n"); -+ return -EINVAL; -+ } -+ -+ return qman_ceetm_lni_set_commit_rate(lni, &token_rate, token_limit); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate_bps); -+ -+int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ query_opts.dcpid = lni->dcp_idx; -+ -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret) { -+ pr_err("The LNI CR rate or limit is not set\n"); -+ return -EINVAL; -+ } -+ token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13; -+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) & -+ 0x1FFF; -+ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate); -+ -+int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni, -+ u64 *bps, u16 *token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_lni_get_commit_rate(lni, &token_rate, token_limit); -+ if (ret) { -+ pr_err("The LNI CR rate or limit is not available\n"); -+ return -EINVAL; -+ } -+ -+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate_bps); -+ -+int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ lni->er_token_rate.whole = token_rate->whole; -+ lni->er_token_rate.fraction = token_rate->fraction; -+ lni->er_token_bucket_limit = token_limit; -+ if (!lni->shaper_enable) -+ return 0; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ query_opts.dcpid = lni->dcp_idx; -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, -+ &query_result); -+ if (ret) { -+ pr_err("Fail to get current LNI shaper setting\n"); -+ return -EINVAL; -+ } -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ config_opts.dcpid = lni->dcp_idx; -+ config_opts.shaper_config.ertcr = cpu_to_be24( -+ (token_rate->whole << 13) | (token_rate->fraction)); -+ config_opts.shaper_config.ertbl = cpu_to_be16(token_limit); -+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; -+ config_opts.shaper_config.oal = query_result.shaper_query.oal; -+ config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr; -+ config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl; -+ config_opts.shaper_config.mps = query_result.shaper_query.mps; -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate); -+ -+int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni, -+ u64 bps, -+ u16 token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); -+ if (ret) { -+ pr_err("Can not convert bps to token rate\n"); -+ return -EINVAL; -+ } -+ return qman_ceetm_lni_set_excess_rate(lni, &token_rate, token_limit); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate_bps); -+ -+int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx); -+ query_opts.dcpid = lni->dcp_idx; -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret) { -+ pr_err("The LNI ER rate or limit is not set\n"); -+ return -EINVAL; -+ } -+ token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13; -+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) & -+ 0x1FFF; -+ *token_limit = be16_to_cpu(query_result.shaper_query.ertbl); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate); -+ -+int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni, -+ u64 *bps, u16 *token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_lni_get_excess_rate(lni, &token_rate, token_limit); -+ if (ret) { -+ pr_err("The LNI ER rate or limit is not available\n"); -+ return -EINVAL; -+ } -+ -+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate_bps); -+ -+#define QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(n) ((15 - n) * 4) -+#define QMAN_CEETM_LNITCFCC_ENABLE 0x8 -+int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni, -+ unsigned int cq_level, -+ int traffic_class) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ u64 lnitcfcc; -+ -+ if ((cq_level > 15) | (traffic_class > 7)) { -+ pr_err("The CQ or traffic class id is out of range\n"); -+ return -EINVAL; -+ } -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx); -+ query_opts.dcpid = lni->dcp_idx; -+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { -+ pr_err("Fail to query tcfcc\n"); -+ return -EINVAL; -+ } -+ -+ lnitcfcc = be64_to_cpu(query_result.tcfc_query.lnitcfcc); -+ if (traffic_class == -1) { -+ /* disable tcfc for this CQ */ -+ lnitcfcc &= ~((u64)QMAN_CEETM_LNITCFCC_ENABLE << -+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level)); -+ } else { -+ lnitcfcc &= ~((u64)0xF << -+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level)); -+ lnitcfcc |= ((u64)(QMAN_CEETM_LNITCFCC_ENABLE | -+ traffic_class)) << -+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level); -+ } -+ config_opts.tcfc_config.lnitcfcc = cpu_to_be64(lnitcfcc); -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx); -+ config_opts.dcpid = lni->dcp_idx; -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_lni_set_tcfcc); -+ -+#define QMAN_CEETM_LNITCFCC_TC_MASK 0x7 -+int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni, unsigned int cq_level, -+ int *traffic_class) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ u8 lnitcfcc; -+ -+ if (cq_level > 15) { -+ pr_err("the CQ level is out of range\n"); -+ return -EINVAL; -+ } -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx); -+ query_opts.dcpid = lni->dcp_idx; -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret) -+ return ret; -+ lnitcfcc = (u8)be64_to_cpu((query_result.tcfc_query.lnitcfcc) >> -+ QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level)); -+ if (lnitcfcc & QMAN_CEETM_LNITCFCC_ENABLE) -+ *traffic_class = lnitcfcc & QMAN_CEETM_LNITCFCC_TC_MASK; -+ else -+ *traffic_class = -1; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_lni_get_tcfcc); -+ -+int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel, -+ struct qm_ceetm_lni *lni) -+{ -+ struct qm_ceetm_channel *p; -+ u32 channel_idx; -+ int ret = 0; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ if (lni->dcp_idx == qm_dc_portal_fman0) { -+ ret = qman_alloc_ceetm0_channel(&channel_idx); -+ } else if (lni->dcp_idx == qm_dc_portal_fman1) { -+ ret = qman_alloc_ceetm1_channel(&channel_idx); -+ } else { -+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", -+ lni->dcp_idx); -+ return -EINVAL; -+ } -+ -+ if (ret) { -+ pr_err("The is no channel available for LNI#%d\n", lni->idx); -+ return -ENODEV; -+ } -+ -+ p = kzalloc(sizeof(*p), GFP_KERNEL); -+ if (!p) -+ return -ENOMEM; -+ p->idx = channel_idx; -+ p->dcp_idx = lni->dcp_idx; -+ p->lni_idx = lni->idx; -+ list_add_tail(&p->node, &lni->channels); -+ INIT_LIST_HEAD(&p->class_queues); -+ INIT_LIST_HEAD(&p->ccgs); -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | -+ channel_idx); -+ config_opts.dcpid = lni->dcp_idx; -+ config_opts.channel_mapping.map_lni_id = lni->idx; -+ config_opts.channel_mapping.map_shaped = 0; -+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) { -+ pr_err("Can't map channel#%d for LNI#%d\n", -+ channel_idx, lni->idx); -+ return -EINVAL; -+ } -+ *channel = p; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_claim); -+ -+int qman_ceetm_channel_release(struct qm_ceetm_channel *channel) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ if (!list_empty(&channel->class_queues)) { -+ pr_err("CEETM channel#%d has class queue unreleased!\n", -+ channel->idx); -+ return -EBUSY; -+ } -+ if (!list_empty(&channel->ccgs)) { -+ pr_err("CEETM channel#%d has ccg unreleased!\n", -+ channel->idx); -+ return -EBUSY; -+ } -+ -+ /* channel->dcp_idx corresponds to known fman validation */ -+ if ((channel->dcp_idx != qm_dc_portal_fman0) && -+ (channel->dcp_idx != qm_dc_portal_fman1)) { -+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", -+ channel->dcp_idx); -+ return -EINVAL; -+ } -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ config_opts.dcpid = channel->dcp_idx; -+ memset(&config_opts.shaper_config, 0, -+ sizeof(config_opts.shaper_config)); -+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) { -+ pr_err("Can't reset channel shapping parameters\n"); -+ return -EINVAL; -+ } -+ -+ if (channel->dcp_idx == qm_dc_portal_fman0) { -+ qman_release_ceetm0_channelid(channel->idx); -+ } else if (channel->dcp_idx == qm_dc_portal_fman1) { -+ qman_release_ceetm1_channelid(channel->idx); -+ } else { -+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", -+ channel->dcp_idx); -+ return -EINVAL; -+ } -+ list_del(&channel->node); -+ kfree(channel); -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_release); -+ -+int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel, -+ int coupled) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ if (channel->shaper_enable == 1) { -+ pr_err("This channel shaper has been enabled!\n"); -+ return -EINVAL; -+ } -+ -+ channel->shaper_enable = 1; -+ channel->shaper_couple = coupled; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ -+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { -+ pr_err("Can't query channel mapping\n"); -+ return -EINVAL; -+ } -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | -+ channel->idx); -+ config_opts.dcpid = channel->dcp_idx; -+ config_opts.channel_mapping.map_lni_id = -+ query_result.channel_mapping_query.map_lni_id; -+ config_opts.channel_mapping.map_shaped = 1; -+ if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) { -+ pr_err("Can't enable shaper for channel #%d\n", channel->idx); -+ return -EINVAL; -+ } -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ config_opts.shaper_config.cpl = coupled; -+ config_opts.shaper_config.crtcr = -+ cpu_to_be24((channel->cr_token_rate.whole -+ << 13) | -+ channel->cr_token_rate.fraction); -+ config_opts.shaper_config.ertcr = -+ cpu_to_be24(channel->er_token_rate.whole -+ << 13 | -+ channel->er_token_rate.fraction); -+ config_opts.shaper_config.crtbl = -+ cpu_to_be16(channel->cr_token_bucket_limit); -+ config_opts.shaper_config.ertbl = -+ cpu_to_be16(channel->er_token_bucket_limit); -+ -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_enable_shaper); -+ -+int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ -+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { -+ pr_err("Can't query channel mapping\n"); -+ return -EINVAL; -+ } -+ -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | -+ channel->idx); -+ config_opts.dcpid = channel->dcp_idx; -+ config_opts.channel_mapping.map_shaped = 0; -+ config_opts.channel_mapping.map_lni_id = -+ query_result.channel_mapping_query.map_lni_id; -+ -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_disable_shaper); -+ -+int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ -+ if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) { -+ pr_err("Can't query channel mapping\n"); -+ return -EINVAL; -+ } -+ -+ return query_result.channel_mapping_query.map_shaped; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_is_shaper_enabled); -+ -+int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret) { -+ pr_err("Fail to get the current channel shaper setting\n"); -+ return -EINVAL; -+ } -+ -+ channel->cr_token_rate.whole = token_rate->whole; -+ channel->cr_token_rate.fraction = token_rate->fraction; -+ channel->cr_token_bucket_limit = token_limit; -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ config_opts.dcpid = channel->dcp_idx; -+ config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole -+ << 13) | (token_rate->fraction)); -+ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit); -+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; -+ config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr; -+ config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl; -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate); -+ -+int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel, -+ u64 bps, u16 token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); -+ if (ret) { -+ pr_err("Can not convert bps to token rate\n"); -+ return -EINVAL; -+ } -+ return qman_ceetm_channel_set_commit_rate(channel, &token_rate, -+ token_limit); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate_bps); -+ -+int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret | !query_result.shaper_query.crtcr | -+ !query_result.shaper_query.crtbl) { -+ pr_err("The channel commit rate or limit is not set\n"); -+ return -EINVAL; -+ } -+ token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13; -+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) & -+ 0x1FFF; -+ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate); -+ -+int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel, -+ u64 *bps, u16 *token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_channel_get_commit_rate(channel, &token_rate, -+ token_limit); -+ if (ret) { -+ pr_err("The channel CR rate or limit is not available\n"); -+ return -EINVAL; -+ } -+ -+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate_bps); -+ -+int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret) { -+ pr_err("Fail to get the current channel shaper setting\n"); -+ return -EINVAL; -+ } -+ -+ channel->er_token_rate.whole = token_rate->whole; -+ channel->er_token_rate.fraction = token_rate->fraction; -+ channel->er_token_bucket_limit = token_limit; -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ config_opts.dcpid = channel->dcp_idx; -+ config_opts.shaper_config.ertcr = cpu_to_be24( -+ (token_rate->whole << 13) | (token_rate->fraction)); -+ config_opts.shaper_config.ertbl = cpu_to_be16(token_limit); -+ config_opts.shaper_config.cpl = query_result.shaper_query.cpl; -+ config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr; -+ config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl; -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate); -+ -+int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel, -+ u64 bps, u16 token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0); -+ if (ret) { -+ pr_err("Can not convert bps to token rate\n"); -+ return -EINVAL; -+ } -+ return qman_ceetm_channel_set_excess_rate(channel, &token_rate, -+ token_limit); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate_bps); -+ -+int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret | !query_result.shaper_query.ertcr | -+ !query_result.shaper_query.ertbl) { -+ pr_err("The channel excess rate or limit is not set\n"); -+ return -EINVAL; -+ } -+ token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13; -+ token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) & -+ 0x1FFF; -+ *token_limit = be16_to_cpu(query_result.shaper_query.ertbl); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate); -+ -+int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel, -+ u64 *bps, u16 *token_limit) -+{ -+ struct qm_ceetm_rate token_rate; -+ int ret; -+ -+ ret = qman_ceetm_channel_get_excess_rate(channel, &token_rate, -+ token_limit); -+ if (ret) { -+ pr_err("The channel ER rate or limit is not available\n"); -+ return -EINVAL; -+ } -+ -+ return qman_ceetm_tokenrate2bps(&token_rate, bps, 0); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate_bps); -+ -+int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel, -+ u16 token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts; -+ -+ if (channel->shaper_enable) { -+ pr_err("This channel is a shaped one\n"); -+ return -EINVAL; -+ } -+ -+ channel->cr_token_bucket_limit = token_limit; -+ config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ config_opts.dcpid = channel->dcp_idx; -+ config_opts.shaper_config.crtbl = cpu_to_be16(token_limit); -+ return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_weight); -+ -+int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel, -+ u16 *token_limit) -+{ -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result; -+ int ret; -+ -+ query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER | -+ channel->idx); -+ query_opts.dcpid = channel->dcp_idx; -+ ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result); -+ if (ret | !query_result.shaper_query.crtbl) { -+ pr_err("This unshaped channel's uFQ wight is unavailable\n"); -+ return -EINVAL; -+ } -+ *token_limit = be16_to_cpu(query_result.shaper_query.crtbl); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_get_weight); -+ -+int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, int group_b, -+ unsigned int prio_a, unsigned int prio_b) -+{ -+ struct qm_mcc_ceetm_class_scheduler_config config_opts; -+ struct qm_mcr_ceetm_class_scheduler_query query_result; -+ int i; -+ -+ if (prio_a > 7) { -+ pr_err("The priority of group A is out of range\n"); -+ return -EINVAL; -+ } -+ if (group_b && (prio_b > 7)) { -+ pr_err("The priority of group B is out of range\n"); -+ return -EINVAL; -+ } -+ -+ if (qman_ceetm_query_class_scheduler(channel, &query_result)) { -+ pr_err("Can't query channel#%d's scheduler!\n", channel->idx); -+ return -EINVAL; -+ } -+ -+ config_opts.cqcid = cpu_to_be16(channel->idx); -+ config_opts.dcpid = channel->dcp_idx; -+ config_opts.gpc_combine_flag = !group_b; -+ config_opts.gpc_prio_a = prio_a; -+ config_opts.gpc_prio_b = prio_b; -+ -+ for (i = 0; i < 8; i++) -+ config_opts.w[i] = query_result.w[i]; -+ config_opts.crem = query_result.crem; -+ config_opts.erem = query_result.erem; -+ -+ return qman_ceetm_configure_class_scheduler(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_group); -+ -+int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, int *group_b, -+ unsigned int *prio_a, unsigned int *prio_b) -+{ -+ struct qm_mcr_ceetm_class_scheduler_query query_result; -+ -+ if (qman_ceetm_query_class_scheduler(channel, &query_result)) { -+ pr_err("Can't query channel#%d's scheduler!\n", channel->idx); -+ return -EINVAL; -+ } -+ *group_b = !query_result.gpc_combine_flag; -+ *prio_a = query_result.gpc_prio_a; -+ *prio_b = query_result.gpc_prio_b; -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_get_group); -+ -+#define GROUP_A_ELIGIBILITY_SET (1 << 8) -+#define GROUP_B_ELIGIBILITY_SET (1 << 9) -+#define CQ_ELIGIBILITY_SET(n) (1 << (7 - n)) -+int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel -+ *channel, int group_b, int cre) -+{ -+ struct qm_mcc_ceetm_class_scheduler_config csch_config; -+ struct qm_mcr_ceetm_class_scheduler_query csch_query; -+ int i; -+ -+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { -+ pr_err("Cannot get the channel %d scheduler setting.\n", -+ channel->idx); -+ return -EINVAL; -+ } -+ csch_config.cqcid = cpu_to_be16(channel->idx); -+ csch_config.dcpid = channel->dcp_idx; -+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; -+ csch_config.gpc_prio_a = csch_query.gpc_prio_a; -+ csch_config.gpc_prio_b = csch_query.gpc_prio_b; -+ -+ for (i = 0; i < 8; i++) -+ csch_config.w[i] = csch_query.w[i]; -+ csch_config.erem = csch_query.erem; -+ if (group_b) -+ csch_config.crem = (be16_to_cpu(csch_query.crem) -+ & ~GROUP_B_ELIGIBILITY_SET) -+ | (cre ? GROUP_B_ELIGIBILITY_SET : 0); -+ else -+ csch_config.crem = (be16_to_cpu(csch_query.crem) -+ & ~GROUP_A_ELIGIBILITY_SET) -+ | (cre ? GROUP_A_ELIGIBILITY_SET : 0); -+ -+ csch_config.crem = cpu_to_be16(csch_config.crem); -+ -+ if (qman_ceetm_configure_class_scheduler(&csch_config)) { -+ pr_err("Cannot config channel %d's scheduler with " -+ "group_%c's cr eligibility\n", channel->idx, -+ group_b ? 'b' : 'a'); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_group_cr_eligibility); -+ -+int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel -+ *channel, int group_b, int ere) -+{ -+ struct qm_mcc_ceetm_class_scheduler_config csch_config; -+ struct qm_mcr_ceetm_class_scheduler_query csch_query; -+ int i; -+ -+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { -+ pr_err("Cannot get the channel %d scheduler setting.\n", -+ channel->idx); -+ return -EINVAL; -+ } -+ csch_config.cqcid = cpu_to_be16(channel->idx); -+ csch_config.dcpid = channel->dcp_idx; -+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; -+ csch_config.gpc_prio_a = csch_query.gpc_prio_a; -+ csch_config.gpc_prio_b = csch_query.gpc_prio_b; -+ -+ for (i = 0; i < 8; i++) -+ csch_config.w[i] = csch_query.w[i]; -+ csch_config.crem = csch_query.crem; -+ if (group_b) -+ csch_config.erem = (be16_to_cpu(csch_query.erem) -+ & ~GROUP_B_ELIGIBILITY_SET) -+ | (ere ? GROUP_B_ELIGIBILITY_SET : 0); -+ else -+ csch_config.erem = (be16_to_cpu(csch_query.erem) -+ & ~GROUP_A_ELIGIBILITY_SET) -+ | (ere ? GROUP_A_ELIGIBILITY_SET : 0); -+ -+ csch_config.erem = cpu_to_be16(csch_config.erem); -+ -+ if (qman_ceetm_configure_class_scheduler(&csch_config)) { -+ pr_err("Cannot config channel %d's scheduler with " -+ "group_%c's er eligibility\n", channel->idx, -+ group_b ? 'b' : 'a'); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_group_er_eligibility); -+ -+int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel, -+ unsigned int idx, int cre) -+{ -+ struct qm_mcc_ceetm_class_scheduler_config csch_config; -+ struct qm_mcr_ceetm_class_scheduler_query csch_query; -+ int i; -+ -+ if (idx > 7) { -+ pr_err("CQ index is out of range\n"); -+ return -EINVAL; -+ } -+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { -+ pr_err("Cannot get the channel %d scheduler setting.\n", -+ channel->idx); -+ return -EINVAL; -+ } -+ csch_config.cqcid = cpu_to_be16(channel->idx); -+ csch_config.dcpid = channel->dcp_idx; -+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; -+ csch_config.gpc_prio_a = csch_query.gpc_prio_a; -+ csch_config.gpc_prio_b = csch_query.gpc_prio_b; -+ for (i = 0; i < 8; i++) -+ csch_config.w[i] = csch_query.w[i]; -+ csch_config.erem = csch_query.erem; -+ csch_config.crem = (be16_to_cpu(csch_query.crem) -+ & ~CQ_ELIGIBILITY_SET(idx)) | -+ (cre ? CQ_ELIGIBILITY_SET(idx) : 0); -+ csch_config.crem = cpu_to_be16(csch_config.crem); -+ if (qman_ceetm_configure_class_scheduler(&csch_config)) { -+ pr_err("Cannot config channel scheduler to set " -+ "cr eligibility mask for CQ#%d\n", idx); -+ return -EINVAL; -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_cq_cr_eligibility); -+ -+int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel, -+ unsigned int idx, int ere) -+{ -+ struct qm_mcc_ceetm_class_scheduler_config csch_config; -+ struct qm_mcr_ceetm_class_scheduler_query csch_query; -+ int i; -+ -+ if (idx > 7) { -+ pr_err("CQ index is out of range\n"); -+ return -EINVAL; -+ } -+ if (qman_ceetm_query_class_scheduler(channel, &csch_query)) { -+ pr_err("Cannot get the channel %d scheduler setting.\n", -+ channel->idx); -+ return -EINVAL; -+ } -+ csch_config.cqcid = cpu_to_be16(channel->idx); -+ csch_config.dcpid = channel->dcp_idx; -+ csch_config.gpc_combine_flag = csch_query.gpc_combine_flag; -+ csch_config.gpc_prio_a = csch_query.gpc_prio_a; -+ csch_config.gpc_prio_b = csch_query.gpc_prio_b; -+ for (i = 0; i < 8; i++) -+ csch_config.w[i] = csch_query.w[i]; -+ csch_config.crem = csch_query.crem; -+ csch_config.erem = (be16_to_cpu(csch_query.erem) -+ & ~CQ_ELIGIBILITY_SET(idx)) | -+ (ere ? CQ_ELIGIBILITY_SET(idx) : 0); -+ csch_config.erem = cpu_to_be16(csch_config.erem); -+ if (qman_ceetm_configure_class_scheduler(&csch_config)) { -+ pr_err("Cannot config channel scheduler to set " -+ "er eligibility mask for CQ#%d\n", idx); -+ return -EINVAL; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_channel_set_cq_er_eligibility); -+ -+int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq, -+ struct qm_ceetm_channel *channel, unsigned int idx, -+ struct qm_ceetm_ccg *ccg) -+{ -+ struct qm_ceetm_cq *p; -+ struct qm_mcc_ceetm_cq_config cq_config; -+ -+ if (idx > 7) { -+ pr_err("The independent class queue id is out of range\n"); -+ return -EINVAL; -+ } -+ -+ list_for_each_entry(p, &channel->class_queues, node) { -+ if (p->idx == idx) { -+ pr_err("The CQ#%d has been claimed!\n", idx); -+ return -EINVAL; -+ } -+ } -+ -+ p = kmalloc(sizeof(*p), GFP_KERNEL); -+ if (!p) { -+ pr_err("Can't allocate memory for CQ#%d!\n", idx); -+ return -ENOMEM; -+ } -+ -+ list_add_tail(&p->node, &channel->class_queues); -+ p->idx = idx; -+ p->is_claimed = 1; -+ p->parent = channel; -+ INIT_LIST_HEAD(&p->bound_lfqids); -+ -+ if (ccg) { -+ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx); -+ cq_config.dcpid = channel->dcp_idx; -+ cq_config.ccgid = cpu_to_be16(ccg->idx); -+ if (qman_ceetm_configure_cq(&cq_config)) { -+ pr_err("Can't configure the CQ#%d with CCGRID#%d\n", -+ idx, ccg->idx); -+ list_del(&p->node); -+ kfree(p); -+ return -EINVAL; -+ } -+ } -+ -+ *cq = p; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cq_claim); -+ -+int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq, -+ struct qm_ceetm_channel *channel, unsigned int idx, -+ struct qm_ceetm_ccg *ccg) -+{ -+ struct qm_ceetm_cq *p; -+ struct qm_mcc_ceetm_cq_config cq_config; -+ -+ if ((idx < 8) || (idx > 15)) { -+ pr_err("This grouped class queue id is out of range\n"); -+ return -EINVAL; -+ } -+ -+ list_for_each_entry(p, &channel->class_queues, node) { -+ if (p->idx == idx) { -+ pr_err("The CQ#%d has been claimed!\n", idx); -+ return -EINVAL; -+ } -+ } -+ -+ p = kmalloc(sizeof(*p), GFP_KERNEL); -+ if (!p) { -+ pr_err("Can't allocate memory for CQ#%d!\n", idx); -+ return -ENOMEM; -+ } -+ -+ list_add_tail(&p->node, &channel->class_queues); -+ p->idx = idx; -+ p->is_claimed = 1; -+ p->parent = channel; -+ INIT_LIST_HEAD(&p->bound_lfqids); -+ -+ if (ccg) { -+ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx); -+ cq_config.dcpid = channel->dcp_idx; -+ cq_config.ccgid = cpu_to_be16(ccg->idx); -+ if (qman_ceetm_configure_cq(&cq_config)) { -+ pr_err("Can't configure the CQ#%d with CCGRID#%d\n", -+ idx, ccg->idx); -+ list_del(&p->node); -+ kfree(p); -+ return -EINVAL; -+ } -+ } -+ *cq = p; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cq_claim_A); -+ -+int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq, -+ struct qm_ceetm_channel *channel, unsigned int idx, -+ struct qm_ceetm_ccg *ccg) -+{ -+ struct qm_ceetm_cq *p; -+ struct qm_mcc_ceetm_cq_config cq_config; -+ -+ if ((idx < 12) || (idx > 15)) { -+ pr_err("This grouped class queue id is out of range\n"); -+ return -EINVAL; -+ } -+ -+ list_for_each_entry(p, &channel->class_queues, node) { -+ if (p->idx == idx) { -+ pr_err("The CQ#%d has been claimed!\n", idx); -+ return -EINVAL; -+ } -+ } -+ -+ p = kmalloc(sizeof(*p), GFP_KERNEL); -+ if (!p) { -+ pr_err("Can't allocate memory for CQ#%d!\n", idx); -+ return -ENOMEM; -+ } -+ -+ list_add_tail(&p->node, &channel->class_queues); -+ p->idx = idx; -+ p->is_claimed = 1; -+ p->parent = channel; -+ INIT_LIST_HEAD(&p->bound_lfqids); -+ -+ if (ccg) { -+ cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx); -+ cq_config.dcpid = channel->dcp_idx; -+ cq_config.ccgid = cpu_to_be16(ccg->idx); -+ if (qman_ceetm_configure_cq(&cq_config)) { -+ pr_err("Can't configure the CQ#%d with CCGRID#%d\n", -+ idx, ccg->idx); -+ list_del(&p->node); -+ kfree(p); -+ return -EINVAL; -+ } -+ } -+ *cq = p; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cq_claim_B); -+ -+int qman_ceetm_cq_release(struct qm_ceetm_cq *cq) -+{ -+ if (!list_empty(&cq->bound_lfqids)) { -+ pr_err("The CQ#%d has unreleased LFQID\n", cq->idx); -+ return -EBUSY; -+ } -+ list_del(&cq->node); -+ qman_ceetm_drain_cq(cq); -+ kfree(cq); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cq_release); -+ -+int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq, -+ struct qm_ceetm_weight_code *weight_code) -+{ -+ struct qm_mcc_ceetm_class_scheduler_config config_opts; -+ struct qm_mcr_ceetm_class_scheduler_query query_result; -+ int i; -+ -+ if (cq->idx < 8) { -+ pr_err("Can not set weight for ungrouped class queue\n"); -+ return -EINVAL; -+ } -+ -+ if (qman_ceetm_query_class_scheduler(cq->parent, &query_result)) { -+ pr_err("Can't query channel#%d's scheduler!\n", -+ cq->parent->idx); -+ return -EINVAL; -+ } -+ -+ config_opts.cqcid = cpu_to_be16(cq->parent->idx); -+ config_opts.dcpid = cq->parent->dcp_idx; -+ config_opts.crem = query_result.crem; -+ config_opts.erem = query_result.erem; -+ config_opts.gpc_combine_flag = query_result.gpc_combine_flag; -+ config_opts.gpc_prio_a = query_result.gpc_prio_a; -+ config_opts.gpc_prio_b = query_result.gpc_prio_b; -+ -+ for (i = 0; i < 8; i++) -+ config_opts.w[i] = query_result.w[i]; -+ config_opts.w[cq->idx - 8] = ((weight_code->y << 3) | -+ (weight_code->x & 0x7)); -+ return qman_ceetm_configure_class_scheduler(&config_opts); -+} -+EXPORT_SYMBOL(qman_ceetm_set_queue_weight); -+ -+int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq, -+ struct qm_ceetm_weight_code *weight_code) -+{ -+ struct qm_mcr_ceetm_class_scheduler_query query_result; -+ -+ if (cq->idx < 8) { -+ pr_err("Can not get weight for ungrouped class queue\n"); -+ return -EINVAL; -+ } -+ -+ if (qman_ceetm_query_class_scheduler(cq->parent, -+ &query_result)) { -+ pr_err("Can't get the weight code for CQ#%d!\n", cq->idx); -+ return -EINVAL; -+ } -+ weight_code->y = query_result.w[cq->idx - 8] >> 3; -+ weight_code->x = query_result.w[cq->idx - 8] & 0x7; -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_get_queue_weight); -+ -+/* The WBFS code is represent as {x,y}, the effect wieght can be calculated as: -+ * effective weight = 2^x / (1 - (y/64)) -+ * = 2^(x+6) / (64 - y) -+ */ -+static void reduce_fraction(u32 *n, u32 *d) -+{ -+ u32 factor = 2; -+ u32 lesser = (*n < *d) ? *n : *d; -+ /* If factor exceeds the square-root of the lesser of *n and *d, -+ * then there's no point continuing. Proof: if there was a factor -+ * bigger than the square root, that would imply there exists -+ * another factor smaller than the square-root with which it -+ * multiplies to give 'lesser' - but that's a contradiction -+ * because the other factor would have already been found and -+ * divided out. -+ */ -+ while ((factor * factor) <= lesser) { -+ /* If 'factor' is a factor of *n and *d, divide them both -+ * by 'factor' as many times as possible. -+ */ -+ while (!(*n % factor) && !(*d % factor)) { -+ *n /= factor; -+ *d /= factor; -+ lesser /= factor; -+ } -+ if (factor == 2) -+ factor = 3; -+ else -+ factor += 2; -+ } -+} -+ -+int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code, -+ u32 *numerator, -+ u32 *denominator) -+{ -+ *numerator = (u32) 1 << (weight_code->x + 6); -+ *denominator = 64 - weight_code->y; -+ reduce_fraction(numerator, denominator); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_wbfs2ratio); -+ -+/* For a given x, the weight is between 2^x (inclusive) and 2^(x+1) (exclusive). -+ * So find 'x' by range, and then estimate 'y' using: -+ * 64 - y = 2^(x + 6) / weight -+ * = 2^(x + 6) / (n/d) -+ * = d * 2^(x+6) / n -+ * y = 64 - (d * 2^(x+6) / n) -+ */ -+int qman_ceetm_ratio2wbfs(u32 numerator, -+ u32 denominator, -+ struct qm_ceetm_weight_code *weight_code, -+ int rounding) -+{ -+ unsigned int y, x = 0; -+ /* search incrementing 'x' until: -+ * weight < 2^(x+1) -+ * n/d < 2^(x+1) -+ * n < d * 2^(x+1) -+ */ -+ while ((x < 8) && (numerator >= (denominator << (x + 1)))) -+ x++; -+ if (x >= 8) -+ return -ERANGE; -+ /* because of the subtraction, use '-rounding' */ -+ y = 64 - ROUNDING(denominator << (x + 6), numerator, -rounding); -+ if (y >= 32) -+ return -ERANGE; -+ weight_code->x = x; -+ weight_code->y = y; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_ratio2wbfs); -+ -+int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio) -+{ -+ struct qm_ceetm_weight_code weight_code; -+ -+ if (qman_ceetm_ratio2wbfs(ratio, 100, &weight_code, 0)) { -+ pr_err("Cannot get wbfs code for cq %x\n", cq->idx); -+ return -EINVAL; -+ } -+ return qman_ceetm_set_queue_weight(cq, &weight_code); -+} -+EXPORT_SYMBOL(qman_ceetm_set_queue_weight_in_ratio); -+ -+int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio) -+{ -+ struct qm_ceetm_weight_code weight_code; -+ u32 n, d; -+ -+ if (qman_ceetm_get_queue_weight(cq, &weight_code)) { -+ pr_err("Cannot query the weight code for cq%x\n", cq->idx); -+ return -EINVAL; -+ } -+ -+ if (qman_ceetm_wbfs2ratio(&weight_code, &n, &d)) { -+ pr_err("Cannot get the ratio with wbfs code\n"); -+ return -EINVAL; -+ } -+ -+ *ratio = (n * 100) / d; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_get_queue_weight_in_ratio); -+ -+int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags, -+ u64 *frame_count, u64 *byte_count) -+{ -+ struct qm_mcr_ceetm_statistics_query result; -+ u16 cid, command_type; -+ enum qm_dc_portal dcp_idx; -+ int ret; -+ -+ cid = cpu_to_be16((cq->parent->idx << 4) | cq->idx); -+ dcp_idx = cq->parent->dcp_idx; -+ if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER) -+ command_type = CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS; -+ else -+ command_type = CEETM_QUERY_DEQUEUE_STATISTICS; -+ -+ ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result); -+ if (ret) { -+ pr_err("Can't query the statistics of CQ#%d!\n", cq->idx); -+ return -EINVAL; -+ } -+ -+ *frame_count = be40_to_cpu(result.frm_cnt); -+ *byte_count = be48_to_cpu(result.byte_cnt); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cq_get_dequeue_statistics); -+ -+int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq) -+{ -+ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread ppxr; -+ int ret; -+ -+ do { -+ ret = qman_ceetm_cq_peek_pop_xsfdrread(cq, 1, 0, &ppxr); -+ if (ret) { -+ pr_err("Failed to pop frame from CQ\n"); -+ return -EINVAL; -+ } -+ } while (!(ppxr.stat & 0x2)); -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_drain_cq); -+ -+#define CEETM_LFQMT_LFQID_MSB 0xF00000 -+#define CEETM_LFQMT_LFQID_LSB 0x000FFF -+int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq, -+ struct qm_ceetm_cq *cq) -+{ -+ struct qm_ceetm_lfq *p; -+ u32 lfqid; -+ int ret = 0; -+ struct qm_mcc_ceetm_lfqmt_config lfqmt_config; -+ -+ if (cq->parent->dcp_idx == qm_dc_portal_fman0) { -+ ret = qman_alloc_ceetm0_lfqid(&lfqid); -+ } else if (cq->parent->dcp_idx == qm_dc_portal_fman1) { -+ ret = qman_alloc_ceetm1_lfqid(&lfqid); -+ } else { -+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", -+ cq->parent->dcp_idx); -+ return -EINVAL; -+ } -+ -+ if (ret) { -+ pr_err("There is no lfqid avalaible for CQ#%d!\n", cq->idx); -+ return -ENODEV; -+ } -+ p = kmalloc(sizeof(*p), GFP_KERNEL); -+ if (!p) -+ return -ENOMEM; -+ p->idx = lfqid; -+ p->dctidx = (u16)(lfqid & CEETM_LFQMT_LFQID_LSB); -+ p->parent = cq->parent; -+ list_add_tail(&p->node, &cq->bound_lfqids); -+ -+ lfqmt_config.lfqid = cpu_to_be24(CEETM_LFQMT_LFQID_MSB | -+ (cq->parent->dcp_idx << 16) | -+ (lfqid & CEETM_LFQMT_LFQID_LSB)); -+ lfqmt_config.cqid = cpu_to_be16((cq->parent->idx << 4) | (cq->idx)); -+ lfqmt_config.dctidx = cpu_to_be16(p->dctidx); -+ if (qman_ceetm_configure_lfqmt(&lfqmt_config)) { -+ pr_err("Can't configure LFQMT for LFQID#%d @ CQ#%d\n", -+ lfqid, cq->idx); -+ list_del(&p->node); -+ kfree(p); -+ return -EINVAL; -+ } -+ *lfq = p; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_lfq_claim); -+ -+int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq) -+{ -+ if (lfq->parent->dcp_idx == qm_dc_portal_fman0) { -+ qman_release_ceetm0_lfqid(lfq->idx); -+ } else if (lfq->parent->dcp_idx == qm_dc_portal_fman1) { -+ qman_release_ceetm1_lfqid(lfq->idx); -+ } else { -+ pr_err("dcp_idx %u does not correspond to a known fman in this driver\n", -+ lfq->parent->dcp_idx); -+ return -EINVAL; -+ } -+ list_del(&lfq->node); -+ kfree(lfq); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_lfq_release); -+ -+int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, u64 context_a, -+ u32 context_b) -+{ -+ struct qm_mcc_ceetm_dct_config dct_config; -+ lfq->context_a = context_a; -+ lfq->context_b = context_b; -+ dct_config.dctidx = cpu_to_be16((u16)lfq->dctidx); -+ dct_config.dcpid = lfq->parent->dcp_idx; -+ dct_config.context_b = cpu_to_be32(context_b); -+ dct_config.context_a = cpu_to_be64(context_a); -+ -+ return qman_ceetm_configure_dct(&dct_config); -+} -+EXPORT_SYMBOL(qman_ceetm_lfq_set_context); -+ -+int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, u64 *context_a, -+ u32 *context_b) -+{ -+ struct qm_mcc_ceetm_dct_query dct_query; -+ struct qm_mcr_ceetm_dct_query query_result; -+ -+ dct_query.dctidx = cpu_to_be16(lfq->dctidx); -+ dct_query.dcpid = lfq->parent->dcp_idx; -+ if (qman_ceetm_query_dct(&dct_query, &query_result)) { -+ pr_err("Can't query LFQID#%d's context!\n", lfq->idx); -+ return -EINVAL; -+ } -+ *context_a = be64_to_cpu(query_result.context_a); -+ *context_b = be32_to_cpu(query_result.context_b); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_lfq_get_context); -+ -+int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq) -+{ -+ spin_lock_init(&fq->fqlock); -+ fq->fqid = lfq->idx; -+ fq->flags = QMAN_FQ_FLAG_NO_MODIFY; -+ if (lfq->ern) -+ fq->cb.ern = lfq->ern; -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ if (unlikely(find_empty_fq_table_entry(&fq->key, fq))) -+ return -ENOMEM; -+#endif -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_create_fq); -+ -+#define MAX_CCG_IDX 0x000F -+int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg, -+ struct qm_ceetm_channel *channel, -+ unsigned int idx, -+ void (*cscn)(struct qm_ceetm_ccg *, -+ void *cb_ctx, -+ int congested), -+ void *cb_ctx) -+{ -+ struct qm_ceetm_ccg *p; -+ -+ if (idx > MAX_CCG_IDX) { -+ pr_err("The given ccg index is out of range\n"); -+ return -EINVAL; -+ } -+ -+ list_for_each_entry(p, &channel->ccgs, node) { -+ if (p->idx == idx) { -+ pr_err("The CCG#%d has been claimed\n", idx); -+ return -EINVAL; -+ } -+ } -+ -+ p = kmalloc(sizeof(*p), GFP_KERNEL); -+ if (!p) { -+ pr_err("Can't allocate memory for CCG#%d!\n", idx); -+ return -ENOMEM; -+ } -+ -+ list_add_tail(&p->node, &channel->ccgs); -+ -+ p->idx = idx; -+ p->parent = channel; -+ p->cb = cscn; -+ p->cb_ctx = cb_ctx; -+ INIT_LIST_HEAD(&p->cb_node); -+ -+ *ccg = p; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_ccg_claim); -+ -+int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg) -+{ -+ unsigned long irqflags __maybe_unused; -+ struct qm_mcc_ceetm_ccgr_config config_opts; -+ int ret = 0; -+ struct qman_portal *p = get_affine_portal(); -+ -+ memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config)); -+ spin_lock_irqsave(&p->ccgr_lock, irqflags); -+ if (!list_empty(&ccg->cb_node)) -+ list_del(&ccg->cb_node); -+ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE | -+ (ccg->parent->idx << 4) | ccg->idx); -+ config_opts.dcpid = ccg->parent->dcp_idx; -+ config_opts.we_mask = cpu_to_be16(QM_CCGR_WE_CSCN_TUPD); -+ config_opts.cm_config.cscn_tupd = cpu_to_be16(PORTAL_IDX(p)); -+ ret = qman_ceetm_configure_ccgr(&config_opts); -+ spin_unlock_irqrestore(&p->ccgr_lock, irqflags); -+ put_affine_portal(); -+ -+ list_del(&ccg->node); -+ kfree(ccg); -+ return ret; -+} -+EXPORT_SYMBOL(qman_ceetm_ccg_release); -+ -+int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, u16 we_mask, -+ const struct qm_ceetm_ccg_params *params) -+{ -+ struct qm_mcc_ceetm_ccgr_config config_opts; -+ unsigned long irqflags __maybe_unused; -+ int ret; -+ struct qman_portal *p; -+ -+ if (((ccg->parent->idx << 4) | ccg->idx) >= (2 * __CGR_NUM)) -+ return -EINVAL; -+ -+ p = get_affine_portal(); -+ -+ memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config)); -+ spin_lock_irqsave(&p->ccgr_lock, irqflags); -+ -+ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE | -+ (ccg->parent->idx << 4) | ccg->idx); -+ config_opts.dcpid = ccg->parent->dcp_idx; -+ config_opts.we_mask = we_mask; -+ if (we_mask & QM_CCGR_WE_CSCN_EN) { -+ config_opts.we_mask |= QM_CCGR_WE_CSCN_TUPD; -+ config_opts.cm_config.cscn_tupd = cpu_to_be16( -+ QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p)); -+ } -+ config_opts.we_mask = cpu_to_be16(config_opts.we_mask); -+ config_opts.cm_config.ctl_wr_en_g = params->wr_en_g; -+ config_opts.cm_config.ctl_wr_en_y = params->wr_en_y; -+ config_opts.cm_config.ctl_wr_en_r = params->wr_en_r; -+ config_opts.cm_config.ctl_td_en = params->td_en; -+ config_opts.cm_config.ctl_td_mode = params->td_mode; -+ config_opts.cm_config.ctl_cscn_en = params->cscn_en; -+ config_opts.cm_config.ctl_mode = params->mode; -+ config_opts.cm_config.oal = params->oal; -+ config_opts.cm_config.cs_thres.hword = -+ cpu_to_be16(params->cs_thres_in.hword); -+ config_opts.cm_config.cs_thres_x.hword = -+ cpu_to_be16(params->cs_thres_out.hword); -+ config_opts.cm_config.td_thres.hword = -+ cpu_to_be16(params->td_thres.hword); -+ config_opts.cm_config.wr_parm_g.word = -+ cpu_to_be32(params->wr_parm_g.word); -+ config_opts.cm_config.wr_parm_y.word = -+ cpu_to_be32(params->wr_parm_y.word); -+ config_opts.cm_config.wr_parm_r.word = -+ cpu_to_be32(params->wr_parm_r.word); -+ ret = qman_ceetm_configure_ccgr(&config_opts); -+ if (ret) { -+ pr_err("Configure CCGR CM failed!\n"); -+ goto release_lock; -+ } -+ -+ if (we_mask & QM_CCGR_WE_CSCN_EN) -+ if (list_empty(&ccg->cb_node)) -+ list_add(&ccg->cb_node, -+ &p->ccgr_cbs[ccg->parent->dcp_idx]); -+release_lock: -+ spin_unlock_irqrestore(&p->ccgr_lock, irqflags); -+ put_affine_portal(); -+ return ret; -+} -+EXPORT_SYMBOL(qman_ceetm_ccg_set); -+ -+#define CEETM_CCGR_CTL_MASK 0x01 -+int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg, -+ struct qm_ceetm_ccg_params *params) -+{ -+ struct qm_mcc_ceetm_ccgr_query query_opts; -+ struct qm_mcr_ceetm_ccgr_query query_result; -+ -+ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY | -+ (ccg->parent->idx << 4) | ccg->idx); -+ query_opts.dcpid = ccg->parent->dcp_idx; -+ -+ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) { -+ pr_err("Can't query CCGR#%d\n", ccg->idx); -+ return -EINVAL; -+ } -+ -+ params->wr_parm_r.word = query_result.cm_query.wr_parm_r.word; -+ params->wr_parm_y.word = query_result.cm_query.wr_parm_y.word; -+ params->wr_parm_g.word = query_result.cm_query.wr_parm_g.word; -+ params->td_thres.hword = query_result.cm_query.td_thres.hword; -+ params->cs_thres_out.hword = query_result.cm_query.cs_thres_x.hword; -+ params->cs_thres_in.hword = query_result.cm_query.cs_thres.hword; -+ params->oal = query_result.cm_query.oal; -+ params->wr_en_g = query_result.cm_query.ctl_wr_en_g; -+ params->wr_en_y = query_result.cm_query.ctl_wr_en_y; -+ params->wr_en_r = query_result.cm_query.ctl_wr_en_r; -+ params->td_en = query_result.cm_query.ctl_td_en; -+ params->td_mode = query_result.cm_query.ctl_td_mode; -+ params->cscn_en = query_result.cm_query.ctl_cscn_en; -+ params->mode = query_result.cm_query.ctl_mode; -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_ccg_get); -+ -+int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags, -+ u64 *frame_count, u64 *byte_count) -+{ -+ struct qm_mcr_ceetm_statistics_query result; -+ u16 cid, command_type; -+ enum qm_dc_portal dcp_idx; -+ int ret; -+ -+ cid = cpu_to_be16((ccg->parent->idx << 4) | ccg->idx); -+ dcp_idx = ccg->parent->dcp_idx; -+ if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER) -+ command_type = CEETM_QUERY_REJECT_CLEAR_STATISTICS; -+ else -+ command_type = CEETM_QUERY_REJECT_STATISTICS; -+ -+ ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result); -+ if (ret) { -+ pr_err("Can't query the statistics of CCG#%d!\n", ccg->idx); -+ return -EINVAL; -+ } -+ -+ *frame_count = be40_to_cpu(result.frm_cnt); -+ *byte_count = be48_to_cpu(result.byte_cnt); -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_ccg_get_reject_statistics); -+ -+int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg, -+ u16 swp_idx, -+ unsigned int *cscn_enabled) -+{ -+ struct qm_mcc_ceetm_ccgr_query query_opts; -+ struct qm_mcr_ceetm_ccgr_query query_result; -+ int i; -+ -+ DPA_ASSERT(swp_idx < 127); -+ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY | -+ (ccg->parent->idx << 4) | ccg->idx); -+ query_opts.dcpid = ccg->parent->dcp_idx; -+ -+ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) { -+ pr_err("Can't query CCGR#%d\n", ccg->idx); -+ return -EINVAL; -+ } -+ -+ i = swp_idx / 32; -+ i = 3 - i; -+ *cscn_enabled = query_result.cm_query.cscn_targ_swp[i] >> -+ (31 - swp_idx % 32); -+ -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cscn_swp_get); -+ -+int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg, -+ u16 dcp_idx, -+ u8 vcgid, -+ unsigned int cscn_enabled, -+ u16 we_mask, -+ const struct qm_ceetm_ccg_params *params) -+{ -+ struct qm_mcc_ceetm_ccgr_config config_opts; -+ int ret; -+ -+ config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE | -+ (ccg->parent->idx << 4) | ccg->idx); -+ config_opts.dcpid = ccg->parent->dcp_idx; -+ config_opts.we_mask = cpu_to_be16(we_mask | QM_CCGR_WE_CSCN_TUPD | -+ QM_CCGR_WE_CDV); -+ config_opts.cm_config.cdv = vcgid; -+ config_opts.cm_config.cscn_tupd = cpu_to_be16((cscn_enabled << 15) | -+ QM_CGR_TARG_UDP_CTRL_DCP | dcp_idx); -+ config_opts.cm_config.ctl_wr_en_g = params->wr_en_g; -+ config_opts.cm_config.ctl_wr_en_y = params->wr_en_y; -+ config_opts.cm_config.ctl_wr_en_r = params->wr_en_r; -+ config_opts.cm_config.ctl_td_en = params->td_en; -+ config_opts.cm_config.ctl_td_mode = params->td_mode; -+ config_opts.cm_config.ctl_cscn_en = params->cscn_en; -+ config_opts.cm_config.ctl_mode = params->mode; -+ config_opts.cm_config.cs_thres.hword = -+ cpu_to_be16(params->cs_thres_in.hword); -+ config_opts.cm_config.cs_thres_x.hword = -+ cpu_to_be16(params->cs_thres_out.hword); -+ config_opts.cm_config.td_thres.hword = -+ cpu_to_be16(params->td_thres.hword); -+ config_opts.cm_config.wr_parm_g.word = -+ cpu_to_be32(params->wr_parm_g.word); -+ config_opts.cm_config.wr_parm_y.word = -+ cpu_to_be32(params->wr_parm_y.word); -+ config_opts.cm_config.wr_parm_r.word = -+ cpu_to_be32(params->wr_parm_r.word); -+ -+ ret = qman_ceetm_configure_ccgr(&config_opts); -+ if (ret) { -+ pr_err("Configure CSCN_TARG_DCP failed!\n"); -+ return -EINVAL; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cscn_dcp_set); -+ -+int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg, -+ u16 dcp_idx, -+ u8 *vcgid, -+ unsigned int *cscn_enabled) -+{ -+ struct qm_mcc_ceetm_ccgr_query query_opts; -+ struct qm_mcr_ceetm_ccgr_query query_result; -+ -+ query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY | -+ (ccg->parent->idx << 4) | ccg->idx); -+ query_opts.dcpid = ccg->parent->dcp_idx; -+ -+ if (qman_ceetm_query_ccgr(&query_opts, &query_result)) { -+ pr_err("Can't query CCGR#%d\n", ccg->idx); -+ return -EINVAL; -+ } -+ -+ *vcgid = query_result.cm_query.cdv; -+ *cscn_enabled = (query_result.cm_query.cscn_targ_dcp >> dcp_idx) & 0x1; -+ return 0; -+} -+EXPORT_SYMBOL(qman_ceetm_cscn_dcp_get); -+ -+int qman_ceetm_querycongestion(struct __qm_mcr_querycongestion *ccg_state, -+ unsigned int dcp_idx) -+{ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ u8 res; -+ int i, j; -+ -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ -+ mcc = qm_mc_start(&p->p); -+ for (i = 0; i < 2; i++) { -+ mcc->ccgr_query.ccgrid = -+ cpu_to_be16(CEETM_QUERY_CONGESTION_STATE | i); -+ mcc->ccgr_query.dcpid = dcp_idx; -+ qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY); -+ -+ while (!(mcr = qm_mc_result(&p->p))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_CEETM_VERB_CCGR_QUERY); -+ res = mcr->result; -+ if (res == QM_MCR_RESULT_OK) { -+ for (j = 0; j < 8; j++) -+ mcr->ccgr_query.congestion_state.state. -+ __state[j] = be32_to_cpu(mcr->ccgr_query. -+ congestion_state.state.__state[j]); -+ *(ccg_state + i) = -+ mcr->ccgr_query.congestion_state.state; -+ } else { -+ pr_err("QUERY CEETM CONGESTION STATE failed\n"); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ return -EIO; -+ } -+ } -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return 0; -+} -+ -+int qman_set_wpm(int wpm_enable) -+{ -+ return qm_set_wpm(wpm_enable); -+} -+EXPORT_SYMBOL(qman_set_wpm); -+ -+int qman_get_wpm(int *wpm_enable) -+{ -+ return qm_get_wpm(wpm_enable); -+} -+EXPORT_SYMBOL(qman_get_wpm); -+ -+int qman_shutdown_fq(u32 fqid) -+{ -+ struct qman_portal *p; -+ unsigned long irqflags __maybe_unused; -+ int ret; -+ struct qm_portal *low_p; -+ p = get_affine_portal(); -+ PORTAL_IRQ_LOCK(p, irqflags); -+ low_p = &p->p; -+ ret = qm_shutdown_fq(&low_p, 1, fqid); -+ PORTAL_IRQ_UNLOCK(p, irqflags); -+ put_affine_portal(); -+ return ret; -+} -+ -+const struct qm_portal_config *qman_get_qm_portal_config( -+ struct qman_portal *portal) -+{ -+ return portal->sharing_redirect ? NULL : portal->config; -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_low.h -@@ -0,0 +1,1427 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "qman_private.h" -+ -+/***************************/ -+/* Portal register assists */ -+/***************************/ -+ -+/* Cache-inhibited register offsets */ -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ -+#define QM_REG_EQCR_PI_CINH 0x0000 -+#define QM_REG_EQCR_CI_CINH 0x0004 -+#define QM_REG_EQCR_ITR 0x0008 -+#define QM_REG_DQRR_PI_CINH 0x0040 -+#define QM_REG_DQRR_CI_CINH 0x0044 -+#define QM_REG_DQRR_ITR 0x0048 -+#define QM_REG_DQRR_DCAP 0x0050 -+#define QM_REG_DQRR_SDQCR 0x0054 -+#define QM_REG_DQRR_VDQCR 0x0058 -+#define QM_REG_DQRR_PDQCR 0x005c -+#define QM_REG_MR_PI_CINH 0x0080 -+#define QM_REG_MR_CI_CINH 0x0084 -+#define QM_REG_MR_ITR 0x0088 -+#define QM_REG_CFG 0x0100 -+#define QM_REG_ISR 0x0e00 -+#define QM_REG_IIR 0x0e0c -+#define QM_REG_ITPR 0x0e14 -+ -+/* Cache-enabled register offsets */ -+#define QM_CL_EQCR 0x0000 -+#define QM_CL_DQRR 0x1000 -+#define QM_CL_MR 0x2000 -+#define QM_CL_EQCR_PI_CENA 0x3000 -+#define QM_CL_EQCR_CI_CENA 0x3100 -+#define QM_CL_DQRR_PI_CENA 0x3200 -+#define QM_CL_DQRR_CI_CENA 0x3300 -+#define QM_CL_MR_PI_CENA 0x3400 -+#define QM_CL_MR_CI_CENA 0x3500 -+#define QM_CL_CR 0x3800 -+#define QM_CL_RR0 0x3900 -+#define QM_CL_RR1 0x3940 -+ -+#endif -+ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ -+#define QM_REG_EQCR_PI_CINH 0x3000 -+#define QM_REG_EQCR_CI_CINH 0x3040 -+#define QM_REG_EQCR_ITR 0x3080 -+#define QM_REG_DQRR_PI_CINH 0x3100 -+#define QM_REG_DQRR_CI_CINH 0x3140 -+#define QM_REG_DQRR_ITR 0x3180 -+#define QM_REG_DQRR_DCAP 0x31C0 -+#define QM_REG_DQRR_SDQCR 0x3200 -+#define QM_REG_DQRR_VDQCR 0x3240 -+#define QM_REG_DQRR_PDQCR 0x3280 -+#define QM_REG_MR_PI_CINH 0x3300 -+#define QM_REG_MR_CI_CINH 0x3340 -+#define QM_REG_MR_ITR 0x3380 -+#define QM_REG_CFG 0x3500 -+#define QM_REG_ISR 0x3600 -+#define QM_REG_IIR 0x36C0 -+#define QM_REG_ITPR 0x3740 -+ -+/* Cache-enabled register offsets */ -+#define QM_CL_EQCR 0x0000 -+#define QM_CL_DQRR 0x1000 -+#define QM_CL_MR 0x2000 -+#define QM_CL_EQCR_PI_CENA 0x3000 -+#define QM_CL_EQCR_CI_CENA 0x3040 -+#define QM_CL_DQRR_PI_CENA 0x3100 -+#define QM_CL_DQRR_CI_CENA 0x3140 -+#define QM_CL_MR_PI_CENA 0x3300 -+#define QM_CL_MR_CI_CENA 0x3340 -+#define QM_CL_CR 0x3800 -+#define QM_CL_RR0 0x3900 -+#define QM_CL_RR1 0x3940 -+ -+#endif -+ -+ -+/* BTW, the drivers (and h/w programming model) already obtain the required -+ * synchronisation for portal accesses via lwsync(), hwsync(), and -+ * data-dependencies. Use of barrier()s or other order-preserving primitives -+ * simply degrade performance. Hence the use of the __raw_*() interfaces, which -+ * simply ensure that the compiler treats the portal registers as volatile (ie. -+ * non-coherent). */ -+ -+/* Cache-inhibited register access. */ -+#define __qm_in(qm, o) be32_to_cpu(__raw_readl((qm)->addr_ci + (o))) -+#define __qm_out(qm, o, val) __raw_writel((cpu_to_be32(val)), \ -+ (qm)->addr_ci + (o)); -+#define qm_in(reg) __qm_in(&portal->addr, QM_REG_##reg) -+#define qm_out(reg, val) __qm_out(&portal->addr, QM_REG_##reg, val) -+ -+/* Cache-enabled (index) register access */ -+#define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->addr_ce + (o)) -+#define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->addr_ce + (o)) -+#define __qm_cl_in(qm, o) be32_to_cpu(__raw_readl((qm)->addr_ce + (o))) -+#define __qm_cl_out(qm, o, val) \ -+ do { \ -+ u32 *__tmpclout = (qm)->addr_ce + (o); \ -+ __raw_writel(cpu_to_be32(val), __tmpclout); \ -+ dcbf(__tmpclout); \ -+ } while (0) -+#define __qm_cl_invalidate(qm, o) dcbi((qm)->addr_ce + (o)) -+#define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA) -+#define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA) -+#define qm_cl_in(reg) __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA) -+#define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val) -+#define qm_cl_invalidate(reg)\ -+ __qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA) -+ -+/* Cache-enabled ring access */ -+#define qm_cl(base, idx) ((void *)base + ((idx) << 6)) -+ -+/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf -+ * analysis, look at using the "extra" bit in the ring index registers to avoid -+ * cyclic issues. */ -+static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last) -+{ -+ /* 'first' is included, 'last' is excluded */ -+ if (first <= last) -+ return last - first; -+ return ringsize + last - first; -+} -+ -+/* Portal modes. -+ * Enum types; -+ * pmode == production mode -+ * cmode == consumption mode, -+ * dmode == h/w dequeue mode. -+ * Enum values use 3 letter codes. First letter matches the portal mode, -+ * remaining two letters indicate; -+ * ci == cache-inhibited portal register -+ * ce == cache-enabled portal register -+ * vb == in-band valid-bit (cache-enabled) -+ * dc == DCA (Discrete Consumption Acknowledgement), DQRR-only -+ * As for "enum qm_dqrr_dmode", it should be self-explanatory. -+ */ -+enum qm_eqcr_pmode { /* matches QCSP_CFG::EPM */ -+ qm_eqcr_pci = 0, /* PI index, cache-inhibited */ -+ qm_eqcr_pce = 1, /* PI index, cache-enabled */ -+ qm_eqcr_pvb = 2 /* valid-bit */ -+}; -+enum qm_dqrr_dmode { /* matches QCSP_CFG::DP */ -+ qm_dqrr_dpush = 0, /* SDQCR + VDQCR */ -+ qm_dqrr_dpull = 1 /* PDQCR */ -+}; -+enum qm_dqrr_pmode { /* s/w-only */ -+ qm_dqrr_pci, /* reads DQRR_PI_CINH */ -+ qm_dqrr_pce, /* reads DQRR_PI_CENA */ -+ qm_dqrr_pvb /* reads valid-bit */ -+}; -+enum qm_dqrr_cmode { /* matches QCSP_CFG::DCM */ -+ qm_dqrr_cci = 0, /* CI index, cache-inhibited */ -+ qm_dqrr_cce = 1, /* CI index, cache-enabled */ -+ qm_dqrr_cdc = 2 /* Discrete Consumption Acknowledgement */ -+}; -+enum qm_mr_pmode { /* s/w-only */ -+ qm_mr_pci, /* reads MR_PI_CINH */ -+ qm_mr_pce, /* reads MR_PI_CENA */ -+ qm_mr_pvb /* reads valid-bit */ -+}; -+enum qm_mr_cmode { /* matches QCSP_CFG::MM */ -+ qm_mr_cci = 0, /* CI index, cache-inhibited */ -+ qm_mr_cce = 1 /* CI index, cache-enabled */ -+}; -+ -+ -+/* ------------------------- */ -+/* --- Portal structures --- */ -+ -+#define QM_EQCR_SIZE 8 -+#define QM_DQRR_SIZE 16 -+#define QM_MR_SIZE 8 -+ -+struct qm_eqcr { -+ struct qm_eqcr_entry *ring, *cursor; -+ u8 ci, available, ithresh, vbit; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ u32 busy; -+ enum qm_eqcr_pmode pmode; -+#endif -+}; -+ -+struct qm_dqrr { -+ const struct qm_dqrr_entry *ring, *cursor; -+ u8 pi, ci, fill, ithresh, vbit; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ enum qm_dqrr_dmode dmode; -+ enum qm_dqrr_pmode pmode; -+ enum qm_dqrr_cmode cmode; -+#endif -+}; -+ -+struct qm_mr { -+ const struct qm_mr_entry *ring, *cursor; -+ u8 pi, ci, fill, ithresh, vbit; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ enum qm_mr_pmode pmode; -+ enum qm_mr_cmode cmode; -+#endif -+}; -+ -+struct qm_mc { -+ struct qm_mc_command *cr; -+ struct qm_mc_result *rr; -+ u8 rridx, vbit; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ enum { -+ /* Can be _mc_start()ed */ -+ qman_mc_idle, -+ /* Can be _mc_commit()ed or _mc_abort()ed */ -+ qman_mc_user, -+ /* Can only be _mc_retry()ed */ -+ qman_mc_hw -+ } state; -+#endif -+}; -+ -+#define QM_PORTAL_ALIGNMENT ____cacheline_aligned -+ -+struct qm_addr { -+ void __iomem *addr_ce; /* cache-enabled */ -+ void __iomem *addr_ci; /* cache-inhibited */ -+}; -+ -+struct qm_portal { -+ /* In the non-CONFIG_FSL_DPA_CHECKING case, the following stuff up to -+ * and including 'mc' fits within a cacheline (yay!). The 'config' part -+ * is setup-only, so isn't a cause for a concern. In other words, don't -+ * rearrange this structure on a whim, there be dragons ... */ -+ struct qm_addr addr; -+ struct qm_eqcr eqcr; -+ struct qm_dqrr dqrr; -+ struct qm_mr mr; -+ struct qm_mc mc; -+} QM_PORTAL_ALIGNMENT; -+ -+ -+/* ---------------- */ -+/* --- EQCR API --- */ -+ -+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */ -+#define EQCR_CARRYCLEAR(p) \ -+ (void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6))) -+ -+/* Bit-wise logic to convert a ring pointer to a ring index */ -+static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e) -+{ -+ return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1); -+} -+ -+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */ -+static inline void EQCR_INC(struct qm_eqcr *eqcr) -+{ -+ /* NB: this is odd-looking, but experiments show that it generates fast -+ * code with essentially no branching overheads. We increment to the -+ * next EQCR pointer and handle overflow and 'vbit'. */ -+ struct qm_eqcr_entry *partial = eqcr->cursor + 1; -+ eqcr->cursor = EQCR_CARRYCLEAR(partial); -+ if (partial != eqcr->cursor) -+ eqcr->vbit ^= QM_EQCR_VERB_VBIT; -+} -+ -+static inline int qm_eqcr_init(struct qm_portal *portal, -+ enum qm_eqcr_pmode pmode, -+ unsigned int eq_stash_thresh, -+ int eq_stash_prio) -+{ -+ /* This use of 'register', as well as all other occurrences, is because -+ * it has been observed to generate much faster code with gcc than is -+ * otherwise the case. */ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ u32 cfg; -+ u8 pi; -+ -+ eqcr->ring = portal->addr.addr_ce + QM_CL_EQCR; -+ eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); -+ qm_cl_invalidate(EQCR_CI); -+ pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); -+ eqcr->cursor = eqcr->ring + pi; -+ eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ? -+ QM_EQCR_VERB_VBIT : 0; -+ eqcr->available = QM_EQCR_SIZE - 1 - -+ qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi); -+ eqcr->ithresh = qm_in(EQCR_ITR); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ eqcr->busy = 0; -+ eqcr->pmode = pmode; -+#endif -+ cfg = (qm_in(CFG) & 0x00ffffff) | -+ (eq_stash_thresh << 28) | /* QCSP_CFG: EST */ -+ (eq_stash_prio << 26) | /* QCSP_CFG: EP */ -+ ((pmode & 0x3) << 24); /* QCSP_CFG::EPM */ -+ qm_out(CFG, cfg); -+ return 0; -+} -+ -+static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal) -+{ -+ return (qm_in(CFG) >> 28) & 0x7; -+} -+ -+static inline void qm_eqcr_finish(struct qm_portal *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ u8 pi, ci; -+ u32 cfg; -+ -+ /* -+ * Disable EQCI stashing because the QMan only -+ * presents the value it previously stashed to -+ * maintain coherency. Setting the stash threshold -+ * to 1 then 0 ensures that QMan has resyncronized -+ * its internal copy so that the portal is clean -+ * when it is reinitialized in the future -+ */ -+ cfg = (qm_in(CFG) & 0x0fffffff) | -+ (1 << 28); /* QCSP_CFG: EST */ -+ qm_out(CFG, cfg); -+ cfg &= 0x0fffffff; /* stash threshold = 0 */ -+ qm_out(CFG, cfg); -+ -+ pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1); -+ ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); -+ -+ /* Refresh EQCR CI cache value */ -+ qm_cl_invalidate(EQCR_CI); -+ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); -+ -+ DPA_ASSERT(!eqcr->busy); -+ if (pi != EQCR_PTR2IDX(eqcr->cursor)) -+ pr_crit("losing uncommited EQCR entries\n"); -+ if (ci != eqcr->ci) -+ pr_crit("missing existing EQCR completions\n"); -+ if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor)) -+ pr_crit("EQCR destroyed unquiesced\n"); -+} -+ -+static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal -+ *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ DPA_ASSERT(!eqcr->busy); -+ if (!eqcr->available) -+ return NULL; -+ -+ -+#ifdef CONFIG_FSL_DPA_CHECKING -+ eqcr->busy = 1; -+#endif -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ dcbz_64(eqcr->cursor); -+#endif -+ return eqcr->cursor; -+} -+ -+static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal -+ *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ u8 diff, old_ci; -+ -+ DPA_ASSERT(!eqcr->busy); -+ if (!eqcr->available) { -+ old_ci = eqcr->ci; -+ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); -+ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); -+ eqcr->available += diff; -+ if (!diff) -+ return NULL; -+ } -+#ifdef CONFIG_FSL_DPA_CHECKING -+ eqcr->busy = 1; -+#endif -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ dcbz_64(eqcr->cursor); -+#endif -+ return eqcr->cursor; -+} -+ -+static inline void qm_eqcr_abort(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; -+ DPA_ASSERT(eqcr->busy); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ eqcr->busy = 0; -+#endif -+} -+ -+static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next( -+ struct qm_portal *portal, u8 myverb) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ DPA_ASSERT(eqcr->busy); -+ DPA_ASSERT(eqcr->pmode != qm_eqcr_pvb); -+ if (eqcr->available == 1) -+ return NULL; -+ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; -+ dcbf(eqcr->cursor); -+ EQCR_INC(eqcr); -+ eqcr->available--; -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ dcbz_64(eqcr->cursor); -+#endif -+ return eqcr->cursor; -+} -+ -+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -+#define EQCR_COMMIT_CHECKS(eqcr) \ -+do { \ -+ DPA_ASSERT(eqcr->busy); \ -+ DPA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0xffffff00)); \ -+ DPA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0xffffff00)); \ -+} while (0) -+#else -+#define EQCR_COMMIT_CHECKS(eqcr) \ -+do { \ -+ DPA_ASSERT(eqcr->busy); \ -+ DPA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & \ -+ cpu_to_be32(0x00ffffff))); \ -+ DPA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & \ -+ cpu_to_be32(0x00ffffff))); \ -+} while (0) -+#endif -+ -+static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ EQCR_COMMIT_CHECKS(eqcr); -+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pci); -+ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; -+ EQCR_INC(eqcr); -+ eqcr->available--; -+ dcbf(eqcr->cursor); -+ hwsync(); -+ qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor)); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ eqcr->busy = 0; -+#endif -+} -+ -+static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; -+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pce); -+ qm_cl_invalidate(EQCR_PI); -+ qm_cl_touch_rw(EQCR_PI); -+} -+ -+static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ EQCR_COMMIT_CHECKS(eqcr); -+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pce); -+ eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit; -+ EQCR_INC(eqcr); -+ eqcr->available--; -+ dcbf(eqcr->cursor); -+ lwsync(); -+ qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor)); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ eqcr->busy = 0; -+#endif -+} -+ -+static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ struct qm_eqcr_entry *eqcursor; -+ EQCR_COMMIT_CHECKS(eqcr); -+ DPA_ASSERT(eqcr->pmode == qm_eqcr_pvb); -+ lwsync(); -+ eqcursor = eqcr->cursor; -+ eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit; -+ dcbf(eqcursor); -+ EQCR_INC(eqcr); -+ eqcr->available--; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ eqcr->busy = 0; -+#endif -+} -+ -+static inline u8 qm_eqcr_cci_update(struct qm_portal *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ u8 diff, old_ci = eqcr->ci; -+ eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1); -+ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); -+ eqcr->available += diff; -+ return diff; -+} -+ -+static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr; -+ qm_cl_touch_ro(EQCR_CI); -+} -+ -+static inline u8 qm_eqcr_cce_update(struct qm_portal *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ u8 diff, old_ci = eqcr->ci; -+ eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1); -+ qm_cl_invalidate(EQCR_CI); -+ diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci); -+ eqcr->available += diff; -+ return diff; -+} -+ -+static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ return eqcr->ithresh; -+} -+ -+static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ eqcr->ithresh = ithresh; -+ qm_out(EQCR_ITR, ithresh); -+} -+ -+static inline u8 qm_eqcr_get_avail(struct qm_portal *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ return eqcr->available; -+} -+ -+static inline u8 qm_eqcr_get_fill(struct qm_portal *portal) -+{ -+ register struct qm_eqcr *eqcr = &portal->eqcr; -+ return QM_EQCR_SIZE - 1 - eqcr->available; -+} -+ -+ -+/* ---------------- */ -+/* --- DQRR API --- */ -+ -+/* FIXME: many possible improvements; -+ * - look at changing the API to use pointer rather than index parameters now -+ * that 'cursor' is a pointer, -+ * - consider moving other parameters to pointer if it could help (ci) -+ */ -+ -+#define DQRR_CARRYCLEAR(p) \ -+ (void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6))) -+ -+static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e) -+{ -+ return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1); -+} -+ -+static inline const struct qm_dqrr_entry *DQRR_INC( -+ const struct qm_dqrr_entry *e) -+{ -+ return DQRR_CARRYCLEAR(e + 1); -+} -+ -+static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf) -+{ -+ qm_out(CFG, (qm_in(CFG) & 0xff0fffff) | -+ ((mf & (QM_DQRR_SIZE - 1)) << 20)); -+} -+ -+static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cci); -+ dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1); -+ qm_out(DQRR_CI_CINH, dqrr->ci); -+} -+ -+static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce); -+ dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1); -+ qm_cl_out(DQRR_CI, dqrr->ci); -+} -+ -+static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); -+ qm_out(DQRR_DCAP, (1 << 8) | /* DQRR_DCAP::S */ -+ ((u32)bitmask << 16)); /* DQRR_DCAP::DCAP_CI */ -+ dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); -+ dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); -+} -+ -+static inline int qm_dqrr_init(struct qm_portal *portal, -+ const struct qm_portal_config *config, -+ enum qm_dqrr_dmode dmode, -+ __maybe_unused enum qm_dqrr_pmode pmode, -+ enum qm_dqrr_cmode cmode, u8 max_fill) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ u32 cfg; -+ -+ /* Make sure the DQRR will be idle when we enable */ -+ qm_out(DQRR_SDQCR, 0); -+ qm_out(DQRR_VDQCR, 0); -+ qm_out(DQRR_PDQCR, 0); -+ dqrr->ring = portal->addr.addr_ce + QM_CL_DQRR; -+ dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); -+ dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); -+ dqrr->cursor = dqrr->ring + dqrr->ci; -+ dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi); -+ dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ? -+ QM_DQRR_VERB_VBIT : 0; -+ dqrr->ithresh = qm_in(DQRR_ITR); -+ -+ /* Free up pending DQRR entries if any as per current DCM */ -+ if (dqrr->fill) { -+ enum qm_dqrr_cmode dcm = (qm_in(CFG) >> 16) & 3; -+ -+#ifdef CONFIG_FSL_DPA_CHECKING -+ dqrr->cmode = dcm; -+#endif -+ switch (dcm) { -+ case qm_dqrr_cci: -+ qm_dqrr_cci_consume(portal, dqrr->fill); -+ break; -+ case qm_dqrr_cce: -+ qm_dqrr_cce_consume(portal, dqrr->fill); -+ break; -+ case qm_dqrr_cdc: -+ qm_dqrr_cdc_consume_n(portal, (QM_DQRR_SIZE - 1)); -+ break; -+ default: -+ DPA_ASSERT(0); -+ } -+ } -+ -+#ifdef CONFIG_FSL_DPA_CHECKING -+ dqrr->dmode = dmode; -+ dqrr->pmode = pmode; -+ dqrr->cmode = cmode; -+#endif -+ /* Invalidate every ring entry before beginning */ -+ for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++) -+ dcbi(qm_cl(dqrr->ring, cfg)); -+ cfg = (qm_in(CFG) & 0xff000f00) | -+ ((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */ -+ ((dmode & 1) << 18) | /* DP */ -+ ((cmode & 3) << 16) | /* DCM */ -+ 0xa0 | /* RE+SE */ -+ (0 ? 0x40 : 0) | /* Ignore RP */ -+ (0 ? 0x10 : 0); /* Ignore SP */ -+ qm_out(CFG, cfg); -+ qm_dqrr_set_maxfill(portal, max_fill); -+ return 0; -+} -+ -+static inline void qm_dqrr_finish(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if ((dqrr->cmode != qm_dqrr_cdc) && -+ (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))) -+ pr_crit("Ignoring completed DQRR entries\n"); -+#endif -+} -+ -+static inline const struct qm_dqrr_entry *qm_dqrr_current( -+ struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ if (!dqrr->fill) -+ return NULL; -+ return dqrr->cursor; -+} -+ -+static inline u8 qm_dqrr_cursor(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ return DQRR_PTR2IDX(dqrr->cursor); -+} -+ -+static inline u8 qm_dqrr_next(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->fill); -+ dqrr->cursor = DQRR_INC(dqrr->cursor); -+ return --dqrr->fill; -+} -+ -+static inline u8 qm_dqrr_pci_update(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ u8 diff, old_pi = dqrr->pi; -+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pci); -+ dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1); -+ diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); -+ dqrr->fill += diff; -+ return diff; -+} -+ -+static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pce); -+ qm_cl_invalidate(DQRR_PI); -+ qm_cl_touch_ro(DQRR_PI); -+} -+ -+static inline u8 qm_dqrr_pce_update(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ u8 diff, old_pi = dqrr->pi; -+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pce); -+ dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1); -+ diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi); -+ dqrr->fill += diff; -+ return diff; -+} -+ -+static inline void qm_dqrr_pvb_update(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi); -+ DPA_ASSERT(dqrr->pmode == qm_dqrr_pvb); -+#if (defined CONFIG_PPC || defined CONFIG_PPC64) && !defined CONFIG_FSL_PAMU -+ /* -+ * On PowerPC platforms if PAMU is not available we need to -+ * manually invalidate the cache. When PAMU is available the -+ * cache is updated by stashing operations generated by QMan -+ */ -+ dcbi(res); -+ dcbt_ro(res); -+#endif -+ -+ /* when accessing 'verb', use __raw_readb() to ensure that compiler -+ * inlining doesn't try to optimise out "excess reads". */ -+ if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) { -+ dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1); -+ if (!dqrr->pi) -+ dqrr->vbit ^= QM_DQRR_VERB_VBIT; -+ dqrr->fill++; -+ } -+} -+ -+ -+static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cci); -+ dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); -+ qm_out(DQRR_CI_CINH, dqrr->ci); -+} -+ -+static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce); -+ qm_cl_invalidate(DQRR_CI); -+ qm_cl_touch_rw(DQRR_CI); -+} -+ -+static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cce); -+ dqrr->ci = DQRR_PTR2IDX(dqrr->cursor); -+ qm_cl_out(DQRR_CI, dqrr->ci); -+} -+ -+static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx, -+ int park) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); -+ DPA_ASSERT(idx < QM_DQRR_SIZE); -+ qm_out(DQRR_DCAP, (0 << 8) | /* S */ -+ ((park ? 1 : 0) << 6) | /* PK */ -+ idx); /* DCAP_CI */ -+} -+ -+static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal, -+ const struct qm_dqrr_entry *dq, -+ int park) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ u8 idx = DQRR_PTR2IDX(dq); -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); -+ DPA_ASSERT((dqrr->ring + idx) == dq); -+ DPA_ASSERT(idx < QM_DQRR_SIZE); -+ qm_out(DQRR_DCAP, (0 << 8) | /* DQRR_DCAP::S */ -+ ((park ? 1 : 0) << 6) | /* DQRR_DCAP::PK */ -+ idx); /* DQRR_DCAP::DCAP_CI */ -+} -+ -+static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); -+ return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1); -+} -+ -+static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); -+ qm_cl_invalidate(DQRR_CI); -+ qm_cl_touch_ro(DQRR_CI); -+} -+ -+static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc); -+ return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1); -+} -+ -+static inline u8 qm_dqrr_get_ci(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc); -+ return dqrr->ci; -+} -+ -+static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx) -+{ -+ __maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc); -+ qm_out(DQRR_DCAP, (0 << 8) | /* S */ -+ (1 << 6) | /* PK */ -+ (idx & (QM_DQRR_SIZE - 1))); /* DCAP_CI */ -+} -+ -+static inline void qm_dqrr_park_current(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc); -+ qm_out(DQRR_DCAP, (0 << 8) | /* S */ -+ (1 << 6) | /* PK */ -+ DQRR_PTR2IDX(dqrr->cursor)); /* DCAP_CI */ -+} -+ -+static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr) -+{ -+ qm_out(DQRR_SDQCR, sdqcr); -+} -+ -+static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal) -+{ -+ return qm_in(DQRR_SDQCR); -+} -+ -+static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr) -+{ -+ qm_out(DQRR_VDQCR, vdqcr); -+} -+ -+static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal) -+{ -+ return qm_in(DQRR_VDQCR); -+} -+ -+static inline void qm_dqrr_pdqcr_set(struct qm_portal *portal, u32 pdqcr) -+{ -+ qm_out(DQRR_PDQCR, pdqcr); -+} -+ -+static inline u32 qm_dqrr_pdqcr_get(struct qm_portal *portal) -+{ -+ return qm_in(DQRR_PDQCR); -+} -+ -+static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal) -+{ -+ register struct qm_dqrr *dqrr = &portal->dqrr; -+ return dqrr->ithresh; -+} -+ -+static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh) -+{ -+ qm_out(DQRR_ITR, ithresh); -+} -+ -+static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal) -+{ -+ return (qm_in(CFG) & 0x00f00000) >> 20; -+} -+ -+ -+/* -------------- */ -+/* --- MR API --- */ -+ -+#define MR_CARRYCLEAR(p) \ -+ (void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6))) -+ -+static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e) -+{ -+ return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1); -+} -+ -+static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e) -+{ -+ return MR_CARRYCLEAR(e + 1); -+} -+ -+static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode, -+ enum qm_mr_cmode cmode) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ u32 cfg; -+ -+ mr->ring = portal->addr.addr_ce + QM_CL_MR; -+ mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1); -+ mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1); -+ mr->cursor = mr->ring + mr->ci; -+ mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi); -+ mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0; -+ mr->ithresh = qm_in(MR_ITR); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mr->pmode = pmode; -+ mr->cmode = cmode; -+#endif -+ cfg = (qm_in(CFG) & 0xfffff0ff) | -+ ((cmode & 1) << 8); /* QCSP_CFG:MM */ -+ qm_out(CFG, cfg); -+ return 0; -+} -+ -+static inline void qm_mr_finish(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ if (mr->ci != MR_PTR2IDX(mr->cursor)) -+ pr_crit("Ignoring completed MR entries\n"); -+} -+ -+static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ if (!mr->fill) -+ return NULL; -+ return mr->cursor; -+} -+ -+static inline u8 qm_mr_cursor(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ return MR_PTR2IDX(mr->cursor); -+} -+ -+static inline u8 qm_mr_next(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ DPA_ASSERT(mr->fill); -+ mr->cursor = MR_INC(mr->cursor); -+ return --mr->fill; -+} -+ -+static inline u8 qm_mr_pci_update(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ u8 diff, old_pi = mr->pi; -+ DPA_ASSERT(mr->pmode == qm_mr_pci); -+ mr->pi = qm_in(MR_PI_CINH); -+ diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi); -+ mr->fill += diff; -+ return diff; -+} -+ -+static inline void qm_mr_pce_prefetch(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_mr *mr = &portal->mr; -+ DPA_ASSERT(mr->pmode == qm_mr_pce); -+ qm_cl_invalidate(MR_PI); -+ qm_cl_touch_ro(MR_PI); -+} -+ -+static inline u8 qm_mr_pce_update(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ u8 diff, old_pi = mr->pi; -+ DPA_ASSERT(mr->pmode == qm_mr_pce); -+ mr->pi = qm_cl_in(MR_PI) & (QM_MR_SIZE - 1); -+ diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi); -+ mr->fill += diff; -+ return diff; -+} -+ -+static inline void qm_mr_pvb_update(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi); -+ DPA_ASSERT(mr->pmode == qm_mr_pvb); -+ /* when accessing 'verb', use __raw_readb() to ensure that compiler -+ * inlining doesn't try to optimise out "excess reads". */ -+ if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) { -+ mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1); -+ if (!mr->pi) -+ mr->vbit ^= QM_MR_VERB_VBIT; -+ mr->fill++; -+ res = MR_INC(res); -+ } -+ dcbit_ro(res); -+} -+ -+static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ DPA_ASSERT(mr->cmode == qm_mr_cci); -+ mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1); -+ qm_out(MR_CI_CINH, mr->ci); -+} -+ -+static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ DPA_ASSERT(mr->cmode == qm_mr_cci); -+ mr->ci = MR_PTR2IDX(mr->cursor); -+ qm_out(MR_CI_CINH, mr->ci); -+} -+ -+static inline void qm_mr_cce_prefetch(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_mr *mr = &portal->mr; -+ DPA_ASSERT(mr->cmode == qm_mr_cce); -+ qm_cl_invalidate(MR_CI); -+ qm_cl_touch_rw(MR_CI); -+} -+ -+static inline void qm_mr_cce_consume(struct qm_portal *portal, u8 num) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ DPA_ASSERT(mr->cmode == qm_mr_cce); -+ mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1); -+ qm_cl_out(MR_CI, mr->ci); -+} -+ -+static inline void qm_mr_cce_consume_to_current(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ DPA_ASSERT(mr->cmode == qm_mr_cce); -+ mr->ci = MR_PTR2IDX(mr->cursor); -+ qm_cl_out(MR_CI, mr->ci); -+} -+ -+static inline u8 qm_mr_get_ci(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ return mr->ci; -+} -+ -+static inline u8 qm_mr_get_ithresh(struct qm_portal *portal) -+{ -+ register struct qm_mr *mr = &portal->mr; -+ return mr->ithresh; -+} -+ -+static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh) -+{ -+ qm_out(MR_ITR, ithresh); -+} -+ -+ -+/* ------------------------------ */ -+/* --- Management command API --- */ -+ -+static inline int qm_mc_init(struct qm_portal *portal) -+{ -+ register struct qm_mc *mc = &portal->mc; -+ mc->cr = portal->addr.addr_ce + QM_CL_CR; -+ mc->rr = portal->addr.addr_ce + QM_CL_RR0; -+ mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) & -+ QM_MCC_VERB_VBIT) ? 0 : 1; -+ mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = qman_mc_idle; -+#endif -+ return 0; -+} -+ -+static inline void qm_mc_finish(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_mc *mc = &portal->mc; -+ DPA_ASSERT(mc->state == qman_mc_idle); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ if (mc->state != qman_mc_idle) -+ pr_crit("Losing incomplete MC command\n"); -+#endif -+} -+ -+static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal) -+{ -+ register struct qm_mc *mc = &portal->mc; -+ DPA_ASSERT(mc->state == qman_mc_idle); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = qman_mc_user; -+#endif -+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64) -+ dcbz_64(mc->cr); -+#endif -+ return mc->cr; -+} -+ -+static inline void qm_mc_abort(struct qm_portal *portal) -+{ -+ __maybe_unused register struct qm_mc *mc = &portal->mc; -+ DPA_ASSERT(mc->state == qman_mc_user); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = qman_mc_idle; -+#endif -+} -+ -+static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb) -+{ -+ register struct qm_mc *mc = &portal->mc; -+ struct qm_mc_result *rr = mc->rr + mc->rridx; -+ DPA_ASSERT(mc->state == qman_mc_user); -+ lwsync(); -+ mc->cr->__dont_write_directly__verb = myverb | mc->vbit; -+ dcbf(mc->cr); -+ dcbit_ro(rr); -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = qman_mc_hw; -+#endif -+} -+ -+static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal) -+{ -+ register struct qm_mc *mc = &portal->mc; -+ struct qm_mc_result *rr = mc->rr + mc->rridx; -+ DPA_ASSERT(mc->state == qman_mc_hw); -+ /* The inactive response register's verb byte always returns zero until -+ * its command is submitted and completed. This includes the valid-bit, -+ * in case you were wondering... */ -+ if (!__raw_readb(&rr->verb)) { -+ dcbit_ro(rr); -+ return NULL; -+ } -+ mc->rridx ^= 1; -+ mc->vbit ^= QM_MCC_VERB_VBIT; -+#ifdef CONFIG_FSL_DPA_CHECKING -+ mc->state = qman_mc_idle; -+#endif -+ return rr; -+} -+ -+ -+/* ------------------------------------- */ -+/* --- Portal interrupt register API --- */ -+ -+static inline int qm_isr_init(__always_unused struct qm_portal *portal) -+{ -+ return 0; -+} -+ -+static inline void qm_isr_finish(__always_unused struct qm_portal *portal) -+{ -+} -+ -+static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod) -+{ -+ qm_out(ITPR, iperiod); -+} -+ -+static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n) -+{ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ return __qm_in(&portal->addr, QM_REG_ISR + (n << 6)); -+#else -+ return __qm_in(&portal->addr, QM_REG_ISR + (n << 2)); -+#endif -+} -+ -+static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n, -+ u32 val) -+{ -+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) -+ __qm_out(&portal->addr, QM_REG_ISR + (n << 6), val); -+#else -+ __qm_out(&portal->addr, QM_REG_ISR + (n << 2), val); -+#endif -+} -+ -+/* Cleanup FQs */ -+static inline int qm_shutdown_fq(struct qm_portal **portal, int portal_count, -+ u32 fqid) -+{ -+ -+ struct qm_mc_command *mcc; -+ struct qm_mc_result *mcr; -+ u8 state; -+ int orl_empty, fq_empty, i, drain = 0; -+ u32 result; -+ u32 channel, wq; -+ u16 dest_wq; -+ -+ /* Determine the state of the FQID */ -+ mcc = qm_mc_start(portal[0]); -+ mcc->queryfq_np.fqid = cpu_to_be32(fqid); -+ qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ_NP); -+ while (!(mcr = qm_mc_result(portal[0]))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP); -+ state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK; -+ if (state == QM_MCR_NP_STATE_OOS) -+ return 0; /* Already OOS, no need to do anymore checks */ -+ -+ /* Query which channel the FQ is using */ -+ mcc = qm_mc_start(portal[0]); -+ mcc->queryfq.fqid = cpu_to_be32(fqid); -+ qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ); -+ while (!(mcr = qm_mc_result(portal[0]))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ); -+ -+ /* Need to store these since the MCR gets reused */ -+ dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq); -+ wq = dest_wq & 0x7; -+ channel = dest_wq>>3; -+ -+ switch (state) { -+ case QM_MCR_NP_STATE_TEN_SCHED: -+ case QM_MCR_NP_STATE_TRU_SCHED: -+ case QM_MCR_NP_STATE_ACTIVE: -+ case QM_MCR_NP_STATE_PARKED: -+ orl_empty = 0; -+ mcc = qm_mc_start(portal[0]); -+ mcc->alterfq.fqid = cpu_to_be32(fqid); -+ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_RETIRE); -+ while (!(mcr = qm_mc_result(portal[0]))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_MCR_VERB_ALTER_RETIRE); -+ result = mcr->result; /* Make a copy as we reuse MCR below */ -+ -+ if (result == QM_MCR_RESULT_PENDING) { -+ /* Need to wait for the FQRN in the message ring, which -+ will only occur once the FQ has been drained. In -+ order for the FQ to drain the portal needs to be set -+ to dequeue from the channel the FQ is scheduled on */ -+ const struct qm_mr_entry *msg; -+ const struct qm_dqrr_entry *dqrr = NULL; -+ int found_fqrn = 0; -+ u16 dequeue_wq = 0; -+ -+ /* Flag that we need to drain FQ */ -+ drain = 1; -+ -+ if (channel >= qm_channel_pool1 && -+ channel < (qm_channel_pool1 + 15)) { -+ /* Pool channel, enable the bit in the portal */ -+ dequeue_wq = (channel - -+ qm_channel_pool1 + 1)<<4 | wq; -+ } else if (channel < qm_channel_pool1) { -+ /* Dedicated channel */ -+ dequeue_wq = wq; -+ } else { -+ pr_info("Cannot recover FQ 0x%x, it is " -+ "scheduled on channel 0x%x", -+ fqid, channel); -+ return -EBUSY; -+ } -+ /* Set the sdqcr to drain this channel */ -+ if (channel < qm_channel_pool1) -+ for (i = 0; i < portal_count; i++) -+ qm_dqrr_sdqcr_set(portal[i], -+ QM_SDQCR_TYPE_ACTIVE | -+ QM_SDQCR_CHANNELS_DEDICATED); -+ else -+ for (i = 0; i < portal_count; i++) -+ qm_dqrr_sdqcr_set( -+ portal[i], -+ QM_SDQCR_TYPE_ACTIVE | -+ QM_SDQCR_CHANNELS_POOL_CONV -+ (channel)); -+ while (!found_fqrn) { -+ /* Keep draining DQRR while checking the MR*/ -+ for (i = 0; i < portal_count; i++) { -+ qm_dqrr_pvb_update(portal[i]); -+ dqrr = qm_dqrr_current(portal[i]); -+ while (dqrr) { -+ qm_dqrr_cdc_consume_1ptr( -+ portal[i], dqrr, 0); -+ qm_dqrr_pvb_update(portal[i]); -+ qm_dqrr_next(portal[i]); -+ dqrr = qm_dqrr_current( -+ portal[i]); -+ } -+ /* Process message ring too */ -+ qm_mr_pvb_update(portal[i]); -+ msg = qm_mr_current(portal[i]); -+ while (msg) { -+ if ((msg->verb & -+ QM_MR_VERB_TYPE_MASK) -+ == QM_MR_VERB_FQRN) -+ found_fqrn = 1; -+ qm_mr_next(portal[i]); -+ qm_mr_cci_consume_to_current( -+ portal[i]); -+ qm_mr_pvb_update(portal[i]); -+ msg = qm_mr_current(portal[i]); -+ } -+ cpu_relax(); -+ } -+ } -+ } -+ if (result != QM_MCR_RESULT_OK && -+ result != QM_MCR_RESULT_PENDING) { -+ /* error */ -+ pr_err("qman_retire_fq failed on FQ 0x%x, result=0x%x\n", -+ fqid, result); -+ return -1; -+ } -+ if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) { -+ /* ORL had no entries, no need to wait until the -+ ERNs come in */ -+ orl_empty = 1; -+ } -+ /* Retirement succeeded, check to see if FQ needs -+ to be drained */ -+ if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) { -+ /* FQ is Not Empty, drain using volatile DQ commands */ -+ fq_empty = 0; -+ do { -+ const struct qm_dqrr_entry *dqrr = NULL; -+ u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3); -+ qm_dqrr_vdqcr_set(portal[0], vdqcr); -+ -+ /* Wait for a dequeue to occur */ -+ while (dqrr == NULL) { -+ qm_dqrr_pvb_update(portal[0]); -+ dqrr = qm_dqrr_current(portal[0]); -+ if (!dqrr) -+ cpu_relax(); -+ } -+ /* Process the dequeues, making sure to -+ empty the ring completely */ -+ while (dqrr) { -+ if (be32_to_cpu(dqrr->fqid) == fqid && -+ dqrr->stat & QM_DQRR_STAT_FQ_EMPTY) -+ fq_empty = 1; -+ qm_dqrr_cdc_consume_1ptr(portal[0], -+ dqrr, 0); -+ qm_dqrr_pvb_update(portal[0]); -+ qm_dqrr_next(portal[0]); -+ dqrr = qm_dqrr_current(portal[0]); -+ } -+ } while (fq_empty == 0); -+ } -+ for (i = 0; i < portal_count; i++) -+ qm_dqrr_sdqcr_set(portal[i], 0); -+ -+ /* Wait for the ORL to have been completely drained */ -+ while (orl_empty == 0) { -+ const struct qm_mr_entry *msg; -+ qm_mr_pvb_update(portal[0]); -+ msg = qm_mr_current(portal[0]); -+ while (msg) { -+ if ((msg->verb & QM_MR_VERB_TYPE_MASK) == -+ QM_MR_VERB_FQRL) -+ orl_empty = 1; -+ qm_mr_next(portal[0]); -+ qm_mr_cci_consume_to_current(portal[0]); -+ qm_mr_pvb_update(portal[0]); -+ msg = qm_mr_current(portal[0]); -+ } -+ cpu_relax(); -+ } -+ mcc = qm_mc_start(portal[0]); -+ mcc->alterfq.fqid = cpu_to_be32(fqid); -+ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS); -+ while (!(mcr = qm_mc_result(portal[0]))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_MCR_VERB_ALTER_OOS); -+ if (mcr->result != QM_MCR_RESULT_OK) { -+ pr_err("OOS after drain Failed on FQID 0x%x, result 0x%x\n", -+ fqid, mcr->result); -+ return -1; -+ } -+ return 0; -+ case QM_MCR_NP_STATE_RETIRED: -+ /* Send OOS Command */ -+ mcc = qm_mc_start(portal[0]); -+ mcc->alterfq.fqid = cpu_to_be32(fqid); -+ qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS); -+ while (!(mcr = qm_mc_result(portal[0]))) -+ cpu_relax(); -+ DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == -+ QM_MCR_VERB_ALTER_OOS); -+ if (mcr->result) { -+ pr_err("OOS Failed on FQID 0x%x\n", fqid); -+ return -1; -+ } -+ return 0; -+ } -+ return -1; -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_private.h -@@ -0,0 +1,398 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "dpa_sys.h" -+#include <linux/fsl_qman.h> -+#include <linux/iommu.h> -+ -+#if defined(CONFIG_FSL_PAMU) -+#include <asm/fsl_pamu_stash.h> -+#endif -+ -+#if !defined(CONFIG_FSL_QMAN_FQ_LOOKUP) && defined(CONFIG_PPC64) -+#error "_PPC64 requires _FSL_QMAN_FQ_LOOKUP" -+#endif -+ -+#define QBMAN_ANY_PORTAL_IDX 0xffffffff -+ /* ----------------- */ -+ /* Congestion Groups */ -+ /* ----------------- */ -+/* This wrapper represents a bit-array for the state of the 256 Qman congestion -+ * groups. Is also used as a *mask* for congestion groups, eg. so we ignore -+ * those that don't concern us. We harness the structure and accessor details -+ * already used in the management command to query congestion groups. */ -+struct qman_cgrs { -+ struct __qm_mcr_querycongestion q; -+}; -+static inline void qman_cgrs_init(struct qman_cgrs *c) -+{ -+ memset(c, 0, sizeof(*c)); -+} -+static inline void qman_cgrs_fill(struct qman_cgrs *c) -+{ -+ memset(c, 0xff, sizeof(*c)); -+} -+static inline int qman_cgrs_get(struct qman_cgrs *c, int num) -+{ -+ return QM_MCR_QUERYCONGESTION(&c->q, num); -+} -+static inline void qman_cgrs_set(struct qman_cgrs *c, int num) -+{ -+ c->q.__state[__CGR_WORD(num)] |= (0x80000000 >> __CGR_SHIFT(num)); -+} -+static inline void qman_cgrs_unset(struct qman_cgrs *c, int num) -+{ -+ c->q.__state[__CGR_WORD(num)] &= ~(0x80000000 >> __CGR_SHIFT(num)); -+} -+static inline int qman_cgrs_next(struct qman_cgrs *c, int num) -+{ -+ while ((++num < __CGR_NUM) && !qman_cgrs_get(c, num)) -+ ; -+ return num; -+} -+static inline void qman_cgrs_cp(struct qman_cgrs *dest, -+ const struct qman_cgrs *src) -+{ -+ *dest = *src; -+} -+static inline void qman_cgrs_and(struct qman_cgrs *dest, -+ const struct qman_cgrs *a, const struct qman_cgrs *b) -+{ -+ int ret; -+ u32 *_d = dest->q.__state; -+ const u32 *_a = a->q.__state; -+ const u32 *_b = b->q.__state; -+ for (ret = 0; ret < 8; ret++) -+ *(_d++) = *(_a++) & *(_b++); -+} -+static inline void qman_cgrs_xor(struct qman_cgrs *dest, -+ const struct qman_cgrs *a, const struct qman_cgrs *b) -+{ -+ int ret; -+ u32 *_d = dest->q.__state; -+ const u32 *_a = a->q.__state; -+ const u32 *_b = b->q.__state; -+ for (ret = 0; ret < 8; ret++) -+ *(_d++) = *(_a++) ^ *(_b++); -+} -+ -+ /* ----------------------- */ -+ /* CEETM Congestion Groups */ -+ /* ----------------------- */ -+/* This wrapper represents a bit-array for the state of the 512 Qman CEETM -+ * congestion groups. -+ */ -+struct qman_ccgrs { -+ struct __qm_mcr_querycongestion q[2]; -+}; -+static inline void qman_ccgrs_init(struct qman_ccgrs *c) -+{ -+ memset(c, 0, sizeof(*c)); -+} -+static inline void qman_ccgrs_fill(struct qman_ccgrs *c) -+{ -+ memset(c, 0xff, sizeof(*c)); -+} -+static inline int qman_ccgrs_get(struct qman_ccgrs *c, int num) -+{ -+ if (num < __CGR_NUM) -+ return QM_MCR_QUERYCONGESTION(&c->q[0], num); -+ else -+ return QM_MCR_QUERYCONGESTION(&c->q[1], (num - __CGR_NUM)); -+} -+static inline int qman_ccgrs_next(struct qman_ccgrs *c, int num) -+{ -+ while ((++num < __CGR_NUM) && !qman_ccgrs_get(c, num)) -+ ; -+ return num; -+} -+static inline void qman_ccgrs_cp(struct qman_ccgrs *dest, -+ const struct qman_ccgrs *src) -+{ -+ *dest = *src; -+} -+static inline void qman_ccgrs_and(struct qman_ccgrs *dest, -+ const struct qman_ccgrs *a, const struct qman_ccgrs *b) -+{ -+ int ret, i; -+ u32 *_d; -+ const u32 *_a, *_b; -+ for (i = 0; i < 2; i++) { -+ _d = dest->q[i].__state; -+ _a = a->q[i].__state; -+ _b = b->q[i].__state; -+ for (ret = 0; ret < 8; ret++) -+ *(_d++) = *(_a++) & *(_b++); -+ } -+} -+static inline void qman_ccgrs_xor(struct qman_ccgrs *dest, -+ const struct qman_ccgrs *a, const struct qman_ccgrs *b) -+{ -+ int ret, i; -+ u32 *_d; -+ const u32 *_a, *_b; -+ for (i = 0; i < 2; i++) { -+ _d = dest->q[i].__state; -+ _a = a->q[i].__state; -+ _b = b->q[i].__state; -+ for (ret = 0; ret < 8; ret++) -+ *(_d++) = *(_a++) ^ *(_b++); -+ } -+} -+ -+/* used by CCSR and portal interrupt code */ -+enum qm_isr_reg { -+ qm_isr_status = 0, -+ qm_isr_enable = 1, -+ qm_isr_disable = 2, -+ qm_isr_inhibit = 3 -+}; -+ -+struct qm_portal_config { -+ /* Corenet portal addresses; -+ * [0]==cache-enabled, [1]==cache-inhibited. */ -+ __iomem void *addr_virt[2]; -+ struct resource addr_phys[2]; -+ struct device dev; -+ struct iommu_domain *iommu_domain; -+ /* Allow these to be joined in lists */ -+ struct list_head list; -+ /* User-visible portal configuration settings */ -+ struct qman_portal_config public_cfg; -+ /* power management saved data */ -+ u32 saved_isdr; -+}; -+ -+/* Revision info (for errata and feature handling) */ -+#define QMAN_REV11 0x0101 -+#define QMAN_REV12 0x0102 -+#define QMAN_REV20 0x0200 -+#define QMAN_REV30 0x0300 -+#define QMAN_REV31 0x0301 -+#define QMAN_REV32 0x0302 -+ -+/* QMan REV_2 register contains the Cfg option */ -+#define QMAN_REV_CFG_0 0x0 -+#define QMAN_REV_CFG_1 0x1 -+#define QMAN_REV_CFG_2 0x2 -+#define QMAN_REV_CFG_3 0x3 -+ -+extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */ -+extern u8 qman_ip_cfg; -+extern u32 qman_clk; -+extern u16 qman_portal_max; -+ -+#ifdef CONFIG_FSL_QMAN_CONFIG -+/* Hooks from qman_driver.c to qman_config.c */ -+int qman_init_ccsr(struct device_node *node); -+void qman_liodn_fixup(u16 channel); -+int qman_set_sdest(u16 channel, unsigned int cpu_idx); -+size_t get_qman_fqd_size(void); -+#else -+static inline size_t get_qman_fqd_size(void) -+{ -+ return (PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ); -+} -+#endif -+ -+int qm_set_wpm(int wpm); -+int qm_get_wpm(int *wpm); -+ -+/* Hooks from qman_driver.c in to qman_high.c */ -+struct qman_portal *qman_create_portal( -+ struct qman_portal *portal, -+ const struct qm_portal_config *config, -+ const struct qman_cgrs *cgrs); -+ -+struct qman_portal *qman_create_affine_portal( -+ const struct qm_portal_config *config, -+ const struct qman_cgrs *cgrs); -+struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect, -+ int cpu); -+const struct qm_portal_config *qman_destroy_affine_portal(void); -+void qman_destroy_portal(struct qman_portal *qm); -+ -+/* Hooks from fsl_usdpaa.c to qman_driver.c */ -+struct qm_portal_config *qm_get_unused_portal(void); -+struct qm_portal_config *qm_get_unused_portal_idx(uint32_t idx); -+ -+void qm_put_unused_portal(struct qm_portal_config *pcfg); -+void qm_set_liodns(struct qm_portal_config *pcfg); -+ -+/* This CGR feature is supported by h/w and required by unit-tests and the -+ * debugfs hooks, so is implemented in the driver. However it allows an explicit -+ * corruption of h/w fields by s/w that are usually incorruptible (because the -+ * counters are usually maintained entirely within h/w). As such, we declare -+ * this API internally. */ -+int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt, -+ struct qm_mcr_cgrtestwrite *result); -+ -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+/* If the fq object pointer is greater than the size of context_b field, -+ * than a lookup table is required. */ -+int qman_setup_fq_lookup_table(size_t num_entries); -+#endif -+ -+ -+/*************************************************/ -+/* QMan s/w corenet portal, low-level i/face */ -+/*************************************************/ -+ -+/* Note: most functions are only used by the high-level interface, so are -+ * inlined from qman_low.h. The stuff below is for use by other parts of the -+ * driver. */ -+ -+/* For qm_dqrr_sdqcr_set(); Choose one SOURCE. Choose one COUNT. Choose one -+ * dequeue TYPE. Choose TOKEN (8-bit). -+ * If SOURCE == CHANNELS, -+ * Choose CHANNELS_DEDICATED and/or CHANNELS_POOL(n). -+ * You can choose DEDICATED_PRECEDENCE if the portal channel should have -+ * priority. -+ * If SOURCE == SPECIFICWQ, -+ * Either select the work-queue ID with SPECIFICWQ_WQ(), or select the -+ * channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the -+ * work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the -+ * same value. -+ */ -+#define QM_SDQCR_SOURCE_CHANNELS 0x0 -+#define QM_SDQCR_SOURCE_SPECIFICWQ 0x40000000 -+#define QM_SDQCR_COUNT_EXACT1 0x0 -+#define QM_SDQCR_COUNT_UPTO3 0x20000000 -+#define QM_SDQCR_DEDICATED_PRECEDENCE 0x10000000 -+#define QM_SDQCR_TYPE_MASK 0x03000000 -+#define QM_SDQCR_TYPE_NULL 0x0 -+#define QM_SDQCR_TYPE_PRIO_QOS 0x01000000 -+#define QM_SDQCR_TYPE_ACTIVE_QOS 0x02000000 -+#define QM_SDQCR_TYPE_ACTIVE 0x03000000 -+#define QM_SDQCR_TOKEN_MASK 0x00ff0000 -+#define QM_SDQCR_TOKEN_SET(v) (((v) & 0xff) << 16) -+#define QM_SDQCR_TOKEN_GET(v) (((v) >> 16) & 0xff) -+#define QM_SDQCR_CHANNELS_DEDICATED 0x00008000 -+#define QM_SDQCR_SPECIFICWQ_MASK 0x000000f7 -+#define QM_SDQCR_SPECIFICWQ_DEDICATED 0x00000000 -+#define QM_SDQCR_SPECIFICWQ_POOL(n) ((n) << 4) -+#define QM_SDQCR_SPECIFICWQ_WQ(n) (n) -+ -+/* For qm_dqrr_vdqcr_set(): use FQID(n) to fill in the frame queue ID */ -+#define QM_VDQCR_FQID_MASK 0x00ffffff -+#define QM_VDQCR_FQID(n) ((n) & QM_VDQCR_FQID_MASK) -+ -+/* For qm_dqrr_pdqcr_set(); Choose one MODE. Choose one COUNT. -+ * If MODE==SCHEDULED -+ * Choose SCHEDULED_CHANNELS or SCHEDULED_SPECIFICWQ. Choose one dequeue TYPE. -+ * If CHANNELS, -+ * Choose CHANNELS_DEDICATED and/or CHANNELS_POOL() channels. -+ * You can choose DEDICATED_PRECEDENCE if the portal channel should have -+ * priority. -+ * If SPECIFICWQ, -+ * Either select the work-queue ID with SPECIFICWQ_WQ(), or select the -+ * channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the -+ * work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the -+ * same value. -+ * If MODE==UNSCHEDULED -+ * Choose FQID(). -+ */ -+#define QM_PDQCR_MODE_SCHEDULED 0x0 -+#define QM_PDQCR_MODE_UNSCHEDULED 0x80000000 -+#define QM_PDQCR_SCHEDULED_CHANNELS 0x0 -+#define QM_PDQCR_SCHEDULED_SPECIFICWQ 0x40000000 -+#define QM_PDQCR_COUNT_EXACT1 0x0 -+#define QM_PDQCR_COUNT_UPTO3 0x20000000 -+#define QM_PDQCR_DEDICATED_PRECEDENCE 0x10000000 -+#define QM_PDQCR_TYPE_MASK 0x03000000 -+#define QM_PDQCR_TYPE_NULL 0x0 -+#define QM_PDQCR_TYPE_PRIO_QOS 0x01000000 -+#define QM_PDQCR_TYPE_ACTIVE_QOS 0x02000000 -+#define QM_PDQCR_TYPE_ACTIVE 0x03000000 -+#define QM_PDQCR_CHANNELS_DEDICATED 0x00008000 -+#define QM_PDQCR_CHANNELS_POOL(n) (0x00008000 >> (n)) -+#define QM_PDQCR_SPECIFICWQ_MASK 0x000000f7 -+#define QM_PDQCR_SPECIFICWQ_DEDICATED 0x00000000 -+#define QM_PDQCR_SPECIFICWQ_POOL(n) ((n) << 4) -+#define QM_PDQCR_SPECIFICWQ_WQ(n) (n) -+#define QM_PDQCR_FQID(n) ((n) & 0xffffff) -+ -+/* Used by all portal interrupt registers except 'inhibit' -+ * Channels with frame availability -+ */ -+#define QM_PIRQ_DQAVAIL 0x0000ffff -+ -+/* The DQAVAIL interrupt fields break down into these bits; */ -+#define QM_DQAVAIL_PORTAL 0x8000 /* Portal channel */ -+#define QM_DQAVAIL_POOL(n) (0x8000 >> (n)) /* Pool channel, n==[1..15] */ -+#define QM_DQAVAIL_MASK 0xffff -+/* This mask contains all the "irqsource" bits visible to API users */ -+#define QM_PIRQ_VISIBLE (QM_PIRQ_SLOW | QM_PIRQ_DQRI) -+ -+/* These are qm_<reg>_<verb>(). So for example, qm_disable_write() means "write -+ * the disable register" rather than "disable the ability to write". */ -+#define qm_isr_status_read(qm) __qm_isr_read(qm, qm_isr_status) -+#define qm_isr_status_clear(qm, m) __qm_isr_write(qm, qm_isr_status, m) -+#define qm_isr_enable_read(qm) __qm_isr_read(qm, qm_isr_enable) -+#define qm_isr_enable_write(qm, v) __qm_isr_write(qm, qm_isr_enable, v) -+#define qm_isr_disable_read(qm) __qm_isr_read(qm, qm_isr_disable) -+#define qm_isr_disable_write(qm, v) __qm_isr_write(qm, qm_isr_disable, v) -+/* TODO: unfortunate name-clash here, reword? */ -+#define qm_isr_inhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 1) -+#define qm_isr_uninhibit(qm) __qm_isr_write(qm, qm_isr_inhibit, 0) -+ -+#ifdef CONFIG_FSL_QMAN_CONFIG -+int qman_have_ccsr(void); -+#else -+#define qman_have_ccsr 0 -+#endif -+ -+__init int qman_init(void); -+__init int qman_resource_init(void); -+ -+/* CEETM related */ -+#define QMAN_CEETM_MAX 2 -+extern u8 num_ceetms; -+extern struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX]; -+int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal); -+int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal); -+int qman_ceetm_set_prescaler(enum qm_dc_portal portal); -+int qman_ceetm_get_prescaler(u16 *pres); -+int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid, -+ struct qm_mcr_ceetm_cq_query *cq_query); -+int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query, -+ struct qm_mcr_ceetm_ccgr_query *response); -+int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num); -+ -+extern void *affine_portals[NR_CPUS]; -+const struct qm_portal_config *qman_get_qm_portal_config( -+ struct qman_portal *portal); -+ -+/* power management */ -+#ifdef CONFIG_SUSPEND -+void suspend_unused_qportal(void); -+void resume_unused_qportal(void); -+#endif ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_test.c -@@ -0,0 +1,57 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "qman_test.h" -+ -+MODULE_AUTHOR("Geoff Thorpe"); -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("Qman testing"); -+ -+static int test_init(void) -+{ -+ int loop = 1; -+ while (loop--) { -+#ifdef CONFIG_FSL_QMAN_TEST_STASH_POTATO -+ qman_test_hotpotato(); -+#endif -+#ifdef CONFIG_FSL_QMAN_TEST_HIGH -+ qman_test_high(); -+#endif -+ } -+ return 0; -+} -+ -+static void test_exit(void) -+{ -+} -+ -+module_init(test_init); -+module_exit(test_exit); ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_test.h -@@ -0,0 +1,45 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <linux/kernel.h> -+#include <linux/errno.h> -+#include <linux/io.h> -+#include <linux/slab.h> -+#include <linux/module.h> -+#include <linux/interrupt.h> -+#include <linux/delay.h> -+#include <linux/sched.h> -+ -+#include <linux/fsl_qman.h> -+ -+void qman_test_hotpotato(void); -+void qman_test_high(void); -+ ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_test_high.c -@@ -0,0 +1,216 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "qman_test.h" -+ -+/*************/ -+/* constants */ -+/*************/ -+ -+#define CGR_ID 27 -+#define POOL_ID 2 -+#define FQ_FLAGS QMAN_FQ_FLAG_DYNAMIC_FQID -+#define NUM_ENQUEUES 10 -+#define NUM_PARTIAL 4 -+#define PORTAL_SDQCR (QM_SDQCR_SOURCE_CHANNELS | \ -+ QM_SDQCR_TYPE_PRIO_QOS | \ -+ QM_SDQCR_TOKEN_SET(0x98) | \ -+ QM_SDQCR_CHANNELS_DEDICATED | \ -+ QM_SDQCR_CHANNELS_POOL(POOL_ID)) -+#define PORTAL_OPAQUE ((void *)0xf00dbeef) -+#define VDQCR_FLAGS (QMAN_VOLATILE_FLAG_WAIT | QMAN_VOLATILE_FLAG_FINISH) -+ -+/*************************************/ -+/* Predeclarations (eg. for fq_base) */ -+/*************************************/ -+ -+static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *, -+ struct qman_fq *, -+ const struct qm_dqrr_entry *); -+static void cb_ern(struct qman_portal *, struct qman_fq *, -+ const struct qm_mr_entry *); -+static void cb_fqs(struct qman_portal *, struct qman_fq *, -+ const struct qm_mr_entry *); -+ -+/***************/ -+/* global vars */ -+/***************/ -+ -+static struct qm_fd fd, fd_dq; -+static struct qman_fq fq_base = { -+ .cb.dqrr = cb_dqrr, -+ .cb.ern = cb_ern, -+ .cb.fqs = cb_fqs -+}; -+static DECLARE_WAIT_QUEUE_HEAD(waitqueue); -+static int retire_complete, sdqcr_complete; -+ -+/**********************/ -+/* internal functions */ -+/**********************/ -+ -+/* Helpers for initialising and "incrementing" a frame descriptor */ -+static void fd_init(struct qm_fd *__fd) -+{ -+ qm_fd_addr_set64(__fd, 0xabdeadbeefLLU); -+ __fd->format = qm_fd_contig_big; -+ __fd->length29 = 0x0000ffff; -+ __fd->cmd = 0xfeedf00d; -+} -+ -+static void fd_inc(struct qm_fd *__fd) -+{ -+ u64 t = qm_fd_addr_get64(__fd); -+ int z = t >> 40; -+ t <<= 1; -+ if (z) -+ t |= 1; -+ qm_fd_addr_set64(__fd, t); -+ __fd->length29--; -+ __fd->cmd++; -+} -+ -+/* The only part of the 'fd' we can't memcmp() is the ppid */ -+static int fd_cmp(const struct qm_fd *a, const struct qm_fd *b) -+{ -+ int r = (qm_fd_addr_get64(a) == qm_fd_addr_get64(b)) ? 0 : -1; -+ if (!r) -+ r = a->format - b->format; -+ if (!r) -+ r = a->opaque - b->opaque; -+ if (!r) -+ r = a->cmd - b->cmd; -+ return r; -+} -+ -+/********/ -+/* test */ -+/********/ -+ -+static void do_enqueues(struct qman_fq *fq) -+{ -+ unsigned int loop; -+ for (loop = 0; loop < NUM_ENQUEUES; loop++) { -+ if (qman_enqueue(fq, &fd, QMAN_ENQUEUE_FLAG_WAIT | -+ (((loop + 1) == NUM_ENQUEUES) ? -+ QMAN_ENQUEUE_FLAG_WAIT_SYNC : 0))) -+ panic("qman_enqueue() failed\n"); -+ fd_inc(&fd); -+ } -+} -+ -+void qman_test_high(void) -+{ -+ unsigned int flags; -+ int res; -+ struct qman_fq *fq = &fq_base; -+ -+ pr_info("qman_test_high starting\n"); -+ fd_init(&fd); -+ fd_init(&fd_dq); -+ -+ /* Initialise (parked) FQ */ -+ if (qman_create_fq(0, FQ_FLAGS, fq)) -+ panic("qman_create_fq() failed\n"); -+ if (qman_init_fq(fq, QMAN_INITFQ_FLAG_LOCAL, NULL)) -+ panic("qman_init_fq() failed\n"); -+ -+ /* Do enqueues + VDQCR, twice. (Parked FQ) */ -+ do_enqueues(fq); -+ pr_info("VDQCR (till-empty);\n"); -+ if (qman_volatile_dequeue(fq, VDQCR_FLAGS, -+ QM_VDQCR_NUMFRAMES_TILLEMPTY)) -+ panic("qman_volatile_dequeue() failed\n"); -+ do_enqueues(fq); -+ pr_info("VDQCR (%d of %d);\n", NUM_PARTIAL, NUM_ENQUEUES); -+ if (qman_volatile_dequeue(fq, VDQCR_FLAGS, -+ QM_VDQCR_NUMFRAMES_SET(NUM_PARTIAL))) -+ panic("qman_volatile_dequeue() failed\n"); -+ pr_info("VDQCR (%d of %d);\n", NUM_ENQUEUES - NUM_PARTIAL, -+ NUM_ENQUEUES); -+ if (qman_volatile_dequeue(fq, VDQCR_FLAGS, -+ QM_VDQCR_NUMFRAMES_SET(NUM_ENQUEUES - NUM_PARTIAL))) -+ panic("qman_volatile_dequeue() failed\n"); -+ -+ do_enqueues(fq); -+ pr_info("scheduled dequeue (till-empty)\n"); -+ if (qman_schedule_fq(fq)) -+ panic("qman_schedule_fq() failed\n"); -+ wait_event(waitqueue, sdqcr_complete); -+ -+ /* Retire and OOS the FQ */ -+ res = qman_retire_fq(fq, &flags); -+ if (res < 0) -+ panic("qman_retire_fq() failed\n"); -+ wait_event(waitqueue, retire_complete); -+ if (flags & QMAN_FQ_STATE_BLOCKOOS) -+ panic("leaking frames\n"); -+ if (qman_oos_fq(fq)) -+ panic("qman_oos_fq() failed\n"); -+ qman_destroy_fq(fq, 0); -+ pr_info("qman_test_high finished\n"); -+} -+ -+static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *p, -+ struct qman_fq *fq, -+ const struct qm_dqrr_entry *dq) -+{ -+ if (fd_cmp(&fd_dq, &dq->fd)) { -+ pr_err("BADNESS: dequeued frame doesn't match;\n"); -+ pr_err("Expected 0x%llx, got 0x%llx\n", -+ (unsigned long long)fd_dq.length29, -+ (unsigned long long)dq->fd.length29); -+ BUG(); -+ } -+ fd_inc(&fd_dq); -+ if (!(dq->stat & QM_DQRR_STAT_UNSCHEDULED) && !fd_cmp(&fd_dq, &fd)) { -+ sdqcr_complete = 1; -+ wake_up(&waitqueue); -+ } -+ return qman_cb_dqrr_consume; -+} -+ -+static void cb_ern(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_mr_entry *msg) -+{ -+ panic("cb_ern() unimplemented"); -+} -+ -+static void cb_fqs(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_mr_entry *msg) -+{ -+ u8 verb = (msg->verb & QM_MR_VERB_TYPE_MASK); -+ if ((verb != QM_MR_VERB_FQRN) && (verb != QM_MR_VERB_FQRNI)) -+ panic("unexpected FQS message"); -+ pr_info("Retirement message received\n"); -+ retire_complete = 1; -+ wake_up(&waitqueue); -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_test_hotpotato.c -@@ -0,0 +1,502 @@ -+/* Copyright 2009-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <linux/kthread.h> -+#include <linux/platform_device.h> -+#include <linux/dma-mapping.h> -+#include "qman_test.h" -+ -+/* Algorithm: -+ * -+ * Each cpu will have HP_PER_CPU "handlers" set up, each of which incorporates -+ * an rx/tx pair of FQ objects (both of which are stashed on dequeue). The -+ * organisation of FQIDs is such that the HP_PER_CPU*NUM_CPUS handlers will -+ * shuttle a "hot potato" frame around them such that every forwarding action -+ * moves it from one cpu to another. (The use of more than one handler per cpu -+ * is to allow enough handlers/FQs to truly test the significance of caching - -+ * ie. when cache-expiries are occurring.) -+ * -+ * The "hot potato" frame content will be HP_NUM_WORDS*4 bytes in size, and the -+ * first and last words of the frame data will undergo a transformation step on -+ * each forwarding action. To achieve this, each handler will be assigned a -+ * 32-bit "mixer", that is produced using a 32-bit LFSR. When a frame is -+ * received by a handler, the mixer of the expected sender is XOR'd into all -+ * words of the entire frame, which is then validated against the original -+ * values. Then, before forwarding, the entire frame is XOR'd with the mixer of -+ * the current handler. Apart from validating that the frame is taking the -+ * expected path, this also provides some quasi-realistic overheads to each -+ * forwarding action - dereferencing *all* the frame data, computation, and -+ * conditional branching. There is a "special" handler designated to act as the -+ * instigator of the test by creating an enqueuing the "hot potato" frame, and -+ * to determine when the test has completed by counting HP_LOOPS iterations. -+ * -+ * Init phases: -+ * -+ * 1. prepare each cpu's 'hp_cpu' struct using on_each_cpu(,,1) and link them -+ * into 'hp_cpu_list'. Specifically, set processor_id, allocate HP_PER_CPU -+ * handlers and link-list them (but do no other handler setup). -+ * -+ * 2. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each -+ * hp_cpu's 'iterator' to point to its first handler. With each loop, -+ * allocate rx/tx FQIDs and mixer values to the hp_cpu's iterator handler -+ * and advance the iterator for the next loop. This includes a final fixup, -+ * which connects the last handler to the first (and which is why phase 2 -+ * and 3 are separate). -+ * -+ * 3. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each -+ * hp_cpu's 'iterator' to point to its first handler. With each loop, -+ * initialise FQ objects and advance the iterator for the next loop. -+ * Moreover, do this initialisation on the cpu it applies to so that Rx FQ -+ * initialisation targets the correct cpu. -+ */ -+ -+/* helper to run something on all cpus (can't use on_each_cpu(), as that invokes -+ * the fn from irq context, which is too restrictive). */ -+struct bstrap { -+ void (*fn)(void); -+ atomic_t started; -+}; -+static int bstrap_fn(void *__bstrap) -+{ -+ struct bstrap *bstrap = __bstrap; -+ atomic_inc(&bstrap->started); -+ bstrap->fn(); -+ while (!kthread_should_stop()) -+ msleep(1); -+ return 0; -+} -+static int on_all_cpus(void (*fn)(void)) -+{ -+ int cpu; -+ for_each_cpu(cpu, cpu_online_mask) { -+ struct bstrap bstrap = { -+ .fn = fn, -+ .started = ATOMIC_INIT(0) -+ }; -+ struct task_struct *k = kthread_create(bstrap_fn, &bstrap, -+ "hotpotato%d", cpu); -+ int ret; -+ if (IS_ERR(k)) -+ return -ENOMEM; -+ kthread_bind(k, cpu); -+ wake_up_process(k); -+ /* If we call kthread_stop() before the "wake up" has had an -+ * effect, then the thread may exit with -EINTR without ever -+ * running the function. So poll until it's started before -+ * requesting it to stop. */ -+ while (!atomic_read(&bstrap.started)) -+ msleep(10); -+ ret = kthread_stop(k); -+ if (ret) -+ return ret; -+ } -+ return 0; -+} -+ -+struct hp_handler { -+ -+ /* The following data is stashed when 'rx' is dequeued; */ -+ /* -------------- */ -+ /* The Rx FQ, dequeues of which will stash the entire hp_handler */ -+ struct qman_fq rx; -+ /* The Tx FQ we should forward to */ -+ struct qman_fq tx; -+ /* The value we XOR post-dequeue, prior to validating */ -+ u32 rx_mixer; -+ /* The value we XOR pre-enqueue, after validating */ -+ u32 tx_mixer; -+ /* what the hotpotato address should be on dequeue */ -+ dma_addr_t addr; -+ u32 *frame_ptr; -+ -+ /* The following data isn't (necessarily) stashed on dequeue; */ -+ /* -------------- */ -+ u32 fqid_rx, fqid_tx; -+ /* list node for linking us into 'hp_cpu' */ -+ struct list_head node; -+ /* Just to check ... */ -+ unsigned int processor_id; -+} ____cacheline_aligned; -+ -+struct hp_cpu { -+ /* identify the cpu we run on; */ -+ unsigned int processor_id; -+ /* root node for the per-cpu list of handlers */ -+ struct list_head handlers; -+ /* list node for linking us into 'hp_cpu_list' */ -+ struct list_head node; -+ /* when repeatedly scanning 'hp_list', each time linking the n'th -+ * handlers together, this is used as per-cpu iterator state */ -+ struct hp_handler *iterator; -+}; -+ -+/* Each cpu has one of these */ -+static DEFINE_PER_CPU(struct hp_cpu, hp_cpus); -+ -+/* links together the hp_cpu structs, in first-come first-serve order. */ -+static LIST_HEAD(hp_cpu_list); -+static spinlock_t hp_lock = __SPIN_LOCK_UNLOCKED(hp_lock); -+ -+static unsigned int hp_cpu_list_length; -+ -+/* the "special" handler, that starts and terminates the test. */ -+static struct hp_handler *special_handler; -+static int loop_counter; -+ -+/* handlers are allocated out of this, so they're properly aligned. */ -+static struct kmem_cache *hp_handler_slab; -+ -+/* this is the frame data */ -+static void *__frame_ptr; -+static u32 *frame_ptr; -+static dma_addr_t frame_dma; -+ -+/* the main function waits on this */ -+static DECLARE_WAIT_QUEUE_HEAD(queue); -+ -+#define HP_PER_CPU 2 -+#define HP_LOOPS 8 -+/* 80 bytes, like a small ethernet frame, and bleeds into a second cacheline */ -+#define HP_NUM_WORDS 80 -+/* First word of the LFSR-based frame data */ -+#define HP_FIRST_WORD 0xabbaf00d -+ -+static inline u32 do_lfsr(u32 prev) -+{ -+ return (prev >> 1) ^ (-(prev & 1u) & 0xd0000001u); -+} -+ -+static void allocate_frame_data(void) -+{ -+ u32 lfsr = HP_FIRST_WORD; -+ int loop; -+ struct platform_device *pdev = platform_device_alloc("foobar", -1); -+ if (!pdev) -+ panic("platform_device_alloc() failed"); -+ if (platform_device_add(pdev)) -+ panic("platform_device_add() failed"); -+ __frame_ptr = kmalloc(4 * HP_NUM_WORDS, GFP_KERNEL); -+ if (!__frame_ptr) -+ panic("kmalloc() failed"); -+ frame_ptr = (void *)(((unsigned long)__frame_ptr + 63) & -+ ~(unsigned long)63); -+ for (loop = 0; loop < HP_NUM_WORDS; loop++) { -+ frame_ptr[loop] = lfsr; -+ lfsr = do_lfsr(lfsr); -+ } -+ frame_dma = dma_map_single(&pdev->dev, frame_ptr, 4 * HP_NUM_WORDS, -+ DMA_BIDIRECTIONAL); -+ platform_device_del(pdev); -+ platform_device_put(pdev); -+} -+ -+static void deallocate_frame_data(void) -+{ -+ kfree(__frame_ptr); -+} -+ -+static inline void process_frame_data(struct hp_handler *handler, -+ const struct qm_fd *fd) -+{ -+ u32 *p = handler->frame_ptr; -+ u32 lfsr = HP_FIRST_WORD; -+ int loop; -+ if (qm_fd_addr_get64(fd) != (handler->addr & 0xffffffffff)) { -+ pr_err("Got 0x%llx expected 0x%llx\n", -+ qm_fd_addr_get64(fd), handler->addr); -+ panic("bad frame address"); -+ } -+ for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) { -+ *p ^= handler->rx_mixer; -+ if (*p != lfsr) -+ panic("corrupt frame data"); -+ *p ^= handler->tx_mixer; -+ lfsr = do_lfsr(lfsr); -+ } -+} -+ -+static enum qman_cb_dqrr_result normal_dqrr(struct qman_portal *portal, -+ struct qman_fq *fq, -+ const struct qm_dqrr_entry *dqrr) -+{ -+ struct hp_handler *handler = (struct hp_handler *)fq; -+ -+ process_frame_data(handler, &dqrr->fd); -+ if (qman_enqueue(&handler->tx, &dqrr->fd, 0)) -+ panic("qman_enqueue() failed"); -+ return qman_cb_dqrr_consume; -+} -+ -+static enum qman_cb_dqrr_result special_dqrr(struct qman_portal *portal, -+ struct qman_fq *fq, -+ const struct qm_dqrr_entry *dqrr) -+{ -+ struct hp_handler *handler = (struct hp_handler *)fq; -+ -+ process_frame_data(handler, &dqrr->fd); -+ if (++loop_counter < HP_LOOPS) { -+ if (qman_enqueue(&handler->tx, &dqrr->fd, 0)) -+ panic("qman_enqueue() failed"); -+ } else { -+ pr_info("Received final (%dth) frame\n", loop_counter); -+ wake_up(&queue); -+ } -+ return qman_cb_dqrr_consume; -+} -+ -+static void create_per_cpu_handlers(void) -+{ -+ struct hp_handler *handler; -+ int loop; -+ struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus); -+ -+ hp_cpu->processor_id = smp_processor_id(); -+ spin_lock(&hp_lock); -+ list_add_tail(&hp_cpu->node, &hp_cpu_list); -+ hp_cpu_list_length++; -+ spin_unlock(&hp_lock); -+ INIT_LIST_HEAD(&hp_cpu->handlers); -+ for (loop = 0; loop < HP_PER_CPU; loop++) { -+ handler = kmem_cache_alloc(hp_handler_slab, GFP_KERNEL); -+ if (!handler) -+ panic("kmem_cache_alloc() failed"); -+ handler->processor_id = hp_cpu->processor_id; -+ handler->addr = frame_dma; -+ handler->frame_ptr = frame_ptr; -+ list_add_tail(&handler->node, &hp_cpu->handlers); -+ } -+ put_cpu_var(hp_cpus); -+} -+ -+static void destroy_per_cpu_handlers(void) -+{ -+ struct list_head *loop, *tmp; -+ struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus); -+ -+ spin_lock(&hp_lock); -+ list_del(&hp_cpu->node); -+ spin_unlock(&hp_lock); -+ list_for_each_safe(loop, tmp, &hp_cpu->handlers) { -+ u32 flags; -+ struct hp_handler *handler = list_entry(loop, struct hp_handler, -+ node); -+ if (qman_retire_fq(&handler->rx, &flags)) -+ panic("qman_retire_fq(rx) failed"); -+ BUG_ON(flags & QMAN_FQ_STATE_BLOCKOOS); -+ if (qman_oos_fq(&handler->rx)) -+ panic("qman_oos_fq(rx) failed"); -+ qman_destroy_fq(&handler->rx, 0); -+ qman_destroy_fq(&handler->tx, 0); -+ qman_release_fqid(handler->fqid_rx); -+ list_del(&handler->node); -+ kmem_cache_free(hp_handler_slab, handler); -+ } -+ put_cpu_var(hp_cpus); -+} -+ -+static inline u8 num_cachelines(u32 offset) -+{ -+ u8 res = (offset + (L1_CACHE_BYTES - 1)) -+ / (L1_CACHE_BYTES); -+ if (res > 3) -+ return 3; -+ return res; -+} -+#define STASH_DATA_CL \ -+ num_cachelines(HP_NUM_WORDS * 4) -+#define STASH_CTX_CL \ -+ num_cachelines(offsetof(struct hp_handler, fqid_rx)) -+ -+static void init_handler(void *__handler) -+{ -+ struct qm_mcc_initfq opts; -+ struct hp_handler *handler = __handler; -+ BUG_ON(handler->processor_id != smp_processor_id()); -+ /* Set up rx */ -+ memset(&handler->rx, 0, sizeof(handler->rx)); -+ if (handler == special_handler) -+ handler->rx.cb.dqrr = special_dqrr; -+ else -+ handler->rx.cb.dqrr = normal_dqrr; -+ if (qman_create_fq(handler->fqid_rx, 0, &handler->rx)) -+ panic("qman_create_fq(rx) failed"); -+ memset(&opts, 0, sizeof(opts)); -+ opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_CONTEXTA; -+ opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING; -+ opts.fqd.context_a.stashing.data_cl = STASH_DATA_CL; -+ opts.fqd.context_a.stashing.context_cl = STASH_CTX_CL; -+ if (qman_init_fq(&handler->rx, QMAN_INITFQ_FLAG_SCHED | -+ QMAN_INITFQ_FLAG_LOCAL, &opts)) -+ panic("qman_init_fq(rx) failed"); -+ /* Set up tx */ -+ memset(&handler->tx, 0, sizeof(handler->tx)); -+ if (qman_create_fq(handler->fqid_tx, QMAN_FQ_FLAG_NO_MODIFY, -+ &handler->tx)) -+ panic("qman_create_fq(tx) failed"); -+} -+ -+static void init_phase2(void) -+{ -+ int loop; -+ u32 fqid = 0; -+ u32 lfsr = 0xdeadbeef; -+ struct hp_cpu *hp_cpu; -+ struct hp_handler *handler; -+ -+ for (loop = 0; loop < HP_PER_CPU; loop++) { -+ list_for_each_entry(hp_cpu, &hp_cpu_list, node) { -+ int ret; -+ if (!loop) -+ hp_cpu->iterator = list_first_entry( -+ &hp_cpu->handlers, -+ struct hp_handler, node); -+ else -+ hp_cpu->iterator = list_entry( -+ hp_cpu->iterator->node.next, -+ struct hp_handler, node); -+ /* Rx FQID is the previous handler's Tx FQID */ -+ hp_cpu->iterator->fqid_rx = fqid; -+ /* Allocate new FQID for Tx */ -+ ret = qman_alloc_fqid(&fqid); -+ if (ret) -+ panic("qman_alloc_fqid() failed"); -+ hp_cpu->iterator->fqid_tx = fqid; -+ /* Rx mixer is the previous handler's Tx mixer */ -+ hp_cpu->iterator->rx_mixer = lfsr; -+ /* Get new mixer for Tx */ -+ lfsr = do_lfsr(lfsr); -+ hp_cpu->iterator->tx_mixer = lfsr; -+ } -+ } -+ /* Fix up the first handler (fqid_rx==0, rx_mixer=0xdeadbeef) */ -+ hp_cpu = list_first_entry(&hp_cpu_list, struct hp_cpu, node); -+ handler = list_first_entry(&hp_cpu->handlers, struct hp_handler, node); -+ BUG_ON((handler->fqid_rx != 0) || (handler->rx_mixer != 0xdeadbeef)); -+ handler->fqid_rx = fqid; -+ handler->rx_mixer = lfsr; -+ /* and tag it as our "special" handler */ -+ special_handler = handler; -+} -+ -+static void init_phase3(void) -+{ -+ int loop; -+ struct hp_cpu *hp_cpu; -+ -+ for (loop = 0; loop < HP_PER_CPU; loop++) { -+ list_for_each_entry(hp_cpu, &hp_cpu_list, node) { -+ if (!loop) -+ hp_cpu->iterator = list_first_entry( -+ &hp_cpu->handlers, -+ struct hp_handler, node); -+ else -+ hp_cpu->iterator = list_entry( -+ hp_cpu->iterator->node.next, -+ struct hp_handler, node); -+ preempt_disable(); -+ if (hp_cpu->processor_id == smp_processor_id()) -+ init_handler(hp_cpu->iterator); -+ else -+ smp_call_function_single(hp_cpu->processor_id, -+ init_handler, hp_cpu->iterator, 1); -+ preempt_enable(); -+ } -+ } -+} -+ -+static void send_first_frame(void *ignore) -+{ -+ u32 *p = special_handler->frame_ptr; -+ u32 lfsr = HP_FIRST_WORD; -+ int loop; -+ struct qm_fd fd; -+ -+ BUG_ON(special_handler->processor_id != smp_processor_id()); -+ memset(&fd, 0, sizeof(fd)); -+ qm_fd_addr_set64(&fd, special_handler->addr); -+ fd.format = qm_fd_contig_big; -+ fd.length29 = HP_NUM_WORDS * 4; -+ for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) { -+ if (*p != lfsr) -+ panic("corrupt frame data"); -+ *p ^= special_handler->tx_mixer; -+ lfsr = do_lfsr(lfsr); -+ } -+ pr_info("Sending first frame\n"); -+ if (qman_enqueue(&special_handler->tx, &fd, 0)) -+ panic("qman_enqueue() failed"); -+} -+ -+void qman_test_hotpotato(void) -+{ -+ if (cpumask_weight(cpu_online_mask) < 2) { -+ pr_info("qman_test_hotpotato, skip - only 1 CPU\n"); -+ return; -+ } -+ -+ pr_info("qman_test_hotpotato starting\n"); -+ -+ hp_cpu_list_length = 0; -+ loop_counter = 0; -+ hp_handler_slab = kmem_cache_create("hp_handler_slab", -+ sizeof(struct hp_handler), L1_CACHE_BYTES, -+ SLAB_HWCACHE_ALIGN, NULL); -+ if (!hp_handler_slab) -+ panic("kmem_cache_create() failed"); -+ -+ allocate_frame_data(); -+ -+ /* Init phase 1 */ -+ pr_info("Creating %d handlers per cpu...\n", HP_PER_CPU); -+ if (on_all_cpus(create_per_cpu_handlers)) -+ panic("on_each_cpu() failed"); -+ pr_info("Number of cpus: %d, total of %d handlers\n", -+ hp_cpu_list_length, hp_cpu_list_length * HP_PER_CPU); -+ -+ init_phase2(); -+ -+ init_phase3(); -+ -+ preempt_disable(); -+ if (special_handler->processor_id == smp_processor_id()) -+ send_first_frame(NULL); -+ else -+ smp_call_function_single(special_handler->processor_id, -+ send_first_frame, NULL, 1); -+ preempt_enable(); -+ -+ wait_event(queue, loop_counter == HP_LOOPS); -+ deallocate_frame_data(); -+ if (on_all_cpus(destroy_per_cpu_handlers)) -+ panic("on_each_cpu() failed"); -+ kmem_cache_destroy(hp_handler_slab); -+ pr_info("qman_test_hotpotato finished\n"); -+} ---- /dev/null -+++ b/drivers/staging/fsl_qbman/qman_utility.c -@@ -0,0 +1,129 @@ -+/* Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "qman_private.h" -+ -+/* ----------------- */ -+/* --- FQID Pool --- */ -+ -+struct qman_fqid_pool { -+ /* Base and size of the FQID range */ -+ u32 fqid_base; -+ u32 total; -+ /* Number of FQIDs currently "allocated" */ -+ u32 used; -+ /* Allocation optimisation. When 'used<total', it is the index of an -+ * available FQID. Otherwise there are no available FQIDs, and this -+ * will be set when the next deallocation occurs. */ -+ u32 next; -+ /* A bit-field representation of the FQID range. */ -+ unsigned long *bits; -+}; -+ -+#define QLONG_BYTES sizeof(unsigned long) -+#define QLONG_BITS (QLONG_BYTES * 8) -+/* Number of 'longs' required for the given number of bits */ -+#define QNUM_LONGS(b) (((b) + QLONG_BITS - 1) / QLONG_BITS) -+/* Shorthand for the number of bytes of same (kmalloc, memset, etc) */ -+#define QNUM_BYTES(b) (QNUM_LONGS(b) * QLONG_BYTES) -+/* And in bits */ -+#define QNUM_BITS(b) (QNUM_LONGS(b) * QLONG_BITS) -+ -+struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num) -+{ -+ struct qman_fqid_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); -+ unsigned int i; -+ -+ BUG_ON(!num); -+ if (!pool) -+ return NULL; -+ pool->fqid_base = fqid_start; -+ pool->total = num; -+ pool->used = 0; -+ pool->next = 0; -+ pool->bits = kzalloc(QNUM_BYTES(num), GFP_KERNEL); -+ if (!pool->bits) { -+ kfree(pool); -+ return NULL; -+ } -+ /* If num is not an even multiple of QLONG_BITS (or even 8, for -+ * byte-oriented searching) then we fill the trailing bits with 1, to -+ * make them look allocated (permanently). */ -+ for (i = num + 1; i < QNUM_BITS(num); i++) -+ set_bit(i, pool->bits); -+ return pool; -+} -+EXPORT_SYMBOL(qman_fqid_pool_create); -+ -+int qman_fqid_pool_destroy(struct qman_fqid_pool *pool) -+{ -+ int ret = pool->used; -+ kfree(pool->bits); -+ kfree(pool); -+ return ret; -+} -+EXPORT_SYMBOL(qman_fqid_pool_destroy); -+ -+int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid) -+{ -+ int ret; -+ if (pool->used == pool->total) -+ return -ENOMEM; -+ *fqid = pool->fqid_base + pool->next; -+ ret = test_and_set_bit(pool->next, pool->bits); -+ BUG_ON(ret); -+ if (++pool->used == pool->total) -+ return 0; -+ pool->next = find_next_zero_bit(pool->bits, pool->total, pool->next); -+ if (pool->next >= pool->total) -+ pool->next = find_first_zero_bit(pool->bits, pool->total); -+ BUG_ON(pool->next >= pool->total); -+ return 0; -+} -+EXPORT_SYMBOL(qman_fqid_pool_alloc); -+ -+void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid) -+{ -+ int ret; -+ -+ fqid -= pool->fqid_base; -+ ret = test_and_clear_bit(fqid, pool->bits); -+ BUG_ON(!ret); -+ if (pool->used-- == pool->total) -+ pool->next = fqid; -+} -+EXPORT_SYMBOL(qman_fqid_pool_free); -+ -+u32 qman_fqid_pool_used(struct qman_fqid_pool *pool) -+{ -+ return pool->used; -+} -+EXPORT_SYMBOL(qman_fqid_pool_used); ---- /dev/null -+++ b/include/linux/fsl_bman.h -@@ -0,0 +1,532 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef FSL_BMAN_H -+#define FSL_BMAN_H -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/* Last updated for v00.79 of the BG */ -+ -+/* Portal processing (interrupt) sources */ -+#define BM_PIRQ_RCRI 0x00000002 /* RCR Ring (below threshold) */ -+#define BM_PIRQ_BSCN 0x00000001 /* Buffer depletion State Change */ -+ -+/* This wrapper represents a bit-array for the depletion state of the 64 Bman -+ * buffer pools. */ -+struct bman_depletion { -+ u32 __state[2]; -+}; -+#define BMAN_DEPLETION_EMPTY { { 0x00000000, 0x00000000 } } -+#define BMAN_DEPLETION_FULL { { 0xffffffff, 0xffffffff } } -+#define __bmdep_word(x) ((x) >> 5) -+#define __bmdep_shift(x) ((x) & 0x1f) -+#define __bmdep_bit(x) (0x80000000 >> __bmdep_shift(x)) -+static inline void bman_depletion_init(struct bman_depletion *c) -+{ -+ c->__state[0] = c->__state[1] = 0; -+} -+static inline void bman_depletion_fill(struct bman_depletion *c) -+{ -+ c->__state[0] = c->__state[1] = ~0; -+} -+static inline int bman_depletion_get(const struct bman_depletion *c, u8 bpid) -+{ -+ return c->__state[__bmdep_word(bpid)] & __bmdep_bit(bpid); -+} -+static inline void bman_depletion_set(struct bman_depletion *c, u8 bpid) -+{ -+ c->__state[__bmdep_word(bpid)] |= __bmdep_bit(bpid); -+} -+static inline void bman_depletion_unset(struct bman_depletion *c, u8 bpid) -+{ -+ c->__state[__bmdep_word(bpid)] &= ~__bmdep_bit(bpid); -+} -+ -+/* ------------------------------------------------------- */ -+/* --- Bman data structures (and associated constants) --- */ -+ -+/* Represents s/w corenet portal mapped data structures */ -+struct bm_rcr_entry; /* RCR (Release Command Ring) entries */ -+struct bm_mc_command; /* MC (Management Command) command */ -+struct bm_mc_result; /* MC result */ -+ -+/* Code-reduction, define a wrapper for 48-bit buffers. In cases where a buffer -+ * pool id specific to this buffer is needed (BM_RCR_VERB_CMD_BPID_MULTI, -+ * BM_MCC_VERB_ACQUIRE), the 'bpid' field is used. */ -+struct bm_buffer { -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 __reserved1; -+ u8 bpid; -+ u16 hi; /* High 16-bits of 48-bit address */ -+ u32 lo; /* Low 32-bits of 48-bit address */ -+#else -+ u32 lo; -+ u16 hi; -+ u8 bpid; -+ u8 __reserved; -+#endif -+ }; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u64 __notaddress:16; -+ u64 addr:48; -+#else -+ u64 addr:48; -+ u64 __notaddress:16; -+#endif -+ }; -+ u64 opaque; -+ }; -+} __aligned(8); -+static inline u64 bm_buffer_get64(const struct bm_buffer *buf) -+{ -+ return buf->addr; -+} -+static inline dma_addr_t bm_buf_addr(const struct bm_buffer *buf) -+{ -+ return (dma_addr_t)buf->addr; -+} -+/* Macro, so we compile better if 'v' isn't always 64-bit */ -+#define bm_buffer_set64(buf, v) \ -+ do { \ -+ struct bm_buffer *__buf931 = (buf); \ -+ __buf931->hi = upper_32_bits(v); \ -+ __buf931->lo = lower_32_bits(v); \ -+ } while (0) -+ -+/* See 1.5.3.5.4: "Release Command" */ -+struct bm_rcr_entry { -+ union { -+ struct { -+ u8 __dont_write_directly__verb; -+ u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */ -+ u8 __reserved1[62]; -+ }; -+ struct bm_buffer bufs[8]; -+ }; -+} __packed; -+#define BM_RCR_VERB_VBIT 0x80 -+#define BM_RCR_VERB_CMD_MASK 0x70 /* one of two values; */ -+#define BM_RCR_VERB_CMD_BPID_SINGLE 0x20 -+#define BM_RCR_VERB_CMD_BPID_MULTI 0x30 -+#define BM_RCR_VERB_BUFCOUNT_MASK 0x0f /* values 1..8 */ -+ -+/* See 1.5.3.1: "Acquire Command" */ -+/* See 1.5.3.2: "Query Command" */ -+struct bm_mcc_acquire { -+ u8 bpid; -+ u8 __reserved1[62]; -+} __packed; -+struct bm_mcc_query { -+ u8 __reserved2[63]; -+} __packed; -+struct bm_mc_command { -+ u8 __dont_write_directly__verb; -+ union { -+ struct bm_mcc_acquire acquire; -+ struct bm_mcc_query query; -+ }; -+} __packed; -+#define BM_MCC_VERB_VBIT 0x80 -+#define BM_MCC_VERB_CMD_MASK 0x70 /* where the verb contains; */ -+#define BM_MCC_VERB_CMD_ACQUIRE 0x10 -+#define BM_MCC_VERB_CMD_QUERY 0x40 -+#define BM_MCC_VERB_ACQUIRE_BUFCOUNT 0x0f /* values 1..8 go here */ -+ -+/* See 1.5.3.3: "Acquire Response" */ -+/* See 1.5.3.4: "Query Response" */ -+struct bm_pool_state { -+ u8 __reserved1[32]; -+ /* "availability state" and "depletion state" */ -+ struct { -+ u8 __reserved1[8]; -+ /* Access using bman_depletion_***() */ -+ struct bman_depletion state; -+ } as, ds; -+}; -+struct bm_mc_result { -+ union { -+ struct { -+ u8 verb; -+ u8 __reserved1[63]; -+ }; -+ union { -+ struct { -+ u8 __reserved1; -+ u8 bpid; -+ u8 __reserved2[62]; -+ }; -+ struct bm_buffer bufs[8]; -+ } acquire; -+ struct bm_pool_state query; -+ }; -+} __packed; -+#define BM_MCR_VERB_VBIT 0x80 -+#define BM_MCR_VERB_CMD_MASK BM_MCC_VERB_CMD_MASK -+#define BM_MCR_VERB_CMD_ACQUIRE BM_MCC_VERB_CMD_ACQUIRE -+#define BM_MCR_VERB_CMD_QUERY BM_MCC_VERB_CMD_QUERY -+#define BM_MCR_VERB_CMD_ERR_INVALID 0x60 -+#define BM_MCR_VERB_CMD_ERR_ECC 0x70 -+#define BM_MCR_VERB_ACQUIRE_BUFCOUNT BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */ -+/* Determine the "availability state" of pool 'p' from a query result 'r' */ -+#define BM_MCR_QUERY_AVAILABILITY(r, p) \ -+ bman_depletion_get(&r->query.as.state, p) -+/* Determine the "depletion state" of pool 'p' from a query result 'r' */ -+#define BM_MCR_QUERY_DEPLETION(r, p) \ -+ bman_depletion_get(&r->query.ds.state, p) -+ -+/*******************************************************************/ -+/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */ -+/*******************************************************************/ -+ -+ /* Portal and Buffer Pools */ -+ /* ----------------------- */ -+/* Represents a managed portal */ -+struct bman_portal; -+ -+/* This object type represents Bman buffer pools. */ -+struct bman_pool; -+ -+struct bman_portal_config { -+ /* This is used for any "core-affine" portals, ie. default portals -+ * associated to the corresponding cpu. -1 implies that there is no core -+ * affinity configured. */ -+ int cpu; -+ /* portal interrupt line */ -+ int irq; -+ /* the unique index of this portal */ -+ u32 index; -+ /* Is this portal shared? (If so, it has coarser locking and demuxes -+ * processing on behalf of other CPUs.) */ -+ int is_shared; -+ /* These are the buffer pool IDs that may be used via this portal. */ -+ struct bman_depletion mask; -+}; -+ -+/* This callback type is used when handling pool depletion entry/exit. The -+ * 'cb_ctx' value is the opaque value associated with the pool object in -+ * bman_new_pool(). 'depleted' is non-zero on depletion-entry, and zero on -+ * depletion-exit. */ -+typedef void (*bman_cb_depletion)(struct bman_portal *bm, -+ struct bman_pool *pool, void *cb_ctx, int depleted); -+ -+/* This struct specifies parameters for a bman_pool object. */ -+struct bman_pool_params { -+ /* index of the buffer pool to encapsulate (0-63), ignored if -+ * BMAN_POOL_FLAG_DYNAMIC_BPID is set. */ -+ u32 bpid; -+ /* bit-mask of BMAN_POOL_FLAG_*** options */ -+ u32 flags; -+ /* depletion-entry/exit callback, if BMAN_POOL_FLAG_DEPLETION is set */ -+ bman_cb_depletion cb; -+ /* opaque user value passed as a parameter to 'cb' */ -+ void *cb_ctx; -+ /* depletion-entry/exit thresholds, if BMAN_POOL_FLAG_THRESH is set. NB: -+ * this is only allowed if BMAN_POOL_FLAG_DYNAMIC_BPID is used *and* -+ * when run in the control plane (which controls Bman CCSR). This array -+ * matches the definition of bm_pool_set(). */ -+ u32 thresholds[4]; -+}; -+ -+/* Flags to bman_new_pool() */ -+#define BMAN_POOL_FLAG_NO_RELEASE 0x00000001 /* can't release to pool */ -+#define BMAN_POOL_FLAG_ONLY_RELEASE 0x00000002 /* can only release to pool */ -+#define BMAN_POOL_FLAG_DEPLETION 0x00000004 /* track depletion entry/exit */ -+#define BMAN_POOL_FLAG_DYNAMIC_BPID 0x00000008 /* (de)allocate bpid */ -+#define BMAN_POOL_FLAG_THRESH 0x00000010 /* set depletion thresholds */ -+#define BMAN_POOL_FLAG_STOCKPILE 0x00000020 /* stockpile to reduce hw ops */ -+ -+/* Flags to bman_release() */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+#define BMAN_RELEASE_FLAG_WAIT 0x00000001 /* wait if RCR is full */ -+#define BMAN_RELEASE_FLAG_WAIT_INT 0x00000002 /* if we wait, interruptible? */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+#define BMAN_RELEASE_FLAG_WAIT_SYNC 0x00000004 /* if wait, until consumed? */ -+#endif -+#endif -+#define BMAN_RELEASE_FLAG_NOW 0x00000008 /* issue immediate release */ -+ -+/* Flags to bman_acquire() */ -+#define BMAN_ACQUIRE_FLAG_STOCKPILE 0x00000001 /* no hw op, stockpile only */ -+ -+ /* Portal Management */ -+ /* ----------------- */ -+/** -+ * bman_get_portal_config - get portal configuration settings -+ * -+ * This returns a read-only view of the current cpu's affine portal settings. -+ */ -+const struct bman_portal_config *bman_get_portal_config(void); -+ -+/** -+ * bman_irqsource_get - return the portal work that is interrupt-driven -+ * -+ * Returns a bitmask of BM_PIRQ_**I processing sources that are currently -+ * enabled for interrupt handling on the current cpu's affine portal. These -+ * sources will trigger the portal interrupt and the interrupt handler (or a -+ * tasklet/bottom-half it defers to) will perform the corresponding processing -+ * work. The bman_poll_***() functions will only process sources that are not in -+ * this bitmask. If the current CPU is sharing a portal hosted on another CPU, -+ * this always returns zero. -+ */ -+u32 bman_irqsource_get(void); -+ -+/** -+ * bman_irqsource_add - add processing sources to be interrupt-driven -+ * @bits: bitmask of BM_PIRQ_**I processing sources -+ * -+ * Adds processing sources that should be interrupt-driven (rather than -+ * processed via bman_poll_***() functions). Returns zero for success, or -+ * -EINVAL if the current CPU is sharing a portal hosted on another CPU. */ -+int bman_irqsource_add(u32 bits); -+ -+/** -+ * bman_irqsource_remove - remove processing sources from being interrupt-driven -+ * @bits: bitmask of BM_PIRQ_**I processing sources -+ * -+ * Removes processing sources from being interrupt-driven, so that they will -+ * instead be processed via bman_poll_***() functions. Returns zero for success, -+ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU. */ -+int bman_irqsource_remove(u32 bits); -+ -+/** -+ * bman_affine_cpus - return a mask of cpus that have affine portals -+ */ -+const cpumask_t *bman_affine_cpus(void); -+ -+/** -+ * bman_poll_slow - process anything that isn't interrupt-driven. -+ * -+ * This function does any portal processing that isn't interrupt-driven. If the -+ * current CPU is sharing a portal hosted on another CPU, this function will -+ * return -EINVAL, otherwise the return value is a bitmask of BM_PIRQ_* sources -+ * indicating what interrupt sources were actually processed by the call. -+ * -+ * NB, unlike the legacy wrapper bman_poll(), this function will -+ * deterministically check for the presence of portal processing work and do it, -+ * which implies some latency even if there's nothing to do. The bman_poll() -+ * wrapper on the other hand (like the qman_poll() wrapper) attenuates this by -+ * checking for (and doing) portal processing infrequently. Ie. such that -+ * qman_poll() and bman_poll() can be called from core-processing loops. Use -+ * bman_poll_slow() when you yourself are deciding when to incur the overhead of -+ * processing. -+ */ -+u32 bman_poll_slow(void); -+ -+/** -+ * bman_poll - process anything that isn't interrupt-driven. -+ * -+ * Dispatcher logic on a cpu can use this to trigger any maintenance of the -+ * affine portal. This function does whatever processing is not triggered by -+ * interrupts. This is a legacy wrapper that can be used in core-processing -+ * loops but mitigates the performance overhead of portal processing by -+ * adaptively bypassing true portal processing most of the time. (Processing is -+ * done once every 10 calls if the previous processing revealed that work needed -+ * to be done, or once very 1000 calls if the previous processing revealed no -+ * work needed doing.) If you wish to control this yourself, call -+ * bman_poll_slow() instead, which always checks for portal processing work. -+ */ -+void bman_poll(void); -+ -+/** -+ * bman_rcr_is_empty - Determine if portal's RCR is empty -+ * -+ * For use in situations where a cpu-affine caller needs to determine when all -+ * releases for the local portal have been processed by Bman but can't use the -+ * BMAN_RELEASE_FLAG_WAIT_SYNC flag to do this from the final bman_release(). -+ * The function forces tracking of RCR consumption (which normally doesn't -+ * happen until release processing needs to find space to put new release -+ * commands), and returns zero if the ring still has unprocessed entries, -+ * non-zero if it is empty. -+ */ -+int bman_rcr_is_empty(void); -+ -+/** -+ * bman_alloc_bpid_range - Allocate a contiguous range of BPIDs -+ * @result: is set by the API to the base BPID of the allocated range -+ * @count: the number of BPIDs required -+ * @align: required alignment of the allocated range -+ * @partial: non-zero if the API can return fewer than @count BPIDs -+ * -+ * Returns the number of buffer pools allocated, or a negative error code. If -+ * @partial is non zero, the allocation request may return a smaller range of -+ * BPs than requested (though alignment will be as requested). If @partial is -+ * zero, the return value will either be 'count' or negative. -+ */ -+int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial); -+static inline int bman_alloc_bpid(u32 *result) -+{ -+ int ret = bman_alloc_bpid_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+ -+/** -+ * bman_release_bpid_range - Release the specified range of buffer pool IDs -+ * @bpid: the base BPID of the range to deallocate -+ * @count: the number of BPIDs in the range -+ * -+ * This function can also be used to seed the allocator with ranges of BPIDs -+ * that it can subsequently allocate from. -+ */ -+void bman_release_bpid_range(u32 bpid, unsigned int count); -+static inline void bman_release_bpid(u32 bpid) -+{ -+ bman_release_bpid_range(bpid, 1); -+} -+ -+int bman_reserve_bpid_range(u32 bpid, unsigned int count); -+static inline int bman_reserve_bpid(u32 bpid) -+{ -+ return bman_reserve_bpid_range(bpid, 1); -+} -+ -+void bman_seed_bpid_range(u32 bpid, unsigned int count); -+ -+ -+int bman_shutdown_pool(u32 bpid); -+ -+ /* Pool management */ -+ /* --------------- */ -+/** -+ * bman_new_pool - Allocates a Buffer Pool object -+ * @params: parameters specifying the buffer pool ID and behaviour -+ * -+ * Creates a pool object for the given @params. A portal and the depletion -+ * callback field of @params are only used if the BMAN_POOL_FLAG_DEPLETION flag -+ * is set. NB, the fields from @params are copied into the new pool object, so -+ * the structure provided by the caller can be released or reused after the -+ * function returns. -+ */ -+struct bman_pool *bman_new_pool(const struct bman_pool_params *params); -+ -+/** -+ * bman_free_pool - Deallocates a Buffer Pool object -+ * @pool: the pool object to release -+ * -+ */ -+void bman_free_pool(struct bman_pool *pool); -+ -+/** -+ * bman_get_params - Returns a pool object's parameters. -+ * @pool: the pool object -+ * -+ * The returned pointer refers to state within the pool object so must not be -+ * modified and can no longer be read once the pool object is destroyed. -+ */ -+const struct bman_pool_params *bman_get_params(const struct bman_pool *pool); -+ -+/** -+ * bman_release - Release buffer(s) to the buffer pool -+ * @pool: the buffer pool object to release to -+ * @bufs: an array of buffers to release -+ * @num: the number of buffers in @bufs (1-8) -+ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options -+ * -+ * Adds the given buffers to RCR entries. If the portal @p was created with the -+ * "COMPACT" flag, then it will be using a compaction algorithm to improve -+ * utilisation of RCR. As such, these buffers may join an existing ring entry -+ * and/or it may not be issued right away so as to allow future releases to join -+ * the same ring entry. Use the BMAN_RELEASE_FLAG_NOW flag to override this -+ * behaviour by committing the RCR entry (or entries) right away. If the RCR -+ * ring is full, the function will return -EBUSY unless BMAN_RELEASE_FLAG_WAIT -+ * is selected, in which case it will sleep waiting for space to become -+ * available in RCR. If the function receives a signal before such time (and -+ * BMAN_RELEASE_FLAG_WAIT_INT is set), the function returns -EINTR. Otherwise, -+ * it returns zero. -+ */ -+int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num, -+ u32 flags); -+ -+/** -+ * bman_acquire - Acquire buffer(s) from a buffer pool -+ * @pool: the buffer pool object to acquire from -+ * @bufs: array for storing the acquired buffers -+ * @num: the number of buffers desired (@bufs is at least this big) -+ * -+ * Issues an "Acquire" command via the portal's management command interface. -+ * The return value will be the number of buffers obtained from the pool, or a -+ * negative error code if a h/w error or pool starvation was encountered. In -+ * the latter case, the content of @bufs is undefined. -+ */ -+int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num, -+ u32 flags); -+ -+/** -+ * bman_flush_stockpile - Flush stockpile buffer(s) to the buffer pool -+ * @pool: the buffer pool object the stockpile belongs -+ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options -+ * -+ * Adds stockpile buffers to RCR entries until the stockpile is empty. -+ * The return value will be a negative error code if a h/w error occurred. -+ * If BMAN_RELEASE_FLAG_NOW flag is passed and RCR ring is full, -+ * -EAGAIN will be returned. -+ */ -+int bman_flush_stockpile(struct bman_pool *pool, u32 flags); -+ -+/** -+ * bman_query_pools - Query all buffer pool states -+ * @state: storage for the queried availability and depletion states -+ */ -+int bman_query_pools(struct bm_pool_state *state); -+ -+#ifdef CONFIG_FSL_BMAN_CONFIG -+/** -+ * bman_query_free_buffers - Query how many free buffers are in buffer pool -+ * @pool: the buffer pool object to query -+ * -+ * Return the number of the free buffers -+ */ -+u32 bman_query_free_buffers(struct bman_pool *pool); -+ -+/** -+ * bman_update_pool_thresholds - Change the buffer pool's depletion thresholds -+ * @pool: the buffer pool object to which the thresholds will be set -+ * @thresholds: the new thresholds -+ */ -+int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds); -+#endif -+ -+/** -+ * The below bman_p_***() variant might be called in a situation that the cpu -+ * which the portal affine to is not online yet. -+ * @bman_portal specifies which portal the API will use. -+*/ -+int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits); -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* FSL_BMAN_H */ ---- /dev/null -+++ b/include/linux/fsl_qman.h -@@ -0,0 +1,3888 @@ -+/* Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef FSL_QMAN_H -+#define FSL_QMAN_H -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+/* Last updated for v00.800 of the BG */ -+ -+/* Hardware constants */ -+#define QM_CHANNEL_SWPORTAL0 0 -+#define QMAN_CHANNEL_POOL1 0x21 -+#define QMAN_CHANNEL_CAAM 0x80 -+#define QMAN_CHANNEL_PME 0xa0 -+#define QMAN_CHANNEL_POOL1_REV3 0x401 -+#define QMAN_CHANNEL_CAAM_REV3 0x840 -+#define QMAN_CHANNEL_PME_REV3 0x860 -+#define QMAN_CHANNEL_DCE 0x8a0 -+#define QMAN_CHANNEL_DCE_QMANREV312 0x880 -+extern u16 qm_channel_pool1; -+extern u16 qm_channel_caam; -+extern u16 qm_channel_pme; -+extern u16 qm_channel_dce; -+enum qm_dc_portal { -+ qm_dc_portal_fman0 = 0, -+ qm_dc_portal_fman1 = 1, -+ qm_dc_portal_caam = 2, -+ qm_dc_portal_pme = 3, -+ qm_dc_portal_rman = 4, -+ qm_dc_portal_dce = 5 -+}; -+ -+/* Portal processing (interrupt) sources */ -+#define QM_PIRQ_CCSCI 0x00200000 /* CEETM Congestion State Change */ -+#define QM_PIRQ_CSCI 0x00100000 /* Congestion State Change */ -+#define QM_PIRQ_EQCI 0x00080000 /* Enqueue Command Committed */ -+#define QM_PIRQ_EQRI 0x00040000 /* EQCR Ring (below threshold) */ -+#define QM_PIRQ_DQRI 0x00020000 /* DQRR Ring (non-empty) */ -+#define QM_PIRQ_MRI 0x00010000 /* MR Ring (non-empty) */ -+/* This mask contains all the interrupt sources that need handling except DQRI, -+ * ie. that if present should trigger slow-path processing. */ -+#define QM_PIRQ_SLOW (QM_PIRQ_CSCI | QM_PIRQ_EQCI | QM_PIRQ_EQRI | \ -+ QM_PIRQ_MRI | QM_PIRQ_CCSCI) -+ -+/* --- Clock speed --- */ -+/* A qman driver instance may or may not know the current qman clock speed. -+ * However, certain CEETM calculations may not be possible if this is not known. -+ * The 'set' function will only succeed (return zero) if the driver did not -+ * already know the clock speed. Likewise, the 'get' function will only succeed -+ * if the driver does know the clock speed (either because it knew when booting, -+ * or was told via 'set'). In cases where software is running on a driver -+ * instance that does not know the clock speed (eg. on a hypervised data-plane), -+ * and the user can obtain the current qman clock speed by other means (eg. from -+ * a message sent from the control-plane), then the 'set' function can be used -+ * to enable rate-calculations in a driver where it would otherwise not be -+ * possible. */ -+int qm_get_clock(u64 *clock_hz); -+int qm_set_clock(u64 clock_hz); -+ -+/* For qman_static_dequeue_*** APIs */ -+#define QM_SDQCR_CHANNELS_POOL_MASK 0x00007fff -+/* for n in [1,15] */ -+#define QM_SDQCR_CHANNELS_POOL(n) (0x00008000 >> (n)) -+/* for conversion from n of qm_channel */ -+static inline u32 QM_SDQCR_CHANNELS_POOL_CONV(u16 channel) -+{ -+ return QM_SDQCR_CHANNELS_POOL(channel + 1 - qm_channel_pool1); -+} -+ -+/* For qman_volatile_dequeue(); Choose one PRECEDENCE. EXACT is optional. Use -+ * NUMFRAMES(n) (6-bit) or NUMFRAMES_TILLEMPTY to fill in the frame-count. Use -+ * FQID(n) to fill in the frame queue ID. */ -+#define QM_VDQCR_PRECEDENCE_VDQCR 0x0 -+#define QM_VDQCR_PRECEDENCE_SDQCR 0x80000000 -+#define QM_VDQCR_EXACT 0x40000000 -+#define QM_VDQCR_NUMFRAMES_MASK 0x3f000000 -+#define QM_VDQCR_NUMFRAMES_SET(n) (((n) & 0x3f) << 24) -+#define QM_VDQCR_NUMFRAMES_GET(n) (((n) >> 24) & 0x3f) -+#define QM_VDQCR_NUMFRAMES_TILLEMPTY QM_VDQCR_NUMFRAMES_SET(0) -+ -+ -+/* ------------------------------------------------------- */ -+/* --- Qman data structures (and associated constants) --- */ -+ -+/* Represents s/w corenet portal mapped data structures */ -+struct qm_eqcr_entry; /* EQCR (EnQueue Command Ring) entries */ -+struct qm_dqrr_entry; /* DQRR (DeQueue Response Ring) entries */ -+struct qm_mr_entry; /* MR (Message Ring) entries */ -+struct qm_mc_command; /* MC (Management Command) command */ -+struct qm_mc_result; /* MC result */ -+ -+/* See David Lapp's "Frame formats" document, "dpateam", Jan 07, 2008 */ -+#define QM_FD_FORMAT_SG 0x4 -+#define QM_FD_FORMAT_LONG 0x2 -+#define QM_FD_FORMAT_COMPOUND 0x1 -+enum qm_fd_format { -+ /* 'contig' implies a contiguous buffer, whereas 'sg' implies a -+ * scatter-gather table. 'big' implies a 29-bit length with no offset -+ * field, otherwise length is 20-bit and offset is 9-bit. 'compound' -+ * implies a s/g-like table, where each entry itself represents a frame -+ * (contiguous or scatter-gather) and the 29-bit "length" is -+ * interpreted purely for congestion calculations, ie. a "congestion -+ * weight". */ -+ qm_fd_contig = 0, -+ qm_fd_contig_big = QM_FD_FORMAT_LONG, -+ qm_fd_sg = QM_FD_FORMAT_SG, -+ qm_fd_sg_big = QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG, -+ qm_fd_compound = QM_FD_FORMAT_COMPOUND -+}; -+ -+/* Capitalised versions are un-typed but can be used in static expressions */ -+#define QM_FD_CONTIG 0 -+#define QM_FD_CONTIG_BIG QM_FD_FORMAT_LONG -+#define QM_FD_SG QM_FD_FORMAT_SG -+#define QM_FD_SG_BIG (QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG) -+#define QM_FD_COMPOUND QM_FD_FORMAT_COMPOUND -+ -+/* See 1.5.1.1: "Frame Descriptor (FD)" */ -+struct qm_fd { -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 dd:2; /* dynamic debug */ -+ u8 liodn_offset:6; -+ u8 bpid:8; /* Buffer Pool ID */ -+ u8 eliodn_offset:4; -+ u8 __reserved:4; -+ u8 addr_hi; /* high 8-bits of 40-bit address */ -+ u32 addr_lo; /* low 32-bits of 40-bit address */ -+#else -+ u32 addr_lo; /* low 32-bits of 40-bit address */ -+ u8 addr_hi; /* high 8-bits of 40-bit address */ -+ u8 __reserved:4; -+ u8 eliodn_offset:4; -+ u8 bpid:8; /* Buffer Pool ID */ -+ u8 liodn_offset:6; -+ u8 dd:2; /* dynamic debug */ -+#endif -+ }; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u64 __notaddress:24; -+ u64 addr:40; -+#else -+ u64 addr:40; -+ u64 __notaddress:24; -+#endif -+ }; -+ u64 opaque_addr; -+ }; -+ /* The 'format' field indicates the interpretation of the remaining 29 -+ * bits of the 32-bit word. For packing reasons, it is duplicated in the -+ * other union elements. Note, union'd structs are difficult to use with -+ * static initialisation under gcc, in which case use the "opaque" form -+ * with one of the macros. */ -+ union { -+ /* For easier/faster copying of this part of the fd (eg. from a -+ * DQRR entry to an EQCR entry) copy 'opaque' */ -+ u32 opaque; -+ /* If 'format' is _contig or _sg, 20b length and 9b offset */ -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ enum qm_fd_format format:3; -+ u16 offset:9; -+ u32 length20:20; -+#else -+ u32 length20:20; -+ u16 offset:9; -+ enum qm_fd_format format:3; -+#endif -+ }; -+ /* If 'format' is _contig_big or _sg_big, 29b length */ -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ enum qm_fd_format _format1:3; -+ u32 length29:29; -+#else -+ u32 length29:29; -+ enum qm_fd_format _format1:3; -+#endif -+ }; -+ /* If 'format' is _compound, 29b "congestion weight" */ -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ enum qm_fd_format _format2:3; -+ u32 cong_weight:29; -+#else -+ u32 cong_weight:29; -+ enum qm_fd_format _format2:3; -+#endif -+ }; -+ }; -+ union { -+ u32 cmd; -+ u32 status; -+ }; -+} __aligned(8); -+#define QM_FD_DD_NULL 0x00 -+#define QM_FD_PID_MASK 0x3f -+static inline u64 qm_fd_addr_get64(const struct qm_fd *fd) -+{ -+ return fd->addr; -+} -+ -+static inline dma_addr_t qm_fd_addr(const struct qm_fd *fd) -+{ -+ return (dma_addr_t)fd->addr; -+} -+/* Macro, so we compile better if 'v' isn't always 64-bit */ -+#define qm_fd_addr_set64(fd, v) \ -+ do { \ -+ struct qm_fd *__fd931 = (fd); \ -+ __fd931->addr = v; \ -+ } while (0) -+ -+/* For static initialisation of FDs (which is complicated by the use of unions -+ * in "struct qm_fd"), use the following macros. Note that; -+ * - 'dd', 'pid' and 'bpid' are ignored because there's no static initialisation -+ * use-case), -+ * - use capitalised QM_FD_*** formats for static initialisation. -+ */ -+#define QM_FD_FMT_20(cmd, addr_hi, addr_lo, fmt, off, len) \ -+ { 0, 0, 0, 0, 0, addr_hi, addr_lo, \ -+ { (((fmt)&0x7) << 29) | (((off)&0x1ff) << 20) | ((len)&0xfffff) }, \ -+ { cmd } } -+#define QM_FD_FMT_29(cmd, addr_hi, addr_lo, fmt, len) \ -+ { 0, 0, 0, 0, 0, addr_hi, addr_lo, \ -+ { (((fmt)&0x7) << 29) | ((len)&0x1fffffff) }, \ -+ { cmd } } -+ -+/* See 2.2.1.3 Multi-Core Datapath Acceleration Architecture */ -+#define QM_SG_OFFSET_MASK 0x1FFF -+struct qm_sg_entry { -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 __reserved1[3]; -+ u8 addr_hi; /* high 8-bits of 40-bit address */ -+ u32 addr_lo; /* low 32-bits of 40-bit address */ -+#else -+ u32 addr_lo; /* low 32-bits of 40-bit address */ -+ u8 addr_hi; /* high 8-bits of 40-bit address */ -+ u8 __reserved1[3]; -+#endif -+ }; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u64 __notaddress:24; -+ u64 addr:40; -+#else -+ u64 addr:40; -+ u64 __notaddress:24; -+#endif -+ }; -+ u64 opaque; -+ }; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 extension:1; /* Extension bit */ -+ u32 final:1; /* Final bit */ -+ u32 length:30; -+#else -+ u32 length:30; -+ u32 final:1; /* Final bit */ -+ u32 extension:1; /* Extension bit */ -+#endif -+ }; -+ u32 sgt_efl; -+ }; -+ u8 __reserved2; -+ u8 bpid; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 __reserved3:3; -+ u16 offset:13; -+#else -+ u16 offset:13; -+ u16 __reserved3:3; -+#endif -+ }; -+ u16 opaque_offset; -+ }; -+} __packed; -+union qm_sg_efl { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 extension:1; /* Extension bit */ -+ u32 final:1; /* Final bit */ -+ u32 length:30; -+#else -+ u32 length:30; -+ u32 final:1; /* Final bit */ -+ u32 extension:1; /* Extension bit */ -+#endif -+ }; -+ u32 efl; -+}; -+static inline dma_addr_t qm_sg_addr(const struct qm_sg_entry *sg) -+{ -+ return (dma_addr_t)be64_to_cpu(sg->opaque) & 0xffffffffffULL; -+} -+static inline u8 qm_sg_entry_get_ext(const struct qm_sg_entry *sg) -+{ -+ union qm_sg_efl u; -+ -+ u.efl = be32_to_cpu(sg->sgt_efl); -+ return u.extension; -+} -+static inline u8 qm_sg_entry_get_final(const struct qm_sg_entry *sg) -+{ -+ union qm_sg_efl u; -+ -+ u.efl = be32_to_cpu(sg->sgt_efl); -+ return u.final; -+} -+static inline u32 qm_sg_entry_get_len(const struct qm_sg_entry *sg) -+{ -+ union qm_sg_efl u; -+ -+ u.efl = be32_to_cpu(sg->sgt_efl); -+ return u.length; -+} -+static inline u8 qm_sg_entry_get_bpid(const struct qm_sg_entry *sg) -+{ -+ return sg->bpid; -+} -+static inline u16 qm_sg_entry_get_offset(const struct qm_sg_entry *sg) -+{ -+ u32 opaque_offset = be16_to_cpu(sg->opaque_offset); -+ -+ return opaque_offset & 0x1fff; -+} -+ -+/* Macro, so we compile better if 'v' isn't always 64-bit */ -+#define qm_sg_entry_set64(sg, v) \ -+ do { \ -+ struct qm_sg_entry *__sg931 = (sg); \ -+ __sg931->opaque = cpu_to_be64(v); \ -+ } while (0) -+#define qm_sg_entry_set_ext(sg, v) \ -+ do { \ -+ union qm_sg_efl __u932; \ -+ __u932.efl = be32_to_cpu((sg)->sgt_efl); \ -+ __u932.extension = v; \ -+ (sg)->sgt_efl = cpu_to_be32(__u932.efl); \ -+ } while (0) -+#define qm_sg_entry_set_final(sg, v) \ -+ do { \ -+ union qm_sg_efl __u933; \ -+ __u933.efl = be32_to_cpu((sg)->sgt_efl); \ -+ __u933.final = v; \ -+ (sg)->sgt_efl = cpu_to_be32(__u933.efl); \ -+ } while (0) -+#define qm_sg_entry_set_len(sg, v) \ -+ do { \ -+ union qm_sg_efl __u934; \ -+ __u934.efl = be32_to_cpu((sg)->sgt_efl); \ -+ __u934.length = v; \ -+ (sg)->sgt_efl = cpu_to_be32(__u934.efl); \ -+ } while (0) -+#define qm_sg_entry_set_bpid(sg, v) \ -+ do { \ -+ struct qm_sg_entry *__u935 = (sg); \ -+ __u935->bpid = v; \ -+ } while (0) -+#define qm_sg_entry_set_offset(sg, v) \ -+ do { \ -+ struct qm_sg_entry *__u936 = (sg); \ -+ __u936->opaque_offset = cpu_to_be16(v); \ -+ } while (0) -+ -+/* See 1.5.8.1: "Enqueue Command" */ -+struct qm_eqcr_entry { -+ u8 __dont_write_directly__verb; -+ u8 dca; -+ u16 seqnum; -+ u32 orp; /* 24-bit */ -+ u32 fqid; /* 24-bit */ -+ u32 tag; -+ struct qm_fd fd; -+ u8 __reserved3[32]; -+} __packed; -+#define QM_EQCR_VERB_VBIT 0x80 -+#define QM_EQCR_VERB_CMD_MASK 0x61 /* but only one value; */ -+#define QM_EQCR_VERB_CMD_ENQUEUE 0x01 -+#define QM_EQCR_VERB_COLOUR_MASK 0x18 /* 4 possible values; */ -+#define QM_EQCR_VERB_COLOUR_GREEN 0x00 -+#define QM_EQCR_VERB_COLOUR_YELLOW 0x08 -+#define QM_EQCR_VERB_COLOUR_RED 0x10 -+#define QM_EQCR_VERB_COLOUR_OVERRIDE 0x18 -+#define QM_EQCR_VERB_INTERRUPT 0x04 /* on command consumption */ -+#define QM_EQCR_VERB_ORP 0x02 /* enable order restoration */ -+#define QM_EQCR_DCA_ENABLE 0x80 -+#define QM_EQCR_DCA_PARK 0x40 -+#define QM_EQCR_DCA_IDXMASK 0x0f /* "DQRR::idx" goes here */ -+#define QM_EQCR_SEQNUM_NESN 0x8000 /* Advance NESN */ -+#define QM_EQCR_SEQNUM_NLIS 0x4000 /* More fragments to come */ -+#define QM_EQCR_SEQNUM_SEQMASK 0x3fff /* sequence number goes here */ -+#define QM_EQCR_FQID_NULL 0 /* eg. for an ORP seqnum hole */ -+ -+/* See 1.5.8.2: "Frame Dequeue Response" */ -+struct qm_dqrr_entry { -+ u8 verb; -+ u8 stat; -+ u16 seqnum; /* 15-bit */ -+ u8 tok; -+ u8 __reserved2[3]; -+ u32 fqid; /* 24-bit */ -+ u32 contextB; -+ struct qm_fd fd; -+ u8 __reserved4[32]; -+}; -+#define QM_DQRR_VERB_VBIT 0x80 -+#define QM_DQRR_VERB_MASK 0x7f /* where the verb contains; */ -+#define QM_DQRR_VERB_FRAME_DEQUEUE 0x60 /* "this format" */ -+#define QM_DQRR_STAT_FQ_EMPTY 0x80 /* FQ empty */ -+#define QM_DQRR_STAT_FQ_HELDACTIVE 0x40 /* FQ held active */ -+#define QM_DQRR_STAT_FQ_FORCEELIGIBLE 0x20 /* FQ was force-eligible'd */ -+#define QM_DQRR_STAT_FD_VALID 0x10 /* has a non-NULL FD */ -+#define QM_DQRR_STAT_UNSCHEDULED 0x02 /* Unscheduled dequeue */ -+#define QM_DQRR_STAT_DQCR_EXPIRED 0x01 /* VDQCR or PDQCR expired*/ -+ -+/* See 1.5.8.3: "ERN Message Response" */ -+/* See 1.5.8.4: "FQ State Change Notification" */ -+struct qm_mr_entry { -+ u8 verb; -+ union { -+ struct { -+ u8 dca; -+ u16 seqnum; -+ u8 rc; /* Rejection Code */ -+ u32 orp:24; -+ u32 fqid; /* 24-bit */ -+ u32 tag; -+ struct qm_fd fd; -+ } __packed ern; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */ -+ u8 __reserved1:3; -+ enum qm_dc_portal portal:3; -+#else -+ enum qm_dc_portal portal:3; -+ u8 __reserved1:3; -+ u8 colour:2; /* See QM_MR_DCERN_COLOUR_* */ -+#endif -+ u16 __reserved2; -+ u8 rc; /* Rejection Code */ -+ u32 __reserved3:24; -+ u32 fqid; /* 24-bit */ -+ u32 tag; -+ struct qm_fd fd; -+ } __packed dcern; -+ struct { -+ u8 fqs; /* Frame Queue Status */ -+ u8 __reserved1[6]; -+ u32 fqid; /* 24-bit */ -+ u32 contextB; -+ u8 __reserved2[16]; -+ } __packed fq; /* FQRN/FQRNI/FQRL/FQPN */ -+ }; -+ u8 __reserved2[32]; -+} __packed; -+#define QM_MR_VERB_VBIT 0x80 -+/* The "ern" VERB bits match QM_EQCR_VERB_*** so aren't reproduced here. ERNs -+ * originating from direct-connect portals ("dcern") use 0x20 as a verb which -+ * would be invalid as a s/w enqueue verb. A s/w ERN can be distinguished from -+ * the other MR types by noting if the 0x20 bit is unset. */ -+#define QM_MR_VERB_TYPE_MASK 0x27 -+#define QM_MR_VERB_DC_ERN 0x20 -+#define QM_MR_VERB_FQRN 0x21 -+#define QM_MR_VERB_FQRNI 0x22 -+#define QM_MR_VERB_FQRL 0x23 -+#define QM_MR_VERB_FQPN 0x24 -+#define QM_MR_RC_MASK 0xf0 /* contains one of; */ -+#define QM_MR_RC_CGR_TAILDROP 0x00 -+#define QM_MR_RC_WRED 0x10 -+#define QM_MR_RC_ERROR 0x20 -+#define QM_MR_RC_ORPWINDOW_EARLY 0x30 -+#define QM_MR_RC_ORPWINDOW_LATE 0x40 -+#define QM_MR_RC_FQ_TAILDROP 0x50 -+#define QM_MR_RC_ORPWINDOW_RETIRED 0x60 -+#define QM_MR_RC_ORP_ZERO 0x70 -+#define QM_MR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */ -+#define QM_MR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */ -+#define QM_MR_DCERN_COLOUR_GREEN 0x00 -+#define QM_MR_DCERN_COLOUR_YELLOW 0x01 -+#define QM_MR_DCERN_COLOUR_RED 0x02 -+#define QM_MR_DCERN_COLOUR_OVERRIDE 0x03 -+ -+/* An identical structure of FQD fields is present in the "Init FQ" command and -+ * the "Query FQ" result, it's suctioned out into the "struct qm_fqd" type. -+ * Within that, the 'stashing' and 'taildrop' pieces are also factored out, the -+ * latter has two inlines to assist with converting to/from the mant+exp -+ * representation. */ -+struct qm_fqd_stashing { -+ /* See QM_STASHING_EXCL_<...> */ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 exclusive; -+ u8 __reserved1:2; -+ /* Numbers of cachelines */ -+ u8 annotation_cl:2; -+ u8 data_cl:2; -+ u8 context_cl:2; -+#else -+ u8 context_cl:2; -+ u8 data_cl:2; -+ u8 annotation_cl:2; -+ u8 __reserved1:2; -+ u8 exclusive; -+#endif -+} __packed; -+struct qm_fqd_taildrop { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 __reserved1:3; -+ u16 mant:8; -+ u16 exp:5; -+#else -+ u16 exp:5; -+ u16 mant:8; -+ u16 __reserved1:3; -+#endif -+} __packed; -+struct qm_fqd_oac { -+ /* See QM_OAC_<...> */ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 oac:2; /* "Overhead Accounting Control" */ -+ u8 __reserved1:6; -+#else -+ u8 __reserved1:6; -+ u8 oac:2; /* "Overhead Accounting Control" */ -+#endif -+ /* Two's-complement value (-128 to +127) */ -+ signed char oal; /* "Overhead Accounting Length" */ -+} __packed; -+struct qm_fqd { -+ union { -+ u8 orpc; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 __reserved1:2; -+ u8 orprws:3; -+ u8 oa:1; -+ u8 olws:2; -+#else -+ u8 olws:2; -+ u8 oa:1; -+ u8 orprws:3; -+ u8 __reserved1:2; -+#endif -+ } __packed; -+ }; -+ u8 cgid; -+ u16 fq_ctrl; /* See QM_FQCTRL_<...> */ -+ union { -+ u16 dest_wq; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 channel:13; /* qm_channel */ -+ u16 wq:3; -+#else -+ u16 wq:3; -+ u16 channel:13; /* qm_channel */ -+#endif -+ } __packed dest; -+ }; -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 __reserved2:1; -+ u16 ics_cred:15; -+#else -+ u16 __reserved2:1; -+ u16 ics_cred:15; -+#endif -+ /* For "Initialize Frame Queue" commands, the write-enable mask -+ * determines whether 'td' or 'oac_init' is observed. For query -+ * commands, this field is always 'td', and 'oac_query' (below) reflects -+ * the Overhead ACcounting values. */ -+ union { -+ struct qm_fqd_taildrop td; -+ struct qm_fqd_oac oac_init; -+ }; -+ u32 context_b; -+ union { -+ /* Treat it as 64-bit opaque */ -+ u64 opaque; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 hi; -+ u32 lo; -+#else -+ u32 lo; -+ u32 hi; -+#endif -+ }; -+ /* Treat it as s/w portal stashing config */ -+ /* See 1.5.6.7.1: "FQD Context_A field used for [...] */ -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ struct qm_fqd_stashing stashing; -+ /* 48-bit address of FQ context to -+ * stash, must be cacheline-aligned */ -+ u16 context_hi; -+ u32 context_lo; -+#else -+ u32 context_lo; -+ u16 context_hi; -+ struct qm_fqd_stashing stashing; -+#endif -+ } __packed; -+ } context_a; -+ struct qm_fqd_oac oac_query; -+} __packed; -+/* 64-bit converters for context_hi/lo */ -+static inline u64 qm_fqd_stashing_get64(const struct qm_fqd *fqd) -+{ -+ return ((u64)fqd->context_a.context_hi << 32) | -+ (u64)fqd->context_a.context_lo; -+} -+static inline dma_addr_t qm_fqd_stashing_addr(const struct qm_fqd *fqd) -+{ -+ return (dma_addr_t)qm_fqd_stashing_get64(fqd); -+} -+static inline u64 qm_fqd_context_a_get64(const struct qm_fqd *fqd) -+{ -+ return ((u64)fqd->context_a.hi << 32) | -+ (u64)fqd->context_a.lo; -+} -+/* Macro, so we compile better when 'v' isn't necessarily 64-bit */ -+#define qm_fqd_stashing_set64(fqd, v) \ -+ do { \ -+ struct qm_fqd *__fqd931 = (fqd); \ -+ __fqd931->context_a.context_hi = upper_32_bits(v); \ -+ __fqd931->context_a.context_lo = lower_32_bits(v); \ -+ } while (0) -+#define qm_fqd_context_a_set64(fqd, v) \ -+ do { \ -+ struct qm_fqd *__fqd931 = (fqd); \ -+ __fqd931->context_a.hi = upper_32_bits(v); \ -+ __fqd931->context_a.lo = lower_32_bits(v); \ -+ } while (0) -+/* convert a threshold value into mant+exp representation */ -+static inline int qm_fqd_taildrop_set(struct qm_fqd_taildrop *td, u32 val, -+ int roundup) -+{ -+ u32 e = 0; -+ int oddbit = 0; -+ if (val > 0xe0000000) -+ return -ERANGE; -+ while (val > 0xff) { -+ oddbit = val & 1; -+ val >>= 1; -+ e++; -+ if (roundup && oddbit) -+ val++; -+ } -+ td->exp = e; -+ td->mant = val; -+ return 0; -+} -+/* and the other direction */ -+static inline u32 qm_fqd_taildrop_get(const struct qm_fqd_taildrop *td) -+{ -+ return (u32)td->mant << td->exp; -+} -+ -+/* See 1.5.2.2: "Frame Queue Descriptor (FQD)" */ -+/* Frame Queue Descriptor (FQD) field 'fq_ctrl' uses these constants */ -+#define QM_FQCTRL_MASK 0x07ff /* 'fq_ctrl' flags; */ -+#define QM_FQCTRL_CGE 0x0400 /* Congestion Group Enable */ -+#define QM_FQCTRL_TDE 0x0200 /* Tail-Drop Enable */ -+#define QM_FQCTRL_ORP 0x0100 /* ORP Enable */ -+#define QM_FQCTRL_CTXASTASHING 0x0080 /* Context-A stashing */ -+#define QM_FQCTRL_CPCSTASH 0x0040 /* CPC Stash Enable */ -+#define QM_FQCTRL_FORCESFDR 0x0008 /* High-priority SFDRs */ -+#define QM_FQCTRL_AVOIDBLOCK 0x0004 /* Don't block active */ -+#define QM_FQCTRL_HOLDACTIVE 0x0002 /* Hold active in portal */ -+#define QM_FQCTRL_PREFERINCACHE 0x0001 /* Aggressively cache FQD */ -+#define QM_FQCTRL_LOCKINCACHE QM_FQCTRL_PREFERINCACHE /* older naming */ -+ -+/* See 1.5.6.7.1: "FQD Context_A field used for [...] */ -+/* Frame Queue Descriptor (FQD) field 'CONTEXT_A' uses these constants */ -+#define QM_STASHING_EXCL_ANNOTATION 0x04 -+#define QM_STASHING_EXCL_DATA 0x02 -+#define QM_STASHING_EXCL_CTX 0x01 -+ -+/* See 1.5.5.3: "Intra Class Scheduling" */ -+/* FQD field 'OAC' (Overhead ACcounting) uses these constants */ -+#define QM_OAC_ICS 0x2 /* Accounting for Intra-Class Scheduling */ -+#define QM_OAC_CG 0x1 /* Accounting for Congestion Groups */ -+ -+/* See 1.5.8.4: "FQ State Change Notification" */ -+/* This struct represents the 32-bit "WR_PARM_[GYR]" parameters in CGR fields -+ * and associated commands/responses. The WRED parameters are calculated from -+ * these fields as follows; -+ * MaxTH = MA * (2 ^ Mn) -+ * Slope = SA / (2 ^ Sn) -+ * MaxP = 4 * (Pn + 1) -+ */ -+struct qm_cgr_wr_parm { -+ union { -+ u32 word; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 MA:8; -+ u32 Mn:5; -+ u32 SA:7; /* must be between 64-127 */ -+ u32 Sn:6; -+ u32 Pn:6; -+#else -+ u32 Pn:6; -+ u32 Sn:6; -+ u32 SA:7; /* must be between 64-127 */ -+ u32 Mn:5; -+ u32 MA:8; -+#endif -+ } __packed; -+ }; -+} __packed; -+/* This struct represents the 13-bit "CS_THRES" CGR field. In the corresponding -+ * management commands, this is padded to a 16-bit structure field, so that's -+ * how we represent it here. The congestion state threshold is calculated from -+ * these fields as follows; -+ * CS threshold = TA * (2 ^ Tn) -+ */ -+struct qm_cgr_cs_thres { -+ union { -+ u16 hword; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 __reserved:3; -+ u16 TA:8; -+ u16 Tn:5; -+#else -+ u16 Tn:5; -+ u16 TA:8; -+ u16 __reserved:3; -+#endif -+ } __packed; -+ }; -+} __packed; -+/* This identical structure of CGR fields is present in the "Init/Modify CGR" -+ * commands and the "Query CGR" result. It's suctioned out here into its own -+ * struct. */ -+struct __qm_mc_cgr { -+ struct qm_cgr_wr_parm wr_parm_g; -+ struct qm_cgr_wr_parm wr_parm_y; -+ struct qm_cgr_wr_parm wr_parm_r; -+ u8 wr_en_g; /* boolean, use QM_CGR_EN */ -+ u8 wr_en_y; /* boolean, use QM_CGR_EN */ -+ u8 wr_en_r; /* boolean, use QM_CGR_EN */ -+ u8 cscn_en; /* boolean, use QM_CGR_EN */ -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */ -+ u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */ -+#else -+ u16 cscn_targ_dcp_low; /* CSCN_TARG_DCP low-16bits */ -+ u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */ -+#endif -+ }; -+ u32 cscn_targ; /* use QM_CGR_TARG_* */ -+ }; -+ u8 cstd_en; /* boolean, use QM_CGR_EN */ -+ u8 cs; /* boolean, only used in query response */ -+ union { -+ /* use qm_cgr_cs_thres_set64() */ -+ struct qm_cgr_cs_thres cs_thres; -+ u16 __cs_thres; -+ }; -+ u8 mode; /* QMAN_CGR_MODE_FRAME not supported in rev1.0 */ -+} __packed; -+#define QM_CGR_EN 0x01 /* For wr_en_*, cscn_en, cstd_en */ -+#define QM_CGR_TARG_UDP_CTRL_WRITE_BIT 0x8000 /* value written to portal bit*/ -+#define QM_CGR_TARG_UDP_CTRL_DCP 0x4000 /* 0: SWP, 1: DCP */ -+#define QM_CGR_TARG_PORTAL(n) (0x80000000 >> (n)) /* s/w portal, 0-9 */ -+#define QM_CGR_TARG_FMAN0 0x00200000 /* direct-connect portal: fman0 */ -+#define QM_CGR_TARG_FMAN1 0x00100000 /* : fman1 */ -+/* Convert CGR thresholds to/from "cs_thres" format */ -+static inline u64 qm_cgr_cs_thres_get64(const struct qm_cgr_cs_thres *th) -+{ -+ return (u64)th->TA << th->Tn; -+} -+static inline int qm_cgr_cs_thres_set64(struct qm_cgr_cs_thres *th, u64 val, -+ int roundup) -+{ -+ u32 e = 0; -+ int oddbit = 0; -+ while (val > 0xff) { -+ oddbit = val & 1; -+ val >>= 1; -+ e++; -+ if (roundup && oddbit) -+ val++; -+ } -+ th->Tn = e; -+ th->TA = val; -+ return 0; -+} -+ -+/* See 1.5.8.5.1: "Initialize FQ" */ -+/* See 1.5.8.5.2: "Query FQ" */ -+/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */ -+/* See 1.5.8.5.4: "Alter FQ State Commands " */ -+/* See 1.5.8.6.1: "Initialize/Modify CGR" */ -+/* See 1.5.8.6.2: "CGR Test Write" */ -+/* See 1.5.8.6.3: "Query CGR" */ -+/* See 1.5.8.6.4: "Query Congestion Group State" */ -+struct qm_mcc_initfq { -+ u8 __reserved1; -+ u16 we_mask; /* Write Enable Mask */ -+ u32 fqid; /* 24-bit */ -+ u16 count; /* Initialises 'count+1' FQDs */ -+ struct qm_fqd fqd; /* the FQD fields go here */ -+ u8 __reserved3[30]; -+} __packed; -+struct qm_mcc_queryfq { -+ u8 __reserved1[3]; -+ u32 fqid; /* 24-bit */ -+ u8 __reserved2[56]; -+} __packed; -+struct qm_mcc_queryfq_np { -+ u8 __reserved1[3]; -+ u32 fqid; /* 24-bit */ -+ u8 __reserved2[56]; -+} __packed; -+struct qm_mcc_alterfq { -+ u8 __reserved1[3]; -+ u32 fqid; /* 24-bit */ -+ u8 __reserved2; -+ u8 count; /* number of consecutive FQID */ -+ u8 __reserved3[10]; -+ u32 context_b; /* frame queue context b */ -+ u8 __reserved4[40]; -+} __packed; -+struct qm_mcc_initcgr { -+ u8 __reserved1; -+ u16 we_mask; /* Write Enable Mask */ -+ struct __qm_mc_cgr cgr; /* CGR fields */ -+ u8 __reserved2[2]; -+ u8 cgid; -+ u8 __reserved4[32]; -+} __packed; -+struct qm_mcc_cgrtestwrite { -+ u8 __reserved1[2]; -+ u8 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ -+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ -+ u8 __reserved2[23]; -+ u8 cgid; -+ u8 __reserved3[32]; -+} __packed; -+struct qm_mcc_querycgr { -+ u8 __reserved1[30]; -+ u8 cgid; -+ u8 __reserved2[32]; -+} __packed; -+struct qm_mcc_querycongestion { -+ u8 __reserved[63]; -+} __packed; -+struct qm_mcc_querywq { -+ u8 __reserved; -+ /* select channel if verb != QUERYWQ_DEDICATED */ -+ union { -+ u16 channel_wq; /* ignores wq (3 lsbits) */ -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 id:13; /* qm_channel */ -+ u16 __reserved1:3; -+#else -+ u16 __reserved1:3; -+ u16 id:13; /* qm_channel */ -+#endif -+ } __packed channel; -+ }; -+ u8 __reserved2[60]; -+} __packed; -+ -+struct qm_mcc_ceetm_lfqmt_config { -+ u8 __reserved1[4]; -+ u32 lfqid:24; -+ u8 __reserved2[2]; -+ u16 cqid; -+ u8 __reserved3[2]; -+ u16 dctidx; -+ u8 __reserved4[48]; -+} __packed; -+ -+struct qm_mcc_ceetm_lfqmt_query { -+ u8 __reserved1[4]; -+ u32 lfqid:24; -+ u8 __reserved2[56]; -+} __packed; -+ -+struct qm_mcc_ceetm_cq_config { -+ u8 __reserved1; -+ u16 cqid; -+ u8 dcpid; -+ u8 __reserved2; -+ u16 ccgid; -+ u8 __reserved3[56]; -+} __packed; -+ -+struct qm_mcc_ceetm_cq_query { -+ u8 __reserved1; -+ u16 cqid; -+ u8 dcpid; -+ u8 __reserved2[59]; -+} __packed; -+ -+struct qm_mcc_ceetm_dct_config { -+ u8 __reserved1; -+ u16 dctidx; -+ u8 dcpid; -+ u8 __reserved2[15]; -+ u32 context_b; -+ u64 context_a; -+ u8 __reserved3[32]; -+} __packed; -+ -+struct qm_mcc_ceetm_dct_query { -+ u8 __reserved1; -+ u16 dctidx; -+ u8 dcpid; -+ u8 __reserved2[59]; -+} __packed; -+ -+struct qm_mcc_ceetm_class_scheduler_config { -+ u8 __reserved1; -+ u16 cqcid; -+ u8 dcpid; -+ u8 __reserved2[6]; -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 gpc_reserved:1; -+ u8 gpc_combine_flag:1; -+ u8 gpc_prio_b:3; -+ u8 gpc_prio_a:3; -+#else -+ u8 gpc_prio_a:3; -+ u8 gpc_prio_b:3; -+ u8 gpc_combine_flag:1; -+ u8 gpc_reserved:1; -+#endif -+ u16 crem; -+ u16 erem; -+ u8 w[8]; -+ u8 __reserved3[40]; -+} __packed; -+ -+struct qm_mcc_ceetm_class_scheduler_query { -+ u8 __reserved1; -+ u16 cqcid; -+ u8 dcpid; -+ u8 __reserved2[59]; -+} __packed; -+ -+#define CEETM_COMMAND_CHANNEL_MAPPING (0 << 12) -+#define CEETM_COMMAND_SP_MAPPING (1 << 12) -+#define CEETM_COMMAND_CHANNEL_SHAPER (2 << 12) -+#define CEETM_COMMAND_LNI_SHAPER (3 << 12) -+#define CEETM_COMMAND_TCFC (4 << 12) -+ -+#define CEETM_CCGRID_MASK 0x01FF -+#define CEETM_CCGR_CM_CONFIGURE (0 << 14) -+#define CEETM_CCGR_DN_CONFIGURE (1 << 14) -+#define CEETM_CCGR_TEST_WRITE (2 << 14) -+#define CEETM_CCGR_CM_QUERY (0 << 14) -+#define CEETM_CCGR_DN_QUERY (1 << 14) -+#define CEETM_CCGR_DN_QUERY_FLUSH (2 << 14) -+#define CEETM_QUERY_CONGESTION_STATE (3 << 14) -+ -+struct qm_mcc_ceetm_mapping_shaper_tcfc_config { -+ u8 __reserved1; -+ u16 cid; -+ u8 dcpid; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 map_shaped:1; -+ u8 map_reserved:4; -+ u8 map_lni_id:3; -+#else -+ u8 map_lni_id:3; -+ u8 map_reserved:4; -+ u8 map_shaped:1; -+#endif -+ u8 __reserved2[58]; -+ } __packed channel_mapping; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 map_reserved:5; -+ u8 map_lni_id:3; -+#else -+ u8 map_lni_id:3; -+ u8 map_reserved:5; -+#endif -+ u8 __reserved2[58]; -+ } __packed sp_mapping; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 cpl:1; -+ u8 cpl_reserved:2; -+ u8 oal:5; -+#else -+ u8 oal:5; -+ u8 cpl_reserved:2; -+ u8 cpl:1; -+#endif -+ u32 crtcr:24; -+ u32 ertcr:24; -+ u16 crtbl; -+ u16 ertbl; -+ u8 mps; /* This will be hardcoded by driver with 60 */ -+ u8 __reserved2[47]; -+ } __packed shaper_config; -+ struct { -+ u8 __reserved2[11]; -+ u64 lnitcfcc; -+ u8 __reserved3[40]; -+ } __packed tcfc_config; -+ }; -+} __packed; -+ -+struct qm_mcc_ceetm_mapping_shaper_tcfc_query { -+ u8 __reserved1; -+ u16 cid; -+ u8 dcpid; -+ u8 __reserved2[59]; -+} __packed; -+ -+struct qm_mcc_ceetm_ccgr_config { -+ u8 __reserved1; -+ u16 ccgrid; -+ u8 dcpid; -+ u8 __reserved2; -+ u16 we_mask; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 ctl_reserved:1; -+ u8 ctl_wr_en_g:1; -+ u8 ctl_wr_en_y:1; -+ u8 ctl_wr_en_r:1; -+ u8 ctl_td_en:1; -+ u8 ctl_td_mode:1; -+ u8 ctl_cscn_en:1; -+ u8 ctl_mode:1; -+#else -+ u8 ctl_mode:1; -+ u8 ctl_cscn_en:1; -+ u8 ctl_td_mode:1; -+ u8 ctl_td_en:1; -+ u8 ctl_wr_en_r:1; -+ u8 ctl_wr_en_y:1; -+ u8 ctl_wr_en_g:1; -+ u8 ctl_reserved:1; -+#endif -+ u8 cdv; -+ u16 cscn_tupd; -+ u8 oal; -+ u8 __reserved3; -+ struct qm_cgr_cs_thres cs_thres; -+ struct qm_cgr_cs_thres cs_thres_x; -+ struct qm_cgr_cs_thres td_thres; -+ struct qm_cgr_wr_parm wr_parm_g; -+ struct qm_cgr_wr_parm wr_parm_y; -+ struct qm_cgr_wr_parm wr_parm_r; -+ } __packed cm_config; -+ struct { -+ u8 dnc; -+ u8 dn0; -+ u8 dn1; -+ u64 dnba:40; -+ u8 __reserved3[2]; -+ u16 dnth_0; -+ u8 __reserved4[2]; -+ u16 dnth_1; -+ u8 __reserved5[8]; -+ } __packed dn_config; -+ struct { -+ u8 __reserved3[3]; -+ u64 i_cnt:40; -+ u8 __reserved4[16]; -+ } __packed test_write; -+ }; -+ u8 __reserved5[32]; -+} __packed; -+ -+struct qm_mcc_ceetm_ccgr_query { -+ u8 __reserved1; -+ u16 ccgrid; -+ u8 dcpid; -+ u8 __reserved2[59]; -+} __packed; -+ -+struct qm_mcc_ceetm_cq_peek_pop_xsfdrread { -+ u8 __reserved1; -+ u16 cqid; -+ u8 dcpid; -+ u8 ct; -+ u16 xsfdr; -+ u8 __reserved2[56]; -+} __packed; -+ -+#define CEETM_QUERY_DEQUEUE_STATISTICS 0x00 -+#define CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS 0x01 -+#define CEETM_WRITE_DEQUEUE_STATISTICS 0x02 -+#define CEETM_QUERY_REJECT_STATISTICS 0x03 -+#define CEETM_QUERY_REJECT_CLEAR_STATISTICS 0x04 -+#define CEETM_WRITE_REJECT_STATISTICS 0x05 -+struct qm_mcc_ceetm_statistics_query_write { -+ u8 __reserved1; -+ u16 cid; -+ u8 dcpid; -+ u8 ct; -+ u8 __reserved2[13]; -+ u64 frm_cnt:40; -+ u8 __reserved3[2]; -+ u64 byte_cnt:48; -+ u8 __reserved[32]; -+} __packed; -+ -+struct qm_mc_command { -+ u8 __dont_write_directly__verb; -+ union { -+ struct qm_mcc_initfq initfq; -+ struct qm_mcc_queryfq queryfq; -+ struct qm_mcc_queryfq_np queryfq_np; -+ struct qm_mcc_alterfq alterfq; -+ struct qm_mcc_initcgr initcgr; -+ struct qm_mcc_cgrtestwrite cgrtestwrite; -+ struct qm_mcc_querycgr querycgr; -+ struct qm_mcc_querycongestion querycongestion; -+ struct qm_mcc_querywq querywq; -+ struct qm_mcc_ceetm_lfqmt_config lfqmt_config; -+ struct qm_mcc_ceetm_lfqmt_query lfqmt_query; -+ struct qm_mcc_ceetm_cq_config cq_config; -+ struct qm_mcc_ceetm_cq_query cq_query; -+ struct qm_mcc_ceetm_dct_config dct_config; -+ struct qm_mcc_ceetm_dct_query dct_query; -+ struct qm_mcc_ceetm_class_scheduler_config csch_config; -+ struct qm_mcc_ceetm_class_scheduler_query csch_query; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_config mst_config; -+ struct qm_mcc_ceetm_mapping_shaper_tcfc_query mst_query; -+ struct qm_mcc_ceetm_ccgr_config ccgr_config; -+ struct qm_mcc_ceetm_ccgr_query ccgr_query; -+ struct qm_mcc_ceetm_cq_peek_pop_xsfdrread cq_ppxr; -+ struct qm_mcc_ceetm_statistics_query_write stats_query_write; -+ }; -+} __packed; -+#define QM_MCC_VERB_VBIT 0x80 -+#define QM_MCC_VERB_MASK 0x7f /* where the verb contains; */ -+#define QM_MCC_VERB_INITFQ_PARKED 0x40 -+#define QM_MCC_VERB_INITFQ_SCHED 0x41 -+#define QM_MCC_VERB_QUERYFQ 0x44 -+#define QM_MCC_VERB_QUERYFQ_NP 0x45 /* "non-programmable" fields */ -+#define QM_MCC_VERB_QUERYWQ 0x46 -+#define QM_MCC_VERB_QUERYWQ_DEDICATED 0x47 -+#define QM_MCC_VERB_ALTER_SCHED 0x48 /* Schedule FQ */ -+#define QM_MCC_VERB_ALTER_FE 0x49 /* Force Eligible FQ */ -+#define QM_MCC_VERB_ALTER_RETIRE 0x4a /* Retire FQ */ -+#define QM_MCC_VERB_ALTER_OOS 0x4b /* Take FQ out of service */ -+#define QM_MCC_VERB_ALTER_FQXON 0x4d /* FQ XON */ -+#define QM_MCC_VERB_ALTER_FQXOFF 0x4e /* FQ XOFF */ -+#define QM_MCC_VERB_INITCGR 0x50 -+#define QM_MCC_VERB_MODIFYCGR 0x51 -+#define QM_MCC_VERB_CGRTESTWRITE 0x52 -+#define QM_MCC_VERB_QUERYCGR 0x58 -+#define QM_MCC_VERB_QUERYCONGESTION 0x59 -+/* INITFQ-specific flags */ -+#define QM_INITFQ_WE_MASK 0x01ff /* 'Write Enable' flags; */ -+#define QM_INITFQ_WE_OAC 0x0100 -+#define QM_INITFQ_WE_ORPC 0x0080 -+#define QM_INITFQ_WE_CGID 0x0040 -+#define QM_INITFQ_WE_FQCTRL 0x0020 -+#define QM_INITFQ_WE_DESTWQ 0x0010 -+#define QM_INITFQ_WE_ICSCRED 0x0008 -+#define QM_INITFQ_WE_TDTHRESH 0x0004 -+#define QM_INITFQ_WE_CONTEXTB 0x0002 -+#define QM_INITFQ_WE_CONTEXTA 0x0001 -+/* INITCGR/MODIFYCGR-specific flags */ -+#define QM_CGR_WE_MASK 0x07ff /* 'Write Enable Mask'; */ -+#define QM_CGR_WE_WR_PARM_G 0x0400 -+#define QM_CGR_WE_WR_PARM_Y 0x0200 -+#define QM_CGR_WE_WR_PARM_R 0x0100 -+#define QM_CGR_WE_WR_EN_G 0x0080 -+#define QM_CGR_WE_WR_EN_Y 0x0040 -+#define QM_CGR_WE_WR_EN_R 0x0020 -+#define QM_CGR_WE_CSCN_EN 0x0010 -+#define QM_CGR_WE_CSCN_TARG 0x0008 -+#define QM_CGR_WE_CSTD_EN 0x0004 -+#define QM_CGR_WE_CS_THRES 0x0002 -+#define QM_CGR_WE_MODE 0x0001 -+ -+/* See 1.5.9.7 CEETM Management Commands */ -+#define QM_CEETM_VERB_LFQMT_CONFIG 0x70 -+#define QM_CEETM_VERB_LFQMT_QUERY 0x71 -+#define QM_CEETM_VERB_CQ_CONFIG 0x72 -+#define QM_CEETM_VERB_CQ_QUERY 0x73 -+#define QM_CEETM_VERB_DCT_CONFIG 0x74 -+#define QM_CEETM_VERB_DCT_QUERY 0x75 -+#define QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG 0x76 -+#define QM_CEETM_VERB_CLASS_SCHEDULER_QUERY 0x77 -+#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG 0x78 -+#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY 0x79 -+#define QM_CEETM_VERB_CCGR_CONFIG 0x7A -+#define QM_CEETM_VERB_CCGR_QUERY 0x7B -+#define QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD 0x7C -+#define QM_CEETM_VERB_STATISTICS_QUERY_WRITE 0x7D -+ -+/* See 1.5.8.5.1: "Initialize FQ" */ -+/* See 1.5.8.5.2: "Query FQ" */ -+/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */ -+/* See 1.5.8.5.4: "Alter FQ State Commands " */ -+/* See 1.5.8.6.1: "Initialize/Modify CGR" */ -+/* See 1.5.8.6.2: "CGR Test Write" */ -+/* See 1.5.8.6.3: "Query CGR" */ -+/* See 1.5.8.6.4: "Query Congestion Group State" */ -+struct qm_mcr_initfq { -+ u8 __reserved1[62]; -+} __packed; -+struct qm_mcr_queryfq { -+ u8 __reserved1[8]; -+ struct qm_fqd fqd; /* the FQD fields are here */ -+ u8 __reserved2[30]; -+} __packed; -+struct qm_mcr_queryfq_np { -+ u8 __reserved1; -+ u8 state; /* QM_MCR_NP_STATE_*** */ -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 __reserved2; -+ u32 fqd_link:24; -+ u16 __reserved3:2; -+ u16 odp_seq:14; -+ u16 __reserved4:2; -+ u16 orp_nesn:14; -+ u16 __reserved5:1; -+ u16 orp_ea_hseq:15; -+ u16 __reserved6:1; -+ u16 orp_ea_tseq:15; -+ u8 __reserved7; -+ u32 orp_ea_hptr:24; -+ u8 __reserved8; -+ u32 orp_ea_tptr:24; -+ u8 __reserved9; -+ u32 pfdr_hptr:24; -+ u8 __reserved10; -+ u32 pfdr_tptr:24; -+ u8 __reserved11[5]; -+ u8 __reserved12:7; -+ u8 is:1; -+ u16 ics_surp; -+ u32 byte_cnt; -+ u8 __reserved13; -+ u32 frm_cnt:24; -+ u32 __reserved14; -+ u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ -+ u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ -+ u16 __reserved15; -+ u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ -+ u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ -+ u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ -+#else -+ u8 __reserved2; -+ u32 fqd_link:24; -+ -+ u16 odp_seq:14; -+ u16 __reserved3:2; -+ -+ u16 orp_nesn:14; -+ u16 __reserved4:2; -+ -+ u16 orp_ea_hseq:15; -+ u16 __reserved5:1; -+ -+ u16 orp_ea_tseq:15; -+ u16 __reserved6:1; -+ -+ u8 __reserved7; -+ u32 orp_ea_hptr:24; -+ -+ u8 __reserved8; -+ u32 orp_ea_tptr:24; -+ -+ u8 __reserved9; -+ u32 pfdr_hptr:24; -+ -+ u8 __reserved10; -+ u32 pfdr_tptr:24; -+ -+ u8 __reserved11[5]; -+ u8 is:1; -+ u8 __reserved12:7; -+ u16 ics_surp; -+ u32 byte_cnt; -+ u8 __reserved13; -+ u32 frm_cnt:24; -+ u32 __reserved14; -+ u16 ra1_sfdr; /* QM_MCR_NP_RA1_*** */ -+ u16 ra2_sfdr; /* QM_MCR_NP_RA2_*** */ -+ u16 __reserved15; -+ u16 od1_sfdr; /* QM_MCR_NP_OD1_*** */ -+ u16 od2_sfdr; /* QM_MCR_NP_OD2_*** */ -+ u16 od3_sfdr; /* QM_MCR_NP_OD3_*** */ -+#endif -+} __packed; -+ -+ -+struct qm_mcr_alterfq { -+ u8 fqs; /* Frame Queue Status */ -+ u8 __reserved1[61]; -+} __packed; -+struct qm_mcr_initcgr { -+ u8 __reserved1[62]; -+} __packed; -+struct qm_mcr_cgrtestwrite { -+ u16 __reserved1; -+ struct __qm_mc_cgr cgr; /* CGR fields */ -+ u8 __reserved2[3]; -+ u32 __reserved3:24; -+ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ -+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ -+ u32 __reserved4:24; -+ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ -+ u32 a_bcnt_lo; /* low 32-bits of 40-bit */ -+ u16 lgt; /* Last Group Tick */ -+ u16 wr_prob_g; -+ u16 wr_prob_y; -+ u16 wr_prob_r; -+ u8 __reserved5[8]; -+} __packed; -+struct qm_mcr_querycgr { -+ u16 __reserved1; -+ struct __qm_mc_cgr cgr; /* CGR fields */ -+ u8 __reserved2[3]; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 __reserved3:24; -+ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ -+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ -+#else -+ u32 i_bcnt_lo; /* low 32-bits of 40-bit */ -+ u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */ -+ u32 __reserved3:24; -+#endif -+ }; -+ u64 i_bcnt; -+ }; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u32 __reserved4:24; -+ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ -+ u32 a_bcnt_lo; /* low 32-bits of 40-bit */ -+#else -+ u32 a_bcnt_lo; /* low 32-bits of 40-bit */ -+ u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */ -+ u32 __reserved4:24; -+#endif -+ }; -+ u64 a_bcnt; -+ }; -+ union { -+ u32 cscn_targ_swp[4]; -+ u8 __reserved5[16]; -+ }; -+} __packed; -+static inline u64 qm_mcr_querycgr_i_get64(const struct qm_mcr_querycgr *q) -+{ -+ return be64_to_cpu(q->i_bcnt); -+} -+static inline u64 qm_mcr_querycgr_a_get64(const struct qm_mcr_querycgr *q) -+{ -+ return be64_to_cpu(q->a_bcnt); -+} -+static inline u64 qm_mcr_cgrtestwrite_i_get64( -+ const struct qm_mcr_cgrtestwrite *q) -+{ -+ return be64_to_cpu(((u64)q->i_bcnt_hi << 32) | (u64)q->i_bcnt_lo); -+} -+static inline u64 qm_mcr_cgrtestwrite_a_get64( -+ const struct qm_mcr_cgrtestwrite *q) -+{ -+ return be64_to_cpu(((u64)q->a_bcnt_hi << 32) | (u64)q->a_bcnt_lo); -+} -+/* Macro, so we compile better if 'v' isn't always 64-bit */ -+#define qm_mcr_querycgr_i_set64(q, v) \ -+ do { \ -+ struct qm_mcr_querycgr *__q931 = (fd); \ -+ __q931->i_bcnt_hi = upper_32_bits(v); \ -+ __q931->i_bcnt_lo = lower_32_bits(v); \ -+ } while (0) -+#define qm_mcr_querycgr_a_set64(q, v) \ -+ do { \ -+ struct qm_mcr_querycgr *__q931 = (fd); \ -+ __q931->a_bcnt_hi = upper_32_bits(v); \ -+ __q931->a_bcnt_lo = lower_32_bits(v); \ -+ } while (0) -+struct __qm_mcr_querycongestion { -+ u32 __state[8]; -+}; -+struct qm_mcr_querycongestion { -+ u8 __reserved[30]; -+ /* Access this struct using QM_MCR_QUERYCONGESTION() */ -+ struct __qm_mcr_querycongestion state; -+} __packed; -+struct qm_mcr_querywq { -+ union { -+ u16 channel_wq; /* ignores wq (3 lsbits) */ -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u16 id:13; /* qm_channel */ -+ u16 __reserved:3; -+#else -+ u16 __reserved:3; -+ u16 id:13; /* qm_channel */ -+#endif -+ } __packed channel; -+ }; -+ u8 __reserved[28]; -+ u32 wq_len[8]; -+} __packed; -+ -+/* QMAN CEETM Management Command Response */ -+struct qm_mcr_ceetm_lfqmt_config { -+ u8 __reserved1[62]; -+} __packed; -+struct qm_mcr_ceetm_lfqmt_query { -+ u8 __reserved1[8]; -+ u16 cqid; -+ u8 __reserved2[2]; -+ u16 dctidx; -+ u8 __reserved3[2]; -+ u16 ccgid; -+ u8 __reserved4[44]; -+} __packed; -+ -+struct qm_mcr_ceetm_cq_config { -+ u8 __reserved1[62]; -+} __packed; -+ -+struct qm_mcr_ceetm_cq_query { -+ u8 __reserved1[4]; -+ u16 ccgid; -+ u16 state; -+ u32 pfdr_hptr:24; -+ u32 pfdr_tptr:24; -+ u16 od1_xsfdr; -+ u16 od2_xsfdr; -+ u16 od3_xsfdr; -+ u16 od4_xsfdr; -+ u16 od5_xsfdr; -+ u16 od6_xsfdr; -+ u16 ra1_xsfdr; -+ u16 ra2_xsfdr; -+ u8 __reserved2; -+ u32 frm_cnt:24; -+ u8 __reserved333[28]; -+} __packed; -+ -+struct qm_mcr_ceetm_dct_config { -+ u8 __reserved1[62]; -+} __packed; -+ -+struct qm_mcr_ceetm_dct_query { -+ u8 __reserved1[18]; -+ u32 context_b; -+ u64 context_a; -+ u8 __reserved2[32]; -+} __packed; -+ -+struct qm_mcr_ceetm_class_scheduler_config { -+ u8 __reserved1[62]; -+} __packed; -+ -+struct qm_mcr_ceetm_class_scheduler_query { -+ u8 __reserved1[9]; -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 gpc_reserved:1; -+ u8 gpc_combine_flag:1; -+ u8 gpc_prio_b:3; -+ u8 gpc_prio_a:3; -+#else -+ u8 gpc_prio_a:3; -+ u8 gpc_prio_b:3; -+ u8 gpc_combine_flag:1; -+ u8 gpc_reserved:1; -+#endif -+ u16 crem; -+ u16 erem; -+ u8 w[8]; -+ u8 __reserved2[5]; -+ u32 wbfslist:24; -+ u32 d8; -+ u32 d9; -+ u32 d10; -+ u32 d11; -+ u32 d12; -+ u32 d13; -+ u32 d14; -+ u32 d15; -+} __packed; -+ -+struct qm_mcr_ceetm_mapping_shaper_tcfc_config { -+ u16 cid; -+ u8 __reserved2[60]; -+} __packed; -+ -+struct qm_mcr_ceetm_mapping_shaper_tcfc_query { -+ u16 cid; -+ u8 __reserved1; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 map_shaped:1; -+ u8 map_reserved:4; -+ u8 map_lni_id:3; -+#else -+ u8 map_lni_id:3; -+ u8 map_reserved:4; -+ u8 map_shaped:1; -+#endif -+ u8 __reserved2[58]; -+ } __packed channel_mapping_query; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 map_reserved:5; -+ u8 map_lni_id:3; -+#else -+ u8 map_lni_id:3; -+ u8 map_reserved:5; -+#endif -+ u8 __reserved2[58]; -+ } __packed sp_mapping_query; -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 cpl:1; -+ u8 cpl_reserved:2; -+ u8 oal:5; -+#else -+ u8 oal:5; -+ u8 cpl_reserved:2; -+ u8 cpl:1; -+#endif -+ u32 crtcr:24; -+ u32 ertcr:24; -+ u16 crtbl; -+ u16 ertbl; -+ u8 mps; -+ u8 __reserved2[15]; -+ u32 crat; -+ u32 erat; -+ u8 __reserved3[24]; -+ } __packed shaper_query; -+ struct { -+ u8 __reserved1[11]; -+ u64 lnitcfcc; -+ u8 __reserved3[40]; -+ } __packed tcfc_query; -+ }; -+} __packed; -+ -+struct qm_mcr_ceetm_ccgr_config { -+ u8 __reserved1[46]; -+ union { -+ u8 __reserved2[8]; -+ struct { -+ u16 timestamp; -+ u16 wr_porb_g; -+ u16 wr_prob_y; -+ u16 wr_prob_r; -+ } __packed test_write; -+ }; -+ u8 __reserved3[8]; -+} __packed; -+ -+struct qm_mcr_ceetm_ccgr_query { -+ u8 __reserved1[6]; -+ union { -+ struct { -+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -+ u8 ctl_reserved:1; -+ u8 ctl_wr_en_g:1; -+ u8 ctl_wr_en_y:1; -+ u8 ctl_wr_en_r:1; -+ u8 ctl_td_en:1; -+ u8 ctl_td_mode:1; -+ u8 ctl_cscn_en:1; -+ u8 ctl_mode:1; -+#else -+ u8 ctl_mode:1; -+ u8 ctl_cscn_en:1; -+ u8 ctl_td_mode:1; -+ u8 ctl_td_en:1; -+ u8 ctl_wr_en_r:1; -+ u8 ctl_wr_en_y:1; -+ u8 ctl_wr_en_g:1; -+ u8 ctl_reserved:1; -+#endif -+ u8 cdv; -+ u8 __reserved2[2]; -+ u8 oal; -+ u8 __reserved3; -+ struct qm_cgr_cs_thres cs_thres; -+ struct qm_cgr_cs_thres cs_thres_x; -+ struct qm_cgr_cs_thres td_thres; -+ struct qm_cgr_wr_parm wr_parm_g; -+ struct qm_cgr_wr_parm wr_parm_y; -+ struct qm_cgr_wr_parm wr_parm_r; -+ u16 cscn_targ_dcp; -+ u8 dcp_lsn; -+ u64 i_cnt:40; -+ u8 __reserved4[3]; -+ u64 a_cnt:40; -+ u32 cscn_targ_swp[4]; -+ } __packed cm_query; -+ struct { -+ u8 dnc; -+ u8 dn0; -+ u8 dn1; -+ u64 dnba:40; -+ u8 __reserved2[2]; -+ u16 dnth_0; -+ u8 __reserved3[2]; -+ u16 dnth_1; -+ u8 __reserved4[10]; -+ u16 dnacc_0; -+ u8 __reserved5[2]; -+ u16 dnacc_1; -+ u8 __reserved6[24]; -+ } __packed dn_query; -+ struct { -+ u8 __reserved2[24]; -+ struct __qm_mcr_querycongestion state; -+ } __packed congestion_state; -+ -+ }; -+} __packed; -+ -+struct qm_mcr_ceetm_cq_peek_pop_xsfdrread { -+ u8 stat; -+ u8 __reserved1[11]; -+ u16 dctidx; -+ struct qm_fd fd; -+ u8 __reserved2[32]; -+} __packed; -+ -+struct qm_mcr_ceetm_statistics_query { -+ u8 __reserved1[17]; -+ u64 frm_cnt:40; -+ u8 __reserved2[2]; -+ u64 byte_cnt:48; -+ u8 __reserved3[32]; -+} __packed; -+ -+struct qm_mc_result { -+ u8 verb; -+ u8 result; -+ union { -+ struct qm_mcr_initfq initfq; -+ struct qm_mcr_queryfq queryfq; -+ struct qm_mcr_queryfq_np queryfq_np; -+ struct qm_mcr_alterfq alterfq; -+ struct qm_mcr_initcgr initcgr; -+ struct qm_mcr_cgrtestwrite cgrtestwrite; -+ struct qm_mcr_querycgr querycgr; -+ struct qm_mcr_querycongestion querycongestion; -+ struct qm_mcr_querywq querywq; -+ struct qm_mcr_ceetm_lfqmt_config lfqmt_config; -+ struct qm_mcr_ceetm_lfqmt_query lfqmt_query; -+ struct qm_mcr_ceetm_cq_config cq_config; -+ struct qm_mcr_ceetm_cq_query cq_query; -+ struct qm_mcr_ceetm_dct_config dct_config; -+ struct qm_mcr_ceetm_dct_query dct_query; -+ struct qm_mcr_ceetm_class_scheduler_config csch_config; -+ struct qm_mcr_ceetm_class_scheduler_query csch_query; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_config mst_config; -+ struct qm_mcr_ceetm_mapping_shaper_tcfc_query mst_query; -+ struct qm_mcr_ceetm_ccgr_config ccgr_config; -+ struct qm_mcr_ceetm_ccgr_query ccgr_query; -+ struct qm_mcr_ceetm_cq_peek_pop_xsfdrread cq_ppxr; -+ struct qm_mcr_ceetm_statistics_query stats_query; -+ }; -+} __packed; -+ -+#define QM_MCR_VERB_RRID 0x80 -+#define QM_MCR_VERB_MASK QM_MCC_VERB_MASK -+#define QM_MCR_VERB_INITFQ_PARKED QM_MCC_VERB_INITFQ_PARKED -+#define QM_MCR_VERB_INITFQ_SCHED QM_MCC_VERB_INITFQ_SCHED -+#define QM_MCR_VERB_QUERYFQ QM_MCC_VERB_QUERYFQ -+#define QM_MCR_VERB_QUERYFQ_NP QM_MCC_VERB_QUERYFQ_NP -+#define QM_MCR_VERB_QUERYWQ QM_MCC_VERB_QUERYWQ -+#define QM_MCR_VERB_QUERYWQ_DEDICATED QM_MCC_VERB_QUERYWQ_DEDICATED -+#define QM_MCR_VERB_ALTER_SCHED QM_MCC_VERB_ALTER_SCHED -+#define QM_MCR_VERB_ALTER_FE QM_MCC_VERB_ALTER_FE -+#define QM_MCR_VERB_ALTER_RETIRE QM_MCC_VERB_ALTER_RETIRE -+#define QM_MCR_VERB_ALTER_OOS QM_MCC_VERB_ALTER_OOS -+#define QM_MCR_RESULT_NULL 0x00 -+#define QM_MCR_RESULT_OK 0xf0 -+#define QM_MCR_RESULT_ERR_FQID 0xf1 -+#define QM_MCR_RESULT_ERR_FQSTATE 0xf2 -+#define QM_MCR_RESULT_ERR_NOTEMPTY 0xf3 /* OOS fails if FQ is !empty */ -+#define QM_MCR_RESULT_ERR_BADCHANNEL 0xf4 -+#define QM_MCR_RESULT_PENDING 0xf8 -+#define QM_MCR_RESULT_ERR_BADCOMMAND 0xff -+#define QM_MCR_NP_STATE_FE 0x10 -+#define QM_MCR_NP_STATE_R 0x08 -+#define QM_MCR_NP_STATE_MASK 0x07 /* Reads FQD::STATE; */ -+#define QM_MCR_NP_STATE_OOS 0x00 -+#define QM_MCR_NP_STATE_RETIRED 0x01 -+#define QM_MCR_NP_STATE_TEN_SCHED 0x02 -+#define QM_MCR_NP_STATE_TRU_SCHED 0x03 -+#define QM_MCR_NP_STATE_PARKED 0x04 -+#define QM_MCR_NP_STATE_ACTIVE 0x05 -+#define QM_MCR_NP_PTR_MASK 0x07ff /* for RA[12] & OD[123] */ -+#define QM_MCR_NP_RA1_NRA(v) (((v) >> 14) & 0x3) /* FQD::NRA */ -+#define QM_MCR_NP_RA2_IT(v) (((v) >> 14) & 0x1) /* FQD::IT */ -+#define QM_MCR_NP_OD1_NOD(v) (((v) >> 14) & 0x3) /* FQD::NOD */ -+#define QM_MCR_NP_OD3_NPC(v) (((v) >> 14) & 0x3) /* FQD::NPC */ -+#define QM_MCR_FQS_ORLPRESENT 0x02 /* ORL fragments to come */ -+#define QM_MCR_FQS_NOTEMPTY 0x01 /* FQ has enqueued frames */ -+/* This extracts the state for congestion group 'n' from a query response. -+ * Eg. -+ * u8 cgr = [...]; -+ * struct qm_mc_result *res = [...]; -+ * printf("congestion group %d congestion state: %d\n", cgr, -+ * QM_MCR_QUERYCONGESTION(&res->querycongestion.state, cgr)); -+ */ -+#define __CGR_WORD(num) (num >> 5) -+#define __CGR_SHIFT(num) (num & 0x1f) -+#define __CGR_NUM (sizeof(struct __qm_mcr_querycongestion) << 3) -+static inline int QM_MCR_QUERYCONGESTION(struct __qm_mcr_querycongestion *p, -+ u8 cgr) -+{ -+ return p->__state[__CGR_WORD(cgr)] & (0x80000000 >> __CGR_SHIFT(cgr)); -+} -+ -+ -+/*********************/ -+/* Utility interface */ -+/*********************/ -+ -+/* Represents an allocator over a range of FQIDs. NB, accesses are not locked, -+ * spinlock them yourself if needed. */ -+struct qman_fqid_pool; -+ -+/* Create/destroy a FQID pool, num must be a multiple of 32. NB, _destroy() -+ * always succeeds, but returns non-zero if there were "leaked" FQID -+ * allocations. */ -+struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num); -+int qman_fqid_pool_destroy(struct qman_fqid_pool *pool); -+/* Alloc/free a FQID from the range. _alloc() returns zero for success. */ -+int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid); -+void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid); -+u32 qman_fqid_pool_used(struct qman_fqid_pool *pool); -+ -+/*******************************************************************/ -+/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */ -+/*******************************************************************/ -+ -+ /* Portal and Frame Queues */ -+ /* ----------------------- */ -+/* Represents a managed portal */ -+struct qman_portal; -+ -+/* This object type represents Qman frame queue descriptors (FQD), it is -+ * cacheline-aligned, and initialised by qman_create_fq(). The structure is -+ * defined further down. */ -+struct qman_fq; -+ -+/* This object type represents a Qman congestion group, it is defined further -+ * down. */ -+struct qman_cgr; -+ -+struct qman_portal_config { -+ /* If the caller enables DQRR stashing (and thus wishes to operate the -+ * portal from only one cpu), this is the logical CPU that the portal -+ * will stash to. Whether stashing is enabled or not, this setting is -+ * also used for any "core-affine" portals, ie. default portals -+ * associated to the corresponding cpu. -1 implies that there is no core -+ * affinity configured. */ -+ int cpu; -+ /* portal interrupt line */ -+ int irq; -+ /* the unique index of this portal */ -+ u32 index; -+ /* Is this portal shared? (If so, it has coarser locking and demuxes -+ * processing on behalf of other CPUs.) */ -+ int is_shared; -+ /* The portal's dedicated channel id, use this value for initialising -+ * frame queues to target this portal when scheduled. */ -+ u16 channel; -+ /* A mask of which pool channels this portal has dequeue access to -+ * (using QM_SDQCR_CHANNELS_POOL(n) for the bitmask) */ -+ u32 pools; -+}; -+ -+/* This enum, and the callback type that returns it, are used when handling -+ * dequeued frames via DQRR. Note that for "null" callbacks registered with the -+ * portal object (for handling dequeues that do not demux because contextB is -+ * NULL), the return value *MUST* be qman_cb_dqrr_consume. */ -+enum qman_cb_dqrr_result { -+ /* DQRR entry can be consumed */ -+ qman_cb_dqrr_consume, -+ /* Like _consume, but requests parking - FQ must be held-active */ -+ qman_cb_dqrr_park, -+ /* Does not consume, for DCA mode only. This allows out-of-order -+ * consumes by explicit calls to qman_dca() and/or the use of implicit -+ * DCA via EQCR entries. */ -+ qman_cb_dqrr_defer, -+ /* Stop processing without consuming this ring entry. Exits the current -+ * qman_poll_dqrr() or interrupt-handling, as appropriate. If within an -+ * interrupt handler, the callback would typically call -+ * qman_irqsource_remove(QM_PIRQ_DQRI) before returning this value, -+ * otherwise the interrupt will reassert immediately. */ -+ qman_cb_dqrr_stop, -+ /* Like qman_cb_dqrr_stop, but consumes the current entry. */ -+ qman_cb_dqrr_consume_stop -+}; -+typedef enum qman_cb_dqrr_result (*qman_cb_dqrr)(struct qman_portal *qm, -+ struct qman_fq *fq, -+ const struct qm_dqrr_entry *dqrr); -+ -+/* This callback type is used when handling ERNs, FQRNs and FQRLs via MR. They -+ * are always consumed after the callback returns. */ -+typedef void (*qman_cb_mr)(struct qman_portal *qm, struct qman_fq *fq, -+ const struct qm_mr_entry *msg); -+ -+/* This callback type is used when handling DCP ERNs */ -+typedef void (*qman_cb_dc_ern)(struct qman_portal *qm, -+ const struct qm_mr_entry *msg); -+ -+/* s/w-visible states. Ie. tentatively scheduled + truly scheduled + active + -+ * held-active + held-suspended are just "sched". Things like "retired" will not -+ * be assumed until it is complete (ie. QMAN_FQ_STATE_CHANGING is set until -+ * then, to indicate it's completing and to gate attempts to retry the retire -+ * command). Note, park commands do not set QMAN_FQ_STATE_CHANGING because it's -+ * technically impossible in the case of enqueue DCAs (which refer to DQRR ring -+ * index rather than the FQ that ring entry corresponds to), so repeated park -+ * commands are allowed (if you're silly enough to try) but won't change FQ -+ * state, and the resulting park notifications move FQs from "sched" to -+ * "parked". */ -+enum qman_fq_state { -+ qman_fq_state_oos, -+ qman_fq_state_parked, -+ qman_fq_state_sched, -+ qman_fq_state_retired -+}; -+ -+/* Frame queue objects (struct qman_fq) are stored within memory passed to -+ * qman_create_fq(), as this allows stashing of caller-provided demux callback -+ * pointers at no extra cost to stashing of (driver-internal) FQ state. If the -+ * caller wishes to add per-FQ state and have it benefit from dequeue-stashing, -+ * they should; -+ * -+ * (a) extend the qman_fq structure with their state; eg. -+ * -+ * // myfq is allocated and driver_fq callbacks filled in; -+ * struct my_fq { -+ * struct qman_fq base; -+ * int an_extra_field; -+ * [ ... add other fields to be associated with each FQ ...] -+ * } *myfq = some_my_fq_allocator(); -+ * struct qman_fq *fq = qman_create_fq(fqid, flags, &myfq->base); -+ * -+ * // in a dequeue callback, access extra fields from 'fq' via a cast; -+ * struct my_fq *myfq = (struct my_fq *)fq; -+ * do_something_with(myfq->an_extra_field); -+ * [...] -+ * -+ * (b) when and if configuring the FQ for context stashing, specify how ever -+ * many cachelines are required to stash 'struct my_fq', to accelerate not -+ * only the Qman driver but the callback as well. -+ */ -+ -+struct qman_fq_cb { -+ qman_cb_dqrr dqrr; /* for dequeued frames */ -+ qman_cb_mr ern; /* for s/w ERNs */ -+ qman_cb_mr fqs; /* frame-queue state changes*/ -+}; -+ -+struct qman_fq { -+ /* Caller of qman_create_fq() provides these demux callbacks */ -+ struct qman_fq_cb cb; -+ /* These are internal to the driver, don't touch. In particular, they -+ * may change, be removed, or extended (so you shouldn't rely on -+ * sizeof(qman_fq) being a constant). */ -+ spinlock_t fqlock; -+ u32 fqid; -+ volatile unsigned long flags; -+ enum qman_fq_state state; -+ int cgr_groupid; -+ struct rb_node node; -+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP -+ u32 key; -+#endif -+}; -+ -+/* This callback type is used when handling congestion group entry/exit. -+ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit. */ -+typedef void (*qman_cb_cgr)(struct qman_portal *qm, -+ struct qman_cgr *cgr, int congested); -+ -+struct qman_cgr { -+ /* Set these prior to qman_create_cgr() */ -+ u32 cgrid; /* 0..255, but u32 to allow specials like -1, 256, etc.*/ -+ qman_cb_cgr cb; -+ /* These are private to the driver */ -+ u16 chan; /* portal channel this object is created on */ -+ struct list_head node; -+}; -+ -+/* Flags to qman_create_fq() */ -+#define QMAN_FQ_FLAG_NO_ENQUEUE 0x00000001 /* can't enqueue */ -+#define QMAN_FQ_FLAG_NO_MODIFY 0x00000002 /* can only enqueue */ -+#define QMAN_FQ_FLAG_TO_DCPORTAL 0x00000004 /* consumed by CAAM/PME/Fman */ -+#define QMAN_FQ_FLAG_LOCKED 0x00000008 /* multi-core locking */ -+#define QMAN_FQ_FLAG_AS_IS 0x00000010 /* query h/w state */ -+#define QMAN_FQ_FLAG_DYNAMIC_FQID 0x00000020 /* (de)allocate fqid */ -+ -+/* Flags to qman_destroy_fq() */ -+#define QMAN_FQ_DESTROY_PARKED 0x00000001 /* FQ can be parked or OOS */ -+ -+/* Flags from qman_fq_state() */ -+#define QMAN_FQ_STATE_CHANGING 0x80000000 /* 'state' is changing */ -+#define QMAN_FQ_STATE_NE 0x40000000 /* retired FQ isn't empty */ -+#define QMAN_FQ_STATE_ORL 0x20000000 /* retired FQ has ORL */ -+#define QMAN_FQ_STATE_BLOCKOOS 0xe0000000 /* if any are set, no OOS */ -+#define QMAN_FQ_STATE_CGR_EN 0x10000000 /* CGR enabled */ -+#define QMAN_FQ_STATE_VDQCR 0x08000000 /* being volatile dequeued */ -+ -+/* Flags to qman_init_fq() */ -+#define QMAN_INITFQ_FLAG_SCHED 0x00000001 /* schedule rather than park */ -+#define QMAN_INITFQ_FLAG_LOCAL 0x00000004 /* set dest portal */ -+ -+/* Flags to qman_volatile_dequeue() */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+#define QMAN_VOLATILE_FLAG_WAIT 0x00000001 /* wait if VDQCR is in use */ -+#define QMAN_VOLATILE_FLAG_WAIT_INT 0x00000002 /* if wait, interruptible? */ -+#define QMAN_VOLATILE_FLAG_FINISH 0x00000004 /* wait till VDQCR completes */ -+#endif -+ -+/* Flags to qman_enqueue(). NB, the strange numbering is to align with hardware, -+ * bit-wise. (NB: the PME API is sensitive to these precise numberings too, so -+ * any change here should be audited in PME.) */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT -+#define QMAN_ENQUEUE_FLAG_WAIT 0x00010000 /* wait if EQCR is full */ -+#define QMAN_ENQUEUE_FLAG_WAIT_INT 0x00020000 /* if wait, interruptible? */ -+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC -+#define QMAN_ENQUEUE_FLAG_WAIT_SYNC 0x00000004 /* if wait, until consumed? */ -+#endif -+#endif -+#define QMAN_ENQUEUE_FLAG_WATCH_CGR 0x00080000 /* watch congestion state */ -+#define QMAN_ENQUEUE_FLAG_DCA 0x00008000 /* perform enqueue-DCA */ -+#define QMAN_ENQUEUE_FLAG_DCA_PARK 0x00004000 /* If DCA, requests park */ -+#define QMAN_ENQUEUE_FLAG_DCA_PTR(p) /* If DCA, p is DQRR entry */ \ -+ (((u32)(p) << 2) & 0x00000f00) -+#define QMAN_ENQUEUE_FLAG_C_GREEN 0x00000000 /* choose one C_*** flag */ -+#define QMAN_ENQUEUE_FLAG_C_YELLOW 0x00000008 -+#define QMAN_ENQUEUE_FLAG_C_RED 0x00000010 -+#define QMAN_ENQUEUE_FLAG_C_OVERRIDE 0x00000018 -+/* For the ORP-specific qman_enqueue_orp() variant; -+ * - this flag indicates "Not Last In Sequence", ie. all but the final fragment -+ * of a frame. */ -+#define QMAN_ENQUEUE_FLAG_NLIS 0x01000000 -+/* - this flag performs no enqueue but fills in an ORP sequence number that -+ * would otherwise block it (eg. if a frame has been dropped). */ -+#define QMAN_ENQUEUE_FLAG_HOLE 0x02000000 -+/* - this flag performs no enqueue but advances NESN to the given sequence -+ * number. */ -+#define QMAN_ENQUEUE_FLAG_NESN 0x04000000 -+ -+/* Flags to qman_modify_cgr() */ -+#define QMAN_CGR_FLAG_USE_INIT 0x00000001 -+#define QMAN_CGR_MODE_FRAME 0x00000001 -+ -+ /* Portal Management */ -+ /* ----------------- */ -+/** -+ * qman_get_portal_config - get portal configuration settings -+ * -+ * This returns a read-only view of the current cpu's affine portal settings. -+ */ -+const struct qman_portal_config *qman_get_portal_config(void); -+ -+/** -+ * qman_irqsource_get - return the portal work that is interrupt-driven -+ * -+ * Returns a bitmask of QM_PIRQ_**I processing sources that are currently -+ * enabled for interrupt handling on the current cpu's affine portal. These -+ * sources will trigger the portal interrupt and the interrupt handler (or a -+ * tasklet/bottom-half it defers to) will perform the corresponding processing -+ * work. The qman_poll_***() functions will only process sources that are not in -+ * this bitmask. If the current CPU is sharing a portal hosted on another CPU, -+ * this always returns zero. -+ */ -+u32 qman_irqsource_get(void); -+ -+/** -+ * qman_irqsource_add - add processing sources to be interrupt-driven -+ * @bits: bitmask of QM_PIRQ_**I processing sources -+ * -+ * Adds processing sources that should be interrupt-driven (rather than -+ * processed via qman_poll_***() functions). Returns zero for success, or -+ * -EINVAL if the current CPU is sharing a portal hosted on another CPU. -+ */ -+int qman_irqsource_add(u32 bits); -+ -+/** -+ * qman_irqsource_remove - remove processing sources from being interrupt-driven -+ * @bits: bitmask of QM_PIRQ_**I processing sources -+ * -+ * Removes processing sources from being interrupt-driven, so that they will -+ * instead be processed via qman_poll_***() functions. Returns zero for success, -+ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU. -+ */ -+int qman_irqsource_remove(u32 bits); -+ -+/** -+ * qman_affine_cpus - return a mask of cpus that have affine portals -+ */ -+const cpumask_t *qman_affine_cpus(void); -+ -+/** -+ * qman_affine_channel - return the channel ID of an portal -+ * @cpu: the cpu whose affine portal is the subject of the query -+ * -+ * If @cpu is -1, the affine portal for the current CPU will be used. It is a -+ * bug to call this function for any value of @cpu (other than -1) that is not a -+ * member of the mask returned from qman_affine_cpus(). -+ */ -+u16 qman_affine_channel(int cpu); -+ -+/** -+ * qman_get_affine_portal - return the portal pointer affine to cpu -+ * @cpu: the cpu whose affine portal is the subject of the query -+ * -+ */ -+void *qman_get_affine_portal(int cpu); -+ -+/** -+ * qman_poll_dqrr - process DQRR (fast-path) entries -+ * @limit: the maximum number of DQRR entries to process -+ * -+ * Use of this function requires that DQRR processing not be interrupt-driven. -+ * Ie. the value returned by qman_irqsource_get() should not include -+ * QM_PIRQ_DQRI. If the current CPU is sharing a portal hosted on another CPU, -+ * this function will return -EINVAL, otherwise the return value is >=0 and -+ * represents the number of DQRR entries processed. -+ */ -+int qman_poll_dqrr(unsigned int limit); -+ -+/** -+ * qman_poll_slow - process anything (except DQRR) that isn't interrupt-driven. -+ * -+ * This function does any portal processing that isn't interrupt-driven. If the -+ * current CPU is sharing a portal hosted on another CPU, this function will -+ * return (u32)-1, otherwise the return value is a bitmask of QM_PIRQ_* sources -+ * indicating what interrupt sources were actually processed by the call. -+ */ -+u32 qman_poll_slow(void); -+ -+/** -+ * qman_poll - legacy wrapper for qman_poll_dqrr() and qman_poll_slow() -+ * -+ * Dispatcher logic on a cpu can use this to trigger any maintenance of the -+ * affine portal. There are two classes of portal processing in question; -+ * fast-path (which involves demuxing dequeue ring (DQRR) entries and tracking -+ * enqueue ring (EQCR) consumption), and slow-path (which involves EQCR -+ * thresholds, congestion state changes, etc). This function does whatever -+ * processing is not triggered by interrupts. -+ * -+ * Note, if DQRR and some slow-path processing are poll-driven (rather than -+ * interrupt-driven) then this function uses a heuristic to determine how often -+ * to run slow-path processing - as slow-path processing introduces at least a -+ * minimum latency each time it is run, whereas fast-path (DQRR) processing is -+ * close to zero-cost if there is no work to be done. Applications can tune this -+ * behaviour themselves by using qman_poll_dqrr() and qman_poll_slow() directly -+ * rather than going via this wrapper. -+ */ -+void qman_poll(void); -+ -+/** -+ * qman_stop_dequeues - Stop h/w dequeuing to the s/w portal -+ * -+ * Disables DQRR processing of the portal. This is reference-counted, so -+ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to -+ * truly re-enable dequeuing. -+ */ -+void qman_stop_dequeues(void); -+ -+/** -+ * qman_start_dequeues - (Re)start h/w dequeuing to the s/w portal -+ * -+ * Enables DQRR processing of the portal. This is reference-counted, so -+ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to -+ * truly re-enable dequeuing. -+ */ -+void qman_start_dequeues(void); -+ -+/** -+ * qman_static_dequeue_add - Add pool channels to the portal SDQCR -+ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n) -+ * -+ * Adds a set of pool channels to the portal's static dequeue command register -+ * (SDQCR). The requested pools are limited to those the portal has dequeue -+ * access to. -+ */ -+void qman_static_dequeue_add(u32 pools); -+ -+/** -+ * qman_static_dequeue_del - Remove pool channels from the portal SDQCR -+ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n) -+ * -+ * Removes a set of pool channels from the portal's static dequeue command -+ * register (SDQCR). The requested pools are limited to those the portal has -+ * dequeue access to. -+ */ -+void qman_static_dequeue_del(u32 pools); -+ -+/** -+ * qman_static_dequeue_get - return the portal's current SDQCR -+ * -+ * Returns the portal's current static dequeue command register (SDQCR). The -+ * entire register is returned, so if only the currently-enabled pool channels -+ * are desired, mask the return value with QM_SDQCR_CHANNELS_POOL_MASK. -+ */ -+u32 qman_static_dequeue_get(void); -+ -+/** -+ * qman_dca - Perform a Discrete Consumption Acknowledgement -+ * @dq: the DQRR entry to be consumed -+ * @park_request: indicates whether the held-active @fq should be parked -+ * -+ * Only allowed in DCA-mode portals, for DQRR entries whose handler callback had -+ * previously returned 'qman_cb_dqrr_defer'. NB, as with the other APIs, this -+ * does not take a 'portal' argument but implies the core affine portal from the -+ * cpu that is currently executing the function. For reasons of locking, this -+ * function must be called from the same CPU as that which processed the DQRR -+ * entry in the first place. -+ */ -+void qman_dca(struct qm_dqrr_entry *dq, int park_request); -+ -+/** -+ * qman_eqcr_is_empty - Determine if portal's EQCR is empty -+ * -+ * For use in situations where a cpu-affine caller needs to determine when all -+ * enqueues for the local portal have been processed by Qman but can't use the -+ * QMAN_ENQUEUE_FLAG_WAIT_SYNC flag to do this from the final qman_enqueue(). -+ * The function forces tracking of EQCR consumption (which normally doesn't -+ * happen until enqueue processing needs to find space to put new enqueue -+ * commands), and returns zero if the ring still has unprocessed entries, -+ * non-zero if it is empty. -+ */ -+int qman_eqcr_is_empty(void); -+ -+/** -+ * qman_set_dc_ern - Set the handler for DCP enqueue rejection notifications -+ * @handler: callback for processing DCP ERNs -+ * @affine: whether this handler is specific to the locally affine portal -+ * -+ * If a hardware block's interface to Qman (ie. its direct-connect portal, or -+ * DCP) is configured not to receive enqueue rejections, then any enqueues -+ * through that DCP that are rejected will be sent to a given software portal. -+ * If @affine is non-zero, then this handler will only be used for DCP ERNs -+ * received on the portal affine to the current CPU. If multiple CPUs share a -+ * portal and they all call this function, they will be setting the handler for -+ * the same portal! If @affine is zero, then this handler will be global to all -+ * portals handled by this instance of the driver. Only those portals that do -+ * not have their own affine handler will use the global handler. -+ */ -+void qman_set_dc_ern(qman_cb_dc_ern handler, int affine); -+ -+ /* FQ management */ -+ /* ------------- */ -+/** -+ * qman_create_fq - Allocates a FQ -+ * @fqid: the index of the FQD to encapsulate, must be "Out of Service" -+ * @flags: bit-mask of QMAN_FQ_FLAG_*** options -+ * @fq: memory for storing the 'fq', with callbacks filled in -+ * -+ * Creates a frame queue object for the given @fqid, unless the -+ * QMAN_FQ_FLAG_DYNAMIC_FQID flag is set in @flags, in which case a FQID is -+ * dynamically allocated (or the function fails if none are available). Once -+ * created, the caller should not touch the memory at 'fq' except as extended to -+ * adjacent memory for user-defined fields (see the definition of "struct -+ * qman_fq" for more info). NO_MODIFY is only intended for enqueuing to -+ * pre-existing frame-queues that aren't to be otherwise interfered with, it -+ * prevents all other modifications to the frame queue. The TO_DCPORTAL flag -+ * causes the driver to honour any contextB modifications requested in the -+ * qm_init_fq() API, as this indicates the frame queue will be consumed by a -+ * direct-connect portal (PME, CAAM, or Fman). When frame queues are consumed by -+ * software portals, the contextB field is controlled by the driver and can't be -+ * modified by the caller. If the AS_IS flag is specified, management commands -+ * will be used on portal @p to query state for frame queue @fqid and construct -+ * a frame queue object based on that, rather than assuming/requiring that it be -+ * Out of Service. -+ */ -+int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq); -+ -+/** -+ * qman_destroy_fq - Deallocates a FQ -+ * @fq: the frame queue object to release -+ * @flags: bit-mask of QMAN_FQ_FREE_*** options -+ * -+ * The memory for this frame queue object ('fq' provided in qman_create_fq()) is -+ * not deallocated but the caller regains ownership, to do with as desired. The -+ * FQ must be in the 'out-of-service' state unless the QMAN_FQ_FREE_PARKED flag -+ * is specified, in which case it may also be in the 'parked' state. -+ */ -+void qman_destroy_fq(struct qman_fq *fq, u32 flags); -+ -+/** -+ * qman_fq_fqid - Queries the frame queue ID of a FQ object -+ * @fq: the frame queue object to query -+ */ -+u32 qman_fq_fqid(struct qman_fq *fq); -+ -+/** -+ * qman_fq_state - Queries the state of a FQ object -+ * @fq: the frame queue object to query -+ * @state: pointer to state enum to return the FQ scheduling state -+ * @flags: pointer to state flags to receive QMAN_FQ_STATE_*** bitmask -+ * -+ * Queries the state of the FQ object, without performing any h/w commands. -+ * This captures the state, as seen by the driver, at the time the function -+ * executes. -+ */ -+void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags); -+ -+/** -+ * qman_init_fq - Initialises FQ fields, leaves the FQ "parked" or "scheduled" -+ * @fq: the frame queue object to modify, must be 'parked' or new. -+ * @flags: bit-mask of QMAN_INITFQ_FLAG_*** options -+ * @opts: the FQ-modification settings, as defined in the low-level API -+ * -+ * The @opts parameter comes from the low-level portal API. Select -+ * QMAN_INITFQ_FLAG_SCHED in @flags to cause the frame queue to be scheduled -+ * rather than parked. NB, @opts can be NULL. -+ * -+ * Note that some fields and options within @opts may be ignored or overwritten -+ * by the driver; -+ * 1. the 'count' and 'fqid' fields are always ignored (this operation only -+ * affects one frame queue: @fq). -+ * 2. the QM_INITFQ_WE_CONTEXTB option of the 'we_mask' field and the associated -+ * 'fqd' structure's 'context_b' field are sometimes overwritten; -+ * - if @fq was not created with QMAN_FQ_FLAG_TO_DCPORTAL, then context_b is -+ * initialised to a value used by the driver for demux. -+ * - if context_b is initialised for demux, so is context_a in case stashing -+ * is requested (see item 4). -+ * (So caller control of context_b is only possible for TO_DCPORTAL frame queue -+ * objects.) -+ * 3. if @flags contains QMAN_INITFQ_FLAG_LOCAL, the 'fqd' structure's -+ * 'dest::channel' field will be overwritten to match the portal used to issue -+ * the command. If the WE_DESTWQ write-enable bit had already been set by the -+ * caller, the channel workqueue will be left as-is, otherwise the write-enable -+ * bit is set and the workqueue is set to a default of 4. If the "LOCAL" flag -+ * isn't set, the destination channel/workqueue fields and the write-enable bit -+ * are left as-is. -+ * 4. if the driver overwrites context_a/b for demux, then if -+ * QM_INITFQ_WE_CONTEXTA is set, the driver will only overwrite -+ * context_a.address fields and will leave the stashing fields provided by the -+ * user alone, otherwise it will zero out the context_a.stashing fields. -+ */ -+int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts); -+ -+/** -+ * qman_schedule_fq - Schedules a FQ -+ * @fq: the frame queue object to schedule, must be 'parked' -+ * -+ * Schedules the frame queue, which must be Parked, which takes it to -+ * Tentatively-Scheduled or Truly-Scheduled depending on its fill-level. -+ */ -+int qman_schedule_fq(struct qman_fq *fq); -+ -+/** -+ * qman_retire_fq - Retires a FQ -+ * @fq: the frame queue object to retire -+ * @flags: FQ flags (as per qman_fq_state) if retirement completes immediately -+ * -+ * Retires the frame queue. This returns zero if it succeeds immediately, +1 if -+ * the retirement was started asynchronously, otherwise it returns negative for -+ * failure. When this function returns zero, @flags is set to indicate whether -+ * the retired FQ is empty and/or whether it has any ORL fragments (to show up -+ * as ERNs). Otherwise the corresponding flags will be known when a subsequent -+ * FQRN message shows up on the portal's message ring. -+ * -+ * NB, if the retirement is asynchronous (the FQ was in the Truly Scheduled or -+ * Active state), the completion will be via the message ring as a FQRN - but -+ * the corresponding callback may occur before this function returns!! Ie. the -+ * caller should be prepared to accept the callback as the function is called, -+ * not only once it has returned. -+ */ -+int qman_retire_fq(struct qman_fq *fq, u32 *flags); -+ -+/** -+ * qman_oos_fq - Puts a FQ "out of service" -+ * @fq: the frame queue object to be put out-of-service, must be 'retired' -+ * -+ * The frame queue must be retired and empty, and if any order restoration list -+ * was released as ERNs at the time of retirement, they must all be consumed. -+ */ -+int qman_oos_fq(struct qman_fq *fq); -+ -+/** -+ * qman_fq_flow_control - Set the XON/XOFF state of a FQ -+ * @fq: the frame queue object to be set to XON/XOFF state, must not be 'oos', -+ * or 'retired' or 'parked' state -+ * @xon: boolean to set fq in XON or XOFF state -+ * -+ * The frame should be in Tentatively Scheduled state or Truly Schedule sate, -+ * otherwise the IFSI interrupt will be asserted. -+ */ -+int qman_fq_flow_control(struct qman_fq *fq, int xon); -+ -+/** -+ * qman_query_fq - Queries FQD fields (via h/w query command) -+ * @fq: the frame queue object to be queried -+ * @fqd: storage for the queried FQD fields -+ */ -+int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd); -+ -+/** -+ * qman_query_fq_np - Queries non-programmable FQD fields -+ * @fq: the frame queue object to be queried -+ * @np: storage for the queried FQD fields -+ */ -+int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np); -+ -+/** -+ * qman_query_wq - Queries work queue lengths -+ * @query_dedicated: If non-zero, query length of WQs in the channel dedicated -+ * to this software portal. Otherwise, query length of WQs in a -+ * channel specified in wq. -+ * @wq: storage for the queried WQs lengths. Also specified the channel to -+ * to query if query_dedicated is zero. -+ */ -+int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq); -+ -+/** -+ * qman_volatile_dequeue - Issue a volatile dequeue command -+ * @fq: the frame queue object to dequeue from -+ * @flags: a bit-mask of QMAN_VOLATILE_FLAG_*** options -+ * @vdqcr: bit mask of QM_VDQCR_*** options, as per qm_dqrr_vdqcr_set() -+ * -+ * Attempts to lock access to the portal's VDQCR volatile dequeue functionality. -+ * The function will block and sleep if QMAN_VOLATILE_FLAG_WAIT is specified and -+ * the VDQCR is already in use, otherwise returns non-zero for failure. If -+ * QMAN_VOLATILE_FLAG_FINISH is specified, the function will only return once -+ * the VDQCR command has finished executing (ie. once the callback for the last -+ * DQRR entry resulting from the VDQCR command has been called). If not using -+ * the FINISH flag, completion can be determined either by detecting the -+ * presence of the QM_DQRR_STAT_UNSCHEDULED and QM_DQRR_STAT_DQCR_EXPIRED bits -+ * in the "stat" field of the "struct qm_dqrr_entry" passed to the FQ's dequeue -+ * callback, or by waiting for the QMAN_FQ_STATE_VDQCR bit to disappear from the -+ * "flags" retrieved from qman_fq_state(). -+ */ -+int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr); -+ -+/** -+ * qman_enqueue - Enqueue a frame to a frame queue -+ * @fq: the frame queue object to enqueue to -+ * @fd: a descriptor of the frame to be enqueued -+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options -+ * -+ * Fills an entry in the EQCR of portal @qm to enqueue the frame described by -+ * @fd. The descriptor details are copied from @fd to the EQCR entry, the 'pid' -+ * field is ignored. The return value is non-zero on error, such as ring full -+ * (and FLAG_WAIT not specified), congestion avoidance (FLAG_WATCH_CGR -+ * specified), etc. If the ring is full and FLAG_WAIT is specified, this -+ * function will block. If FLAG_INTERRUPT is set, the EQCI bit of the portal -+ * interrupt will assert when Qman consumes the EQCR entry (subject to "status -+ * disable", "enable", and "inhibit" registers). If FLAG_DCA is set, Qman will -+ * perform an implied "discrete consumption acknowledgement" on the dequeue -+ * ring's (DQRR) entry, at the ring index specified by the FLAG_DCA_IDX(x) -+ * macro. (As an alternative to issuing explicit DCA actions on DQRR entries, -+ * this implicit DCA can delay the release of a "held active" frame queue -+ * corresponding to a DQRR entry until Qman consumes the EQCR entry - providing -+ * order-preservation semantics in packet-forwarding scenarios.) If FLAG_DCA is -+ * set, then FLAG_DCA_PARK can also be set to imply that the DQRR consumption -+ * acknowledgement should "park request" the "held active" frame queue. Ie. -+ * when the portal eventually releases that frame queue, it will be left in the -+ * Parked state rather than Tentatively Scheduled or Truly Scheduled. If the -+ * portal is watching congestion groups, the QMAN_ENQUEUE_FLAG_WATCH_CGR flag -+ * is requested, and the FQ is a member of a congestion group, then this -+ * function returns -EAGAIN if the congestion group is currently congested. -+ * Note, this does not eliminate ERNs, as the async interface means we can be -+ * sending enqueue commands to an un-congested FQ that becomes congested before -+ * the enqueue commands are processed, but it does minimise needless thrashing -+ * of an already busy hardware resource by throttling many of the to-be-dropped -+ * enqueues "at the source". -+ */ -+int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags); -+ -+typedef int (*qman_cb_precommit) (void *arg); -+/** -+ * qman_enqueue_precommit - Enqueue a frame to a frame queue and call cb -+ * @fq: the frame queue object to enqueue to -+ * @fd: a descriptor of the frame to be enqueued -+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options -+ * @cb: user supplied callback function to invoke before writing commit verb. -+ * @cb_arg: callback function argument -+ * -+ * This is similar to qman_enqueue except that it will invoke a user supplied -+ * callback function just before writng the commit verb. This is useful -+ * when the user want to do something *just before* enqueuing the request and -+ * the enqueue can't fail. -+ */ -+int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd, -+ u32 flags, qman_cb_precommit cb, void *cb_arg); -+ -+/** -+ * qman_enqueue_orp - Enqueue a frame to a frame queue using an ORP -+ * @fq: the frame queue object to enqueue to -+ * @fd: a descriptor of the frame to be enqueued -+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options -+ * @orp: the frame queue object used as an order restoration point. -+ * @orp_seqnum: the sequence number of this frame in the order restoration path -+ * -+ * Similar to qman_enqueue(), but with the addition of an Order Restoration -+ * Point (@orp) and corresponding sequence number (@orp_seqnum) for this -+ * enqueue operation to employ order restoration. Each frame queue object acts -+ * as an Order Definition Point (ODP) by providing each frame dequeued from it -+ * with an incrementing sequence number, this value is generally ignored unless -+ * that sequence of dequeued frames will need order restoration later. Each -+ * frame queue object also encapsulates an Order Restoration Point (ORP), which -+ * is a re-assembly context for re-ordering frames relative to their sequence -+ * numbers as they are enqueued. The ORP does not have to be within the frame -+ * queue that receives the enqueued frame, in fact it is usually the frame -+ * queue from which the frames were originally dequeued. For the purposes of -+ * order restoration, multiple frames (or "fragments") can be enqueued for a -+ * single sequence number by setting the QMAN_ENQUEUE_FLAG_NLIS flag for all -+ * enqueues except the final fragment of a given sequence number. Ordering -+ * between sequence numbers is guaranteed, even if fragments of different -+ * sequence numbers are interlaced with one another. Fragments of the same -+ * sequence number will retain the order in which they are enqueued. If no -+ * enqueue is to performed, QMAN_ENQUEUE_FLAG_HOLE indicates that the given -+ * sequence number is to be "skipped" by the ORP logic (eg. if a frame has been -+ * dropped from a sequence), or QMAN_ENQUEUE_FLAG_NESN indicates that the given -+ * sequence number should become the ORP's "Next Expected Sequence Number". -+ * -+ * Side note: a frame queue object can be used purely as an ORP, without -+ * carrying any frames at all. Care should be taken not to deallocate a frame -+ * queue object that is being actively used as an ORP, as a future allocation -+ * of the frame queue object may start using the internal ORP before the -+ * previous use has finished. -+ */ -+int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags, -+ struct qman_fq *orp, u16 orp_seqnum); -+ -+/** -+ * qman_alloc_fqid_range - Allocate a contiguous range of FQIDs -+ * @result: is set by the API to the base FQID of the allocated range -+ * @count: the number of FQIDs required -+ * @align: required alignment of the allocated range -+ * @partial: non-zero if the API can return fewer than @count FQIDs -+ * -+ * Returns the number of frame queues allocated, or a negative error code. If -+ * @partial is non zero, the allocation request may return a smaller range of -+ * FQs than requested (though alignment will be as requested). If @partial is -+ * zero, the return value will either be 'count' or negative. -+ */ -+int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial); -+static inline int qman_alloc_fqid(u32 *result) -+{ -+ int ret = qman_alloc_fqid_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+ -+/** -+ * qman_release_fqid_range - Release the specified range of frame queue IDs -+ * @fqid: the base FQID of the range to deallocate -+ * @count: the number of FQIDs in the range -+ * -+ * This function can also be used to seed the allocator with ranges of FQIDs -+ * that it can subsequently allocate from. -+ */ -+void qman_release_fqid_range(u32 fqid, unsigned int count); -+static inline void qman_release_fqid(u32 fqid) -+{ -+ qman_release_fqid_range(fqid, 1); -+} -+ -+void qman_seed_fqid_range(u32 fqid, unsigned int count); -+ -+ -+int qman_shutdown_fq(u32 fqid); -+ -+/** -+ * qman_reserve_fqid_range - Reserve the specified range of frame queue IDs -+ * @fqid: the base FQID of the range to deallocate -+ * @count: the number of FQIDs in the range -+ */ -+int qman_reserve_fqid_range(u32 fqid, unsigned int count); -+static inline int qman_reserve_fqid(u32 fqid) -+{ -+ return qman_reserve_fqid_range(fqid, 1); -+} -+ -+ /* Pool-channel management */ -+ /* ----------------------- */ -+/** -+ * qman_alloc_pool_range - Allocate a contiguous range of pool-channel IDs -+ * @result: is set by the API to the base pool-channel ID of the allocated range -+ * @count: the number of pool-channel IDs required -+ * @align: required alignment of the allocated range -+ * @partial: non-zero if the API can return fewer than @count -+ * -+ * Returns the number of pool-channel IDs allocated, or a negative error code. -+ * If @partial is non zero, the allocation request may return a smaller range of -+ * than requested (though alignment will be as requested). If @partial is zero, -+ * the return value will either be 'count' or negative. -+ */ -+int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial); -+static inline int qman_alloc_pool(u32 *result) -+{ -+ int ret = qman_alloc_pool_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+ -+/** -+ * qman_release_pool_range - Release the specified range of pool-channel IDs -+ * @id: the base pool-channel ID of the range to deallocate -+ * @count: the number of pool-channel IDs in the range -+ */ -+void qman_release_pool_range(u32 id, unsigned int count); -+static inline void qman_release_pool(u32 id) -+{ -+ qman_release_pool_range(id, 1); -+} -+ -+/** -+ * qman_reserve_pool_range - Reserve the specified range of pool-channel IDs -+ * @id: the base pool-channel ID of the range to reserve -+ * @count: the number of pool-channel IDs in the range -+ */ -+int qman_reserve_pool_range(u32 id, unsigned int count); -+static inline int qman_reserve_pool(u32 id) -+{ -+ return qman_reserve_pool_range(id, 1); -+} -+ -+void qman_seed_pool_range(u32 id, unsigned int count); -+ -+ /* CGR management */ -+ /* -------------- */ -+/** -+ * qman_create_cgr - Register a congestion group object -+ * @cgr: the 'cgr' object, with fields filled in -+ * @flags: QMAN_CGR_FLAG_* values -+ * @opts: optional state of CGR settings -+ * -+ * Registers this object to receiving congestion entry/exit callbacks on the -+ * portal affine to the cpu portal on which this API is executed. If opts is -+ * NULL then only the callback (cgr->cb) function is registered. If @flags -+ * contains QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset -+ * any unspecified parameters) will be used rather than a modify hw hardware -+ * (which only modifies the specified parameters). -+ */ -+int qman_create_cgr(struct qman_cgr *cgr, u32 flags, -+ struct qm_mcc_initcgr *opts); -+ -+/** -+ * qman_create_cgr_to_dcp - Register a congestion group object to DCP portal -+ * @cgr: the 'cgr' object, with fields filled in -+ * @flags: QMAN_CGR_FLAG_* values -+ * @dcp_portal: the DCP portal to which the cgr object is registered. -+ * @opts: optional state of CGR settings -+ * -+ */ -+int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal, -+ struct qm_mcc_initcgr *opts); -+ -+/** -+ * qman_delete_cgr - Deregisters a congestion group object -+ * @cgr: the 'cgr' object to deregister -+ * -+ * "Unplugs" this CGR object from the portal affine to the cpu on which this API -+ * is executed. This must be excuted on the same affine portal on which it was -+ * created. -+ */ -+int qman_delete_cgr(struct qman_cgr *cgr); -+ -+/** -+ * qman_delete_cgr_safe - Deregisters a congestion group object from any CPU -+ * @cgr: the 'cgr' object to deregister -+ * -+ * This will select the proper CPU and run there qman_delete_cgr(). -+ */ -+void qman_delete_cgr_safe(struct qman_cgr *cgr); -+ -+/** -+ * qman_modify_cgr - Modify CGR fields -+ * @cgr: the 'cgr' object to modify -+ * @flags: QMAN_CGR_FLAG_* values -+ * @opts: the CGR-modification settings -+ * -+ * The @opts parameter comes from the low-level portal API, and can be NULL. -+ * Note that some fields and options within @opts may be ignored or overwritten -+ * by the driver, in particular the 'cgrid' field is ignored (this operation -+ * only affects the given CGR object). If @flags contains -+ * QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset any -+ * unspecified parameters) will be used rather than a modify hw hardware (which -+ * only modifies the specified parameters). -+ */ -+int qman_modify_cgr(struct qman_cgr *cgr, u32 flags, -+ struct qm_mcc_initcgr *opts); -+ -+/** -+* qman_query_cgr - Queries CGR fields -+* @cgr: the 'cgr' object to query -+* @result: storage for the queried congestion group record -+*/ -+int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *result); -+ -+/** -+ * qman_query_congestion - Queries the state of all congestion groups -+ * @congestion: storage for the queried state of all congestion groups -+ */ -+int qman_query_congestion(struct qm_mcr_querycongestion *congestion); -+ -+/** -+ * qman_alloc_cgrid_range - Allocate a contiguous range of CGR IDs -+ * @result: is set by the API to the base CGR ID of the allocated range -+ * @count: the number of CGR IDs required -+ * @align: required alignment of the allocated range -+ * @partial: non-zero if the API can return fewer than @count -+ * -+ * Returns the number of CGR IDs allocated, or a negative error code. -+ * If @partial is non zero, the allocation request may return a smaller range of -+ * than requested (though alignment will be as requested). If @partial is zero, -+ * the return value will either be 'count' or negative. -+ */ -+int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial); -+static inline int qman_alloc_cgrid(u32 *result) -+{ -+ int ret = qman_alloc_cgrid_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+ -+/** -+ * qman_release_cgrid_range - Release the specified range of CGR IDs -+ * @id: the base CGR ID of the range to deallocate -+ * @count: the number of CGR IDs in the range -+ */ -+void qman_release_cgrid_range(u32 id, unsigned int count); -+static inline void qman_release_cgrid(u32 id) -+{ -+ qman_release_cgrid_range(id, 1); -+} -+ -+/** -+ * qman_reserve_cgrid_range - Reserve the specified range of CGR ID -+ * @id: the base CGR ID of the range to reserve -+ * @count: the number of CGR IDs in the range -+ */ -+int qman_reserve_cgrid_range(u32 id, unsigned int count); -+static inline int qman_reserve_cgrid(u32 id) -+{ -+ return qman_reserve_cgrid_range(id, 1); -+} -+ -+void qman_seed_cgrid_range(u32 id, unsigned int count); -+ -+ -+ /* Helpers */ -+ /* ------- */ -+/** -+ * qman_poll_fq_for_init - Check if an FQ has been initialised from OOS -+ * @fqid: the FQID that will be initialised by other s/w -+ * -+ * In many situations, a FQID is provided for communication between s/w -+ * entities, and whilst the consumer is responsible for initialising and -+ * scheduling the FQ, the producer(s) generally create a wrapper FQ object using -+ * and only call qman_enqueue() (no FQ initialisation, scheduling, etc). Ie; -+ * qman_create_fq(..., QMAN_FQ_FLAG_NO_MODIFY, ...); -+ * However, data can not be enqueued to the FQ until it is initialised out of -+ * the OOS state - this function polls for that condition. It is particularly -+ * useful for users of IPC functions - each endpoint's Rx FQ is the other -+ * endpoint's Tx FQ, so each side can initialise and schedule their Rx FQ object -+ * and then use this API on the (NO_MODIFY) Tx FQ object in order to -+ * synchronise. The function returns zero for success, +1 if the FQ is still in -+ * the OOS state, or negative if there was an error. -+ */ -+static inline int qman_poll_fq_for_init(struct qman_fq *fq) -+{ -+ struct qm_mcr_queryfq_np np; -+ int err; -+ err = qman_query_fq_np(fq, &np); -+ if (err) -+ return err; -+ if ((np.state & QM_MCR_NP_STATE_MASK) == QM_MCR_NP_STATE_OOS) -+ return 1; -+ return 0; -+} -+ -+ /* -------------- */ -+ /* CEETM :: types */ -+ /* -------------- */ -+/** -+ * Token Rate Structure -+ * Shaping rates are based on a "credit" system and a pre-configured h/w -+ * internal timer. The following type represents a shaper "rate" parameter as a -+ * fractional number of "tokens". Here's how it works. This (fractional) number -+ * of tokens is added to the shaper's "credit" every time the h/w timer elapses -+ * (up to a limit which is set by another shaper parameter). Every time a frame -+ * is enqueued through a shaper, the shaper deducts as many tokens as there are -+ * bytes of data in the enqueued frame. A shaper will not allow itself to -+ * enqueue any frames if its token count is negative. As such; -+ * -+ * The rate at which data is enqueued is limited by the -+ * rate at which tokens are added. -+ * -+ * Therefore if the user knows the period between these h/w timer updates in -+ * seconds, they can calculate the maximum traffic rate of the shaper (in -+ * bytes-per-second) from the token rate. And vice versa, they can calculate -+ * the token rate to use in order to achieve a given traffic rate. -+ */ -+struct qm_ceetm_rate { -+ /* The token rate is; whole + (fraction/8192) */ -+ u32 whole:11; /* 0..2047 */ -+ u32 fraction:13; /* 0..8191 */ -+}; -+ -+struct qm_ceetm_weight_code { -+ /* The weight code is; 5 msbits + 3 lsbits */ -+ u8 y:5; -+ u8 x:3; -+}; -+ -+struct qm_ceetm { -+ unsigned int idx; -+ struct list_head sub_portals; -+ struct list_head lnis; -+ unsigned int sp_range[2]; -+ unsigned int lni_range[2]; -+}; -+ -+struct qm_ceetm_sp { -+ struct list_head node; -+ unsigned int idx; -+ unsigned int dcp_idx; -+ int is_claimed; -+ struct qm_ceetm_lni *lni; -+}; -+ -+/* Logical Network Interface */ -+struct qm_ceetm_lni { -+ struct list_head node; -+ unsigned int idx; -+ unsigned int dcp_idx; -+ int is_claimed; -+ struct qm_ceetm_sp *sp; -+ struct list_head channels; -+ int shaper_enable; -+ int shaper_couple; -+ int oal; -+ struct qm_ceetm_rate cr_token_rate; -+ struct qm_ceetm_rate er_token_rate; -+ u16 cr_token_bucket_limit; -+ u16 er_token_bucket_limit; -+}; -+ -+/* Class Queue Channel */ -+struct qm_ceetm_channel { -+ struct list_head node; -+ unsigned int idx; -+ unsigned int lni_idx; -+ unsigned int dcp_idx; -+ struct list_head class_queues; -+ struct list_head ccgs; -+ u8 shaper_enable; -+ u8 shaper_couple; -+ struct qm_ceetm_rate cr_token_rate; -+ struct qm_ceetm_rate er_token_rate; -+ u16 cr_token_bucket_limit; -+ u16 er_token_bucket_limit; -+}; -+ -+struct qm_ceetm_ccg; -+ -+/* This callback type is used when handling congestion entry/exit. The -+ * 'cb_ctx' value is the opaque value associated with ccg object. -+ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit. -+ */ -+typedef void (*qman_cb_ccgr)(struct qm_ceetm_ccg *ccg, void *cb_ctx, -+ int congested); -+ -+/* Class Congestion Group */ -+struct qm_ceetm_ccg { -+ struct qm_ceetm_channel *parent; -+ struct list_head node; -+ struct list_head cb_node; -+ qman_cb_ccgr cb; -+ void *cb_ctx; -+ unsigned int idx; -+}; -+ -+/* Class Queue */ -+struct qm_ceetm_cq { -+ struct qm_ceetm_channel *parent; -+ struct qm_ceetm_ccg *ccg; -+ struct list_head node; -+ unsigned int idx; -+ int is_claimed; -+ struct list_head bound_lfqids; -+ struct list_head binding_node; -+}; -+ -+/* Logical Frame Queue */ -+struct qm_ceetm_lfq { -+ struct qm_ceetm_channel *parent; -+ struct list_head node; -+ unsigned int idx; -+ unsigned int dctidx; -+ u64 context_a; -+ u32 context_b; -+ qman_cb_mr ern; -+}; -+ -+/** -+ * qman_ceetm_bps2tokenrate - Given a desired rate 'bps' measured in bps -+ * (ie. bits-per-second), compute the 'token_rate' fraction that best -+ * approximates that rate. -+ * @bps: the desired shaper rate in bps. -+ * @token_rate: the output token rate computed with the given kbps. -+ * @rounding: dictates how to round if an exact conversion is not possible; if -+ * it is negative then 'token_rate' will round down to the highest value that -+ * does not exceed the desired rate, if it is positive then 'token_rate' will -+ * round up to the lowest value that is greater than or equal to the desired -+ * rate, and if it is zero then it will round to the nearest approximation, -+ * whether that be up or down. -+ * -+ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available. -+ */ -+int qman_ceetm_bps2tokenrate(u64 bps, -+ struct qm_ceetm_rate *token_rate, -+ int rounding); -+ -+/** -+ * qman_ceetm_tokenrate2bps - Given a 'token_rate', compute the -+ * corresponding number of 'bps'. -+ * @token_rate: the input desired token_rate fraction. -+ * @bps: the output shaper rate in bps computed with the give token rate. -+ * @rounding: has the same semantics as the previous function. -+ * -+ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available. -+ */ -+int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, -+ u64 *bps, -+ int rounding); -+ -+int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align, -+ int partial); -+static inline int qman_alloc_ceetm0_channel(u32 *result) -+{ -+ int ret = qman_alloc_ceetm0_channel_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+void qman_release_ceetm0_channel_range(u32 channelid, u32 count); -+static inline void qman_release_ceetm0_channelid(u32 channelid) -+{ -+ qman_release_ceetm0_channel_range(channelid, 1); -+} -+ -+int qman_reserve_ceetm0_channel_range(u32 channelid, u32 count); -+static inline int qman_reserve_ceetm0_channelid(u32 channelid) -+{ -+ return qman_reserve_ceetm0_channel_range(channelid, 1); -+} -+ -+void qman_seed_ceetm0_channel_range(u32 channelid, u32 count); -+ -+ -+int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align, -+ int partial); -+static inline int qman_alloc_ceetm1_channel(u32 *result) -+{ -+ int ret = qman_alloc_ceetm1_channel_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+void qman_release_ceetm1_channel_range(u32 channelid, u32 count); -+static inline void qman_release_ceetm1_channelid(u32 channelid) -+{ -+ qman_release_ceetm1_channel_range(channelid, 1); -+} -+int qman_reserve_ceetm1_channel_range(u32 channelid, u32 count); -+static inline int qman_reserve_ceetm1_channelid(u32 channelid) -+{ -+ return qman_reserve_ceetm1_channel_range(channelid, 1); -+} -+ -+void qman_seed_ceetm1_channel_range(u32 channelid, u32 count); -+ -+ -+int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align, -+ int partial); -+static inline int qman_alloc_ceetm0_lfqid(u32 *result) -+{ -+ int ret = qman_alloc_ceetm0_lfqid_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count); -+static inline void qman_release_ceetm0_lfqid(u32 lfqid) -+{ -+ qman_release_ceetm0_lfqid_range(lfqid, 1); -+} -+int qman_reserve_ceetm0_lfqid_range(u32 lfqid, u32 count); -+static inline int qman_reserve_ceetm0_lfqid(u32 lfqid) -+{ -+ return qman_reserve_ceetm0_lfqid_range(lfqid, 1); -+} -+ -+void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count); -+ -+ -+int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align, -+ int partial); -+static inline int qman_alloc_ceetm1_lfqid(u32 *result) -+{ -+ int ret = qman_alloc_ceetm1_lfqid_range(result, 1, 0, 0); -+ return (ret > 0) ? 0 : ret; -+} -+void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count); -+static inline void qman_release_ceetm1_lfqid(u32 lfqid) -+{ -+ qman_release_ceetm1_lfqid_range(lfqid, 1); -+} -+int qman_reserve_ceetm1_lfqid_range(u32 lfqid, u32 count); -+static inline int qman_reserve_ceetm1_lfqid(u32 lfqid) -+{ -+ return qman_reserve_ceetm1_lfqid_range(lfqid, 1); -+} -+ -+void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count); -+ -+ -+ /* ----------------------------- */ -+ /* CEETM :: sub-portals */ -+ /* ----------------------------- */ -+ -+/** -+ * qman_ceetm_sp_claim - Claims the given sub-portal, provided it is available -+ * to us and configured for traffic-management. -+ * @sp: the returned sub-portal object, if successful. -+ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM -+ * instance), -+ * @sp_idx" is the desired sub-portal index from 0 to 15. -+ * -+ * Returns zero for success, or -ENODEV if the sub-portal is in use, or -EINVAL -+ * if the sp_idx is out of range. -+ * -+ * Note that if there are multiple driver domains (eg. a linux kernel versus -+ * user-space drivers in USDPAA, or multiple guests running under a hypervisor) -+ * then a sub-portal may be accessible by more than one instance of a qman -+ * driver and so it may be claimed multiple times. If this is the case, it is -+ * up to the system architect to prevent conflicting configuration actions -+ * coming from the different driver domains. The qman drivers do not have any -+ * behind-the-scenes coordination to prevent this from happening. -+ */ -+int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, -+ enum qm_dc_portal dcp_idx, -+ unsigned int sp_idx); -+ -+/** -+ * qman_ceetm_sp_release - Releases a previously claimed sub-portal. -+ * @sp: the sub-portal to be released. -+ * -+ * Returns 0 for success, or -EBUSY for failure if the dependencies are not -+ * released. -+ */ -+int qman_ceetm_sp_release(struct qm_ceetm_sp *sp); -+ -+ /* ----------------------------------- */ -+ /* CEETM :: logical network interfaces */ -+ /* ----------------------------------- */ -+ -+/** -+ * qman_ceetm_lni_claim - Claims an unclaimed LNI. -+ * @lni: the returned LNI object, if successful. -+ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM -+ * instance) -+ * @lni_idx: is the desired LNI index. -+ * -+ * Returns zero for success, or -EINVAL on failure, which will happen if the LNI -+ * is not available or has already been claimed (and not yet successfully -+ * released), or lni_dix is out of range. -+ * -+ * Note that there may be multiple driver domains (or instances) that need to -+ * transmit out the same LNI, so this claim is only guaranteeing exclusivity -+ * within the domain of the driver being called. See qman_ceetm_sp_claim() and -+ * qman_ceetm_sp_get_lni() for more information. -+ */ -+int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, -+ enum qm_dc_portal dcp_id, -+ unsigned int lni_idx); -+ -+/** -+ * qman_ceetm_lni_releaes - Releases a previously claimed LNI. -+ * @lni: the lni needs to be released. -+ * -+ * This will only succeed if all dependent objects have been released. -+ * Returns zero for success, or -EBUSY if the dependencies are not released. -+ */ -+int qman_ceetm_lni_release(struct qm_ceetm_lni *lni); -+ -+/** -+ * qman_ceetm_sp_set_lni -+ * qman_ceetm_sp_get_lni - Set/get the LNI that the sub-portal is currently -+ * mapped to. -+ * @sp: the given sub-portal. -+ * @lni(in "set"function): the LNI object which the sp will be mappaed to. -+ * @lni_idx(in "get" function): the LNI index which the sp is mapped to. -+ * -+ * Returns zero for success, or -EINVAL for the "set" function when this sp-lni -+ * mapping has been set, or configure mapping command returns error, and -+ * -EINVAL for "get" function when this sp-lni mapping is not set or the query -+ * mapping command returns error. -+ * -+ * This may be useful in situations where multiple driver domains have access -+ * to the same sub-portals in order to all be able to transmit out the same -+ * physical interface (perhaps they're on different IP addresses or VPNs, so -+ * Fman is splitting Rx traffic and here we need to converge Tx traffic). In -+ * that case, a control-plane is likely to use qman_ceetm_lni_claim() followed -+ * by qman_ceetm_sp_set_lni() to configure the sub-portal, and other domains -+ * are likely to use qman_ceetm_sp_get_lni() followed by qman_ceetm_lni_claim() -+ * in order to determine the LNI that the control-plane had assigned. This is -+ * why the "get" returns an index, whereas the "set" takes an (already claimed) -+ * LNI object. -+ */ -+int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, -+ struct qm_ceetm_lni *lni); -+int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, -+ unsigned int *lni_idx); -+ -+/** -+ * qman_ceetm_lni_enable_shaper -+ * qman_ceetm_lni_disable_shaper - Enables/disables shaping on the LNI. -+ * @lni: the given LNI. -+ * @coupled: indicates whether CR and ER are coupled. -+ * @oal: the overhead accounting length which is added to the actual length of -+ * each frame when performing shaper calculations. -+ * -+ * When the number of (unused) committed-rate tokens reach the committed-rate -+ * token limit, 'coupled' indicates whether surplus tokens should be added to -+ * the excess-rate token count (up to the excess-rate token limit). -+ * When LNI is claimed, the shaper is disabled by default. The enable function -+ * will turn on this shaper for this lni. -+ * Whenever a claimed LNI is first enabled for shaping, its committed and -+ * excess token rates and limits are zero, so will need to be changed to do -+ * anything useful. The shaper can subsequently be enabled/disabled without -+ * resetting the shaping parameters, but the shaping parameters will be reset -+ * when the LNI is released. -+ * -+ * Returns zero for success, or errno for "enable" function in the cases as: -+ * a) -EINVAL if the shaper is already enabled, -+ * b) -EIO if the configure shaper command returns error. -+ * For "disable" function, returns: -+ * a) -EINVAL if the shaper is has already disabled. -+ * b) -EIO if calling configure shaper command returns error. -+ */ -+int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled, -+ int oal); -+int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni); -+ -+/** -+ * qman_ceetm_lni_is_shaper_enabled - Check LNI shaper status -+ * @lni: the give LNI -+ */ -+int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni); -+ -+/** -+ * qman_ceetm_lni_set_commit_rate -+ * qman_ceetm_lni_get_commit_rate -+ * qman_ceetm_lni_set_excess_rate -+ * qman_ceetm_lni_get_excess_rate - Set/get the shaper CR/ER token rate and -+ * token limit for the given LNI. -+ * @lni: the given LNI. -+ * @token_rate: the desired token rate for "set" fuction, or the token rate of -+ * the LNI queried by "get" function. -+ * @token_limit: the desired token bucket limit for "set" function, or the token -+ * limit of the given LNI queried by "get" function. -+ * -+ * Returns zero for success. The "set" function returns -EINVAL if the given -+ * LNI is unshapped or -EIO if the configure shaper command returns error. -+ * The "get" function returns -EINVAL if the token rate or the token limit is -+ * not set or the query command returns error. -+ */ -+int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit); -+int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit); -+int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit); -+int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit); -+/** -+ * qman_ceetm_lni_set_commit_rate_bps -+ * qman_ceetm_lni_get_commit_rate_bps -+ * qman_ceetm_lni_set_excess_rate_bps -+ * qman_ceetm_lni_get_excess_rate_bps - Set/get the shaper CR/ER rate -+ * and token limit for the given LNI. -+ * @lni: the given LNI. -+ * @bps: the desired shaping rate in bps for "set" fuction, or the shaping rate -+ * of the LNI queried by "get" function. -+ * @token_limit: the desired token bucket limit for "set" function, or the token -+ * limit of the given LNI queried by "get" function. -+ * -+ * Returns zero for success. The "set" function returns -EINVAL if the given -+ * LNI is unshapped or -EIO if the configure shaper command returns error. -+ * The "get" function returns -EINVAL if the token rate or the token limit is -+ * not set or the query command returns error. -+ */ -+int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni, -+ u64 bps, -+ u16 token_limit); -+int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni, -+ u64 *bps, u16 *token_limit); -+int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni, -+ u64 bps, -+ u16 token_limit); -+int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni, -+ u64 *bps, u16 *token_limit); -+ -+/** -+ * qman_ceetm_lni_set_tcfcc -+ * qman_ceetm_lni_get_tcfcc - Configure/query "Traffic Class Flow Control". -+ * @lni: the given LNI. -+ * @cq_level: is between 0 and 15, representing individual class queue levels -+ * (CQ0 to CQ7 for every channel) and grouped class queue levels (CQ8 to CQ15 -+ * for every channel). -+ * @traffic_class: is between 0 and 7 when associating a given class queue level -+ * to a traffic class, or -1 when disabling traffic class flow control for this -+ * class queue level. -+ * -+ * Return zero for success, or -EINVAL if the cq_level or traffic_class is out -+ * of range as indicated above, or -EIO if the configure/query tcfcc command -+ * returns error. -+ * -+ * Refer to the section of QMan CEETM traffic class flow control in the -+ * Reference Manual. -+ */ -+int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni, -+ unsigned int cq_level, -+ int traffic_class); -+int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni, -+ unsigned int cq_level, -+ int *traffic_class); -+ -+ /* ----------------------------- */ -+ /* CEETM :: class queue channels */ -+ /* ----------------------------- */ -+ -+/** -+ * qman_ceetm_channel_claim - Claims an unclaimed CQ channel that is mapped to -+ * the given LNI. -+ * @channel: the returned class queue channel object, if successful. -+ * @lni: the LNI that the channel belongs to. -+ * -+ * Channels are always initially "unshaped". -+ * -+ * Return zero for success, or -ENODEV if there is no channel available(all 32 -+ * channels are claimed) or -EINVAL if the channel mapping command returns -+ * error. -+ */ -+int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel, -+ struct qm_ceetm_lni *lni); -+ -+/** -+ * qman_ceetm_channel_release - Releases a previously claimed CQ channel. -+ * @channel: the channel needs to be released. -+ * -+ * Returns zero for success, or -EBUSY if the dependencies are still in use. -+ * -+ * Note any shaping of the channel will be cleared to leave it in an unshaped -+ * state. -+ */ -+int qman_ceetm_channel_release(struct qm_ceetm_channel *channel); -+ -+/** -+ * qman_ceetm_channel_enable_shaper -+ * qman_ceetm_channel_disable_shaper - Enables/disables shaping on the channel. -+ * @channel: the given channel. -+ * @coupled: indicates whether surplus CR tokens should be added to the -+ * excess-rate token count (up to the excess-rate token limit) when the number -+ * of (unused) committed-rate tokens reach the committed_rate token limit. -+ * -+ * Whenever a claimed channel is first enabled for shaping, its committed and -+ * excess token rates and limits are zero, so will need to be changed to do -+ * anything useful. The shaper can subsequently be enabled/disabled without -+ * resetting the shaping parameters, but the shaping parameters will be reset -+ * when the channel is released. -+ * -+ * Return 0 for success, or -EINVAL for failure, in the case that the channel -+ * shaper has been enabled/disabled or the management command returns error. -+ */ -+int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel, -+ int coupled); -+int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel); -+ -+/** -+ * qman_ceetm_channel_is_shaper_enabled - Check channel shaper status. -+ * @channel: the give channel. -+ */ -+int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel); -+ -+/** -+ * qman_ceetm_channel_set_commit_rate -+ * qman_ceetm_channel_get_commit_rate -+ * qman_ceetm_channel_set_excess_rate -+ * qman_ceetm_channel_get_excess_rate - Set/get channel CR/ER shaper parameters. -+ * @channel: the given channel. -+ * @token_rate: the desired token rate for "set" function, or the queried token -+ * rate for "get" function. -+ * @token_limit: the desired token limit for "set" function, or the queried -+ * token limit for "get" function. -+ * -+ * Return zero for success. The "set" function returns -EINVAL if the channel -+ * is unshaped, or -EIO if the configure shapper command returns error. The -+ * "get" function returns -EINVAL if token rate of token limit is not set, or -+ * the query shaper command returns error. -+ */ -+int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit); -+int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit); -+int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel, -+ const struct qm_ceetm_rate *token_rate, -+ u16 token_limit); -+int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel, -+ struct qm_ceetm_rate *token_rate, -+ u16 *token_limit); -+/** -+ * qman_ceetm_channel_set_commit_rate_bps -+ * qman_ceetm_channel_get_commit_rate_bps -+ * qman_ceetm_channel_set_excess_rate_bps -+ * qman_ceetm_channel_get_excess_rate_bps - Set/get channel CR/ER shaper -+ * parameters. -+ * @channel: the given channel. -+ * @token_rate: the desired shaper rate in bps for "set" function, or the -+ * shaper rate in bps for "get" function. -+ * @token_limit: the desired token limit for "set" function, or the queried -+ * token limit for "get" function. -+ * -+ * Return zero for success. The "set" function returns -EINVAL if the channel -+ * is unshaped, or -EIO if the configure shapper command returns error. The -+ * "get" function returns -EINVAL if token rate of token limit is not set, or -+ * the query shaper command returns error. -+ */ -+int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel, -+ u64 bps, u16 token_limit); -+int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel, -+ u64 *bps, u16 *token_limit); -+int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel, -+ u64 bps, u16 token_limit); -+int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel, -+ u64 *bps, u16 *token_limit); -+ -+/** -+ * qman_ceetm_channel_set_weight -+ * qman_ceetm_channel_get_weight - Set/get the weight for unshaped channel -+ * @channel: the given channel. -+ * @token_limit: the desired token limit as the weight of the unshaped channel -+ * for "set" function, or the queried token limit for "get" function. -+ * -+ * The algorithm of unshaped fair queuing (uFQ) is used for unshaped channel. -+ * It allows the unshaped channels to be included in the CR time eligible list, -+ * and thus use the configured CR token limit value as their fair queuing -+ * weight. -+ * -+ * Return zero for success, or -EINVAL if the channel is a shaped channel or -+ * the management command returns error. -+ */ -+int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel, -+ u16 token_limit); -+int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel, -+ u16 *token_limit); -+ -+/** -+ * qman_ceetm_channel_set_group -+ * qman_ceetm_channel_get_group - Set/get the grouping of the class scheduler. -+ * @channel: the given channel. -+ * @group_b: indicates whether there is group B in this channel. -+ * @prio_a: the priority of group A. -+ * @prio_b: the priority of group B. -+ * -+ * There are 8 individual class queues (CQ0-CQ7), and 8 grouped class queues -+ * (CQ8-CQ15). If 'group_b' is zero, then all the grouped class queues are in -+ * group A, otherwise they are split into group A (CQ8-11) and group B -+ * (CQ12-C15). The individual class queues and the group(s) are in strict -+ * priority order relative to each other. Within the group(s), the scheduling -+ * is not strict priority order, but the result of scheduling within a group -+ * is in strict priority order relative to the other class queues in the -+ * channel. 'prio_a' and 'prio_b' control the priority order of the groups -+ * relative to the individual class queues, and take values from 0-7. Eg. if -+ * 'group_b' is non-zero, 'prio_a' is 2 and 'prio_b' is 6, then the strict -+ * priority order would be; -+ * CQ0, CQ1, CQ2, GROUPA, CQ3, CQ4, CQ5, CQ6, GROUPB, CQ7 -+ * -+ * Return 0 for success. For "set" function, returns -EINVAL if prio_a or -+ * prio_b are out of the range 0 - 7 (priority of group A or group B can not -+ * be 0, CQ0 is always the highest class queue in this channel.), or -EIO if -+ * the configure scheduler command returns error. For "get" function, return -+ * -EINVAL if the query scheduler command returns error. -+ */ -+int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, -+ int group_b, -+ unsigned int prio_a, -+ unsigned int prio_b); -+int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, -+ int *group_b, -+ unsigned int *prio_a, -+ unsigned int *prio_b); -+ -+/** -+ * qman_ceetm_channel_set_group_cr_eligibility -+ * qman_ceetm_channel_set_group_er_eligibility - Set channel group eligibility -+ * @channel: the given channel object -+ * @group_b: indicates whether there is group B in this channel. -+ * @cre: the commit rate eligibility, 1 for enable, 0 for disable. -+ * -+ * Return zero for success, or -EINVAL if eligibility setting fails. -+*/ -+int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel -+ *channel, int group_b, int cre); -+int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel -+ *channel, int group_b, int ere); -+ -+/** -+ * qman_ceetm_channel_set_cq_cr_eligibility -+ * qman_ceetm_channel_set_cq_er_eligibility - Set channel cq eligibility -+ * @channel: the given channel object -+ * @idx: is from 0 to 7 (representing CQ0 to CQ7). -+ * @cre: the commit rate eligibility, 1 for enable, 0 for disable. -+ * -+ * Return zero for success, or -EINVAL if eligibility setting fails. -+*/ -+int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel, -+ unsigned int idx, int cre); -+int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel, -+ unsigned int idx, int ere); -+ -+ /* --------------------- */ -+ /* CEETM :: class queues */ -+ /* --------------------- */ -+ -+/** -+ * qman_ceetm_cq_claim - Claims an individual class queue. -+ * @cq: the returned class queue object, if successful. -+ * @channel: the class queue channel. -+ * @idx: is from 0 to 7 (representing CQ0 to CQ7). -+ * @ccg: represents the class congestion group that this class queue should be -+ * subscribed to, or NULL if no congestion group membership is desired. -+ * -+ * Returns zero for success, or -EINVAL if @idx is out of range 0 - 7 or -+ * if this class queue has been claimed, or configure class queue command -+ * returns error, or returns -ENOMEM if allocating CQ memory fails. -+ */ -+int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq, -+ struct qm_ceetm_channel *channel, -+ unsigned int idx, -+ struct qm_ceetm_ccg *ccg); -+ -+/** -+ * qman_ceetm_cq_claim_A - Claims a class queue group A. -+ * @cq: the returned class queue object, if successful. -+ * @channel: the class queue channel. -+ * @idx: is from 8 to 15 if only group A exits, otherwise, it is from 8 to 11. -+ * @ccg: represents the class congestion group that this class queue should be -+ * subscribed to, or NULL if no congestion group membership is desired. -+ * -+ * Return zero for success, or -EINVAL if @idx is out the range or if -+ * this class queue has been claimed or configure class queue command returns -+ * error, or returns -ENOMEM if allocating CQ memory fails. -+ */ -+int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq, -+ struct qm_ceetm_channel *channel, -+ unsigned int idx, -+ struct qm_ceetm_ccg *ccg); -+ -+/** -+ * qman_ceetm_cq_claim_B - Claims a class queue group B. -+ * @cq: the returned class queue object, if successful. -+ * @channel: the class queue channel. -+ * @idx: is from 0 to 3 (CQ12 to CQ15). -+ * @ccg: represents the class congestion group that this class queue should be -+ * subscribed to, or NULL if no congestion group membership is desired. -+ * -+ * Return zero for success, or -EINVAL if @idx is out the range or if -+ * this class queue has been claimed or configure class queue command returns -+ * error, or returns -ENOMEM if allocating CQ memory fails. -+ */ -+int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq, -+ struct qm_ceetm_channel *channel, -+ unsigned int idx, -+ struct qm_ceetm_ccg *ccg); -+ -+/** -+ * qman_ceetm_cq_release - Releases a previously claimed class queue. -+ * @cq: The class queue to be released. -+ * -+ * Return zero for success, or -EBUSY if the dependent objects (eg. logical -+ * FQIDs) have not been released. -+ */ -+int qman_ceetm_cq_release(struct qm_ceetm_cq *cq); -+ -+/** -+ * qman_ceetm_set_queue_weight -+ * qman_ceetm_get_queue_weight - Configure/query the weight of a grouped class -+ * queue. -+ * @cq: the given class queue. -+ * @weight_code: the desired weight code to set for the given class queue for -+ * "set" function or the queired weight code for "get" function. -+ * -+ * Grouped class queues have a default weight code of zero, which corresponds to -+ * a scheduler weighting of 1. This function can be used to modify a grouped -+ * class queue to another weight, (Use the helpers qman_ceetm_wbfs2ratio() -+ * and qman_ceetm_ratio2wbfs() to convert between these 'weight_code' values -+ * and the corresponding sharing weight.) -+ * -+ * Returns zero for success, or -EIO if the configure weight command returns -+ * error for "set" function, or -EINVAL if the query command returns -+ * error for "get" function. -+ * See section "CEETM Weighted Scheduling among Grouped Classes" in Reference -+ * Manual for weight and weight code. -+ */ -+int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq, -+ struct qm_ceetm_weight_code *weight_code); -+int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq, -+ struct qm_ceetm_weight_code *weight_code); -+ -+/** -+ * qman_ceetm_set_queue_weight_in_ratio -+ * qman_ceetm_get_queue_weight_in_ratio - Configure/query the weight of a -+ * grouped class queue. -+ * @cq: the given class queue. -+ * @ratio: the weight in ratio. It should be the real ratio number multiplied -+ * by 100 to get rid of fraction. -+ * -+ * Returns zero for success, or -EIO if the configure weight command returns -+ * error for "set" function, or -EINVAL if the query command returns -+ * error for "get" function. -+ */ -+int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio); -+int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio); -+ -+/* Weights are encoded using a pseudo-exponential scheme. The weight codes 0, -+ * 32, 64, [...] correspond to weights of 1, 2, 4, [...]. The weights -+ * corresponding to intermediate weight codes are calculated using linear -+ * interpolation on the inverted values. Or put another way, the inverse weights -+ * for each 32nd weight code are 1, 1/2, 1/4, [...], and so the intervals -+ * between these are divided linearly into 32 intermediate values, the inverses -+ * of which form the remaining weight codes. -+ * -+ * The Weighted Bandwidth Fair Scheduling (WBFS) algorithm provides a form of -+ * scheduling within a group of class queues (group A or B). Weights are used to -+ * normalise the class queues to an underlying BFS algorithm where all class -+ * queues are assumed to require "equal bandwidth". So the weights referred to -+ * by the weight codes act as divisors on the size of frames being enqueued. Ie. -+ * one class queue in a group is assigned a weight of 2 whilst the other class -+ * queues in the group keep the default weight of 1, then the WBFS scheduler -+ * will effectively treat all frames enqueued on the weight-2 class queue as -+ * having half the number of bytes they really have. Ie. if all other things are -+ * equal, that class queue would get twice as much bytes-per-second bandwidth as -+ * the others. So weights should be chosen to provide bandwidth ratios between -+ * members of the same class queue group. These weights have no bearing on -+ * behaviour outside that group's WBFS mechanism though. -+ */ -+ -+/** -+ * qman_ceetm_wbfs2ratio - Given a weight code ('wbfs'), an accurate fractional -+ * representation of the corresponding weight is given (in order to not lose -+ * any precision). -+ * @weight_code: The given weight code in WBFS. -+ * @numerator: the numerator part of the weight computed by the weight code. -+ * @denominator: the denominator part of the weight computed by the weight code -+ * -+ * Returns zero for success or -EINVAL if the given weight code is illegal. -+ */ -+int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code, -+ u32 *numerator, -+ u32 *denominator); -+/** -+ * qman_ceetm_ratio2wbfs - Given a weight, find the nearest possible weight code -+ * If the user needs to know how close this is, convert the resulting weight -+ * code back to a weight and compare. -+ * @numerator: numerator part of the given weight. -+ * @denominator: denominator part of the given weight. -+ * @weight_code: the weight code computed from the given weight. -+ * -+ * Returns zero for success, or -ERANGE if "numerator/denominator" is outside -+ * the range of weights. -+ */ -+int qman_ceetm_ratio2wbfs(u32 numerator, -+ u32 denominator, -+ struct qm_ceetm_weight_code *weight_code, -+ int rounding); -+ -+#define QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER 0x1 -+/** -+ * qman_ceetm_cq_get_dequeue_statistics - Get the statistics provided by CEETM -+ * CQ counters. -+ * @cq: the given CQ object. -+ * @flags: indicates whether the statistics counter will be cleared after query. -+ * @frame_count: The number of the frames that have been counted since the -+ * counter was cleared last time. -+ * @byte_count: the number of bytes in all frames that have been counted. -+ * -+ * Return zero for success or -EINVAL if query statistics command returns error. -+ * -+ */ -+int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags, -+ u64 *frame_count, u64 *byte_count); -+ -+/** -+ * qman_ceetm_drain_cq - drain the CQ till it is empty. -+ * @cq: the give CQ object. -+ * Return 0 for success or -EINVAL for unsuccessful command to empty CQ. -+ */ -+int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq); -+ -+ /* ---------------------- */ -+ /* CEETM :: logical FQIDs */ -+ /* ---------------------- */ -+/** -+ * qman_ceetm_lfq_claim - Claims an unused logical FQID, associates it with -+ * the given class queue. -+ * @lfq: the returned lfq object, if successful. -+ * @cq: the class queue which needs to claim a LFQID. -+ * -+ * Return zero for success, or -ENODEV if no LFQID is available or -ENOMEM if -+ * allocating memory for lfq fails, or -EINVAL if configuring LFQMT fails. -+ */ -+int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq, -+ struct qm_ceetm_cq *cq); -+ -+/** -+ * qman_ceetm_lfq_release - Releases a previously claimed logical FQID. -+ * @lfq: the lfq to be released. -+ * -+ * Return zero for success. -+ */ -+int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq); -+ -+/** -+ * qman_ceetm_lfq_set_context -+ * qman_ceetm_lfq_get_context - Set/get the context_a/context_b pair to the -+ * "dequeue context table" associated with the logical FQID. -+ * @lfq: the given logical FQ object. -+ * @context_a: contextA of the dequeue context. -+ * @context_b: contextB of the dequeue context. -+ * -+ * Returns zero for success, or -EINVAL if there is error to set/get the -+ * context pair. -+ */ -+int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, -+ u64 context_a, -+ u32 context_b); -+int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, -+ u64 *context_a, -+ u32 *context_b); -+ -+/** -+ * qman_ceetm_create_fq - Initialise a FQ object for the LFQ. -+ * @lfq: the given logic fq. -+ * @fq: the fq object created for the given logic fq. -+ * -+ * The FQ object can be used in qman_enqueue() and qman_enqueue_orp() APIs to -+ * target a logical FQID (and the class queue it is associated with). -+ * Note that this FQ object can only be used for enqueues, and -+ * in the case of qman_enqueue_orp() it can not be used as the 'orp' parameter, -+ * only as 'fq'. This FQ object can not (and shouldn't) be destroyed, it is only -+ * valid as long as the underlying 'lfq' remains claimed. It is the user's -+ * responsibility to ensure that the underlying 'lfq' is not released until any -+ * enqueues to this FQ object have completed. The only field the user needs to -+ * fill in is fq->cb.ern, as that enqueue rejection handler is the callback that -+ * could conceivably be called on this FQ object. This API can be called -+ * multiple times to create multiple FQ objects referring to the same logical -+ * FQID, and any enqueue rejections will respect the callback of the object that -+ * issued the enqueue (and will identify the object via the parameter passed to -+ * the callback too). There is no 'flags' parameter to this API as there is for -+ * qman_create_fq() - the created FQ object behaves as though qman_create_fq() -+ * had been called with the single flag QMAN_FQ_FLAG_NO_MODIFY. -+ * -+ * Returns 0 for success. -+ */ -+int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq); -+ -+ /* -------------------------------- */ -+ /* CEETM :: class congestion groups */ -+ /* -------------------------------- */ -+ -+/** -+ * qman_ceetm_ccg_claim - Claims an unused CCG. -+ * @ccg: the returned CCG object, if successful. -+ * @channel: the given class queue channel -+ * @cscn: the callback function of this CCG. -+ * @cb_ctx: the corresponding context to be used used if state change -+ * notifications are later enabled for this CCG. -+ * -+ * The congestion group is local to the given class queue channel, so only -+ * class queues within the channel can be associated with that congestion group. -+ * The association of class queues to congestion groups occurs when the class -+ * queues are claimed, see qman_ceetm_cq_claim() and related functions. -+ * Congestion groups are in a "zero" state when initially claimed, and they are -+ * returned to that state when released. -+ * -+ * Return zero for success, or -EINVAL if no CCG in the channel is available. -+ */ -+int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg, -+ struct qm_ceetm_channel *channel, -+ unsigned int idx, -+ void (*cscn)(struct qm_ceetm_ccg *, -+ void *cb_ctx, -+ int congested), -+ void *cb_ctx); -+ -+/** -+ * qman_ceetm_ccg_release - Releases a previously claimed CCG. -+ * @ccg: the given ccg. -+ * -+ * Returns zero for success, or -EBUSY if the given ccg's dependent objects -+ * (class queues that are associated with the CCG) have not been released. -+ */ -+int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg); -+ -+/* This struct is used to specify attributes for a CCG. The 'we_mask' field -+ * controls which CCG attributes are to be updated, and the remainder specify -+ * the values for those attributes. A CCG counts either frames or the bytes -+ * within those frames, but not both ('mode'). A CCG can optionally cause -+ * enqueues to be rejected, due to tail-drop or WRED, or both (they are -+ * independent options, 'td_en' and 'wr_en_g,wr_en_y,wr_en_r'). Tail-drop can be -+ * level-triggered due to a single threshold ('td_thres') or edge-triggered due -+ * to a "congestion state", but not both ('td_mode'). Congestion state has -+ * distinct entry and exit thresholds ('cs_thres_in' and 'cs_thres_out'), and -+ * notifications can be sent to software the CCG goes in to and out of this -+ * congested state ('cscn_en'). */ -+struct qm_ceetm_ccg_params { -+ /* Boolean fields together in a single bitfield struct */ -+ struct { -+ /* Whether to count bytes or frames. 1==frames */ -+ u8 mode:1; -+ /* En/disable tail-drop. 1==enable */ -+ u8 td_en:1; -+ /* Tail-drop on congestion-state or threshold. 1=threshold */ -+ u8 td_mode:1; -+ /* Generate congestion state change notifications. 1==enable */ -+ u8 cscn_en:1; -+ /* Enable WRED rejections (per colour). 1==enable */ -+ u8 wr_en_g:1; -+ u8 wr_en_y:1; -+ u8 wr_en_r:1; -+ } __packed; -+ /* Tail-drop threshold. See qm_cgr_thres_[gs]et64(). */ -+ struct qm_cgr_cs_thres td_thres; -+ /* Congestion state thresholds, for entry and exit. */ -+ struct qm_cgr_cs_thres cs_thres_in; -+ struct qm_cgr_cs_thres cs_thres_out; -+ /* Overhead accounting length. Per-packet "tax", from -128 to +127 */ -+ signed char oal; -+ /* Congestion state change notification for DCP portal, virtual CCGID*/ -+ /* WRED parameters. */ -+ struct qm_cgr_wr_parm wr_parm_g; -+ struct qm_cgr_wr_parm wr_parm_y; -+ struct qm_cgr_wr_parm wr_parm_r; -+}; -+/* Bits used in 'we_mask' to qman_ceetm_ccg_set(), controls which attributes of -+ * the CCGR are to be updated. */ -+#define QM_CCGR_WE_MODE 0x0001 /* mode (bytes/frames) */ -+#define QM_CCGR_WE_CS_THRES_IN 0x0002 /* congestion state entry threshold */ -+#define QM_CCGR_WE_TD_EN 0x0004 /* congestion state tail-drop enable */ -+#define QM_CCGR_WE_CSCN_TUPD 0x0008 /* CSCN target update */ -+#define QM_CCGR_WE_CSCN_EN 0x0010 /* congestion notification enable */ -+#define QM_CCGR_WE_WR_EN_R 0x0020 /* WRED enable - red */ -+#define QM_CCGR_WE_WR_EN_Y 0x0040 /* WRED enable - yellow */ -+#define QM_CCGR_WE_WR_EN_G 0x0080 /* WRED enable - green */ -+#define QM_CCGR_WE_WR_PARM_R 0x0100 /* WRED parameters - red */ -+#define QM_CCGR_WE_WR_PARM_Y 0x0200 /* WRED parameters - yellow */ -+#define QM_CCGR_WE_WR_PARM_G 0x0400 /* WRED parameters - green */ -+#define QM_CCGR_WE_OAL 0x0800 /* overhead accounting length */ -+#define QM_CCGR_WE_CS_THRES_OUT 0x1000 /* congestion state exit threshold */ -+#define QM_CCGR_WE_TD_THRES 0x2000 /* tail-drop threshold */ -+#define QM_CCGR_WE_TD_MODE 0x4000 /* tail-drop mode (state/threshold) */ -+#define QM_CCGR_WE_CDV 0x8000 /* cdv */ -+ -+/** -+ * qman_ceetm_ccg_set -+ * qman_ceetm_ccg_get - Configure/query a subset of CCG attributes. -+ * @ccg: the given CCG object. -+ * @we_mask: the write enable mask. -+ * @params: the parameters setting for this ccg -+ * -+ * Return 0 for success, or -EIO if configure ccg command returns error for -+ * "set" function, or -EINVAL if query ccg command returns error for "get" -+ * function. -+ */ -+int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, -+ u16 we_mask, -+ const struct qm_ceetm_ccg_params *params); -+int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg, -+ struct qm_ceetm_ccg_params *params); -+ -+/** qman_ceetm_cscn_swp_set - Add or remove a software portal from the target -+ * mask. -+ * qman_ceetm_cscn_swp_get - Query whether a given software portal index is -+ * in the cscn target mask. -+ * @ccg: the give CCG object. -+ * @swp_idx: the index of the software portal. -+ * @cscn_enabled: 1: Set the swp to be cscn target. 0: remove the swp from -+ * the target mask. -+ * @we_mask: the write enable mask. -+ * @params: the parameters setting for this ccg -+ * -+ * Return 0 for success, or -EINVAL if command in set/get function fails. -+ */ -+int qman_ceetm_cscn_swp_set(struct qm_ceetm_ccg *ccg, -+ u16 swp_idx, -+ unsigned int cscn_enabled, -+ u16 we_mask, -+ const struct qm_ceetm_ccg_params *params); -+int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg, -+ u16 swp_idx, -+ unsigned int *cscn_enabled); -+ -+/** qman_ceetm_cscn_dcp_set - Add or remove a direct connect portal from the\ -+ * target mask. -+ * qman_ceetm_cscn_dcp_get - Query whether a given direct connect portal index -+ * is in the cscn target mask. -+ * @ccg: the give CCG object. -+ * @dcp_idx: the index of the direct connect portal. -+ * @vcgid: congestion state change notification for dcp portal, virtual CGID. -+ * @cscn_enabled: 1: Set the dcp to be cscn target. 0: remove the dcp from -+ * the target mask. -+ * @we_mask: the write enable mask. -+ * @params: the parameters setting for this ccg -+ * -+ * Return 0 for success, or -EINVAL if command in set/get function fails. -+ */ -+int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg, -+ u16 dcp_idx, -+ u8 vcgid, -+ unsigned int cscn_enabled, -+ u16 we_mask, -+ const struct qm_ceetm_ccg_params *params); -+int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg, -+ u16 dcp_idx, -+ u8 *vcgid, -+ unsigned int *cscn_enabled); -+ -+/** -+ * qman_ceetm_ccg_get_reject_statistics - Get the statistics provided by -+ * CEETM CCG counters. -+ * @ccg: the given CCG object. -+ * @flags: indicates whether the statistics counter will be cleared after query. -+ * @frame_count: The number of the frames that have been counted since the -+ * counter was cleared last time. -+ * @byte_count: the number of bytes in all frames that have been counted. -+ * -+ * Return zero for success or -EINVAL if query statistics command returns error. -+ * -+ */ -+int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags, -+ u64 *frame_count, u64 *byte_count); -+ -+/** -+ * qman_ceetm_query_lfqmt - Query the logical frame queue mapping table -+ * @lfqid: Logical Frame Queue ID -+ * @lfqmt_query: Results of the query command -+ * -+ * Returns zero for success or -EIO if the query command returns error. -+ * -+ */ -+int qman_ceetm_query_lfqmt(int lfqid, -+ struct qm_mcr_ceetm_lfqmt_query *lfqmt_query); -+ -+/** -+ * qman_ceetm_query_write_statistics - Query (and optionally write) statistics -+ * @cid: Target ID (CQID or CCGRID) -+ * @dcp_idx: CEETM portal ID -+ * @command_type: One of the following: -+ * 0 = Query dequeue statistics. CID carries the CQID to be queried. -+ * 1 = Query and clear dequeue statistics. CID carries the CQID to be queried -+ * 2 = Write dequeue statistics. CID carries the CQID to be written. -+ * 3 = Query reject statistics. CID carries the CCGRID to be queried. -+ * 4 = Query and clear reject statistics. CID carries the CCGRID to be queried -+ * 5 = Write reject statistics. CID carries the CCGRID to be written -+ * @frame_count: Frame count value to be written if this is a write command -+ * @byte_count: Bytes count value to be written if this is a write command -+ * -+ * Returns zero for success or -EIO if the query command returns error. -+ */ -+int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx, -+ u16 command_type, u64 frame_count, -+ u64 byte_count); -+ -+/** -+ * qman_set_wpm - Set waterfall power management -+ * -+ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm. -+ * -+ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not -+ * accessible. -+ */ -+int qman_set_wpm(int wpm_enable); -+ -+/** -+ * qman_get_wpm - Query the waterfall power management setting -+ * -+ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm. -+ * -+ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not -+ * accessible. -+ */ -+int qman_get_wpm(int *wpm_enable); -+ -+/* The below qman_p_***() variants might be called in a migration situation -+ * (e.g. cpu hotplug). They are used to continue accessing the portal that -+ * execution was affine to prior to migration. -+ * @qman_portal specifies which portal the APIs will use. -+*/ -+const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal -+ *p); -+int qman_p_irqsource_add(struct qman_portal *p, u32 bits); -+int qman_p_irqsource_remove(struct qman_portal *p, u32 bits); -+int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit); -+u32 qman_p_poll_slow(struct qman_portal *p); -+void qman_p_poll(struct qman_portal *p); -+void qman_p_stop_dequeues(struct qman_portal *p); -+void qman_p_start_dequeues(struct qman_portal *p); -+void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools); -+void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools); -+u32 qman_p_static_dequeue_get(struct qman_portal *p); -+void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq, -+ int park_request); -+int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq, -+ u32 flags __maybe_unused, u32 vdqcr); -+int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_fd *fd, u32 flags); -+int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_fd *fd, u32 flags, -+ struct qman_fq *orp, u16 orp_seqnum); -+int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq, -+ const struct qm_fd *fd, u32 flags, -+ qman_cb_precommit cb, void *cb_arg); -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* FSL_QMAN_H */ ---- /dev/null -+++ b/include/linux/fsl_usdpaa.h -@@ -0,0 +1,372 @@ -+/* Copyright 2011-2012 Freescale Semiconductor, Inc. -+ * -+ * This file is licensed under the terms of the GNU General Public License -+ * version 2. This program is licensed "as is" without any warranty of any -+ * kind, whether express or implied. -+ */ -+ -+#ifndef FSL_USDPAA_H -+#define FSL_USDPAA_H -+ -+#ifdef __cplusplus -+extern "C" { -+#endif -+ -+#include <linux/uaccess.h> -+#include <linux/ioctl.h> -+#include <linux/fsl_qman.h> /* For "enum qm_channel" */ -+#include <linux/compat.h> -+ -+#ifdef CONFIG_FSL_USDPAA -+ -+/******************************/ -+/* Allocation of resource IDs */ -+/******************************/ -+ -+/* This enum is used to distinguish between the type of underlying object being -+ * manipulated. */ -+enum usdpaa_id_type { -+ usdpaa_id_fqid, -+ usdpaa_id_bpid, -+ usdpaa_id_qpool, -+ usdpaa_id_cgrid, -+ usdpaa_id_ceetm0_lfqid, -+ usdpaa_id_ceetm0_channelid, -+ usdpaa_id_ceetm1_lfqid, -+ usdpaa_id_ceetm1_channelid, -+ usdpaa_id_max /* <-- not a valid type, represents the number of types */ -+}; -+#define USDPAA_IOCTL_MAGIC 'u' -+struct usdpaa_ioctl_id_alloc { -+ uint32_t base; /* Return value, the start of the allocated range */ -+ enum usdpaa_id_type id_type; /* what kind of resource(s) to allocate */ -+ uint32_t num; /* how many IDs to allocate (and return value) */ -+ uint32_t align; /* must be a power of 2, 0 is treated like 1 */ -+ int partial; /* whether to allow less than 'num' */ -+}; -+struct usdpaa_ioctl_id_release { -+ /* Input; */ -+ enum usdpaa_id_type id_type; -+ uint32_t base; -+ uint32_t num; -+}; -+struct usdpaa_ioctl_id_reserve { -+ enum usdpaa_id_type id_type; -+ uint32_t base; -+ uint32_t num; -+}; -+ -+ -+/* ioctl() commands */ -+#define USDPAA_IOCTL_ID_ALLOC \ -+ _IOWR(USDPAA_IOCTL_MAGIC, 0x01, struct usdpaa_ioctl_id_alloc) -+#define USDPAA_IOCTL_ID_RELEASE \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x02, struct usdpaa_ioctl_id_release) -+#define USDPAA_IOCTL_ID_RESERVE \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x0A, struct usdpaa_ioctl_id_reserve) -+ -+/**********************/ -+/* Mapping DMA memory */ -+/**********************/ -+ -+/* Maximum length for a map name, including NULL-terminator */ -+#define USDPAA_DMA_NAME_MAX 16 -+/* Flags for requesting DMA maps. Maps are private+unnamed or sharable+named. -+ * For a sharable and named map, specify _SHARED (whether creating one or -+ * binding to an existing one). If _SHARED is specified and _CREATE is not, then -+ * the mapping must already exist. If _SHARED and _CREATE are specified and the -+ * mapping doesn't already exist, it will be created. If _SHARED and _CREATE are -+ * specified and the mapping already exists, the mapping will fail unless _LAZY -+ * is specified. When mapping to a pre-existing sharable map, the length must be -+ * an exact match. Lengths must be a power-of-4 multiple of page size. -+ * -+ * Note that this does not actually map the memory to user-space, that is done -+ * by a subsequent mmap() using the page offset returned from this ioctl(). The -+ * ioctl() is what gives the process permission to do this, and a page-offset -+ * with which to do so. -+ */ -+#define USDPAA_DMA_FLAG_SHARE 0x01 -+#define USDPAA_DMA_FLAG_CREATE 0x02 -+#define USDPAA_DMA_FLAG_LAZY 0x04 -+#define USDPAA_DMA_FLAG_RDONLY 0x08 -+struct usdpaa_ioctl_dma_map { -+ /* Output parameters - virtual and physical addresses */ -+ void *ptr; -+ uint64_t phys_addr; -+ /* Input parameter, the length of the region to be created (or if -+ * mapping an existing region, this must match it). Must be a power-of-4 -+ * multiple of page size. */ -+ uint64_t len; -+ /* Input parameter, the USDPAA_DMA_FLAG_* settings. */ -+ uint32_t flags; -+ /* If _FLAG_SHARE is specified, the name of the region to be created (or -+ * of the existing mapping to use). */ -+ char name[USDPAA_DMA_NAME_MAX]; -+ /* If this ioctl() creates the mapping, this is an input parameter -+ * stating whether the region supports locking. If mapping an existing -+ * region, this is a return value indicating the same thing. */ -+ int has_locking; -+ /* In the case of a successful map with _CREATE and _LAZY, this return -+ * value indicates whether we created the mapped region or whether it -+ * already existed. */ -+ int did_create; -+}; -+ -+#ifdef CONFIG_COMPAT -+struct usdpaa_ioctl_dma_map_compat { -+ /* Output parameters - virtual and physical addresses */ -+ compat_uptr_t ptr; -+ uint64_t phys_addr; -+ /* Input parameter, the length of the region to be created (or if -+ * mapping an existing region, this must match it). Must be a power-of-4 -+ * multiple of page size. */ -+ uint64_t len; -+ /* Input parameter, the USDPAA_DMA_FLAG_* settings. */ -+ uint32_t flags; -+ /* If _FLAG_SHARE is specified, the name of the region to be created (or -+ * of the existing mapping to use). */ -+ char name[USDPAA_DMA_NAME_MAX]; -+ /* If this ioctl() creates the mapping, this is an input parameter -+ * stating whether the region supports locking. If mapping an existing -+ * region, this is a return value indicating the same thing. */ -+ int has_locking; -+ /* In the case of a successful map with _CREATE and _LAZY, this return -+ * value indicates whether we created the mapped region or whether it -+ * already existed. */ -+ int did_create; -+}; -+ -+#define USDPAA_IOCTL_DMA_MAP_COMPAT \ -+ _IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map_compat) -+#endif -+ -+ -+#define USDPAA_IOCTL_DMA_MAP \ -+ _IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map) -+/* munmap() does not remove the DMA map, just the user-space mapping to it. -+ * This ioctl will do both (though you can munmap() before calling the ioctl -+ * too). */ -+#define USDPAA_IOCTL_DMA_UNMAP \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x04, unsigned char) -+/* We implement a cross-process locking scheme per DMA map. Call this ioctl() -+ * with a mmap()'d address, and the process will (interruptible) sleep if the -+ * lock is already held by another process. Process destruction will -+ * automatically clean up any held locks. */ -+#define USDPAA_IOCTL_DMA_LOCK \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x05, unsigned char) -+#define USDPAA_IOCTL_DMA_UNLOCK \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x06, unsigned char) -+ -+/***************************************/ -+/* Mapping and using QMan/BMan portals */ -+/***************************************/ -+enum usdpaa_portal_type { -+ usdpaa_portal_qman, -+ usdpaa_portal_bman, -+}; -+ -+#define QBMAN_ANY_PORTAL_IDX 0xffffffff -+ -+struct usdpaa_ioctl_portal_map { -+ /* Input parameter, is a qman or bman portal required. */ -+ -+ enum usdpaa_portal_type type; -+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX -+ for don't care. The portal index will be populated by the -+ driver when the ioctl() successfully completes */ -+ uint32_t index; -+ -+ /* Return value if the map succeeds, this gives the mapped -+ * cache-inhibited (cinh) and cache-enabled (cena) addresses. */ -+ struct usdpaa_portal_map { -+ void *cinh; -+ void *cena; -+ } addr; -+ /* Qman-specific return values */ -+ uint16_t channel; -+ uint32_t pools; -+}; -+ -+#ifdef CONFIG_COMPAT -+struct compat_usdpaa_ioctl_portal_map { -+ /* Input parameter, is a qman or bman portal required. */ -+ enum usdpaa_portal_type type; -+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX -+ for don't care. The portal index will be populated by the -+ driver when the ioctl() successfully completes */ -+ uint32_t index; -+ /* Return value if the map succeeds, this gives the mapped -+ * cache-inhibited (cinh) and cache-enabled (cena) addresses. */ -+ struct usdpaa_portal_map_compat { -+ compat_uptr_t cinh; -+ compat_uptr_t cena; -+ } addr; -+ /* Qman-specific return values */ -+ uint16_t channel; -+ uint32_t pools; -+}; -+#define USDPAA_IOCTL_PORTAL_MAP_COMPAT \ -+ _IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct compat_usdpaa_ioctl_portal_map) -+#define USDPAA_IOCTL_PORTAL_UNMAP_COMPAT \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map_compat) -+#endif -+ -+#define USDPAA_IOCTL_PORTAL_MAP \ -+ _IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct usdpaa_ioctl_portal_map) -+#define USDPAA_IOCTL_PORTAL_UNMAP \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map) -+ -+struct usdpaa_ioctl_irq_map { -+ enum usdpaa_portal_type type; /* Type of portal to map */ -+ int fd; /* File descriptor that contains the portal */ -+ void *portal_cinh; /* Cache inhibited area to identify the portal */ -+}; -+ -+#define USDPAA_IOCTL_PORTAL_IRQ_MAP \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x09, struct usdpaa_ioctl_irq_map) -+ -+#ifdef CONFIG_COMPAT -+ -+struct compat_ioctl_irq_map { -+ enum usdpaa_portal_type type; /* Type of portal to map */ -+ compat_int_t fd; /* File descriptor that contains the portal */ -+ compat_uptr_t portal_cinh; /* Used identify the portal */}; -+ -+#define USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT \ -+ _IOW(USDPAA_IOCTL_MAGIC, 0x09, struct compat_ioctl_irq_map) -+#endif -+ -+/* ioctl to query the amount of DMA memory used in the system */ -+struct usdpaa_ioctl_dma_used { -+ uint64_t free_bytes; -+ uint64_t total_bytes; -+}; -+#define USDPAA_IOCTL_DMA_USED \ -+ _IOR(USDPAA_IOCTL_MAGIC, 0x0B, struct usdpaa_ioctl_dma_used) -+ -+/* ioctl to allocate a raw portal */ -+struct usdpaa_ioctl_raw_portal { -+ /* inputs */ -+ enum usdpaa_portal_type type; /* Type of portal to allocate */ -+ -+ /* set to non zero to turn on stashing */ -+ uint8_t enable_stash; -+ /* Stashing attributes for the portal */ -+ uint32_t cpu; -+ uint32_t cache; -+ uint32_t window; -+ -+ /* Specifies the stash request queue this portal should use */ -+ uint8_t sdest; -+ -+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX -+ * for don't care. The portal index will be populated by the -+ * driver when the ioctl() successfully completes */ -+ uint32_t index; -+ -+ /* outputs */ -+ uint64_t cinh; -+ uint64_t cena; -+}; -+ -+#define USDPAA_IOCTL_ALLOC_RAW_PORTAL \ -+ _IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct usdpaa_ioctl_raw_portal) -+ -+#define USDPAA_IOCTL_FREE_RAW_PORTAL \ -+ _IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct usdpaa_ioctl_raw_portal) -+ -+#ifdef CONFIG_COMPAT -+ -+struct compat_ioctl_raw_portal { -+ /* inputs */ -+ enum usdpaa_portal_type type; /* Type of portal to allocate */ -+ -+ /* set to non zero to turn on stashing */ -+ uint8_t enable_stash; -+ /* Stashing attributes for the portal */ -+ uint32_t cpu; -+ uint32_t cache; -+ uint32_t window; -+ /* Specifies the stash request queue this portal should use */ -+ uint8_t sdest; -+ -+ /* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX -+ * for don't care. The portal index will be populated by the -+ * driver when the ioctl() successfully completes */ -+ uint32_t index; -+ -+ /* outputs */ -+ uint64_t cinh; -+ uint64_t cena; -+}; -+ -+#define USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT \ -+ _IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct compat_ioctl_raw_portal) -+ -+#define USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT \ -+ _IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct compat_ioctl_raw_portal) -+ -+#endif -+ -+#ifdef __KERNEL__ -+ -+/* Early-boot hook */ -+int __init fsl_usdpaa_init_early(void); -+ -+/* Fault-handling in arch/powerpc/mm/mem.c gives USDPAA an opportunity to detect -+ * faults within its ranges via this hook. */ -+int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size); -+ -+#endif /* __KERNEL__ */ -+ -+#endif /* CONFIG_FSL_USDPAA */ -+ -+#ifdef __KERNEL__ -+/* This interface is needed in a few places and though it's not specific to -+ * USDPAA as such, creating a new header for it doesn't make any sense. The -+ * qbman kernel driver implements this interface and uses it as the backend for -+ * both the FQID and BPID allocators. The fsl_usdpaa driver also uses this -+ * interface for tracking per-process allocations handed out to user-space. */ -+struct dpa_alloc { -+ struct list_head free; -+ spinlock_t lock; -+ struct list_head used; -+}; -+#define DECLARE_DPA_ALLOC(name) \ -+ struct dpa_alloc name = { \ -+ .free = { \ -+ .prev = &name.free, \ -+ .next = &name.free \ -+ }, \ -+ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ -+ .used = { \ -+ .prev = &name.used, \ -+ .next = &name.used \ -+ } \ -+ } -+static inline void dpa_alloc_init(struct dpa_alloc *alloc) -+{ -+ INIT_LIST_HEAD(&alloc->free); -+ INIT_LIST_HEAD(&alloc->used); -+ spin_lock_init(&alloc->lock); -+} -+int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align, -+ int partial); -+void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count); -+void dpa_alloc_seed(struct dpa_alloc *alloc, u32 fqid, u32 count); -+ -+/* Like 'new' but specifies the desired range, returns -ENOMEM if the entire -+ * desired range is not available, or 0 for success. */ -+int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base_id, u32 count); -+/* Pops and returns contiguous ranges from the allocator. Returns -ENOMEM when -+ * 'alloc' is empty. */ -+int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count); -+/* Returns 1 if the specified id is alloced, 0 otherwise */ -+int dpa_alloc_check(struct dpa_alloc *list, u32 id); -+#endif /* __KERNEL__ */ -+ -+#ifdef __cplusplus -+} -+#endif -+ -+#endif /* FSL_USDPAA_H */ |