diff options
Diffstat (limited to 'arch/arm/kernel')
75 files changed, 25488 insertions, 0 deletions
diff --git a/arch/arm/kernel/.gitignore b/arch/arm/kernel/.gitignore new file mode 100644 index 00000000..c5f676c3 --- /dev/null +++ b/arch/arm/kernel/.gitignore @@ -0,0 +1 @@ +vmlinux.lds diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile new file mode 100644 index 00000000..a5b31af5 --- /dev/null +++ b/arch/arm/kernel/Makefile @@ -0,0 +1,73 @@ +# +# Makefile for the linux kernel. +# + +CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET) +AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET) + +ifdef CONFIG_FUNCTION_TRACER +CFLAGS_REMOVE_ftrace.o = -pg +endif + +CFLAGS_REMOVE_return_address.o = -pg + +# Object file lists. + +obj-y := elf.o entry-armv.o entry-common.o irq.o \ + process.o ptrace.o return_address.o setup.o signal.o \ + sys_arm.o stacktrace.o time.o traps.o + +obj-$(CONFIG_DEPRECATED_PARAM_STRUCT) += compat.o + +obj-$(CONFIG_LEDS) += leds.o +obj-$(CONFIG_OC_ETM) += etm.o + +obj-$(CONFIG_ISA_DMA_API) += dma.o +obj-$(CONFIG_ARCH_ACORN) += ecard.o +obj-$(CONFIG_FIQ) += fiq.o fiqasm.o +obj-$(CONFIG_MODULES) += armksyms.o module.o +obj-$(CONFIG_ARTHUR) += arthur.o +obj-$(CONFIG_ISA_DMA) += dma-isa.o +obj-$(CONFIG_PCI) += bios32.o isa.o +obj-$(CONFIG_PM_SLEEP) += sleep.o +obj-$(CONFIG_HAVE_SCHED_CLOCK) += sched_clock.o +obj-$(CONFIG_SMP) += smp.o smp_tlb.o +obj-$(CONFIG_HAVE_ARM_SCU) += smp_scu.o +obj-$(CONFIG_HAVE_ARM_TWD) += smp_twd.o +obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o +obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o +obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o +obj-$(CONFIG_KPROBES) += kprobes.o kprobes-decode.o +obj-$(CONFIG_ATAGS_PROC) += atags.o +obj-$(CONFIG_OABI_COMPAT) += sys_oabi-compat.o +obj-$(CONFIG_ARM_THUMBEE) += thumbee.o +obj-$(CONFIG_KGDB) += kgdb.o +obj-$(CONFIG_ARM_UNWIND) += unwind.o +obj-$(CONFIG_HAVE_TCM) += tcm.o +obj-$(CONFIG_OF) += devtree.o +obj-$(CONFIG_CRASH_DUMP) += crash_dump.o +obj-$(CONFIG_SWP_EMULATE) += swp_emulate.o +CFLAGS_swp_emulate.o := -Wa,-march=armv7-a +obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o + +obj-$(CONFIG_CRUNCH) += crunch.o crunch-bits.o +AFLAGS_crunch-bits.o := -Wa,-mcpu=ep9312 + +obj-$(CONFIG_CPU_XSCALE) += xscale-cp0.o +obj-$(CONFIG_CPU_XSC3) += xscale-cp0.o +obj-$(CONFIG_CPU_MOHAWK) += xscale-cp0.o +obj-$(CONFIG_CPU_PJ4) += pj4-cp0.o +obj-$(CONFIG_IWMMXT) += iwmmxt.o +obj-$(CONFIG_CPU_HAS_PMU) += pmu.o +obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o +AFLAGS_iwmmxt.o := -Wa,-mcpu=iwmmxt + +ifneq ($(CONFIG_ARCH_EBSA110),y) + obj-y += io.o +endif + +head-y := head$(MMUEXT).o +obj-$(CONFIG_DEBUG_LL) += debug.o +obj-$(CONFIG_EARLY_PRINTK) += early_printk.o + +extra-y := $(head-y) init_task.o vmlinux.lds diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c new file mode 100644 index 00000000..acca35ae --- /dev/null +++ b/arch/arm/kernel/armksyms.c @@ -0,0 +1,170 @@ +/* + * linux/arch/arm/kernel/armksyms.c + * + * Copyright (C) 2000 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/string.h> +#include <linux/cryptohash.h> +#include <linux/delay.h> +#include <linux/in6.h> +#include <linux/syscalls.h> +#include <linux/uaccess.h> +#include <linux/io.h> + +#include <asm/checksum.h> +#include <asm/system.h> +#include <asm/ftrace.h> + +/* + * libgcc functions - functions that are used internally by the + * compiler... (prototypes are not correct though, but that + * doesn't really matter since they're not versioned). + */ +extern void __ashldi3(void); +extern void __ashrdi3(void); +extern void __divsi3(void); +extern void __lshrdi3(void); +extern void __modsi3(void); +extern void __muldi3(void); +extern void __ucmpdi2(void); +extern void __udivsi3(void); +extern void __umodsi3(void); +extern void __do_div64(void); + +extern void __aeabi_idiv(void); +extern void __aeabi_idivmod(void); +extern void __aeabi_lasr(void); +extern void __aeabi_llsl(void); +extern void __aeabi_llsr(void); +extern void __aeabi_lmul(void); +extern void __aeabi_uidiv(void); +extern void __aeabi_uidivmod(void); +extern void __aeabi_ulcmp(void); + +extern void fpundefinstr(void); + + +EXPORT_SYMBOL(__backtrace); + + /* platform dependent support */ +EXPORT_SYMBOL(__udelay); +EXPORT_SYMBOL(__const_udelay); + + /* networking */ +EXPORT_SYMBOL(csum_partial); +EXPORT_SYMBOL(csum_partial_copy_from_user); +EXPORT_SYMBOL(csum_partial_copy_nocheck); +EXPORT_SYMBOL(__csum_ipv6_magic); + + /* io */ +#ifndef __raw_readsb +EXPORT_SYMBOL(__raw_readsb); +#endif +#ifndef __raw_readsw +EXPORT_SYMBOL(__raw_readsw); +#endif +#ifndef __raw_readsl +EXPORT_SYMBOL(__raw_readsl); +#endif +#ifndef __raw_writesb +EXPORT_SYMBOL(__raw_writesb); +#endif +#ifndef __raw_writesw +EXPORT_SYMBOL(__raw_writesw); +#endif +#ifndef __raw_writesl +EXPORT_SYMBOL(__raw_writesl); +#endif + + /* string / mem functions */ +EXPORT_SYMBOL(strchr); +EXPORT_SYMBOL(strrchr); +EXPORT_SYMBOL(memset); +EXPORT_SYMBOL(memcpy); +EXPORT_SYMBOL(memmove); +EXPORT_SYMBOL(memchr); +EXPORT_SYMBOL(__memzero); + + /* user mem (segment) */ +EXPORT_SYMBOL(__strnlen_user); +EXPORT_SYMBOL(__strncpy_from_user); + +#ifdef CONFIG_MMU +EXPORT_SYMBOL(copy_page); + +EXPORT_SYMBOL(__copy_from_user); +EXPORT_SYMBOL(__copy_to_user); +EXPORT_SYMBOL(__clear_user); + +EXPORT_SYMBOL(__get_user_1); +EXPORT_SYMBOL(__get_user_2); +EXPORT_SYMBOL(__get_user_4); + +EXPORT_SYMBOL(__put_user_1); +EXPORT_SYMBOL(__put_user_2); +EXPORT_SYMBOL(__put_user_4); +EXPORT_SYMBOL(__put_user_8); +#endif + + /* crypto hash */ +EXPORT_SYMBOL(sha_transform); + + /* gcc lib functions */ +EXPORT_SYMBOL(__ashldi3); +EXPORT_SYMBOL(__ashrdi3); +EXPORT_SYMBOL(__divsi3); +EXPORT_SYMBOL(__lshrdi3); +EXPORT_SYMBOL(__modsi3); +EXPORT_SYMBOL(__muldi3); +EXPORT_SYMBOL(__ucmpdi2); +EXPORT_SYMBOL(__udivsi3); +EXPORT_SYMBOL(__umodsi3); +EXPORT_SYMBOL(__do_div64); + +#ifdef CONFIG_AEABI +EXPORT_SYMBOL(__aeabi_idiv); +EXPORT_SYMBOL(__aeabi_idivmod); +EXPORT_SYMBOL(__aeabi_lasr); +EXPORT_SYMBOL(__aeabi_llsl); +EXPORT_SYMBOL(__aeabi_llsr); +EXPORT_SYMBOL(__aeabi_lmul); +EXPORT_SYMBOL(__aeabi_uidiv); +EXPORT_SYMBOL(__aeabi_uidivmod); +EXPORT_SYMBOL(__aeabi_ulcmp); +#endif + + /* bitops */ +EXPORT_SYMBOL(_set_bit); +EXPORT_SYMBOL(_test_and_set_bit); +EXPORT_SYMBOL(_clear_bit); +EXPORT_SYMBOL(_test_and_clear_bit); +EXPORT_SYMBOL(_change_bit); +EXPORT_SYMBOL(_test_and_change_bit); +EXPORT_SYMBOL(_find_first_zero_bit_le); +EXPORT_SYMBOL(_find_next_zero_bit_le); +EXPORT_SYMBOL(_find_first_bit_le); +EXPORT_SYMBOL(_find_next_bit_le); + +#ifdef __ARMEB__ +EXPORT_SYMBOL(_find_first_zero_bit_be); +EXPORT_SYMBOL(_find_next_zero_bit_be); +EXPORT_SYMBOL(_find_first_bit_be); +EXPORT_SYMBOL(_find_next_bit_be); +#endif + +#ifdef CONFIG_FUNCTION_TRACER +#ifdef CONFIG_OLD_MCOUNT +EXPORT_SYMBOL(mcount); +#endif +EXPORT_SYMBOL(__gnu_mcount_nc); +#endif + +#ifdef CONFIG_ARM_PATCH_PHYS_VIRT +EXPORT_SYMBOL(__pv_phys_offset); +#endif diff --git a/arch/arm/kernel/arthur.c b/arch/arm/kernel/arthur.c new file mode 100644 index 00000000..321c5291 --- /dev/null +++ b/arch/arm/kernel/arthur.c @@ -0,0 +1,94 @@ +/* + * linux/arch/arm/kernel/arthur.c + * + * Copyright (C) 1998, 1999, 2000, 2001 Philip Blundell + * + * Arthur personality + */ + +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include <linux/module.h> +#include <linux/personality.h> +#include <linux/stddef.h> +#include <linux/signal.h> +#include <linux/init.h> +#include <linux/sched.h> + +#include <asm/ptrace.h> + +/* Arthur doesn't have many signals, and a lot of those that it does + have don't map easily to any Linux equivalent. Never mind. */ + +#define ARTHUR_SIGABRT 1 +#define ARTHUR_SIGFPE 2 +#define ARTHUR_SIGILL 3 +#define ARTHUR_SIGINT 4 +#define ARTHUR_SIGSEGV 5 +#define ARTHUR_SIGTERM 6 +#define ARTHUR_SIGSTAK 7 +#define ARTHUR_SIGUSR1 8 +#define ARTHUR_SIGUSR2 9 +#define ARTHUR_SIGOSERROR 10 + +static unsigned long arthur_to_linux_signals[32] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 +}; + +static unsigned long linux_to_arthur_signals[32] = { + 0, -1, ARTHUR_SIGINT, -1, + ARTHUR_SIGILL, 5, ARTHUR_SIGABRT, 7, + ARTHUR_SIGFPE, 9, ARTHUR_SIGUSR1, ARTHUR_SIGSEGV, + ARTHUR_SIGUSR2, 13, 14, ARTHUR_SIGTERM, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31 +}; + +static void arthur_lcall7(int nr, struct pt_regs *regs) +{ + struct siginfo info; + info.si_signo = SIGSWI; + info.si_errno = nr; + /* Bounce it to the emulator */ + send_sig_info(SIGSWI, &info, current); +} + +static struct exec_domain arthur_exec_domain = { + .name = "Arthur", + .handler = arthur_lcall7, + .pers_low = PER_RISCOS, + .pers_high = PER_RISCOS, + .signal_map = arthur_to_linux_signals, + .signal_invmap = linux_to_arthur_signals, + .module = THIS_MODULE, +}; + +/* + * We could do with some locking to stop Arthur being removed while + * processes are using it. + */ + +static int __init arthur_init(void) +{ + return register_exec_domain(&arthur_exec_domain); +} + +static void __exit arthur_exit(void) +{ + unregister_exec_domain(&arthur_exec_domain); +} + +module_init(arthur_init); +module_exit(arthur_exit); + +MODULE_LICENSE("GPL"); diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c new file mode 100644 index 00000000..927522cf --- /dev/null +++ b/arch/arm/kernel/asm-offsets.c @@ -0,0 +1,133 @@ +/* + * Copyright (C) 1995-2003 Russell King + * 2001-2002 Keith Owens + * + * Generate definitions needed by assembly language modules. + * This code generates raw asm output which is post-processed to extract + * and format the required data. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/dma-mapping.h> +#include <asm/cacheflush.h> +#include <asm/glue-df.h> +#include <asm/glue-pf.h> +#include <asm/mach/arch.h> +#include <asm/thread_info.h> +#include <asm/memory.h> +#include <asm/procinfo.h> +#include <linux/kbuild.h> + +/* + * Make sure that the compiler and target are compatible. + */ +#if defined(__APCS_26__) +#error Sorry, your compiler targets APCS-26 but this kernel requires APCS-32 +#endif +/* + * GCC 3.0, 3.1: general bad code generation. + * GCC 3.2.0: incorrect function argument offset calculation. + * GCC 3.2.x: miscompiles NEW_AUX_ENT in fs/binfmt_elf.c + * (http://gcc.gnu.org/PR8896) and incorrect structure + * initialisation in fs/jffs2/erase.c + */ +#if (__GNUC__ == 3 && __GNUC_MINOR__ < 3) +#error Your compiler is too buggy; it is known to miscompile kernels. +#error Known good compilers: 3.3 +#endif + +int main(void) +{ + DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); +#ifdef CONFIG_CC_STACKPROTECTOR + DEFINE(TSK_STACK_CANARY, offsetof(struct task_struct, stack_canary)); +#endif + BLANK(); + DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); + DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); + DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); + DEFINE(TI_TASK, offsetof(struct thread_info, task)); + DEFINE(TI_EXEC_DOMAIN, offsetof(struct thread_info, exec_domain)); + DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); + DEFINE(TI_CPU_DOMAIN, offsetof(struct thread_info, cpu_domain)); + DEFINE(TI_CPU_SAVE, offsetof(struct thread_info, cpu_context)); + DEFINE(TI_USED_CP, offsetof(struct thread_info, used_cp)); + DEFINE(TI_TP_VALUE, offsetof(struct thread_info, tp_value)); + DEFINE(TI_FPSTATE, offsetof(struct thread_info, fpstate)); + DEFINE(TI_VFPSTATE, offsetof(struct thread_info, vfpstate)); +#ifdef CONFIG_ARM_THUMBEE + DEFINE(TI_THUMBEE_STATE, offsetof(struct thread_info, thumbee_state)); +#endif +#ifdef CONFIG_IWMMXT + DEFINE(TI_IWMMXT_STATE, offsetof(struct thread_info, fpstate.iwmmxt)); +#endif +#ifdef CONFIG_CRUNCH + DEFINE(TI_CRUNCH_STATE, offsetof(struct thread_info, crunchstate)); +#endif + BLANK(); + DEFINE(S_R0, offsetof(struct pt_regs, ARM_r0)); + DEFINE(S_R1, offsetof(struct pt_regs, ARM_r1)); + DEFINE(S_R2, offsetof(struct pt_regs, ARM_r2)); + DEFINE(S_R3, offsetof(struct pt_regs, ARM_r3)); + DEFINE(S_R4, offsetof(struct pt_regs, ARM_r4)); + DEFINE(S_R5, offsetof(struct pt_regs, ARM_r5)); + DEFINE(S_R6, offsetof(struct pt_regs, ARM_r6)); + DEFINE(S_R7, offsetof(struct pt_regs, ARM_r7)); + DEFINE(S_R8, offsetof(struct pt_regs, ARM_r8)); + DEFINE(S_R9, offsetof(struct pt_regs, ARM_r9)); + DEFINE(S_R10, offsetof(struct pt_regs, ARM_r10)); + DEFINE(S_FP, offsetof(struct pt_regs, ARM_fp)); + DEFINE(S_IP, offsetof(struct pt_regs, ARM_ip)); + DEFINE(S_SP, offsetof(struct pt_regs, ARM_sp)); + DEFINE(S_LR, offsetof(struct pt_regs, ARM_lr)); + DEFINE(S_PC, offsetof(struct pt_regs, ARM_pc)); + DEFINE(S_PSR, offsetof(struct pt_regs, ARM_cpsr)); + DEFINE(S_OLD_R0, offsetof(struct pt_regs, ARM_ORIG_r0)); + DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); + BLANK(); +#ifdef CONFIG_CPU_HAS_ASID + DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id)); + BLANK(); +#endif + DEFINE(VMA_VM_MM, offsetof(struct vm_area_struct, vm_mm)); + DEFINE(VMA_VM_FLAGS, offsetof(struct vm_area_struct, vm_flags)); + BLANK(); + DEFINE(VM_EXEC, VM_EXEC); + BLANK(); + DEFINE(PAGE_SZ, PAGE_SIZE); + BLANK(); + DEFINE(SYS_ERROR0, 0x9f0000); + BLANK(); + DEFINE(SIZEOF_MACHINE_DESC, sizeof(struct machine_desc)); + DEFINE(MACHINFO_TYPE, offsetof(struct machine_desc, nr)); + DEFINE(MACHINFO_NAME, offsetof(struct machine_desc, name)); + BLANK(); + DEFINE(PROC_INFO_SZ, sizeof(struct proc_info_list)); + DEFINE(PROCINFO_INITFUNC, offsetof(struct proc_info_list, __cpu_flush)); + DEFINE(PROCINFO_MM_MMUFLAGS, offsetof(struct proc_info_list, __cpu_mm_mmu_flags)); + DEFINE(PROCINFO_IO_MMUFLAGS, offsetof(struct proc_info_list, __cpu_io_mmu_flags)); + BLANK(); +#ifdef MULTI_DABORT + DEFINE(PROCESSOR_DABT_FUNC, offsetof(struct processor, _data_abort)); +#endif +#ifdef MULTI_PABORT + DEFINE(PROCESSOR_PABT_FUNC, offsetof(struct processor, _prefetch_abort)); +#endif +#ifdef MULTI_CPU + DEFINE(CPU_SLEEP_SIZE, offsetof(struct processor, suspend_size)); + DEFINE(CPU_DO_SUSPEND, offsetof(struct processor, do_suspend)); + DEFINE(CPU_DO_RESUME, offsetof(struct processor, do_resume)); +#endif +#ifdef MULTI_CACHE + DEFINE(CACHE_FLUSH_KERN_ALL, offsetof(struct cpu_cache_fns, flush_kern_all)); +#endif + BLANK(); + DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL); + DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); + DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); + return 0; +} diff --git a/arch/arm/kernel/atags.c b/arch/arm/kernel/atags.c new file mode 100644 index 00000000..42a1a141 --- /dev/null +++ b/arch/arm/kernel/atags.c @@ -0,0 +1,83 @@ +#include <linux/slab.h> +#include <linux/proc_fs.h> +#include <asm/setup.h> +#include <asm/types.h> +#include <asm/page.h> + +struct buffer { + size_t size; + char data[]; +}; + +static int +read_buffer(char* page, char** start, off_t off, int count, + int* eof, void* data) +{ + struct buffer *buffer = (struct buffer *)data; + + if (off >= buffer->size) { + *eof = 1; + return 0; + } + + count = min((int) (buffer->size - off), count); + + memcpy(page, &buffer->data[off], count); + + return count; +} + +#define BOOT_PARAMS_SIZE 1536 +static char __initdata atags_copy[BOOT_PARAMS_SIZE]; + +void __init save_atags(const struct tag *tags) +{ + memcpy(atags_copy, tags, sizeof(atags_copy)); +} + +static int __init init_atags_procfs(void) +{ + /* + * This cannot go into save_atags() because kmalloc and proc don't work + * yet when it is called. + */ + struct proc_dir_entry *tags_entry; + struct tag *tag = (struct tag *)atags_copy; + struct buffer *b; + size_t size; + + if (tag->hdr.tag != ATAG_CORE) { + printk(KERN_INFO "No ATAGs?"); + return -EINVAL; + } + + for (; tag->hdr.size; tag = tag_next(tag)) + ; + + /* include the terminating ATAG_NONE */ + size = (char *)tag - atags_copy + sizeof(struct tag_header); + + WARN_ON(tag->hdr.tag != ATAG_NONE); + + b = kmalloc(sizeof(*b) + size, GFP_KERNEL); + if (!b) + goto nomem; + + b->size = size; + memcpy(b->data, atags_copy, size); + + tags_entry = create_proc_read_entry("atags", 0400, + NULL, read_buffer, b); + + if (!tags_entry) + goto nomem; + + return 0; + +nomem: + kfree(b); + printk(KERN_ERR "Exporting ATAGs: not enough memory\n"); + + return -ENOMEM; +} +arch_initcall(init_atags_procfs); diff --git a/arch/arm/kernel/atags.h b/arch/arm/kernel/atags.h new file mode 100644 index 00000000..e5f028d2 --- /dev/null +++ b/arch/arm/kernel/atags.h @@ -0,0 +1,5 @@ +#ifdef CONFIG_ATAGS_PROC +extern void save_atags(struct tag *tags); +#else +static inline void save_atags(struct tag *tags) { } +#endif diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c new file mode 100644 index 00000000..e4ee050a --- /dev/null +++ b/arch/arm/kernel/bios32.c @@ -0,0 +1,681 @@ +/* + * linux/arch/arm/kernel/bios32.c + * + * PCI bios-type initialisation for PCI machines + * + * Bits taken from various places. + */ +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/init.h> +#include <linux/io.h> + +#include <asm/mach-types.h> +#include <asm/mach/pci.h> + +static int debug_pci; +static int use_firmware; + +/* + * We can't use pci_find_device() here since we are + * called from interrupt context. + */ +static void pcibios_bus_report_status(struct pci_bus *bus, u_int status_mask, int warn) +{ + struct pci_dev *dev; + + list_for_each_entry(dev, &bus->devices, bus_list) { + u16 status; + + /* + * ignore host bridge - we handle + * that separately + */ + if (dev->bus->number == 0 && dev->devfn == 0) + continue; + + pci_read_config_word(dev, PCI_STATUS, &status); + if (status == 0xffff) + continue; + + if ((status & status_mask) == 0) + continue; + + /* clear the status errors */ + pci_write_config_word(dev, PCI_STATUS, status & status_mask); + + if (warn) + printk("(%s: %04X) ", pci_name(dev), status); + } + + list_for_each_entry(dev, &bus->devices, bus_list) + if (dev->subordinate) + pcibios_bus_report_status(dev->subordinate, status_mask, warn); +} + +void pcibios_report_status(u_int status_mask, int warn) +{ + struct list_head *l; + + list_for_each(l, &pci_root_buses) { + struct pci_bus *bus = pci_bus_b(l); + + pcibios_bus_report_status(bus, status_mask, warn); + } +} + +/* + * We don't use this to fix the device, but initialisation of it. + * It's not the correct use for this, but it works. + * Note that the arbiter/ISA bridge appears to be buggy, specifically in + * the following area: + * 1. park on CPU + * 2. ISA bridge ping-pong + * 3. ISA bridge master handling of target RETRY + * + * Bug 3 is responsible for the sound DMA grinding to a halt. We now + * live with bug 2. + */ +static void __devinit pci_fixup_83c553(struct pci_dev *dev) +{ + /* + * Set memory region to start at address 0, and enable IO + */ + pci_write_config_dword(dev, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_SPACE_MEMORY); + pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_IO); + + dev->resource[0].end -= dev->resource[0].start; + dev->resource[0].start = 0; + + /* + * All memory requests from ISA to be channelled to PCI + */ + pci_write_config_byte(dev, 0x48, 0xff); + + /* + * Enable ping-pong on bus master to ISA bridge transactions. + * This improves the sound DMA substantially. The fixed + * priority arbiter also helps (see below). + */ + pci_write_config_byte(dev, 0x42, 0x01); + + /* + * Enable PCI retry + */ + pci_write_config_byte(dev, 0x40, 0x22); + + /* + * We used to set the arbiter to "park on last master" (bit + * 1 set), but unfortunately the CyberPro does not park the + * bus. We must therefore park on CPU. Unfortunately, this + * may trigger yet another bug in the 553. + */ + pci_write_config_byte(dev, 0x83, 0x02); + + /* + * Make the ISA DMA request lowest priority, and disable + * rotating priorities completely. + */ + pci_write_config_byte(dev, 0x80, 0x11); + pci_write_config_byte(dev, 0x81, 0x00); + + /* + * Route INTA input to IRQ 11, and set IRQ11 to be level + * sensitive. + */ + pci_write_config_word(dev, 0x44, 0xb000); + outb(0x08, 0x4d1); +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND, PCI_DEVICE_ID_WINBOND_83C553, pci_fixup_83c553); + +static void __devinit pci_fixup_unassign(struct pci_dev *dev) +{ + dev->resource[0].end -= dev->resource[0].start; + dev->resource[0].start = 0; +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_WINBOND2, PCI_DEVICE_ID_WINBOND2_89C940F, pci_fixup_unassign); + +/* + * Prevent the PCI layer from seeing the resources allocated to this device + * if it is the host bridge by marking it as such. These resources are of + * no consequence to the PCI layer (they are handled elsewhere). + */ +static void __devinit pci_fixup_dec21285(struct pci_dev *dev) +{ + int i; + + if (dev->devfn == 0) { + dev->class &= 0xff; + dev->class |= PCI_CLASS_BRIDGE_HOST << 8; + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + dev->resource[i].start = 0; + dev->resource[i].end = 0; + dev->resource[i].flags = 0; + } + } +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_21285, pci_fixup_dec21285); + +/* + * PCI IDE controllers use non-standard I/O port decoding, respect it. + */ +static void __devinit pci_fixup_ide_bases(struct pci_dev *dev) +{ + struct resource *r; + int i; + + if ((dev->class >> 8) != PCI_CLASS_STORAGE_IDE) + return; + + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + r = dev->resource + i; + if ((r->start & ~0x80) == 0x374) { + r->start |= 2; + r->end = r->start; + } + } +} +DECLARE_PCI_FIXUP_HEADER(PCI_ANY_ID, PCI_ANY_ID, pci_fixup_ide_bases); + +/* + * Put the DEC21142 to sleep + */ +static void __devinit pci_fixup_dec21142(struct pci_dev *dev) +{ + pci_write_config_dword(dev, 0x40, 0x80000000); +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_21142, pci_fixup_dec21142); + +/* + * The CY82C693 needs some rather major fixups to ensure that it does + * the right thing. Idea from the Alpha people, with a few additions. + * + * We ensure that the IDE base registers are set to 1f0/3f4 for the + * primary bus, and 170/374 for the secondary bus. Also, hide them + * from the PCI subsystem view as well so we won't try to perform + * our own auto-configuration on them. + * + * In addition, we ensure that the PCI IDE interrupts are routed to + * IRQ 14 and IRQ 15 respectively. + * + * The above gets us to a point where the IDE on this device is + * functional. However, The CY82C693U _does not work_ in bus + * master mode without locking the PCI bus solid. + */ +static void __devinit pci_fixup_cy82c693(struct pci_dev *dev) +{ + if ((dev->class >> 8) == PCI_CLASS_STORAGE_IDE) { + u32 base0, base1; + + if (dev->class & 0x80) { /* primary */ + base0 = 0x1f0; + base1 = 0x3f4; + } else { /* secondary */ + base0 = 0x170; + base1 = 0x374; + } + + pci_write_config_dword(dev, PCI_BASE_ADDRESS_0, + base0 | PCI_BASE_ADDRESS_SPACE_IO); + pci_write_config_dword(dev, PCI_BASE_ADDRESS_1, + base1 | PCI_BASE_ADDRESS_SPACE_IO); + + dev->resource[0].start = 0; + dev->resource[0].end = 0; + dev->resource[0].flags = 0; + + dev->resource[1].start = 0; + dev->resource[1].end = 0; + dev->resource[1].flags = 0; + } else if (PCI_FUNC(dev->devfn) == 0) { + /* + * Setup IDE IRQ routing. + */ + pci_write_config_byte(dev, 0x4b, 14); + pci_write_config_byte(dev, 0x4c, 15); + + /* + * Disable FREQACK handshake, enable USB. + */ + pci_write_config_byte(dev, 0x4d, 0x41); + + /* + * Enable PCI retry, and PCI post-write buffer. + */ + pci_write_config_byte(dev, 0x44, 0x17); + + /* + * Enable ISA master and DMA post write buffering. + */ + pci_write_config_byte(dev, 0x45, 0x03); + } +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_CONTAQ, PCI_DEVICE_ID_CONTAQ_82C693, pci_fixup_cy82c693); + +static void __init pci_fixup_it8152(struct pci_dev *dev) +{ + int i; + /* fixup for ITE 8152 devices */ + /* FIXME: add defines for class 0x68000 and 0x80103 */ + if ((dev->class >> 8) == PCI_CLASS_BRIDGE_HOST || + dev->class == 0x68000 || + dev->class == 0x80103) { + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + dev->resource[i].start = 0; + dev->resource[i].end = 0; + dev->resource[i].flags = 0; + } + } +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_ITE, PCI_DEVICE_ID_ITE_8152, pci_fixup_it8152); + + + +void __devinit pcibios_update_irq(struct pci_dev *dev, int irq) +{ + if (debug_pci) + printk("PCI: Assigning IRQ %02d to %s\n", irq, pci_name(dev)); + pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq); +} + +/* + * If the bus contains any of these devices, then we must not turn on + * parity checking of any kind. Currently this is CyberPro 20x0 only. + */ +static inline int pdev_bad_for_parity(struct pci_dev *dev) +{ + return ((dev->vendor == PCI_VENDOR_ID_INTERG && + (dev->device == PCI_DEVICE_ID_INTERG_2000 || + dev->device == PCI_DEVICE_ID_INTERG_2010)) || + (dev->vendor == PCI_VENDOR_ID_ITE && + dev->device == PCI_DEVICE_ID_ITE_8152)); + +} + +/* + * Adjust the device resources from bus-centric to Linux-centric. + */ +static void __devinit +pdev_fixup_device_resources(struct pci_sys_data *root, struct pci_dev *dev) +{ + resource_size_t offset; + int i; + + for (i = 0; i < PCI_NUM_RESOURCES; i++) { + if (dev->resource[i].start == 0) + continue; + if (dev->resource[i].flags & IORESOURCE_MEM) + offset = root->mem_offset; + else + offset = root->io_offset; + + dev->resource[i].start += offset; + dev->resource[i].end += offset; + } +} + +static void __devinit +pbus_assign_bus_resources(struct pci_bus *bus, struct pci_sys_data *root) +{ + struct pci_dev *dev = bus->self; + int i; + + if (!dev) { + /* + * Assign root bus resources. + */ + for (i = 0; i < 3; i++) + bus->resource[i] = root->resource[i]; + } +} + +/* + * pcibios_fixup_bus - Called after each bus is probed, + * but before its children are examined. + */ +void pcibios_fixup_bus(struct pci_bus *bus) +{ + struct pci_sys_data *root = bus->sysdata; + struct pci_dev *dev; + u16 features = PCI_COMMAND_SERR | PCI_COMMAND_PARITY | PCI_COMMAND_FAST_BACK; + + pbus_assign_bus_resources(bus, root); + + /* + * Walk the devices on this bus, working out what we can + * and can't support. + */ + list_for_each_entry(dev, &bus->devices, bus_list) { + u16 status; + + pdev_fixup_device_resources(root, dev); + + pci_read_config_word(dev, PCI_STATUS, &status); + + /* + * If any device on this bus does not support fast back + * to back transfers, then the bus as a whole is not able + * to support them. Having fast back to back transfers + * on saves us one PCI cycle per transaction. + */ + if (!(status & PCI_STATUS_FAST_BACK)) + features &= ~PCI_COMMAND_FAST_BACK; + + if (pdev_bad_for_parity(dev)) + features &= ~(PCI_COMMAND_SERR | PCI_COMMAND_PARITY); + + switch (dev->class >> 8) { + case PCI_CLASS_BRIDGE_PCI: + pci_read_config_word(dev, PCI_BRIDGE_CONTROL, &status); + status |= PCI_BRIDGE_CTL_PARITY|PCI_BRIDGE_CTL_MASTER_ABORT; + status &= ~(PCI_BRIDGE_CTL_BUS_RESET|PCI_BRIDGE_CTL_FAST_BACK); + pci_write_config_word(dev, PCI_BRIDGE_CONTROL, status); + break; + + case PCI_CLASS_BRIDGE_CARDBUS: + pci_read_config_word(dev, PCI_CB_BRIDGE_CONTROL, &status); + status |= PCI_CB_BRIDGE_CTL_PARITY|PCI_CB_BRIDGE_CTL_MASTER_ABORT; + pci_write_config_word(dev, PCI_CB_BRIDGE_CONTROL, status); + break; + } + } + + /* + * Now walk the devices again, this time setting them up. + */ + list_for_each_entry(dev, &bus->devices, bus_list) { + u16 cmd; + + pci_read_config_word(dev, PCI_COMMAND, &cmd); + cmd |= features; + pci_write_config_word(dev, PCI_COMMAND, cmd); + + pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, + L1_CACHE_BYTES >> 2); + } + + /* + * Propagate the flags to the PCI bridge. + */ + if (bus->self && bus->self->hdr_type == PCI_HEADER_TYPE_BRIDGE) { + if (features & PCI_COMMAND_FAST_BACK) + bus->bridge_ctl |= PCI_BRIDGE_CTL_FAST_BACK; + if (features & PCI_COMMAND_PARITY) + bus->bridge_ctl |= PCI_BRIDGE_CTL_PARITY; + } + + /* + * Report what we did for this bus + */ + printk(KERN_INFO "PCI: bus%d: Fast back to back transfers %sabled\n", + bus->number, (features & PCI_COMMAND_FAST_BACK) ? "en" : "dis"); +} + +/* + * Convert from Linux-centric to bus-centric addresses for bridge devices. + */ +void +pcibios_resource_to_bus(struct pci_dev *dev, struct pci_bus_region *region, + struct resource *res) +{ + struct pci_sys_data *root = dev->sysdata; + unsigned long offset = 0; + + if (res->flags & IORESOURCE_IO) + offset = root->io_offset; + if (res->flags & IORESOURCE_MEM) + offset = root->mem_offset; + + region->start = res->start - offset; + region->end = res->end - offset; +} + +void __devinit +pcibios_bus_to_resource(struct pci_dev *dev, struct resource *res, + struct pci_bus_region *region) +{ + struct pci_sys_data *root = dev->sysdata; + unsigned long offset = 0; + + if (res->flags & IORESOURCE_IO) + offset = root->io_offset; + if (res->flags & IORESOURCE_MEM) + offset = root->mem_offset; + + res->start = region->start + offset; + res->end = region->end + offset; +} + +#ifdef CONFIG_HOTPLUG +EXPORT_SYMBOL(pcibios_fixup_bus); +EXPORT_SYMBOL(pcibios_resource_to_bus); +EXPORT_SYMBOL(pcibios_bus_to_resource); +#endif + +/* + * Swizzle the device pin each time we cross a bridge. + * This might update pin and returns the slot number. + */ +static u8 __devinit pcibios_swizzle(struct pci_dev *dev, u8 *pin) +{ + struct pci_sys_data *sys = dev->sysdata; + int slot = 0, oldpin = *pin; + + if (sys->swizzle) + slot = sys->swizzle(dev, pin); + + if (debug_pci) + printk("PCI: %s swizzling pin %d => pin %d slot %d\n", + pci_name(dev), oldpin, *pin, slot); + + return slot; +} + +/* + * Map a slot/pin to an IRQ. + */ +static int pcibios_map_irq(struct pci_dev *dev, u8 slot, u8 pin) +{ + struct pci_sys_data *sys = dev->sysdata; + int irq = -1; + + if (sys->map_irq) + irq = sys->map_irq(dev, slot, pin); + + if (debug_pci) + printk("PCI: %s mapping slot %d pin %d => irq %d\n", + pci_name(dev), slot, pin, irq); + + return irq; +} + +static void __init pcibios_init_hw(struct hw_pci *hw) +{ + struct pci_sys_data *sys = NULL; + int ret; + int nr, busnr; + + for (nr = busnr = 0; nr < hw->nr_controllers; nr++) { + sys = kzalloc(sizeof(struct pci_sys_data), GFP_KERNEL); + if (!sys) + panic("PCI: unable to allocate sys data!"); + +#ifdef CONFIG_PCI_DOMAINS + sys->domain = hw->domain; +#endif + sys->hw = hw; + sys->busnr = busnr; + sys->swizzle = hw->swizzle; + sys->map_irq = hw->map_irq; + sys->resource[0] = &ioport_resource; + sys->resource[1] = &iomem_resource; + + ret = hw->setup(nr, sys); + + if (ret > 0) { + sys->bus = hw->scan(nr, sys); + + if (!sys->bus) + panic("PCI: unable to scan bus!"); + + busnr = sys->bus->subordinate + 1; + + list_add(&sys->node, &hw->buses); + } else { + kfree(sys); + if (ret < 0) + break; + } + } +} + +void __init pci_common_init(struct hw_pci *hw) +{ + struct pci_sys_data *sys; + + INIT_LIST_HEAD(&hw->buses); + + if (hw->preinit) + hw->preinit(); + pcibios_init_hw(hw); + if (hw->postinit) + hw->postinit(); + + pci_fixup_irqs(pcibios_swizzle, pcibios_map_irq); + + list_for_each_entry(sys, &hw->buses, node) { + struct pci_bus *bus = sys->bus; + + if (!use_firmware) { + /* + * Size the bridge windows. + */ + pci_bus_size_bridges(bus); + + /* + * Assign resources. + */ + pci_bus_assign_resources(bus); + + /* + * Enable bridges + */ + pci_enable_bridges(bus); + } + + /* + * Tell drivers about devices found. + */ + pci_bus_add_devices(bus); + } +} + +char * __init pcibios_setup(char *str) +{ + if (!strcmp(str, "debug")) { + debug_pci = 1; + return NULL; + } else if (!strcmp(str, "firmware")) { + use_firmware = 1; + return NULL; + } + return str; +} + +/* + * From arch/i386/kernel/pci-i386.c: + * + * We need to avoid collisions with `mirrored' VGA ports + * and other strange ISA hardware, so we always want the + * addresses to be allocated in the 0x000-0x0ff region + * modulo 0x400. + * + * Why? Because some silly external IO cards only decode + * the low 10 bits of the IO address. The 0x00-0xff region + * is reserved for motherboard devices that decode all 16 + * bits, so it's ok to allocate at, say, 0x2800-0x28ff, + * but we want to try to avoid allocating at 0x2900-0x2bff + * which might be mirrored at 0x0100-0x03ff.. + */ +resource_size_t pcibios_align_resource(void *data, const struct resource *res, + resource_size_t size, resource_size_t align) +{ + resource_size_t start = res->start; + + if (res->flags & IORESOURCE_IO && start & 0x300) + start = (start + 0x3ff) & ~0x3ff; + + start = (start + align - 1) & ~(align - 1); + + return start; +} + +/** + * pcibios_enable_device - Enable I/O and memory. + * @dev: PCI device to be enabled + */ +int pcibios_enable_device(struct pci_dev *dev, int mask) +{ + u16 cmd, old_cmd; + int idx; + struct resource *r; + + pci_read_config_word(dev, PCI_COMMAND, &cmd); + old_cmd = cmd; + for (idx = 0; idx < 6; idx++) { + /* Only set up the requested stuff */ + if (!(mask & (1 << idx))) + continue; + + r = dev->resource + idx; + if (!r->start && r->end) { + printk(KERN_ERR "PCI: Device %s not available because" + " of resource collisions\n", pci_name(dev)); + return -EINVAL; + } + if (r->flags & IORESOURCE_IO) + cmd |= PCI_COMMAND_IO; + if (r->flags & IORESOURCE_MEM) + cmd |= PCI_COMMAND_MEMORY; + } + + /* + * Bridges (eg, cardbus bridges) need to be fully enabled + */ + if ((dev->class >> 16) == PCI_BASE_CLASS_BRIDGE) + cmd |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY; + + if (cmd != old_cmd) { + printk("PCI: enabling device %s (%04x -> %04x)\n", + pci_name(dev), old_cmd, cmd); + pci_write_config_word(dev, PCI_COMMAND, cmd); + } + return 0; +} + +int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, + enum pci_mmap_state mmap_state, int write_combine) +{ + struct pci_sys_data *root = dev->sysdata; + unsigned long phys; + + if (mmap_state == pci_mmap_io) { + return -EINVAL; + } else { + phys = vma->vm_pgoff + (root->mem_offset >> PAGE_SHIFT); + } + + /* + * Mark this as IO + */ + vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); + + if (remap_pfn_range(vma, vma->vm_start, phys, + vma->vm_end - vma->vm_start, + vma->vm_page_prot)) + return -EAGAIN; + + return 0; +} diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S new file mode 100644 index 00000000..80f7896c --- /dev/null +++ b/arch/arm/kernel/calls.S @@ -0,0 +1,394 @@ +/* + * linux/arch/arm/kernel/calls.S + * + * Copyright (C) 1995-2005 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This file is included thrice in entry-common.S + */ +/* 0 */ CALL(sys_restart_syscall) + CALL(sys_exit) + CALL(sys_fork_wrapper) + CALL(sys_read) + CALL(sys_write) +/* 5 */ CALL(sys_open) + CALL(sys_close) + CALL(sys_ni_syscall) /* was sys_waitpid */ + CALL(sys_creat) + CALL(sys_link) +/* 10 */ CALL(sys_unlink) + CALL(sys_execve_wrapper) + CALL(sys_chdir) + CALL(OBSOLETE(sys_time)) /* used by libc4 */ + CALL(sys_mknod) +/* 15 */ CALL(sys_chmod) + CALL(sys_lchown16) + CALL(sys_ni_syscall) /* was sys_break */ + CALL(sys_ni_syscall) /* was sys_stat */ + CALL(sys_lseek) +/* 20 */ CALL(sys_getpid) + CALL(sys_mount) + CALL(OBSOLETE(sys_oldumount)) /* used by libc4 */ + CALL(sys_setuid16) + CALL(sys_getuid16) +/* 25 */ CALL(OBSOLETE(sys_stime)) + CALL(sys_ptrace) + CALL(OBSOLETE(sys_alarm)) /* used by libc4 */ + CALL(sys_ni_syscall) /* was sys_fstat */ + CALL(sys_pause) +/* 30 */ CALL(OBSOLETE(sys_utime)) /* used by libc4 */ + CALL(sys_ni_syscall) /* was sys_stty */ + CALL(sys_ni_syscall) /* was sys_getty */ + CALL(sys_access) + CALL(sys_nice) +/* 35 */ CALL(sys_ni_syscall) /* was sys_ftime */ + CALL(sys_sync) + CALL(sys_kill) + CALL(sys_rename) + CALL(sys_mkdir) +/* 40 */ CALL(sys_rmdir) + CALL(sys_dup) + CALL(sys_pipe) + CALL(sys_times) + CALL(sys_ni_syscall) /* was sys_prof */ +/* 45 */ CALL(sys_brk) + CALL(sys_setgid16) + CALL(sys_getgid16) + CALL(sys_ni_syscall) /* was sys_signal */ + CALL(sys_geteuid16) +/* 50 */ CALL(sys_getegid16) + CALL(sys_acct) + CALL(sys_umount) + CALL(sys_ni_syscall) /* was sys_lock */ + CALL(sys_ioctl) +/* 55 */ CALL(sys_fcntl) + CALL(sys_ni_syscall) /* was sys_mpx */ + CALL(sys_setpgid) + CALL(sys_ni_syscall) /* was sys_ulimit */ + CALL(sys_ni_syscall) /* was sys_olduname */ +/* 60 */ CALL(sys_umask) + CALL(sys_chroot) + CALL(sys_ustat) + CALL(sys_dup2) + CALL(sys_getppid) +/* 65 */ CALL(sys_getpgrp) + CALL(sys_setsid) + CALL(sys_sigaction) + CALL(sys_ni_syscall) /* was sys_sgetmask */ + CALL(sys_ni_syscall) /* was sys_ssetmask */ +/* 70 */ CALL(sys_setreuid16) + CALL(sys_setregid16) + CALL(sys_sigsuspend) + CALL(sys_sigpending) + CALL(sys_sethostname) +/* 75 */ CALL(sys_setrlimit) + CALL(OBSOLETE(sys_old_getrlimit)) /* used by libc4 */ + CALL(sys_getrusage) + CALL(sys_gettimeofday) + CALL(sys_settimeofday) +/* 80 */ CALL(sys_getgroups16) + CALL(sys_setgroups16) + CALL(OBSOLETE(sys_old_select)) /* used by libc4 */ + CALL(sys_symlink) + CALL(sys_ni_syscall) /* was sys_lstat */ +/* 85 */ CALL(sys_readlink) + CALL(sys_uselib) + CALL(sys_swapon) + CALL(sys_reboot) + CALL(OBSOLETE(sys_old_readdir)) /* used by libc4 */ +/* 90 */ CALL(OBSOLETE(sys_old_mmap)) /* used by libc4 */ + CALL(sys_munmap) + CALL(sys_truncate) + CALL(sys_ftruncate) + CALL(sys_fchmod) +/* 95 */ CALL(sys_fchown16) + CALL(sys_getpriority) + CALL(sys_setpriority) + CALL(sys_ni_syscall) /* was sys_profil */ + CALL(sys_statfs) +/* 100 */ CALL(sys_fstatfs) + CALL(sys_ni_syscall) /* sys_ioperm */ + CALL(OBSOLETE(ABI(sys_socketcall, sys_oabi_socketcall))) + CALL(sys_syslog) + CALL(sys_setitimer) +/* 105 */ CALL(sys_getitimer) + CALL(sys_newstat) + CALL(sys_newlstat) + CALL(sys_newfstat) + CALL(sys_ni_syscall) /* was sys_uname */ +/* 110 */ CALL(sys_ni_syscall) /* was sys_iopl */ + CALL(sys_vhangup) + CALL(sys_ni_syscall) + CALL(OBSOLETE(sys_syscall)) /* call a syscall */ + CALL(sys_wait4) +/* 115 */ CALL(sys_swapoff) + CALL(sys_sysinfo) + CALL(OBSOLETE(ABI(sys_ipc, sys_oabi_ipc))) + CALL(sys_fsync) + CALL(sys_sigreturn_wrapper) +/* 120 */ CALL(sys_clone_wrapper) + CALL(sys_setdomainname) + CALL(sys_newuname) + CALL(sys_ni_syscall) /* modify_ldt */ + CALL(sys_adjtimex) +/* 125 */ CALL(sys_mprotect) + CALL(sys_sigprocmask) + CALL(sys_ni_syscall) /* was sys_create_module */ + CALL(sys_init_module) + CALL(sys_delete_module) +/* 130 */ CALL(sys_ni_syscall) /* was sys_get_kernel_syms */ + CALL(sys_quotactl) + CALL(sys_getpgid) + CALL(sys_fchdir) + CALL(sys_bdflush) +/* 135 */ CALL(sys_sysfs) + CALL(sys_personality) + CALL(sys_ni_syscall) /* reserved for afs_syscall */ + CALL(sys_setfsuid16) + CALL(sys_setfsgid16) +/* 140 */ CALL(sys_llseek) + CALL(sys_getdents) + CALL(sys_select) + CALL(sys_flock) + CALL(sys_msync) +/* 145 */ CALL(sys_readv) + CALL(sys_writev) + CALL(sys_getsid) + CALL(sys_fdatasync) + CALL(sys_sysctl) +/* 150 */ CALL(sys_mlock) + CALL(sys_munlock) + CALL(sys_mlockall) + CALL(sys_munlockall) + CALL(sys_sched_setparam) +/* 155 */ CALL(sys_sched_getparam) + CALL(sys_sched_setscheduler) + CALL(sys_sched_getscheduler) + CALL(sys_sched_yield) + CALL(sys_sched_get_priority_max) +/* 160 */ CALL(sys_sched_get_priority_min) + CALL(sys_sched_rr_get_interval) + CALL(sys_nanosleep) + CALL(sys_mremap) + CALL(sys_setresuid16) +/* 165 */ CALL(sys_getresuid16) + CALL(sys_ni_syscall) /* vm86 */ + CALL(sys_ni_syscall) /* was sys_query_module */ + CALL(sys_poll) + CALL(sys_nfsservctl) +/* 170 */ CALL(sys_setresgid16) + CALL(sys_getresgid16) + CALL(sys_prctl) + CALL(sys_rt_sigreturn_wrapper) + CALL(sys_rt_sigaction) +/* 175 */ CALL(sys_rt_sigprocmask) + CALL(sys_rt_sigpending) + CALL(sys_rt_sigtimedwait) + CALL(sys_rt_sigqueueinfo) + CALL(sys_rt_sigsuspend) +/* 180 */ CALL(ABI(sys_pread64, sys_oabi_pread64)) + CALL(ABI(sys_pwrite64, sys_oabi_pwrite64)) + CALL(sys_chown16) + CALL(sys_getcwd) + CALL(sys_capget) +/* 185 */ CALL(sys_capset) + CALL(sys_sigaltstack_wrapper) + CALL(sys_sendfile) + CALL(sys_ni_syscall) /* getpmsg */ + CALL(sys_ni_syscall) /* putpmsg */ +/* 190 */ CALL(sys_vfork_wrapper) + CALL(sys_getrlimit) + CALL(sys_mmap2) + CALL(ABI(sys_truncate64, sys_oabi_truncate64)) + CALL(ABI(sys_ftruncate64, sys_oabi_ftruncate64)) +/* 195 */ CALL(ABI(sys_stat64, sys_oabi_stat64)) + CALL(ABI(sys_lstat64, sys_oabi_lstat64)) + CALL(ABI(sys_fstat64, sys_oabi_fstat64)) + CALL(sys_lchown) + CALL(sys_getuid) +/* 200 */ CALL(sys_getgid) + CALL(sys_geteuid) + CALL(sys_getegid) + CALL(sys_setreuid) + CALL(sys_setregid) +/* 205 */ CALL(sys_getgroups) + CALL(sys_setgroups) + CALL(sys_fchown) + CALL(sys_setresuid) + CALL(sys_getresuid) +/* 210 */ CALL(sys_setresgid) + CALL(sys_getresgid) + CALL(sys_chown) + CALL(sys_setuid) + CALL(sys_setgid) +/* 215 */ CALL(sys_setfsuid) + CALL(sys_setfsgid) + CALL(sys_getdents64) + CALL(sys_pivot_root) + CALL(sys_mincore) +/* 220 */ CALL(sys_madvise) + CALL(ABI(sys_fcntl64, sys_oabi_fcntl64)) + CALL(sys_ni_syscall) /* TUX */ + CALL(sys_ni_syscall) + CALL(sys_gettid) +/* 225 */ CALL(ABI(sys_readahead, sys_oabi_readahead)) + CALL(sys_setxattr) + CALL(sys_lsetxattr) + CALL(sys_fsetxattr) + CALL(sys_getxattr) +/* 230 */ CALL(sys_lgetxattr) + CALL(sys_fgetxattr) + CALL(sys_listxattr) + CALL(sys_llistxattr) + CALL(sys_flistxattr) +/* 235 */ CALL(sys_removexattr) + CALL(sys_lremovexattr) + CALL(sys_fremovexattr) + CALL(sys_tkill) + CALL(sys_sendfile64) +/* 240 */ CALL(sys_futex) + CALL(sys_sched_setaffinity) + CALL(sys_sched_getaffinity) + CALL(sys_io_setup) + CALL(sys_io_destroy) +/* 245 */ CALL(sys_io_getevents) + CALL(sys_io_submit) + CALL(sys_io_cancel) + CALL(sys_exit_group) + CALL(sys_lookup_dcookie) +/* 250 */ CALL(sys_epoll_create) + CALL(ABI(sys_epoll_ctl, sys_oabi_epoll_ctl)) + CALL(ABI(sys_epoll_wait, sys_oabi_epoll_wait)) + CALL(sys_remap_file_pages) + CALL(sys_ni_syscall) /* sys_set_thread_area */ +/* 255 */ CALL(sys_ni_syscall) /* sys_get_thread_area */ + CALL(sys_set_tid_address) + CALL(sys_timer_create) + CALL(sys_timer_settime) + CALL(sys_timer_gettime) +/* 260 */ CALL(sys_timer_getoverrun) + CALL(sys_timer_delete) + CALL(sys_clock_settime) + CALL(sys_clock_gettime) + CALL(sys_clock_getres) +/* 265 */ CALL(sys_clock_nanosleep) + CALL(sys_statfs64_wrapper) + CALL(sys_fstatfs64_wrapper) + CALL(sys_tgkill) + CALL(sys_utimes) +/* 270 */ CALL(sys_arm_fadvise64_64) + CALL(sys_pciconfig_iobase) + CALL(sys_pciconfig_read) + CALL(sys_pciconfig_write) + CALL(sys_mq_open) +/* 275 */ CALL(sys_mq_unlink) + CALL(sys_mq_timedsend) + CALL(sys_mq_timedreceive) + CALL(sys_mq_notify) + CALL(sys_mq_getsetattr) +/* 280 */ CALL(sys_waitid) + CALL(sys_socket) + CALL(ABI(sys_bind, sys_oabi_bind)) + CALL(ABI(sys_connect, sys_oabi_connect)) + CALL(sys_listen) +/* 285 */ CALL(sys_accept) + CALL(sys_getsockname) + CALL(sys_getpeername) + CALL(sys_socketpair) + CALL(sys_send) +/* 290 */ CALL(ABI(sys_sendto, sys_oabi_sendto)) + CALL(sys_recv) + CALL(sys_recvfrom) + CALL(sys_shutdown) + CALL(sys_setsockopt) +/* 295 */ CALL(sys_getsockopt) + CALL(ABI(sys_sendmsg, sys_oabi_sendmsg)) + CALL(sys_recvmsg) + CALL(ABI(sys_semop, sys_oabi_semop)) + CALL(sys_semget) +/* 300 */ CALL(sys_semctl) + CALL(sys_msgsnd) + CALL(sys_msgrcv) + CALL(sys_msgget) + CALL(sys_msgctl) +/* 305 */ CALL(sys_shmat) + CALL(sys_shmdt) + CALL(sys_shmget) + CALL(sys_shmctl) + CALL(sys_add_key) +/* 310 */ CALL(sys_request_key) + CALL(sys_keyctl) + CALL(ABI(sys_semtimedop, sys_oabi_semtimedop)) +/* vserver */ CALL(sys_ni_syscall) + CALL(sys_ioprio_set) +/* 315 */ CALL(sys_ioprio_get) + CALL(sys_inotify_init) + CALL(sys_inotify_add_watch) + CALL(sys_inotify_rm_watch) + CALL(sys_mbind) +/* 320 */ CALL(sys_get_mempolicy) + CALL(sys_set_mempolicy) + CALL(sys_openat) + CALL(sys_mkdirat) + CALL(sys_mknodat) +/* 325 */ CALL(sys_fchownat) + CALL(sys_futimesat) + CALL(ABI(sys_fstatat64, sys_oabi_fstatat64)) + CALL(sys_unlinkat) + CALL(sys_renameat) +/* 330 */ CALL(sys_linkat) + CALL(sys_symlinkat) + CALL(sys_readlinkat) + CALL(sys_fchmodat) + CALL(sys_faccessat) +/* 335 */ CALL(sys_pselect6) + CALL(sys_ppoll) + CALL(sys_unshare) + CALL(sys_set_robust_list) + CALL(sys_get_robust_list) +/* 340 */ CALL(sys_splice) + CALL(sys_sync_file_range2) + CALL(sys_tee) + CALL(sys_vmsplice) + CALL(sys_move_pages) +/* 345 */ CALL(sys_getcpu) + CALL(sys_epoll_pwait) + CALL(sys_kexec_load) + CALL(sys_utimensat) + CALL(sys_signalfd) +/* 350 */ CALL(sys_timerfd_create) + CALL(sys_eventfd) + CALL(sys_fallocate) + CALL(sys_timerfd_settime) + CALL(sys_timerfd_gettime) +/* 355 */ CALL(sys_signalfd4) + CALL(sys_eventfd2) + CALL(sys_epoll_create1) + CALL(sys_dup3) + CALL(sys_pipe2) +/* 360 */ CALL(sys_inotify_init1) + CALL(sys_preadv) + CALL(sys_pwritev) + CALL(sys_rt_tgsigqueueinfo) + CALL(sys_perf_event_open) +/* 365 */ CALL(sys_recvmmsg) + CALL(sys_accept4) + CALL(sys_fanotify_init) + CALL(sys_fanotify_mark) + CALL(sys_prlimit64) +/* 370 */ CALL(sys_name_to_handle_at) + CALL(sys_open_by_handle_at) + CALL(sys_clock_adjtime) + CALL(sys_syncfs) + CALL(sys_sendmmsg) +/* 375 */ CALL(sys_setns) +#ifndef syscalls_counted +.equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls +#define syscalls_counted +#endif +.rept syscalls_padding + CALL(sys_ni_syscall) +.endr diff --git a/arch/arm/kernel/compat.c b/arch/arm/kernel/compat.c new file mode 100644 index 00000000..92565231 --- /dev/null +++ b/arch/arm/kernel/compat.c @@ -0,0 +1,219 @@ +/* + * linux/arch/arm/kernel/compat.c + * + * Copyright (C) 2001 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * We keep the old params compatibility cruft in one place (here) + * so we don't end up with lots of mess around other places. + * + * NOTE: + * The old struct param_struct is deprecated, but it will be kept in + * the kernel for 5 years from now (2001). This will allow boot loaders + * to convert to the new struct tag way. + */ +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/string.h> +#include <linux/init.h> + +#include <asm/setup.h> +#include <asm/mach-types.h> +#include <asm/page.h> + +#include <asm/mach/arch.h> + +#include "compat.h" + +/* + * Usage: + * - do not go blindly adding fields, add them at the end + * - when adding fields, don't rely on the address until + * a patch from me has been released + * - unused fields should be zero (for future expansion) + * - this structure is relatively short-lived - only + * guaranteed to contain useful data in setup_arch() + * + * This is the old deprecated way to pass parameters to the kernel + */ +struct param_struct { + union { + struct { + unsigned long page_size; /* 0 */ + unsigned long nr_pages; /* 4 */ + unsigned long ramdisk_size; /* 8 */ + unsigned long flags; /* 12 */ +#define FLAG_READONLY 1 +#define FLAG_RDLOAD 4 +#define FLAG_RDPROMPT 8 + unsigned long rootdev; /* 16 */ + unsigned long video_num_cols; /* 20 */ + unsigned long video_num_rows; /* 24 */ + unsigned long video_x; /* 28 */ + unsigned long video_y; /* 32 */ + unsigned long memc_control_reg; /* 36 */ + unsigned char sounddefault; /* 40 */ + unsigned char adfsdrives; /* 41 */ + unsigned char bytes_per_char_h; /* 42 */ + unsigned char bytes_per_char_v; /* 43 */ + unsigned long pages_in_bank[4]; /* 44 */ + unsigned long pages_in_vram; /* 60 */ + unsigned long initrd_start; /* 64 */ + unsigned long initrd_size; /* 68 */ + unsigned long rd_start; /* 72 */ + unsigned long system_rev; /* 76 */ + unsigned long system_serial_low; /* 80 */ + unsigned long system_serial_high; /* 84 */ + unsigned long mem_fclk_21285; /* 88 */ + } s; + char unused[256]; + } u1; + union { + char paths[8][128]; + struct { + unsigned long magic; + char n[1024 - sizeof(unsigned long)]; + } s; + } u2; + char commandline[COMMAND_LINE_SIZE]; +}; + +static struct tag * __init memtag(struct tag *tag, unsigned long start, unsigned long size) +{ + tag = tag_next(tag); + tag->hdr.tag = ATAG_MEM; + tag->hdr.size = tag_size(tag_mem32); + tag->u.mem.size = size; + tag->u.mem.start = start; + + return tag; +} + +static void __init build_tag_list(struct param_struct *params, void *taglist) +{ + struct tag *tag = taglist; + + if (params->u1.s.page_size != PAGE_SIZE) { + printk(KERN_WARNING "Warning: bad configuration page, " + "trying to continue\n"); + return; + } + + printk(KERN_DEBUG "Converting old-style param struct to taglist\n"); + +#ifdef CONFIG_ARCH_NETWINDER + if (params->u1.s.nr_pages != 0x02000 && + params->u1.s.nr_pages != 0x04000 && + params->u1.s.nr_pages != 0x08000 && + params->u1.s.nr_pages != 0x10000) { + printk(KERN_WARNING "Warning: bad NeTTrom parameters " + "detected, using defaults\n"); + + params->u1.s.nr_pages = 0x1000; /* 16MB */ + params->u1.s.ramdisk_size = 0; + params->u1.s.flags = FLAG_READONLY; + params->u1.s.initrd_start = 0; + params->u1.s.initrd_size = 0; + params->u1.s.rd_start = 0; + } +#endif + + tag->hdr.tag = ATAG_CORE; + tag->hdr.size = tag_size(tag_core); + tag->u.core.flags = params->u1.s.flags & FLAG_READONLY; + tag->u.core.pagesize = params->u1.s.page_size; + tag->u.core.rootdev = params->u1.s.rootdev; + + tag = tag_next(tag); + tag->hdr.tag = ATAG_RAMDISK; + tag->hdr.size = tag_size(tag_ramdisk); + tag->u.ramdisk.flags = (params->u1.s.flags & FLAG_RDLOAD ? 1 : 0) | + (params->u1.s.flags & FLAG_RDPROMPT ? 2 : 0); + tag->u.ramdisk.size = params->u1.s.ramdisk_size; + tag->u.ramdisk.start = params->u1.s.rd_start; + + tag = tag_next(tag); + tag->hdr.tag = ATAG_INITRD; + tag->hdr.size = tag_size(tag_initrd); + tag->u.initrd.start = params->u1.s.initrd_start; + tag->u.initrd.size = params->u1.s.initrd_size; + + tag = tag_next(tag); + tag->hdr.tag = ATAG_SERIAL; + tag->hdr.size = tag_size(tag_serialnr); + tag->u.serialnr.low = params->u1.s.system_serial_low; + tag->u.serialnr.high = params->u1.s.system_serial_high; + + tag = tag_next(tag); + tag->hdr.tag = ATAG_REVISION; + tag->hdr.size = tag_size(tag_revision); + tag->u.revision.rev = params->u1.s.system_rev; + +#ifdef CONFIG_ARCH_ACORN + if (machine_is_riscpc()) { + int i; + for (i = 0; i < 4; i++) + tag = memtag(tag, PHYS_OFFSET + (i << 26), + params->u1.s.pages_in_bank[i] * PAGE_SIZE); + } else +#endif + tag = memtag(tag, PHYS_OFFSET, params->u1.s.nr_pages * PAGE_SIZE); + +#ifdef CONFIG_FOOTBRIDGE + if (params->u1.s.mem_fclk_21285) { + tag = tag_next(tag); + tag->hdr.tag = ATAG_MEMCLK; + tag->hdr.size = tag_size(tag_memclk); + tag->u.memclk.fmemclk = params->u1.s.mem_fclk_21285; + } +#endif + +#ifdef CONFIG_ARCH_EBSA285 + if (machine_is_ebsa285()) { + tag = tag_next(tag); + tag->hdr.tag = ATAG_VIDEOTEXT; + tag->hdr.size = tag_size(tag_videotext); + tag->u.videotext.x = params->u1.s.video_x; + tag->u.videotext.y = params->u1.s.video_y; + tag->u.videotext.video_page = 0; + tag->u.videotext.video_mode = 0; + tag->u.videotext.video_cols = params->u1.s.video_num_cols; + tag->u.videotext.video_ega_bx = 0; + tag->u.videotext.video_lines = params->u1.s.video_num_rows; + tag->u.videotext.video_isvga = 1; + tag->u.videotext.video_points = 8; + } +#endif + +#ifdef CONFIG_ARCH_ACORN + tag = tag_next(tag); + tag->hdr.tag = ATAG_ACORN; + tag->hdr.size = tag_size(tag_acorn); + tag->u.acorn.memc_control_reg = params->u1.s.memc_control_reg; + tag->u.acorn.vram_pages = params->u1.s.pages_in_vram; + tag->u.acorn.sounddefault = params->u1.s.sounddefault; + tag->u.acorn.adfsdrives = params->u1.s.adfsdrives; +#endif + + tag = tag_next(tag); + tag->hdr.tag = ATAG_CMDLINE; + tag->hdr.size = (strlen(params->commandline) + 3 + + sizeof(struct tag_header)) >> 2; + strcpy(tag->u.cmdline.cmdline, params->commandline); + + tag = tag_next(tag); + tag->hdr.tag = ATAG_NONE; + tag->hdr.size = 0; + + memmove(params, taglist, ((int)tag) - ((int)taglist) + + sizeof(struct tag_header)); +} + +void __init convert_to_tag_list(struct tag *tags) +{ + struct param_struct *params = (struct param_struct *)tags; + build_tag_list(params, ¶ms->u2); +} diff --git a/arch/arm/kernel/compat.h b/arch/arm/kernel/compat.h new file mode 100644 index 00000000..39264ab1 --- /dev/null +++ b/arch/arm/kernel/compat.h @@ -0,0 +1,11 @@ +/* + * linux/arch/arm/kernel/compat.h + * + * Copyright (C) 2001 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. +*/ + +extern void convert_to_tag_list(struct tag *tags); diff --git a/arch/arm/kernel/crash_dump.c b/arch/arm/kernel/crash_dump.c new file mode 100644 index 00000000..90c50d4b --- /dev/null +++ b/arch/arm/kernel/crash_dump.c @@ -0,0 +1,57 @@ +/* + * arch/arm/kernel/crash_dump.c + * + * Copyright (C) 2010 Nokia Corporation. + * Author: Mika Westerberg + * + * This code is taken from arch/x86/kernel/crash_dump_64.c + * Created by: Hariprasad Nellitheertha (hari@in.ibm.com) + * Copyright (C) IBM Corporation, 2004. All rights reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/errno.h> +#include <linux/crash_dump.h> +#include <linux/uaccess.h> +#include <linux/io.h> + +/** + * copy_oldmem_page() - copy one page from old kernel memory + * @pfn: page frame number to be copied + * @buf: buffer where the copied page is placed + * @csize: number of bytes to copy + * @offset: offset in bytes into the page + * @userbuf: if set, @buf is int he user address space + * + * This function copies one page from old kernel memory into buffer pointed by + * @buf. If @buf is in userspace, set @userbuf to %1. Returns number of bytes + * copied or negative error in case of failure. + */ +ssize_t copy_oldmem_page(unsigned long pfn, char *buf, + size_t csize, unsigned long offset, + int userbuf) +{ + void *vaddr; + + if (!csize) + return 0; + + vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE); + if (!vaddr) + return -ENOMEM; + + if (userbuf) { + if (copy_to_user(buf, vaddr + offset, csize)) { + iounmap(vaddr); + return -EFAULT; + } + } else { + memcpy(buf, vaddr + offset, csize); + } + + iounmap(vaddr); + return csize; +} diff --git a/arch/arm/kernel/crunch-bits.S b/arch/arm/kernel/crunch-bits.S new file mode 100644 index 00000000..0ec9bb48 --- /dev/null +++ b/arch/arm/kernel/crunch-bits.S @@ -0,0 +1,305 @@ +/* + * arch/arm/kernel/crunch-bits.S + * Cirrus MaverickCrunch context switching and handling + * + * Copyright (C) 2006 Lennert Buytenhek <buytenh@wantstofly.org> + * + * Shamelessly stolen from the iWMMXt code by Nicolas Pitre, which is + * Copyright (c) 2003-2004, MontaVista Software, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/ptrace.h> +#include <asm/thread_info.h> +#include <asm/asm-offsets.h> +#include <mach/ep93xx-regs.h> + +/* + * We can't use hex constants here due to a bug in gas. + */ +#define CRUNCH_MVDX0 0 +#define CRUNCH_MVDX1 8 +#define CRUNCH_MVDX2 16 +#define CRUNCH_MVDX3 24 +#define CRUNCH_MVDX4 32 +#define CRUNCH_MVDX5 40 +#define CRUNCH_MVDX6 48 +#define CRUNCH_MVDX7 56 +#define CRUNCH_MVDX8 64 +#define CRUNCH_MVDX9 72 +#define CRUNCH_MVDX10 80 +#define CRUNCH_MVDX11 88 +#define CRUNCH_MVDX12 96 +#define CRUNCH_MVDX13 104 +#define CRUNCH_MVDX14 112 +#define CRUNCH_MVDX15 120 +#define CRUNCH_MVAX0L 128 +#define CRUNCH_MVAX0M 132 +#define CRUNCH_MVAX0H 136 +#define CRUNCH_MVAX1L 140 +#define CRUNCH_MVAX1M 144 +#define CRUNCH_MVAX1H 148 +#define CRUNCH_MVAX2L 152 +#define CRUNCH_MVAX2M 156 +#define CRUNCH_MVAX2H 160 +#define CRUNCH_MVAX3L 164 +#define CRUNCH_MVAX3M 168 +#define CRUNCH_MVAX3H 172 +#define CRUNCH_DSPSC 176 + +#define CRUNCH_SIZE 184 + + .text + +/* + * Lazy switching of crunch coprocessor context + * + * r10 = struct thread_info pointer + * r9 = ret_from_exception + * lr = undefined instr exit + * + * called from prefetch exception handler with interrupts disabled + */ +ENTRY(crunch_task_enable) + ldr r8, =(EP93XX_APB_VIRT_BASE + 0x00130000) @ syscon addr + + ldr r1, [r8, #0x80] + tst r1, #0x00800000 @ access to crunch enabled? + movne pc, lr @ if so no business here + mov r3, #0xaa @ unlock syscon swlock + str r3, [r8, #0xc0] + orr r1, r1, #0x00800000 @ enable access to crunch + str r1, [r8, #0x80] + + ldr r3, =crunch_owner + add r0, r10, #TI_CRUNCH_STATE @ get task crunch save area + ldr r2, [sp, #60] @ current task pc value + ldr r1, [r3] @ get current crunch owner + str r0, [r3] @ this task now owns crunch + sub r2, r2, #4 @ adjust pc back + str r2, [sp, #60] + + ldr r2, [r8, #0x80] + mov r2, r2 @ flush out enable (@@@) + + teq r1, #0 @ test for last ownership + mov lr, r9 @ normal exit from exception + beq crunch_load @ no owner, skip save + +crunch_save: + cfstr64 mvdx0, [r1, #CRUNCH_MVDX0] @ save 64b registers + cfstr64 mvdx1, [r1, #CRUNCH_MVDX1] + cfstr64 mvdx2, [r1, #CRUNCH_MVDX2] + cfstr64 mvdx3, [r1, #CRUNCH_MVDX3] + cfstr64 mvdx4, [r1, #CRUNCH_MVDX4] + cfstr64 mvdx5, [r1, #CRUNCH_MVDX5] + cfstr64 mvdx6, [r1, #CRUNCH_MVDX6] + cfstr64 mvdx7, [r1, #CRUNCH_MVDX7] + cfstr64 mvdx8, [r1, #CRUNCH_MVDX8] + cfstr64 mvdx9, [r1, #CRUNCH_MVDX9] + cfstr64 mvdx10, [r1, #CRUNCH_MVDX10] + cfstr64 mvdx11, [r1, #CRUNCH_MVDX11] + cfstr64 mvdx12, [r1, #CRUNCH_MVDX12] + cfstr64 mvdx13, [r1, #CRUNCH_MVDX13] + cfstr64 mvdx14, [r1, #CRUNCH_MVDX14] + cfstr64 mvdx15, [r1, #CRUNCH_MVDX15] + +#ifdef __ARMEB__ +#error fix me for ARMEB +#endif + + cfmv32al mvfx0, mvax0 @ save 72b accumulators + cfstr32 mvfx0, [r1, #CRUNCH_MVAX0L] + cfmv32am mvfx0, mvax0 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX0M] + cfmv32ah mvfx0, mvax0 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX0H] + cfmv32al mvfx0, mvax1 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX1L] + cfmv32am mvfx0, mvax1 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX1M] + cfmv32ah mvfx0, mvax1 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX1H] + cfmv32al mvfx0, mvax2 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX2L] + cfmv32am mvfx0, mvax2 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX2M] + cfmv32ah mvfx0, mvax2 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX2H] + cfmv32al mvfx0, mvax3 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX3L] + cfmv32am mvfx0, mvax3 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX3M] + cfmv32ah mvfx0, mvax3 + cfstr32 mvfx0, [r1, #CRUNCH_MVAX3H] + + cfmv32sc mvdx0, dspsc @ save status word + cfstr64 mvdx0, [r1, #CRUNCH_DSPSC] + + teq r0, #0 @ anything to load? + cfldr64eq mvdx0, [r1, #CRUNCH_MVDX0] @ mvdx0 was clobbered + moveq pc, lr + +crunch_load: + cfldr64 mvdx0, [r0, #CRUNCH_DSPSC] @ load status word + cfmvsc32 dspsc, mvdx0 + + cfldr32 mvfx0, [r0, #CRUNCH_MVAX0L] @ load 72b accumulators + cfmval32 mvax0, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX0M] + cfmvam32 mvax0, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX0H] + cfmvah32 mvax0, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX1L] + cfmval32 mvax1, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX1M] + cfmvam32 mvax1, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX1H] + cfmvah32 mvax1, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX2L] + cfmval32 mvax2, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX2M] + cfmvam32 mvax2, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX2H] + cfmvah32 mvax2, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX3L] + cfmval32 mvax3, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX3M] + cfmvam32 mvax3, mvfx0 + cfldr32 mvfx0, [r0, #CRUNCH_MVAX3H] + cfmvah32 mvax3, mvfx0 + + cfldr64 mvdx0, [r0, #CRUNCH_MVDX0] @ load 64b registers + cfldr64 mvdx1, [r0, #CRUNCH_MVDX1] + cfldr64 mvdx2, [r0, #CRUNCH_MVDX2] + cfldr64 mvdx3, [r0, #CRUNCH_MVDX3] + cfldr64 mvdx4, [r0, #CRUNCH_MVDX4] + cfldr64 mvdx5, [r0, #CRUNCH_MVDX5] + cfldr64 mvdx6, [r0, #CRUNCH_MVDX6] + cfldr64 mvdx7, [r0, #CRUNCH_MVDX7] + cfldr64 mvdx8, [r0, #CRUNCH_MVDX8] + cfldr64 mvdx9, [r0, #CRUNCH_MVDX9] + cfldr64 mvdx10, [r0, #CRUNCH_MVDX10] + cfldr64 mvdx11, [r0, #CRUNCH_MVDX11] + cfldr64 mvdx12, [r0, #CRUNCH_MVDX12] + cfldr64 mvdx13, [r0, #CRUNCH_MVDX13] + cfldr64 mvdx14, [r0, #CRUNCH_MVDX14] + cfldr64 mvdx15, [r0, #CRUNCH_MVDX15] + + mov pc, lr + +/* + * Back up crunch regs to save area and disable access to them + * (mainly for gdb or sleep mode usage) + * + * r0 = struct thread_info pointer of target task or NULL for any + */ +ENTRY(crunch_task_disable) + stmfd sp!, {r4, r5, lr} + + mrs ip, cpsr + orr r2, ip, #PSR_I_BIT @ disable interrupts + msr cpsr_c, r2 + + ldr r4, =(EP93XX_APB_VIRT_BASE + 0x00130000) @ syscon addr + + ldr r3, =crunch_owner + add r2, r0, #TI_CRUNCH_STATE @ get task crunch save area + ldr r1, [r3] @ get current crunch owner + teq r1, #0 @ any current owner? + beq 1f @ no: quit + teq r0, #0 @ any owner? + teqne r1, r2 @ or specified one? + bne 1f @ no: quit + + ldr r5, [r4, #0x80] @ enable access to crunch + mov r2, #0xaa + str r2, [r4, #0xc0] + orr r5, r5, #0x00800000 + str r5, [r4, #0x80] + + mov r0, #0 @ nothing to load + str r0, [r3] @ no more current owner + ldr r2, [r4, #0x80] @ flush out enable (@@@) + mov r2, r2 + bl crunch_save + + mov r2, #0xaa @ disable access to crunch + str r2, [r4, #0xc0] + bic r5, r5, #0x00800000 + str r5, [r4, #0x80] + ldr r5, [r4, #0x80] @ flush out enable (@@@) + mov r5, r5 + +1: msr cpsr_c, ip @ restore interrupt mode + ldmfd sp!, {r4, r5, pc} + +/* + * Copy crunch state to given memory address + * + * r0 = struct thread_info pointer of target task + * r1 = memory address where to store crunch state + * + * this is called mainly in the creation of signal stack frames + */ +ENTRY(crunch_task_copy) + mrs ip, cpsr + orr r2, ip, #PSR_I_BIT @ disable interrupts + msr cpsr_c, r2 + + ldr r3, =crunch_owner + add r2, r0, #TI_CRUNCH_STATE @ get task crunch save area + ldr r3, [r3] @ get current crunch owner + teq r2, r3 @ does this task own it... + beq 1f + + @ current crunch values are in the task save area + msr cpsr_c, ip @ restore interrupt mode + mov r0, r1 + mov r1, r2 + mov r2, #CRUNCH_SIZE + b memcpy + +1: @ this task owns crunch regs -- grab a copy from there + mov r0, #0 @ nothing to load + mov r3, lr @ preserve return address + bl crunch_save + msr cpsr_c, ip @ restore interrupt mode + mov pc, r3 + +/* + * Restore crunch state from given memory address + * + * r0 = struct thread_info pointer of target task + * r1 = memory address where to get crunch state from + * + * this is used to restore crunch state when unwinding a signal stack frame + */ +ENTRY(crunch_task_restore) + mrs ip, cpsr + orr r2, ip, #PSR_I_BIT @ disable interrupts + msr cpsr_c, r2 + + ldr r3, =crunch_owner + add r2, r0, #TI_CRUNCH_STATE @ get task crunch save area + ldr r3, [r3] @ get current crunch owner + teq r2, r3 @ does this task own it... + beq 1f + + @ this task doesn't own crunch regs -- use its save area + msr cpsr_c, ip @ restore interrupt mode + mov r0, r2 + mov r2, #CRUNCH_SIZE + b memcpy + +1: @ this task owns crunch regs -- load them directly + mov r0, r1 + mov r1, #0 @ nothing to save + mov r3, lr @ preserve return address + bl crunch_load + msr cpsr_c, ip @ restore interrupt mode + mov pc, r3 diff --git a/arch/arm/kernel/crunch.c b/arch/arm/kernel/crunch.c new file mode 100644 index 00000000..25ef223b --- /dev/null +++ b/arch/arm/kernel/crunch.c @@ -0,0 +1,88 @@ +/* + * arch/arm/kernel/crunch.c + * Cirrus MaverickCrunch context switching and handling + * + * Copyright (C) 2006 Lennert Buytenhek <buytenh@wantstofly.org> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/signal.h> +#include <linux/sched.h> +#include <linux/init.h> +#include <linux/io.h> +#include <mach/ep93xx-regs.h> +#include <asm/thread_notify.h> + +struct crunch_state *crunch_owner; + +void crunch_task_release(struct thread_info *thread) +{ + local_irq_disable(); + if (crunch_owner == &thread->crunchstate) + crunch_owner = NULL; + local_irq_enable(); +} + +static int crunch_enabled(u32 devcfg) +{ + return !!(devcfg & EP93XX_SYSCON_DEVCFG_CPENA); +} + +static int crunch_do(struct notifier_block *self, unsigned long cmd, void *t) +{ + struct thread_info *thread = (struct thread_info *)t; + struct crunch_state *crunch_state; + u32 devcfg; + + crunch_state = &thread->crunchstate; + + switch (cmd) { + case THREAD_NOTIFY_FLUSH: + memset(crunch_state, 0, sizeof(*crunch_state)); + + /* + * FALLTHROUGH: Ensure we don't try to overwrite our newly + * initialised state information on the first fault. + */ + + case THREAD_NOTIFY_EXIT: + crunch_task_release(thread); + break; + + case THREAD_NOTIFY_SWITCH: + devcfg = __raw_readl(EP93XX_SYSCON_DEVCFG); + if (crunch_enabled(devcfg) || crunch_owner == crunch_state) { + /* + * We don't use ep93xx_syscon_swlocked_write() here + * because we are on the context switch path and + * preemption is already disabled. + */ + devcfg ^= EP93XX_SYSCON_DEVCFG_CPENA; + __raw_writel(0xaa, EP93XX_SYSCON_SWLOCK); + __raw_writel(devcfg, EP93XX_SYSCON_DEVCFG); + } + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block crunch_notifier_block = { + .notifier_call = crunch_do, +}; + +static int __init crunch_init(void) +{ + thread_register_notifier(&crunch_notifier_block); + elf_hwcap |= HWCAP_CRUNCH; + + return 0; +} + +late_initcall(crunch_init); diff --git a/arch/arm/kernel/debug.S b/arch/arm/kernel/debug.S new file mode 100644 index 00000000..bcd66e00 --- /dev/null +++ b/arch/arm/kernel/debug.S @@ -0,0 +1,179 @@ +/* + * linux/arch/arm/kernel/debug.S + * + * Copyright (C) 1994-1999 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * 32-bit debugging code + */ +#include <linux/linkage.h> + + .text + +/* + * Some debugging routines (useful if you've got MM problems and + * printk isn't working). For DEBUGGING ONLY!!! Do not leave + * references to these in a production kernel! + */ + +#if defined(CONFIG_DEBUG_ICEDCC) + @@ debug using ARM EmbeddedICE DCC channel + + .macro addruart, rp, rv + .endm + +#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) || defined(CONFIG_CPU_V7) + + .macro senduart, rd, rx + mcr p14, 0, \rd, c0, c5, 0 + .endm + + .macro busyuart, rd, rx +1001: + mrc p14, 0, \rx, c0, c1, 0 + tst \rx, #0x20000000 + beq 1001b + .endm + + .macro waituart, rd, rx + mov \rd, #0x2000000 +1001: + subs \rd, \rd, #1 + bmi 1002f + mrc p14, 0, \rx, c0, c1, 0 + tst \rx, #0x20000000 + bne 1001b +1002: + .endm + +#elif defined(CONFIG_CPU_XSCALE) + + .macro senduart, rd, rx + mcr p14, 0, \rd, c8, c0, 0 + .endm + + .macro busyuart, rd, rx +1001: + mrc p14, 0, \rx, c14, c0, 0 + tst \rx, #0x10000000 + beq 1001b + .endm + + .macro waituart, rd, rx + mov \rd, #0x10000000 +1001: + subs \rd, \rd, #1 + bmi 1002f + mrc p14, 0, \rx, c14, c0, 0 + tst \rx, #0x10000000 + bne 1001b +1002: + .endm + +#else + + .macro senduart, rd, rx + mcr p14, 0, \rd, c1, c0, 0 + .endm + + .macro busyuart, rd, rx +1001: + mrc p14, 0, \rx, c0, c0, 0 + tst \rx, #2 + beq 1001b + + .endm + + .macro waituart, rd, rx + mov \rd, #0x2000000 +1001: + subs \rd, \rd, #1 + bmi 1002f + mrc p14, 0, \rx, c0, c0, 0 + tst \rx, #2 + bne 1001b +1002: + .endm + +#endif /* CONFIG_CPU_V6 */ + +#else +#include <mach/debug-macro.S> +#endif /* CONFIG_DEBUG_ICEDCC */ + +#ifdef CONFIG_MMU + .macro addruart_current, rx, tmp1, tmp2 + addruart \tmp1, \tmp2 + mrc p15, 0, \rx, c1, c0 + tst \rx, #1 + moveq \rx, \tmp1 + movne \rx, \tmp2 + .endm + +#else /* !CONFIG_MMU */ + .macro addruart_current, rx, tmp1, tmp2 + addruart \rx, \tmp1 + .endm + +#endif /* CONFIG_MMU */ + +/* + * Useful debugging routines + */ +ENTRY(printhex8) + mov r1, #8 + b printhex +ENDPROC(printhex8) + +ENTRY(printhex4) + mov r1, #4 + b printhex +ENDPROC(printhex4) + +ENTRY(printhex2) + mov r1, #2 +printhex: adr r2, hexbuf + add r3, r2, r1 + mov r1, #0 + strb r1, [r3] +1: and r1, r0, #15 + mov r0, r0, lsr #4 + cmp r1, #10 + addlt r1, r1, #'0' + addge r1, r1, #'a' - 10 + strb r1, [r3, #-1]! + teq r3, r2 + bne 1b + mov r0, r2 + b printascii +ENDPROC(printhex2) + + .ltorg + +ENTRY(printascii) + addruart_current r3, r1, r2 + b 2f +1: waituart r2, r3 + senduart r1, r3 + busyuart r2, r3 + teq r1, #'\n' + moveq r1, #'\r' + beq 1b +2: teq r0, #0 + ldrneb r1, [r0], #1 + teqne r1, #0 + bne 1b + mov pc, lr +ENDPROC(printascii) + +ENTRY(printch) + addruart_current r3, r1, r2 + mov r1, r0 + mov r0, #0 + b 1b +ENDPROC(printch) + +hexbuf: .space 16 diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c new file mode 100644 index 00000000..0cdd7b45 --- /dev/null +++ b/arch/arm/kernel/devtree.c @@ -0,0 +1,148 @@ +/* + * linux/arch/arm/kernel/devtree.c + * + * Copyright (C) 2009 Canonical Ltd. <jeremy.kerr@canonical.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/errno.h> +#include <linux/types.h> +#include <linux/bootmem.h> +#include <linux/memblock.h> +#include <linux/of.h> +#include <linux/of_fdt.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> + +#include <asm/setup.h> +#include <asm/page.h> +#include <asm/mach/arch.h> +#include <asm/mach-types.h> + +void __init early_init_dt_add_memory_arch(u64 base, u64 size) +{ + arm_add_memory(base, size); +} + +void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align) +{ + return alloc_bootmem_align(size, align); +} + +void __init arm_dt_memblock_reserve(void) +{ + u64 *reserve_map, base, size; + + if (!initial_boot_params) + return; + + /* Reserve the dtb region */ + memblock_reserve(virt_to_phys(initial_boot_params), + be32_to_cpu(initial_boot_params->totalsize)); + + /* + * Process the reserve map. This will probably overlap the initrd + * and dtb locations which are already reserved, but overlaping + * doesn't hurt anything + */ + reserve_map = ((void*)initial_boot_params) + + be32_to_cpu(initial_boot_params->off_mem_rsvmap); + while (1) { + base = be64_to_cpup(reserve_map++); + size = be64_to_cpup(reserve_map++); + if (!size) + break; + memblock_reserve(base, size); + } +} + +/** + * setup_machine_fdt - Machine setup when an dtb was passed to the kernel + * @dt_phys: physical address of dt blob + * + * If a dtb was passed to the kernel in r2, then use it to choose the + * correct machine_desc and to setup the system. + */ +struct machine_desc * __init setup_machine_fdt(unsigned int dt_phys) +{ + struct boot_param_header *devtree; + struct machine_desc *mdesc, *mdesc_best = NULL; + unsigned int score, mdesc_score = ~1; + unsigned long dt_root; + const char *model; + + if (!dt_phys) + return NULL; + + devtree = phys_to_virt(dt_phys); + + /* check device tree validity */ + if (be32_to_cpu(devtree->magic) != OF_DT_HEADER) + return NULL; + + /* Search the mdescs for the 'best' compatible value match */ + initial_boot_params = devtree; + dt_root = of_get_flat_dt_root(); + for_each_machine_desc(mdesc) { + score = of_flat_dt_match(dt_root, mdesc->dt_compat); + if (score > 0 && score < mdesc_score) { + mdesc_best = mdesc; + mdesc_score = score; + } + } + if (!mdesc_best) { + const char *prop; + long size; + + early_print("\nError: unrecognized/unsupported " + "device tree compatible list:\n[ "); + + prop = of_get_flat_dt_prop(dt_root, "compatible", &size); + while (size > 0) { + early_print("'%s' ", prop); + size -= strlen(prop) + 1; + prop += strlen(prop) + 1; + } + early_print("]\n\n"); + + dump_machine_table(); /* does not return */ + } + + model = of_get_flat_dt_prop(dt_root, "model", NULL); + if (!model) + model = of_get_flat_dt_prop(dt_root, "compatible", NULL); + if (!model) + model = "<unknown>"; + pr_info("Machine: %s, model: %s\n", mdesc_best->name, model); + + /* Retrieve various information from the /chosen node */ + of_scan_flat_dt(early_init_dt_scan_chosen, boot_command_line); + /* Initialize {size,address}-cells info */ + of_scan_flat_dt(early_init_dt_scan_root, NULL); + /* Setup memory, calling early_init_dt_add_memory_arch */ + of_scan_flat_dt(early_init_dt_scan_memory, NULL); + + /* Change machine number to match the mdesc we're using */ + __machine_arch_type = mdesc_best->nr; + + return mdesc_best; +} + +/** + * irq_create_of_mapping - Hook to resolve OF irq specifier into a Linux irq# + * + * Currently the mapping mechanism is trivial; simple flat hwirq numbers are + * mapped 1:1 onto Linux irq numbers. Cascaded irq controllers are not + * supported. + */ +unsigned int irq_create_of_mapping(struct device_node *controller, + const u32 *intspec, unsigned int intsize) +{ + return intspec[0]; +} +EXPORT_SYMBOL_GPL(irq_create_of_mapping); diff --git a/arch/arm/kernel/dma-isa.c b/arch/arm/kernel/dma-isa.c new file mode 100644 index 00000000..360bb6d7 --- /dev/null +++ b/arch/arm/kernel/dma-isa.c @@ -0,0 +1,222 @@ +/* + * linux/arch/arm/kernel/dma-isa.c + * + * Copyright (C) 1999-2000 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * ISA DMA primitives + * Taken from various sources, including: + * linux/include/asm/dma.h: Defines for using and allocating dma channels. + * Written by Hennus Bergman, 1992. + * High DMA channel support & info by Hannu Savolainen and John Boyd, + * Nov. 1992. + * arch/arm/kernel/dma-ebsa285.c + * Copyright (C) 1998 Phil Blundell + */ +#include <linux/ioport.h> +#include <linux/init.h> +#include <linux/dma-mapping.h> +#include <linux/io.h> + +#include <asm/dma.h> +#include <asm/mach/dma.h> + +#define ISA_DMA_MASK 0 +#define ISA_DMA_MODE 1 +#define ISA_DMA_CLRFF 2 +#define ISA_DMA_PGHI 3 +#define ISA_DMA_PGLO 4 +#define ISA_DMA_ADDR 5 +#define ISA_DMA_COUNT 6 + +static unsigned int isa_dma_port[8][7] = { + /* MASK MODE CLRFF PAGE_HI PAGE_LO ADDR COUNT */ + { 0x0a, 0x0b, 0x0c, 0x487, 0x087, 0x00, 0x01 }, + { 0x0a, 0x0b, 0x0c, 0x483, 0x083, 0x02, 0x03 }, + { 0x0a, 0x0b, 0x0c, 0x481, 0x081, 0x04, 0x05 }, + { 0x0a, 0x0b, 0x0c, 0x482, 0x082, 0x06, 0x07 }, + { 0xd4, 0xd6, 0xd8, 0x000, 0x000, 0xc0, 0xc2 }, + { 0xd4, 0xd6, 0xd8, 0x48b, 0x08b, 0xc4, 0xc6 }, + { 0xd4, 0xd6, 0xd8, 0x489, 0x089, 0xc8, 0xca }, + { 0xd4, 0xd6, 0xd8, 0x48a, 0x08a, 0xcc, 0xce } +}; + +static int isa_get_dma_residue(unsigned int chan, dma_t *dma) +{ + unsigned int io_port = isa_dma_port[chan][ISA_DMA_COUNT]; + int count; + + count = 1 + inb(io_port); + count |= inb(io_port) << 8; + + return chan < 4 ? count : (count << 1); +} + +static void isa_enable_dma(unsigned int chan, dma_t *dma) +{ + if (dma->invalid) { + unsigned long address, length; + unsigned int mode; + enum dma_data_direction direction; + + mode = (chan & 3) | dma->dma_mode; + switch (dma->dma_mode & DMA_MODE_MASK) { + case DMA_MODE_READ: + direction = DMA_FROM_DEVICE; + break; + + case DMA_MODE_WRITE: + direction = DMA_TO_DEVICE; + break; + + case DMA_MODE_CASCADE: + direction = DMA_BIDIRECTIONAL; + break; + + default: + direction = DMA_NONE; + break; + } + + if (!dma->sg) { + /* + * Cope with ISA-style drivers which expect cache + * coherence. + */ + dma->sg = &dma->buf; + dma->sgcount = 1; + dma->buf.length = dma->count; + dma->buf.dma_address = dma_map_single(NULL, + dma->addr, dma->count, + direction); + } + + address = dma->buf.dma_address; + length = dma->buf.length - 1; + + outb(address >> 16, isa_dma_port[chan][ISA_DMA_PGLO]); + outb(address >> 24, isa_dma_port[chan][ISA_DMA_PGHI]); + + if (chan >= 4) { + address >>= 1; + length >>= 1; + } + + outb(0, isa_dma_port[chan][ISA_DMA_CLRFF]); + + outb(address, isa_dma_port[chan][ISA_DMA_ADDR]); + outb(address >> 8, isa_dma_port[chan][ISA_DMA_ADDR]); + + outb(length, isa_dma_port[chan][ISA_DMA_COUNT]); + outb(length >> 8, isa_dma_port[chan][ISA_DMA_COUNT]); + + outb(mode, isa_dma_port[chan][ISA_DMA_MODE]); + dma->invalid = 0; + } + outb(chan & 3, isa_dma_port[chan][ISA_DMA_MASK]); +} + +static void isa_disable_dma(unsigned int chan, dma_t *dma) +{ + outb(chan | 4, isa_dma_port[chan][ISA_DMA_MASK]); +} + +static struct dma_ops isa_dma_ops = { + .type = "ISA", + .enable = isa_enable_dma, + .disable = isa_disable_dma, + .residue = isa_get_dma_residue, +}; + +static struct resource dma_resources[] = { { + .name = "dma1", + .start = 0x0000, + .end = 0x000f +}, { + .name = "dma low page", + .start = 0x0080, + .end = 0x008f +}, { + .name = "dma2", + .start = 0x00c0, + .end = 0x00df +}, { + .name = "dma high page", + .start = 0x0480, + .end = 0x048f +} }; + +static dma_t isa_dma[8]; + +/* + * ISA DMA always starts at channel 0 + */ +void __init isa_init_dma(void) +{ + /* + * Try to autodetect presence of an ISA DMA controller. + * We do some minimal initialisation, and check that + * channel 0's DMA address registers are writeable. + */ + outb(0xff, 0x0d); + outb(0xff, 0xda); + + /* + * Write high and low address, and then read them back + * in the same order. + */ + outb(0x55, 0x00); + outb(0xaa, 0x00); + + if (inb(0) == 0x55 && inb(0) == 0xaa) { + unsigned int chan, i; + + for (chan = 0; chan < 8; chan++) { + isa_dma[chan].d_ops = &isa_dma_ops; + isa_disable_dma(chan, NULL); + } + + outb(0x40, 0x0b); + outb(0x41, 0x0b); + outb(0x42, 0x0b); + outb(0x43, 0x0b); + + outb(0xc0, 0xd6); + outb(0x41, 0xd6); + outb(0x42, 0xd6); + outb(0x43, 0xd6); + + outb(0, 0xd4); + + outb(0x10, 0x08); + outb(0x10, 0xd0); + + /* + * Is this correct? According to my documentation, it + * doesn't appear to be. It should be: + * outb(0x3f, 0x40b); outb(0x3f, 0x4d6); + */ + outb(0x30, 0x40b); + outb(0x31, 0x40b); + outb(0x32, 0x40b); + outb(0x33, 0x40b); + outb(0x31, 0x4d6); + outb(0x32, 0x4d6); + outb(0x33, 0x4d6); + + for (i = 0; i < ARRAY_SIZE(dma_resources); i++) + request_resource(&ioport_resource, dma_resources + i); + + for (chan = 0; chan < 8; chan++) { + int ret = isa_dma_add(chan, &isa_dma[chan]); + if (ret) + printk(KERN_ERR "ISADMA%u: unable to register: %d\n", + chan, ret); + } + + request_dma(DMA_ISA_CASCADE, "cascade"); + } +} diff --git a/arch/arm/kernel/dma.c b/arch/arm/kernel/dma.c new file mode 100644 index 00000000..2c4a185f --- /dev/null +++ b/arch/arm/kernel/dma.c @@ -0,0 +1,302 @@ +/* + * linux/arch/arm/kernel/dma.c + * + * Copyright (C) 1995-2000 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Front-end to the DMA handling. This handles the allocation/freeing + * of DMA channels, and provides a unified interface to the machines + * DMA facilities. + */ +#include <linux/module.h> +#include <linux/init.h> +#include <linux/spinlock.h> +#include <linux/errno.h> +#include <linux/scatterlist.h> +#include <linux/seq_file.h> +#include <linux/proc_fs.h> + +#include <asm/dma.h> + +#include <asm/mach/dma.h> + +DEFINE_SPINLOCK(dma_spin_lock); +EXPORT_SYMBOL(dma_spin_lock); + +static dma_t *dma_chan[MAX_DMA_CHANNELS]; + +static inline dma_t *dma_channel(unsigned int chan) +{ + if (chan >= MAX_DMA_CHANNELS) + return NULL; + + return dma_chan[chan]; +} + +int __init isa_dma_add(unsigned int chan, dma_t *dma) +{ + if (!dma->d_ops) + return -EINVAL; + + sg_init_table(&dma->buf, 1); + + if (dma_chan[chan]) + return -EBUSY; + dma_chan[chan] = dma; + return 0; +} + +/* + * Request DMA channel + * + * On certain platforms, we have to allocate an interrupt as well... + */ +int request_dma(unsigned int chan, const char *device_id) +{ + dma_t *dma = dma_channel(chan); + int ret; + + if (!dma) + goto bad_dma; + + if (xchg(&dma->lock, 1) != 0) + goto busy; + + dma->device_id = device_id; + dma->active = 0; + dma->invalid = 1; + + ret = 0; + if (dma->d_ops->request) + ret = dma->d_ops->request(chan, dma); + + if (ret) + xchg(&dma->lock, 0); + + return ret; + +bad_dma: + printk(KERN_ERR "dma: trying to allocate DMA%d\n", chan); + return -EINVAL; + +busy: + return -EBUSY; +} +EXPORT_SYMBOL(request_dma); + +/* + * Free DMA channel + * + * On certain platforms, we have to free interrupt as well... + */ +void free_dma(unsigned int chan) +{ + dma_t *dma = dma_channel(chan); + + if (!dma) + goto bad_dma; + + if (dma->active) { + printk(KERN_ERR "dma%d: freeing active DMA\n", chan); + dma->d_ops->disable(chan, dma); + dma->active = 0; + } + + if (xchg(&dma->lock, 0) != 0) { + if (dma->d_ops->free) + dma->d_ops->free(chan, dma); + return; + } + + printk(KERN_ERR "dma%d: trying to free free DMA\n", chan); + return; + +bad_dma: + printk(KERN_ERR "dma: trying to free DMA%d\n", chan); +} +EXPORT_SYMBOL(free_dma); + +/* Set DMA Scatter-Gather list + */ +void set_dma_sg (unsigned int chan, struct scatterlist *sg, int nr_sg) +{ + dma_t *dma = dma_channel(chan); + + if (dma->active) + printk(KERN_ERR "dma%d: altering DMA SG while " + "DMA active\n", chan); + + dma->sg = sg; + dma->sgcount = nr_sg; + dma->invalid = 1; +} +EXPORT_SYMBOL(set_dma_sg); + +/* Set DMA address + * + * Copy address to the structure, and set the invalid bit + */ +void __set_dma_addr (unsigned int chan, void *addr) +{ + dma_t *dma = dma_channel(chan); + + if (dma->active) + printk(KERN_ERR "dma%d: altering DMA address while " + "DMA active\n", chan); + + dma->sg = NULL; + dma->addr = addr; + dma->invalid = 1; +} +EXPORT_SYMBOL(__set_dma_addr); + +/* Set DMA byte count + * + * Copy address to the structure, and set the invalid bit + */ +void set_dma_count (unsigned int chan, unsigned long count) +{ + dma_t *dma = dma_channel(chan); + + if (dma->active) + printk(KERN_ERR "dma%d: altering DMA count while " + "DMA active\n", chan); + + dma->sg = NULL; + dma->count = count; + dma->invalid = 1; +} +EXPORT_SYMBOL(set_dma_count); + +/* Set DMA direction mode + */ +void set_dma_mode (unsigned int chan, unsigned int mode) +{ + dma_t *dma = dma_channel(chan); + + if (dma->active) + printk(KERN_ERR "dma%d: altering DMA mode while " + "DMA active\n", chan); + + dma->dma_mode = mode; + dma->invalid = 1; +} +EXPORT_SYMBOL(set_dma_mode); + +/* Enable DMA channel + */ +void enable_dma (unsigned int chan) +{ + dma_t *dma = dma_channel(chan); + + if (!dma->lock) + goto free_dma; + + if (dma->active == 0) { + dma->active = 1; + dma->d_ops->enable(chan, dma); + } + return; + +free_dma: + printk(KERN_ERR "dma%d: trying to enable free DMA\n", chan); + BUG(); +} +EXPORT_SYMBOL(enable_dma); + +/* Disable DMA channel + */ +void disable_dma (unsigned int chan) +{ + dma_t *dma = dma_channel(chan); + + if (!dma->lock) + goto free_dma; + + if (dma->active == 1) { + dma->active = 0; + dma->d_ops->disable(chan, dma); + } + return; + +free_dma: + printk(KERN_ERR "dma%d: trying to disable free DMA\n", chan); + BUG(); +} +EXPORT_SYMBOL(disable_dma); + +/* + * Is the specified DMA channel active? + */ +int dma_channel_active(unsigned int chan) +{ + dma_t *dma = dma_channel(chan); + return dma->active; +} +EXPORT_SYMBOL(dma_channel_active); + +void set_dma_page(unsigned int chan, char pagenr) +{ + printk(KERN_ERR "dma%d: trying to set_dma_page\n", chan); +} +EXPORT_SYMBOL(set_dma_page); + +void set_dma_speed(unsigned int chan, int cycle_ns) +{ + dma_t *dma = dma_channel(chan); + int ret = 0; + + if (dma->d_ops->setspeed) + ret = dma->d_ops->setspeed(chan, dma, cycle_ns); + dma->speed = ret; +} +EXPORT_SYMBOL(set_dma_speed); + +int get_dma_residue(unsigned int chan) +{ + dma_t *dma = dma_channel(chan); + int ret = 0; + + if (dma->d_ops->residue) + ret = dma->d_ops->residue(chan, dma); + + return ret; +} +EXPORT_SYMBOL(get_dma_residue); + +#ifdef CONFIG_PROC_FS +static int proc_dma_show(struct seq_file *m, void *v) +{ + int i; + + for (i = 0 ; i < MAX_DMA_CHANNELS ; i++) { + dma_t *dma = dma_channel(i); + if (dma && dma->lock) + seq_printf(m, "%2d: %s\n", i, dma->device_id); + } + return 0; +} + +static int proc_dma_open(struct inode *inode, struct file *file) +{ + return single_open(file, proc_dma_show, NULL); +} + +static const struct file_operations proc_dma_operations = { + .open = proc_dma_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __init proc_dma_init(void) +{ + proc_create("dma", 0, NULL, &proc_dma_operations); + return 0; +} + +__initcall(proc_dma_init); +#endif diff --git a/arch/arm/kernel/early_printk.c b/arch/arm/kernel/early_printk.c new file mode 100644 index 00000000..85aa2b29 --- /dev/null +++ b/arch/arm/kernel/early_printk.c @@ -0,0 +1,57 @@ +/* + * linux/arch/arm/kernel/early_printk.c + * + * Copyright (C) 2009 Sascha Hauer <s.hauer@pengutronix.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/kernel.h> +#include <linux/console.h> +#include <linux/init.h> + +extern void printch(int); + +static void early_write(const char *s, unsigned n) +{ + while (n-- > 0) { + if (*s == '\n') + printch('\r'); + printch(*s); + s++; + } +} + +static void early_console_write(struct console *con, const char *s, unsigned n) +{ + early_write(s, n); +} + +static struct console early_console = { + .name = "earlycon", + .write = early_console_write, + .flags = CON_PRINTBUFFER | CON_BOOT, + .index = -1, +}; + +asmlinkage void early_printk(const char *fmt, ...) +{ + char buf[512]; + int n; + va_list ap; + + va_start(ap, fmt); + n = vscnprintf(buf, sizeof(buf), fmt, ap); + early_write(buf, n); + va_end(ap); +} + +static int __init setup_early_printk(char *buf) +{ + register_console(&early_console); + return 0; +} + +early_param("earlyprintk", setup_early_printk); diff --git a/arch/arm/kernel/ecard.c b/arch/arm/kernel/ecard.c new file mode 100644 index 00000000..d1650011 --- /dev/null +++ b/arch/arm/kernel/ecard.c @@ -0,0 +1,1241 @@ +/* + * linux/arch/arm/kernel/ecard.c + * + * Copyright 1995-2001 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Find all installed expansion cards, and handle interrupts from them. + * + * Created from information from Acorns RiscOS3 PRMs + * + * 08-Dec-1996 RMK Added code for the 9'th expansion card - the ether + * podule slot. + * 06-May-1997 RMK Added blacklist for cards whose loader doesn't work. + * 12-Sep-1997 RMK Created new handling of interrupt enables/disables + * - cards can now register their own routine to control + * interrupts (recommended). + * 29-Sep-1997 RMK Expansion card interrupt hardware not being re-enabled + * on reset from Linux. (Caused cards not to respond + * under RiscOS without hard reset). + * 15-Feb-1998 RMK Added DMA support + * 12-Sep-1998 RMK Added EASI support + * 10-Jan-1999 RMK Run loaders in a simulated RISC OS environment. + * 17-Apr-1999 RMK Support for EASI Type C cycles. + */ +#define ECARD_C + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/completion.h> +#include <linux/reboot.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/proc_fs.h> +#include <linux/seq_file.h> +#include <linux/device.h> +#include <linux/init.h> +#include <linux/mutex.h> +#include <linux/kthread.h> +#include <linux/io.h> + +#include <asm/dma.h> +#include <asm/ecard.h> +#include <mach/hardware.h> +#include <asm/irq.h> +#include <asm/mmu_context.h> +#include <asm/mach/irq.h> +#include <asm/tlbflush.h> + +#include "ecard.h" + +#ifndef CONFIG_ARCH_RPC +#define HAVE_EXPMASK +#endif + +struct ecard_request { + void (*fn)(struct ecard_request *); + ecard_t *ec; + unsigned int address; + unsigned int length; + unsigned int use_loader; + void *buffer; + struct completion *complete; +}; + +struct expcard_blacklist { + unsigned short manufacturer; + unsigned short product; + const char *type; +}; + +static ecard_t *cards; +static ecard_t *slot_to_expcard[MAX_ECARDS]; +static unsigned int ectcr; +#ifdef HAS_EXPMASK +static unsigned int have_expmask; +#endif + +/* List of descriptions of cards which don't have an extended + * identification, or chunk directories containing a description. + */ +static struct expcard_blacklist __initdata blacklist[] = { + { MANU_ACORN, PROD_ACORN_ETHER1, "Acorn Ether1" } +}; + +asmlinkage extern int +ecard_loader_reset(unsigned long base, loader_t loader); +asmlinkage extern int +ecard_loader_read(int off, unsigned long base, loader_t loader); + +static inline unsigned short ecard_getu16(unsigned char *v) +{ + return v[0] | v[1] << 8; +} + +static inline signed long ecard_gets24(unsigned char *v) +{ + return v[0] | v[1] << 8 | v[2] << 16 | ((v[2] & 0x80) ? 0xff000000 : 0); +} + +static inline ecard_t *slot_to_ecard(unsigned int slot) +{ + return slot < MAX_ECARDS ? slot_to_expcard[slot] : NULL; +} + +/* ===================== Expansion card daemon ======================== */ +/* + * Since the loader programs on the expansion cards need to be run + * in a specific environment, create a separate task with this + * environment up, and pass requests to this task as and when we + * need to. + * + * This should allow 99% of loaders to be called from Linux. + * + * From a security standpoint, we trust the card vendors. This + * may be a misplaced trust. + */ +static void ecard_task_reset(struct ecard_request *req) +{ + struct expansion_card *ec = req->ec; + struct resource *res; + + res = ec->slot_no == 8 + ? &ec->resource[ECARD_RES_MEMC] + : ec->easi + ? &ec->resource[ECARD_RES_EASI] + : &ec->resource[ECARD_RES_IOCSYNC]; + + ecard_loader_reset(res->start, ec->loader); +} + +static void ecard_task_readbytes(struct ecard_request *req) +{ + struct expansion_card *ec = req->ec; + unsigned char *buf = req->buffer; + unsigned int len = req->length; + unsigned int off = req->address; + + if (ec->slot_no == 8) { + void __iomem *base = (void __iomem *) + ec->resource[ECARD_RES_MEMC].start; + + /* + * The card maintains an index which increments the address + * into a 4096-byte page on each access. We need to keep + * track of the counter. + */ + static unsigned int index; + unsigned int page; + + page = (off >> 12) * 4; + if (page > 256 * 4) + return; + + off &= 4095; + + /* + * If we are reading offset 0, or our current index is + * greater than the offset, reset the hardware index counter. + */ + if (off == 0 || index > off) { + writeb(0, base); + index = 0; + } + + /* + * Increment the hardware index counter until we get to the + * required offset. The read bytes are discarded. + */ + while (index < off) { + readb(base + page); + index += 1; + } + + while (len--) { + *buf++ = readb(base + page); + index += 1; + } + } else { + unsigned long base = (ec->easi + ? &ec->resource[ECARD_RES_EASI] + : &ec->resource[ECARD_RES_IOCSYNC])->start; + void __iomem *pbase = (void __iomem *)base; + + if (!req->use_loader || !ec->loader) { + off *= 4; + while (len--) { + *buf++ = readb(pbase + off); + off += 4; + } + } else { + while(len--) { + /* + * The following is required by some + * expansion card loader programs. + */ + *(unsigned long *)0x108 = 0; + *buf++ = ecard_loader_read(off++, base, + ec->loader); + } + } + } + +} + +static DECLARE_WAIT_QUEUE_HEAD(ecard_wait); +static struct ecard_request *ecard_req; +static DEFINE_MUTEX(ecard_mutex); + +/* + * Set up the expansion card daemon's page tables. + */ +static void ecard_init_pgtables(struct mm_struct *mm) +{ + struct vm_area_struct vma; + + /* We want to set up the page tables for the following mapping: + * Virtual Physical + * 0x03000000 0x03000000 + * 0x03010000 unmapped + * 0x03210000 0x03210000 + * 0x03400000 unmapped + * 0x08000000 0x08000000 + * 0x10000000 unmapped + * + * FIXME: we don't follow this 100% yet. + */ + pgd_t *src_pgd, *dst_pgd; + + src_pgd = pgd_offset(mm, (unsigned long)IO_BASE); + dst_pgd = pgd_offset(mm, IO_START); + + memcpy(dst_pgd, src_pgd, sizeof(pgd_t) * (IO_SIZE / PGDIR_SIZE)); + + src_pgd = pgd_offset(mm, EASI_BASE); + dst_pgd = pgd_offset(mm, EASI_START); + + memcpy(dst_pgd, src_pgd, sizeof(pgd_t) * (EASI_SIZE / PGDIR_SIZE)); + + vma.vm_mm = mm; + + flush_tlb_range(&vma, IO_START, IO_START + IO_SIZE); + flush_tlb_range(&vma, EASI_START, EASI_START + EASI_SIZE); +} + +static int ecard_init_mm(void) +{ + struct mm_struct * mm = mm_alloc(); + struct mm_struct *active_mm = current->active_mm; + + if (!mm) + return -ENOMEM; + + current->mm = mm; + current->active_mm = mm; + activate_mm(active_mm, mm); + mmdrop(active_mm); + ecard_init_pgtables(mm); + return 0; +} + +static int +ecard_task(void * unused) +{ + /* + * Allocate a mm. We're not a lazy-TLB kernel task since we need + * to set page table entries where the user space would be. Note + * that this also creates the page tables. Failure is not an + * option here. + */ + if (ecard_init_mm()) + panic("kecardd: unable to alloc mm\n"); + + while (1) { + struct ecard_request *req; + + wait_event_interruptible(ecard_wait, ecard_req != NULL); + + req = xchg(&ecard_req, NULL); + if (req != NULL) { + req->fn(req); + complete(req->complete); + } + } +} + +/* + * Wake the expansion card daemon to action our request. + * + * FIXME: The test here is not sufficient to detect if the + * kcardd is running. + */ +static void ecard_call(struct ecard_request *req) +{ + DECLARE_COMPLETION_ONSTACK(completion); + + req->complete = &completion; + + mutex_lock(&ecard_mutex); + ecard_req = req; + wake_up(&ecard_wait); + + /* + * Now wait for kecardd to run. + */ + wait_for_completion(&completion); + mutex_unlock(&ecard_mutex); +} + +/* ======================= Mid-level card control ===================== */ + +static void +ecard_readbytes(void *addr, ecard_t *ec, int off, int len, int useld) +{ + struct ecard_request req; + + req.fn = ecard_task_readbytes; + req.ec = ec; + req.address = off; + req.length = len; + req.use_loader = useld; + req.buffer = addr; + + ecard_call(&req); +} + +int ecard_readchunk(struct in_chunk_dir *cd, ecard_t *ec, int id, int num) +{ + struct ex_chunk_dir excd; + int index = 16; + int useld = 0; + + if (!ec->cid.cd) + return 0; + + while(1) { + ecard_readbytes(&excd, ec, index, 8, useld); + index += 8; + if (c_id(&excd) == 0) { + if (!useld && ec->loader) { + useld = 1; + index = 0; + continue; + } + return 0; + } + if (c_id(&excd) == 0xf0) { /* link */ + index = c_start(&excd); + continue; + } + if (c_id(&excd) == 0x80) { /* loader */ + if (!ec->loader) { + ec->loader = kmalloc(c_len(&excd), + GFP_KERNEL); + if (ec->loader) + ecard_readbytes(ec->loader, ec, + (int)c_start(&excd), + c_len(&excd), useld); + else + return 0; + } + continue; + } + if (c_id(&excd) == id && num-- == 0) + break; + } + + if (c_id(&excd) & 0x80) { + switch (c_id(&excd) & 0x70) { + case 0x70: + ecard_readbytes((unsigned char *)excd.d.string, ec, + (int)c_start(&excd), c_len(&excd), + useld); + break; + case 0x00: + break; + } + } + cd->start_offset = c_start(&excd); + memcpy(cd->d.string, excd.d.string, 256); + return 1; +} + +/* ======================= Interrupt control ============================ */ + +static void ecard_def_irq_enable(ecard_t *ec, int irqnr) +{ +#ifdef HAS_EXPMASK + if (irqnr < 4 && have_expmask) { + have_expmask |= 1 << irqnr; + __raw_writeb(have_expmask, EXPMASK_ENABLE); + } +#endif +} + +static void ecard_def_irq_disable(ecard_t *ec, int irqnr) +{ +#ifdef HAS_EXPMASK + if (irqnr < 4 && have_expmask) { + have_expmask &= ~(1 << irqnr); + __raw_writeb(have_expmask, EXPMASK_ENABLE); + } +#endif +} + +static int ecard_def_irq_pending(ecard_t *ec) +{ + return !ec->irqmask || readb(ec->irqaddr) & ec->irqmask; +} + +static void ecard_def_fiq_enable(ecard_t *ec, int fiqnr) +{ + panic("ecard_def_fiq_enable called - impossible"); +} + +static void ecard_def_fiq_disable(ecard_t *ec, int fiqnr) +{ + panic("ecard_def_fiq_disable called - impossible"); +} + +static int ecard_def_fiq_pending(ecard_t *ec) +{ + return !ec->fiqmask || readb(ec->fiqaddr) & ec->fiqmask; +} + +static expansioncard_ops_t ecard_default_ops = { + ecard_def_irq_enable, + ecard_def_irq_disable, + ecard_def_irq_pending, + ecard_def_fiq_enable, + ecard_def_fiq_disable, + ecard_def_fiq_pending +}; + +/* + * Enable and disable interrupts from expansion cards. + * (interrupts are disabled for these functions). + * + * They are not meant to be called directly, but via enable/disable_irq. + */ +static void ecard_irq_unmask(struct irq_data *d) +{ + ecard_t *ec = slot_to_ecard(d->irq - 32); + + if (ec) { + if (!ec->ops) + ec->ops = &ecard_default_ops; + + if (ec->claimed && ec->ops->irqenable) + ec->ops->irqenable(ec, d->irq); + else + printk(KERN_ERR "ecard: rejecting request to " + "enable IRQs for %d\n", d->irq); + } +} + +static void ecard_irq_mask(struct irq_data *d) +{ + ecard_t *ec = slot_to_ecard(d->irq - 32); + + if (ec) { + if (!ec->ops) + ec->ops = &ecard_default_ops; + + if (ec->ops && ec->ops->irqdisable) + ec->ops->irqdisable(ec, d->irq); + } +} + +static struct irq_chip ecard_chip = { + .name = "ECARD", + .irq_ack = ecard_irq_mask, + .irq_mask = ecard_irq_mask, + .irq_unmask = ecard_irq_unmask, +}; + +void ecard_enablefiq(unsigned int fiqnr) +{ + ecard_t *ec = slot_to_ecard(fiqnr); + + if (ec) { + if (!ec->ops) + ec->ops = &ecard_default_ops; + + if (ec->claimed && ec->ops->fiqenable) + ec->ops->fiqenable(ec, fiqnr); + else + printk(KERN_ERR "ecard: rejecting request to " + "enable FIQs for %d\n", fiqnr); + } +} + +void ecard_disablefiq(unsigned int fiqnr) +{ + ecard_t *ec = slot_to_ecard(fiqnr); + + if (ec) { + if (!ec->ops) + ec->ops = &ecard_default_ops; + + if (ec->ops->fiqdisable) + ec->ops->fiqdisable(ec, fiqnr); + } +} + +static void ecard_dump_irq_state(void) +{ + ecard_t *ec; + + printk("Expansion card IRQ state:\n"); + + for (ec = cards; ec; ec = ec->next) { + if (ec->slot_no == 8) + continue; + + printk(" %d: %sclaimed, ", + ec->slot_no, ec->claimed ? "" : "not "); + + if (ec->ops && ec->ops->irqpending && + ec->ops != &ecard_default_ops) + printk("irq %spending\n", + ec->ops->irqpending(ec) ? "" : "not "); + else + printk("irqaddr %p, mask = %02X, status = %02X\n", + ec->irqaddr, ec->irqmask, readb(ec->irqaddr)); + } +} + +static void ecard_check_lockup(struct irq_desc *desc) +{ + static unsigned long last; + static int lockup; + + /* + * If the timer interrupt has not run since the last million + * unrecognised expansion card interrupts, then there is + * something seriously wrong. Disable the expansion card + * interrupts so at least we can continue. + * + * Maybe we ought to start a timer to re-enable them some time + * later? + */ + if (last == jiffies) { + lockup += 1; + if (lockup > 1000000) { + printk(KERN_ERR "\nInterrupt lockup detected - " + "disabling all expansion card interrupts\n"); + + desc->irq_data.chip->irq_mask(&desc->irq_data); + ecard_dump_irq_state(); + } + } else + lockup = 0; + + /* + * If we did not recognise the source of this interrupt, + * warn the user, but don't flood the user with these messages. + */ + if (!last || time_after(jiffies, last + 5*HZ)) { + last = jiffies; + printk(KERN_WARNING "Unrecognised interrupt from backplane\n"); + ecard_dump_irq_state(); + } +} + +static void +ecard_irq_handler(unsigned int irq, struct irq_desc *desc) +{ + ecard_t *ec; + int called = 0; + + desc->irq_data.chip->irq_mask(&desc->irq_data); + for (ec = cards; ec; ec = ec->next) { + int pending; + + if (!ec->claimed || ec->irq == NO_IRQ || ec->slot_no == 8) + continue; + + if (ec->ops && ec->ops->irqpending) + pending = ec->ops->irqpending(ec); + else + pending = ecard_default_ops.irqpending(ec); + + if (pending) { + generic_handle_irq(ec->irq); + called ++; + } + } + desc->irq_data.chip->irq_unmask(&desc->irq_data); + + if (called == 0) + ecard_check_lockup(desc); +} + +#ifdef HAS_EXPMASK +static unsigned char priority_masks[] = +{ + 0xf0, 0xf1, 0xf3, 0xf7, 0xff, 0xff, 0xff, 0xff +}; + +static unsigned char first_set[] = +{ + 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01, 0x00, + 0x03, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01, 0x00 +}; + +static void +ecard_irqexp_handler(unsigned int irq, struct irq_desc *desc) +{ + const unsigned int statusmask = 15; + unsigned int status; + + status = __raw_readb(EXPMASK_STATUS) & statusmask; + if (status) { + unsigned int slot = first_set[status]; + ecard_t *ec = slot_to_ecard(slot); + + if (ec->claimed) { + /* + * this ugly code is so that we can operate a + * prioritorising system: + * + * Card 0 highest priority + * Card 1 + * Card 2 + * Card 3 lowest priority + * + * Serial cards should go in 0/1, ethernet/scsi in 2/3 + * otherwise you will lose serial data at high speeds! + */ + generic_handle_irq(ec->irq); + } else { + printk(KERN_WARNING "card%d: interrupt from unclaimed " + "card???\n", slot); + have_expmask &= ~(1 << slot); + __raw_writeb(have_expmask, EXPMASK_ENABLE); + } + } else + printk(KERN_WARNING "Wild interrupt from backplane (masks)\n"); +} + +static int __init ecard_probeirqhw(void) +{ + ecard_t *ec; + int found; + + __raw_writeb(0x00, EXPMASK_ENABLE); + __raw_writeb(0xff, EXPMASK_STATUS); + found = (__raw_readb(EXPMASK_STATUS) & 15) == 0; + __raw_writeb(0xff, EXPMASK_ENABLE); + + if (found) { + printk(KERN_DEBUG "Expansion card interrupt " + "management hardware found\n"); + + /* for each card present, set a bit to '1' */ + have_expmask = 0x80000000; + + for (ec = cards; ec; ec = ec->next) + have_expmask |= 1 << ec->slot_no; + + __raw_writeb(have_expmask, EXPMASK_ENABLE); + } + + return found; +} +#else +#define ecard_irqexp_handler NULL +#define ecard_probeirqhw() (0) +#endif + +#ifndef IO_EC_MEMC8_BASE +#define IO_EC_MEMC8_BASE 0 +#endif + +static unsigned int __ecard_address(ecard_t *ec, card_type_t type, card_speed_t speed) +{ + unsigned long address = 0; + int slot = ec->slot_no; + + if (ec->slot_no == 8) + return IO_EC_MEMC8_BASE; + + ectcr &= ~(1 << slot); + + switch (type) { + case ECARD_MEMC: + if (slot < 4) + address = IO_EC_MEMC_BASE + (slot << 12); + break; + + case ECARD_IOC: + if (slot < 4) + address = IO_EC_IOC_BASE + (slot << 12); +#ifdef IO_EC_IOC4_BASE + else + address = IO_EC_IOC4_BASE + ((slot - 4) << 12); +#endif + if (address) + address += speed << 17; + break; + +#ifdef IO_EC_EASI_BASE + case ECARD_EASI: + address = IO_EC_EASI_BASE + (slot << 22); + if (speed == ECARD_FAST) + ectcr |= 1 << slot; + break; +#endif + default: + break; + } + +#ifdef IOMD_ECTCR + iomd_writeb(ectcr, IOMD_ECTCR); +#endif + return address; +} + +static int ecard_prints(struct seq_file *m, ecard_t *ec) +{ + seq_printf(m, " %d: %s ", ec->slot_no, ec->easi ? "EASI" : " "); + + if (ec->cid.id == 0) { + struct in_chunk_dir incd; + + seq_printf(m, "[%04X:%04X] ", + ec->cid.manufacturer, ec->cid.product); + + if (!ec->card_desc && ec->cid.cd && + ecard_readchunk(&incd, ec, 0xf5, 0)) { + ec->card_desc = kmalloc(strlen(incd.d.string)+1, GFP_KERNEL); + + if (ec->card_desc) + strcpy((char *)ec->card_desc, incd.d.string); + } + + seq_printf(m, "%s\n", ec->card_desc ? ec->card_desc : "*unknown*"); + } else + seq_printf(m, "Simple card %d\n", ec->cid.id); + + return 0; +} + +static int ecard_devices_proc_show(struct seq_file *m, void *v) +{ + ecard_t *ec = cards; + + while (ec) { + ecard_prints(m, ec); + ec = ec->next; + } + return 0; +} + +static int ecard_devices_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, ecard_devices_proc_show, NULL); +} + +static const struct file_operations bus_ecard_proc_fops = { + .owner = THIS_MODULE, + .open = ecard_devices_proc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static struct proc_dir_entry *proc_bus_ecard_dir = NULL; + +static void ecard_proc_init(void) +{ + proc_bus_ecard_dir = proc_mkdir("bus/ecard", NULL); + proc_create("devices", 0, proc_bus_ecard_dir, &bus_ecard_proc_fops); +} + +#define ec_set_resource(ec,nr,st,sz) \ + do { \ + (ec)->resource[nr].name = dev_name(&ec->dev); \ + (ec)->resource[nr].start = st; \ + (ec)->resource[nr].end = (st) + (sz) - 1; \ + (ec)->resource[nr].flags = IORESOURCE_MEM; \ + } while (0) + +static void __init ecard_free_card(struct expansion_card *ec) +{ + int i; + + for (i = 0; i < ECARD_NUM_RESOURCES; i++) + if (ec->resource[i].flags) + release_resource(&ec->resource[i]); + + kfree(ec); +} + +static struct expansion_card *__init ecard_alloc_card(int type, int slot) +{ + struct expansion_card *ec; + unsigned long base; + int i; + + ec = kzalloc(sizeof(ecard_t), GFP_KERNEL); + if (!ec) { + ec = ERR_PTR(-ENOMEM); + goto nomem; + } + + ec->slot_no = slot; + ec->easi = type == ECARD_EASI; + ec->irq = NO_IRQ; + ec->fiq = NO_IRQ; + ec->dma = NO_DMA; + ec->ops = &ecard_default_ops; + + dev_set_name(&ec->dev, "ecard%d", slot); + ec->dev.parent = NULL; + ec->dev.bus = &ecard_bus_type; + ec->dev.dma_mask = &ec->dma_mask; + ec->dma_mask = (u64)0xffffffff; + ec->dev.coherent_dma_mask = ec->dma_mask; + + if (slot < 4) { + ec_set_resource(ec, ECARD_RES_MEMC, + PODSLOT_MEMC_BASE + (slot << 14), + PODSLOT_MEMC_SIZE); + base = PODSLOT_IOC0_BASE + (slot << 14); + } else + base = PODSLOT_IOC4_BASE + ((slot - 4) << 14); + +#ifdef CONFIG_ARCH_RPC + if (slot < 8) { + ec_set_resource(ec, ECARD_RES_EASI, + PODSLOT_EASI_BASE + (slot << 24), + PODSLOT_EASI_SIZE); + } + + if (slot == 8) { + ec_set_resource(ec, ECARD_RES_MEMC, NETSLOT_BASE, NETSLOT_SIZE); + } else +#endif + + for (i = 0; i <= ECARD_RES_IOCSYNC - ECARD_RES_IOCSLOW; i++) + ec_set_resource(ec, i + ECARD_RES_IOCSLOW, + base + (i << 19), PODSLOT_IOC_SIZE); + + for (i = 0; i < ECARD_NUM_RESOURCES; i++) { + if (ec->resource[i].flags && + request_resource(&iomem_resource, &ec->resource[i])) { + dev_err(&ec->dev, "resource(s) not available\n"); + ec->resource[i].end -= ec->resource[i].start; + ec->resource[i].start = 0; + ec->resource[i].flags = 0; + } + } + + nomem: + return ec; +} + +static ssize_t ecard_show_irq(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct expansion_card *ec = ECARD_DEV(dev); + return sprintf(buf, "%u\n", ec->irq); +} + +static ssize_t ecard_show_dma(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct expansion_card *ec = ECARD_DEV(dev); + return sprintf(buf, "%u\n", ec->dma); +} + +static ssize_t ecard_show_resources(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct expansion_card *ec = ECARD_DEV(dev); + char *str = buf; + int i; + + for (i = 0; i < ECARD_NUM_RESOURCES; i++) + str += sprintf(str, "%08x %08x %08lx\n", + ec->resource[i].start, + ec->resource[i].end, + ec->resource[i].flags); + + return str - buf; +} + +static ssize_t ecard_show_vendor(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct expansion_card *ec = ECARD_DEV(dev); + return sprintf(buf, "%u\n", ec->cid.manufacturer); +} + +static ssize_t ecard_show_device(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct expansion_card *ec = ECARD_DEV(dev); + return sprintf(buf, "%u\n", ec->cid.product); +} + +static ssize_t ecard_show_type(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct expansion_card *ec = ECARD_DEV(dev); + return sprintf(buf, "%s\n", ec->easi ? "EASI" : "IOC"); +} + +static struct device_attribute ecard_dev_attrs[] = { + __ATTR(device, S_IRUGO, ecard_show_device, NULL), + __ATTR(dma, S_IRUGO, ecard_show_dma, NULL), + __ATTR(irq, S_IRUGO, ecard_show_irq, NULL), + __ATTR(resource, S_IRUGO, ecard_show_resources, NULL), + __ATTR(type, S_IRUGO, ecard_show_type, NULL), + __ATTR(vendor, S_IRUGO, ecard_show_vendor, NULL), + __ATTR_NULL, +}; + + +int ecard_request_resources(struct expansion_card *ec) +{ + int i, err = 0; + + for (i = 0; i < ECARD_NUM_RESOURCES; i++) { + if (ecard_resource_end(ec, i) && + !request_mem_region(ecard_resource_start(ec, i), + ecard_resource_len(ec, i), + ec->dev.driver->name)) { + err = -EBUSY; + break; + } + } + + if (err) { + while (i--) + if (ecard_resource_end(ec, i)) + release_mem_region(ecard_resource_start(ec, i), + ecard_resource_len(ec, i)); + } + return err; +} +EXPORT_SYMBOL(ecard_request_resources); + +void ecard_release_resources(struct expansion_card *ec) +{ + int i; + + for (i = 0; i < ECARD_NUM_RESOURCES; i++) + if (ecard_resource_end(ec, i)) + release_mem_region(ecard_resource_start(ec, i), + ecard_resource_len(ec, i)); +} +EXPORT_SYMBOL(ecard_release_resources); + +void ecard_setirq(struct expansion_card *ec, const struct expansion_card_ops *ops, void *irq_data) +{ + ec->irq_data = irq_data; + barrier(); + ec->ops = ops; +} +EXPORT_SYMBOL(ecard_setirq); + +void __iomem *ecardm_iomap(struct expansion_card *ec, unsigned int res, + unsigned long offset, unsigned long maxsize) +{ + unsigned long start = ecard_resource_start(ec, res); + unsigned long end = ecard_resource_end(ec, res); + + if (offset > (end - start)) + return NULL; + + start += offset; + if (maxsize && end - start > maxsize) + end = start + maxsize; + + return devm_ioremap(&ec->dev, start, end - start); +} +EXPORT_SYMBOL(ecardm_iomap); + +/* + * Probe for an expansion card. + * + * If bit 1 of the first byte of the card is set, then the + * card does not exist. + */ +static int __init +ecard_probe(int slot, card_type_t type) +{ + ecard_t **ecp; + ecard_t *ec; + struct ex_ecid cid; + int i, rc; + + ec = ecard_alloc_card(type, slot); + if (IS_ERR(ec)) { + rc = PTR_ERR(ec); + goto nomem; + } + + rc = -ENODEV; + if ((ec->podaddr = __ecard_address(ec, type, ECARD_SYNC)) == 0) + goto nodev; + + cid.r_zero = 1; + ecard_readbytes(&cid, ec, 0, 16, 0); + if (cid.r_zero) + goto nodev; + + ec->cid.id = cid.r_id; + ec->cid.cd = cid.r_cd; + ec->cid.is = cid.r_is; + ec->cid.w = cid.r_w; + ec->cid.manufacturer = ecard_getu16(cid.r_manu); + ec->cid.product = ecard_getu16(cid.r_prod); + ec->cid.country = cid.r_country; + ec->cid.irqmask = cid.r_irqmask; + ec->cid.irqoff = ecard_gets24(cid.r_irqoff); + ec->cid.fiqmask = cid.r_fiqmask; + ec->cid.fiqoff = ecard_gets24(cid.r_fiqoff); + ec->fiqaddr = + ec->irqaddr = (void __iomem *)ioaddr(ec->podaddr); + + if (ec->cid.is) { + ec->irqmask = ec->cid.irqmask; + ec->irqaddr += ec->cid.irqoff; + ec->fiqmask = ec->cid.fiqmask; + ec->fiqaddr += ec->cid.fiqoff; + } else { + ec->irqmask = 1; + ec->fiqmask = 4; + } + + for (i = 0; i < ARRAY_SIZE(blacklist); i++) + if (blacklist[i].manufacturer == ec->cid.manufacturer && + blacklist[i].product == ec->cid.product) { + ec->card_desc = blacklist[i].type; + break; + } + + /* + * hook the interrupt handlers + */ + if (slot < 8) { + ec->irq = 32 + slot; + irq_set_chip_and_handler(ec->irq, &ecard_chip, + handle_level_irq); + set_irq_flags(ec->irq, IRQF_VALID); + } + +#ifdef IO_EC_MEMC8_BASE + if (slot == 8) + ec->irq = 11; +#endif +#ifdef CONFIG_ARCH_RPC + /* On RiscPC, only first two slots have DMA capability */ + if (slot < 2) + ec->dma = 2 + slot; +#endif + + for (ecp = &cards; *ecp; ecp = &(*ecp)->next); + + *ecp = ec; + slot_to_expcard[slot] = ec; + + device_register(&ec->dev); + + return 0; + + nodev: + ecard_free_card(ec); + nomem: + return rc; +} + +/* + * Initialise the expansion card system. + * Locate all hardware - interrupt management and + * actual cards. + */ +static int __init ecard_init(void) +{ + struct task_struct *task; + int slot, irqhw; + + task = kthread_run(ecard_task, NULL, "kecardd"); + if (IS_ERR(task)) { + printk(KERN_ERR "Ecard: unable to create kernel thread: %ld\n", + PTR_ERR(task)); + return PTR_ERR(task); + } + + printk("Probing expansion cards\n"); + + for (slot = 0; slot < 8; slot ++) { + if (ecard_probe(slot, ECARD_EASI) == -ENODEV) + ecard_probe(slot, ECARD_IOC); + } + +#ifdef IO_EC_MEMC8_BASE + ecard_probe(8, ECARD_IOC); +#endif + + irqhw = ecard_probeirqhw(); + + irq_set_chained_handler(IRQ_EXPANSIONCARD, + irqhw ? ecard_irqexp_handler : ecard_irq_handler); + + ecard_proc_init(); + + return 0; +} + +subsys_initcall(ecard_init); + +/* + * ECARD "bus" + */ +static const struct ecard_id * +ecard_match_device(const struct ecard_id *ids, struct expansion_card *ec) +{ + int i; + + for (i = 0; ids[i].manufacturer != 65535; i++) + if (ec->cid.manufacturer == ids[i].manufacturer && + ec->cid.product == ids[i].product) + return ids + i; + + return NULL; +} + +static int ecard_drv_probe(struct device *dev) +{ + struct expansion_card *ec = ECARD_DEV(dev); + struct ecard_driver *drv = ECARD_DRV(dev->driver); + const struct ecard_id *id; + int ret; + + id = ecard_match_device(drv->id_table, ec); + + ec->claimed = 1; + ret = drv->probe(ec, id); + if (ret) + ec->claimed = 0; + return ret; +} + +static int ecard_drv_remove(struct device *dev) +{ + struct expansion_card *ec = ECARD_DEV(dev); + struct ecard_driver *drv = ECARD_DRV(dev->driver); + + drv->remove(ec); + ec->claimed = 0; + + /* + * Restore the default operations. We ensure that the + * ops are set before we change the data. + */ + ec->ops = &ecard_default_ops; + barrier(); + ec->irq_data = NULL; + + return 0; +} + +/* + * Before rebooting, we must make sure that the expansion card is in a + * sensible state, so it can be re-detected. This means that the first + * page of the ROM must be visible. We call the expansion cards reset + * handler, if any. + */ +static void ecard_drv_shutdown(struct device *dev) +{ + struct expansion_card *ec = ECARD_DEV(dev); + struct ecard_driver *drv = ECARD_DRV(dev->driver); + struct ecard_request req; + + if (dev->driver) { + if (drv->shutdown) + drv->shutdown(ec); + ec->claimed = 0; + } + + /* + * If this card has a loader, call the reset handler. + */ + if (ec->loader) { + req.fn = ecard_task_reset; + req.ec = ec; + ecard_call(&req); + } +} + +int ecard_register_driver(struct ecard_driver *drv) +{ + drv->drv.bus = &ecard_bus_type; + + return driver_register(&drv->drv); +} + +void ecard_remove_driver(struct ecard_driver *drv) +{ + driver_unregister(&drv->drv); +} + +static int ecard_match(struct device *_dev, struct device_driver *_drv) +{ + struct expansion_card *ec = ECARD_DEV(_dev); + struct ecard_driver *drv = ECARD_DRV(_drv); + int ret; + + if (drv->id_table) { + ret = ecard_match_device(drv->id_table, ec) != NULL; + } else { + ret = ec->cid.id == drv->id; + } + + return ret; +} + +struct bus_type ecard_bus_type = { + .name = "ecard", + .dev_attrs = ecard_dev_attrs, + .match = ecard_match, + .probe = ecard_drv_probe, + .remove = ecard_drv_remove, + .shutdown = ecard_drv_shutdown, +}; + +static int ecard_bus_init(void) +{ + return bus_register(&ecard_bus_type); +} + +postcore_initcall(ecard_bus_init); + +EXPORT_SYMBOL(ecard_readchunk); +EXPORT_SYMBOL(ecard_register_driver); +EXPORT_SYMBOL(ecard_remove_driver); +EXPORT_SYMBOL(ecard_bus_type); diff --git a/arch/arm/kernel/ecard.h b/arch/arm/kernel/ecard.h new file mode 100644 index 00000000..4642d436 --- /dev/null +++ b/arch/arm/kernel/ecard.h @@ -0,0 +1,69 @@ +/* + * ecard.h + * + * Copyright 2007 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* Definitions internal to ecard.c - for it's use only!! + * + * External expansion card header as read from the card + */ +struct ex_ecid { + unsigned char r_irq:1; + unsigned char r_zero:1; + unsigned char r_fiq:1; + unsigned char r_id:4; + unsigned char r_a:1; + + unsigned char r_cd:1; + unsigned char r_is:1; + unsigned char r_w:2; + unsigned char r_r1:4; + + unsigned char r_r2:8; + + unsigned char r_prod[2]; + + unsigned char r_manu[2]; + + unsigned char r_country; + + unsigned char r_fiqmask; + unsigned char r_fiqoff[3]; + + unsigned char r_irqmask; + unsigned char r_irqoff[3]; +}; + +/* + * Chunk directory entry as read from the card + */ +struct ex_chunk_dir { + unsigned char r_id; + unsigned char r_len[3]; + unsigned long r_start; + union { + char string[256]; + char data[1]; + } d; +#define c_id(x) ((x)->r_id) +#define c_len(x) ((x)->r_len[0]|((x)->r_len[1]<<8)|((x)->r_len[2]<<16)) +#define c_start(x) ((x)->r_start) +}; + +typedef enum ecard_type { /* Cards address space */ + ECARD_IOC, + ECARD_MEMC, + ECARD_EASI +} card_type_t; + +typedef enum { /* Speed for ECARD_IOC space */ + ECARD_SLOW = 0, + ECARD_MEDIUM = 1, + ECARD_FAST = 2, + ECARD_SYNC = 3 +} card_speed_t; diff --git a/arch/arm/kernel/elf.c b/arch/arm/kernel/elf.c new file mode 100644 index 00000000..9b05c6a0 --- /dev/null +++ b/arch/arm/kernel/elf.c @@ -0,0 +1,90 @@ +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/personality.h> +#include <linux/binfmts.h> +#include <linux/elf.h> + +int elf_check_arch(const struct elf32_hdr *x) +{ + unsigned int eflags; + + /* Make sure it's an ARM executable */ + if (x->e_machine != EM_ARM) + return 0; + + /* Make sure the entry address is reasonable */ + if (x->e_entry & 1) { + if (!(elf_hwcap & HWCAP_THUMB)) + return 0; + } else if (x->e_entry & 3) + return 0; + + eflags = x->e_flags; + if ((eflags & EF_ARM_EABI_MASK) == EF_ARM_EABI_UNKNOWN) { + unsigned int flt_fmt; + + /* APCS26 is only allowed if the CPU supports it */ + if ((eflags & EF_ARM_APCS_26) && !(elf_hwcap & HWCAP_26BIT)) + return 0; + + flt_fmt = eflags & (EF_ARM_VFP_FLOAT | EF_ARM_SOFT_FLOAT); + + /* VFP requires the supporting code */ + if (flt_fmt == EF_ARM_VFP_FLOAT && !(elf_hwcap & HWCAP_VFP)) + return 0; + } + return 1; +} +EXPORT_SYMBOL(elf_check_arch); + +void elf_set_personality(const struct elf32_hdr *x) +{ + unsigned int eflags = x->e_flags; + unsigned int personality = current->personality & ~PER_MASK; + + /* + * We only support Linux ELF executables, so always set the + * personality to LINUX. + */ + personality |= PER_LINUX; + + /* + * APCS-26 is only valid for OABI executables + */ + if ((eflags & EF_ARM_EABI_MASK) == EF_ARM_EABI_UNKNOWN && + (eflags & EF_ARM_APCS_26)) + personality &= ~ADDR_LIMIT_32BIT; + else + personality |= ADDR_LIMIT_32BIT; + + set_personality(personality); + + /* + * Since the FPA coprocessor uses CP1 and CP2, and iWMMXt uses CP0 + * and CP1, we only enable access to the iWMMXt coprocessor if the + * binary is EABI or softfloat (and thus, guaranteed not to use + * FPA instructions.) + */ + if (elf_hwcap & HWCAP_IWMMXT && + eflags & (EF_ARM_EABI_MASK | EF_ARM_SOFT_FLOAT)) { + set_thread_flag(TIF_USING_IWMMXT); + } else { + clear_thread_flag(TIF_USING_IWMMXT); + } +} +EXPORT_SYMBOL(elf_set_personality); + +/* + * Set READ_IMPLIES_EXEC if: + * - the binary requires an executable stack + * - we're running on a CPU which doesn't support NX. + */ +int arm_elf_read_implies_exec(const struct elf32_hdr *x, int executable_stack) +{ + if (executable_stack != EXSTACK_DISABLE_X) + return 1; + if (cpu_architecture() < CPU_ARCH_ARMv6) + return 1; + return 0; +} +EXPORT_SYMBOL(arm_elf_read_implies_exec); diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S new file mode 100644 index 00000000..2cd00764 --- /dev/null +++ b/arch/arm/kernel/entry-armv.S @@ -0,0 +1,1242 @@ +/* + * linux/arch/arm/kernel/entry-armv.S + * + * Copyright (C) 1996,1997,1998 Russell King. + * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk) + * nommu support by Hyok S. Choi (hyok.choi@samsung.com) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Low-level vector interface routines + * + * Note: there is a StrongARM bug in the STMIA rn, {regs}^ instruction + * that causes it to save wrong values... Be aware! + */ + +#include <asm/memory.h> +#include <asm/glue-df.h> +#include <asm/glue-pf.h> +#include <asm/vfpmacros.h> +#include <mach/entry-macro.S> +#include <asm/thread_notify.h> +#include <asm/unwind.h> +#include <asm/unistd.h> +#include <asm/tls.h> + +#include "entry-header.S" +#include <asm/entry-macro-multi.S> + +/* + * Interrupt handling. Preserves r7, r8, r9 + */ + .macro irq_handler +#ifdef CONFIG_MULTI_IRQ_HANDLER + ldr r5, =handle_arch_irq + mov r0, sp + ldr r5, [r5] + adr lr, BSYM(9997f) + teq r5, #0 + movne pc, r5 +#endif + arch_irq_handler_default +9997: + .endm + +#ifdef CONFIG_KPROBES + .section .kprobes.text,"ax",%progbits +#else + .text +#endif + +/* + * Invalid mode handlers + */ + .macro inv_entry, reason + sub sp, sp, #S_FRAME_SIZE + ARM( stmib sp, {r1 - lr} ) + THUMB( stmia sp, {r0 - r12} ) + THUMB( str sp, [sp, #S_SP] ) + THUMB( str lr, [sp, #S_LR] ) + mov r1, #\reason + .endm + +__pabt_invalid: + inv_entry BAD_PREFETCH + b common_invalid +ENDPROC(__pabt_invalid) + +__dabt_invalid: + inv_entry BAD_DATA + b common_invalid +ENDPROC(__dabt_invalid) + +__irq_invalid: + inv_entry BAD_IRQ + b common_invalid +ENDPROC(__irq_invalid) + +__und_invalid: + inv_entry BAD_UNDEFINSTR + + @ + @ XXX fall through to common_invalid + @ + +@ +@ common_invalid - generic code for failed exception (re-entrant version of handlers) +@ +common_invalid: + zero_fp + + ldmia r0, {r4 - r6} + add r0, sp, #S_PC @ here for interlock avoidance + mov r7, #-1 @ "" "" "" "" + str r4, [sp] @ save preserved r0 + stmia r0, {r5 - r7} @ lr_<exception>, + @ cpsr_<exception>, "old_r0" + + mov r0, sp + b bad_mode +ENDPROC(__und_invalid) + +/* + * SVC mode handlers + */ + +#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) +#define SPFIX(code...) code +#else +#define SPFIX(code...) +#endif + + .macro svc_entry, stack_hole=0 + UNWIND(.fnstart ) + UNWIND(.save {r0 - pc} ) + sub sp, sp, #(S_FRAME_SIZE + \stack_hole - 4) +#ifdef CONFIG_THUMB2_KERNEL + SPFIX( str r0, [sp] ) @ temporarily saved + SPFIX( mov r0, sp ) + SPFIX( tst r0, #4 ) @ test original stack alignment + SPFIX( ldr r0, [sp] ) @ restored +#else + SPFIX( tst sp, #4 ) +#endif + SPFIX( subeq sp, sp, #4 ) + stmia sp, {r1 - r12} + + ldmia r0, {r1 - r3} + add r5, sp, #S_SP - 4 @ here for interlock avoidance + mov r4, #-1 @ "" "" "" "" + add r0, sp, #(S_FRAME_SIZE + \stack_hole - 4) + SPFIX( addeq r0, r0, #4 ) + str r1, [sp, #-4]! @ save the "real" r0 copied + @ from the exception stack + + mov r1, lr + + @ + @ We are now ready to fill in the remaining blanks on the stack: + @ + @ r0 - sp_svc + @ r1 - lr_svc + @ r2 - lr_<exception>, already fixed up for correct return/restart + @ r3 - spsr_<exception> + @ r4 - orig_r0 (see pt_regs definition in ptrace.h) + @ + stmia r5, {r0 - r4} + .endm + + .align 5 +__dabt_svc: + svc_entry + + @ + @ get ready to re-enable interrupts if appropriate + @ + mrs r9, cpsr + tst r3, #PSR_I_BIT + biceq r9, r9, #PSR_I_BIT + + @ + @ Call the processor-specific abort handler: + @ + @ r2 - aborted context pc + @ r3 - aborted context cpsr + @ + @ The abort handler must return the aborted address in r0, and + @ the fault status register in r1. r9 must be preserved. + @ +#ifdef MULTI_DABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_DABT_FUNC] +#else + bl CPU_DABORT_HANDLER +#endif + + @ + @ set desired IRQ state, then call main handler + @ + debug_entry r1 + msr cpsr_c, r9 + mov r2, sp + bl do_DataAbort + + @ + @ IRQs off again before pulling preserved data off the stack + @ + disable_irq_notrace + + @ + @ restore SPSR and restart the instruction + @ + ldr r2, [sp, #S_PSR] + svc_exit r2 @ return from exception + UNWIND(.fnend ) +ENDPROC(__dabt_svc) + + .align 5 +__irq_svc: + svc_entry + +#ifdef CONFIG_TRACE_IRQFLAGS + bl trace_hardirqs_off +#endif +#ifdef CONFIG_PREEMPT + get_thread_info tsk + ldr r8, [tsk, #TI_PREEMPT] @ get preempt count + add r7, r8, #1 @ increment it + str r7, [tsk, #TI_PREEMPT] +#endif + + irq_handler +#ifdef CONFIG_PREEMPT + str r8, [tsk, #TI_PREEMPT] @ restore preempt count + ldr r0, [tsk, #TI_FLAGS] @ get flags + teq r8, #0 @ if preempt count != 0 + movne r0, #0 @ force flags to 0 + tst r0, #_TIF_NEED_RESCHED + blne svc_preempt +#endif + ldr r4, [sp, #S_PSR] @ irqs are already disabled +#ifdef CONFIG_TRACE_IRQFLAGS + tst r4, #PSR_I_BIT + bleq trace_hardirqs_on +#endif + svc_exit r4 @ return from exception + UNWIND(.fnend ) +ENDPROC(__irq_svc) + + .ltorg + +#ifdef CONFIG_PREEMPT +svc_preempt: + mov r8, lr +1: bl preempt_schedule_irq @ irq en/disable is done inside + ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS + tst r0, #_TIF_NEED_RESCHED + moveq pc, r8 @ go again + b 1b +#endif + + .align 5 +__und_svc: +#ifdef CONFIG_KPROBES + @ If a kprobe is about to simulate a "stmdb sp..." instruction, + @ it obviously needs free stack space which then will belong to + @ the saved context. + svc_entry 64 +#else + svc_entry +#endif + + @ + @ call emulation code, which returns using r9 if it has emulated + @ the instruction, or the more conventional lr if we are to treat + @ this as a real undefined instruction + @ + @ r0 - instruction + @ +#ifndef CONFIG_THUMB2_KERNEL + ldr r0, [r2, #-4] +#else + ldrh r0, [r2, #-2] @ Thumb instruction at LR - 2 + and r9, r0, #0xf800 + cmp r9, #0xe800 @ 32-bit instruction if xx >= 0 + ldrhhs r9, [r2] @ bottom 16 bits + orrhs r0, r9, r0, lsl #16 +#endif + adr r9, BSYM(1f) + bl call_fpe + + mov r0, sp @ struct pt_regs *regs + bl do_undefinstr + + @ + @ IRQs off again before pulling preserved data off the stack + @ +1: disable_irq_notrace + + @ + @ restore SPSR and restart the instruction + @ + ldr r2, [sp, #S_PSR] @ Get SVC cpsr + svc_exit r2 @ return from exception + UNWIND(.fnend ) +ENDPROC(__und_svc) + + .align 5 +__pabt_svc: + svc_entry + + @ + @ re-enable interrupts if appropriate + @ + mrs r9, cpsr + tst r3, #PSR_I_BIT + biceq r9, r9, #PSR_I_BIT + + mov r0, r2 @ pass address of aborted instruction. +#ifdef MULTI_PABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_PABT_FUNC] +#else + bl CPU_PABORT_HANDLER +#endif + debug_entry r1 + msr cpsr_c, r9 @ Maybe enable interrupts + mov r2, sp @ regs + bl do_PrefetchAbort @ call abort handler + + @ + @ IRQs off again before pulling preserved data off the stack + @ + disable_irq_notrace + + @ + @ restore SPSR and restart the instruction + @ + ldr r2, [sp, #S_PSR] + svc_exit r2 @ return from exception + UNWIND(.fnend ) +ENDPROC(__pabt_svc) + + .align 5 +.LCcralign: + .word cr_alignment +#ifdef MULTI_DABORT +.LCprocfns: + .word processor +#endif +.LCfp: + .word fp_enter + +/* + * User mode handlers + * + * EABI note: sp_svc is always 64-bit aligned here, so should S_FRAME_SIZE + */ + +#if defined(CONFIG_AEABI) && (__LINUX_ARM_ARCH__ >= 5) && (S_FRAME_SIZE & 7) +#error "sizeof(struct pt_regs) must be a multiple of 8" +#endif + + .macro usr_entry + UNWIND(.fnstart ) + UNWIND(.cantunwind ) @ don't unwind the user space + sub sp, sp, #S_FRAME_SIZE + ARM( stmib sp, {r1 - r12} ) + THUMB( stmia sp, {r0 - r12} ) + + ldmia r0, {r1 - r3} + add r0, sp, #S_PC @ here for interlock avoidance + mov r4, #-1 @ "" "" "" "" + + str r1, [sp] @ save the "real" r0 copied + @ from the exception stack + + @ + @ We are now ready to fill in the remaining blanks on the stack: + @ + @ r2 - lr_<exception>, already fixed up for correct return/restart + @ r3 - spsr_<exception> + @ r4 - orig_r0 (see pt_regs definition in ptrace.h) + @ + @ Also, separately save sp_usr and lr_usr + @ + stmia r0, {r2 - r4} + ARM( stmdb r0, {sp, lr}^ ) + THUMB( store_user_sp_lr r0, r1, S_SP - S_PC ) + + @ + @ Enable the alignment trap while in kernel mode + @ + alignment_trap r0 + + @ + @ Clear FP to mark the first stack frame + @ + zero_fp + .endm + + .macro kuser_cmpxchg_check +#if __LINUX_ARM_ARCH__ < 6 && !defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) +#ifndef CONFIG_MMU +#warning "NPTL on non MMU needs fixing" +#else + @ Make sure our user space atomic helper is restarted + @ if it was interrupted in a critical region. Here we + @ perform a quick test inline since it should be false + @ 99.9999% of the time. The rest is done out of line. + cmp r2, #TASK_SIZE + blhs kuser_cmpxchg_fixup +#endif +#endif + .endm + + .align 5 +__dabt_usr: + usr_entry + kuser_cmpxchg_check + + @ + @ Call the processor-specific abort handler: + @ + @ r2 - aborted context pc + @ r3 - aborted context cpsr + @ + @ The abort handler must return the aborted address in r0, and + @ the fault status register in r1. + @ +#ifdef MULTI_DABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_DABT_FUNC] +#else + bl CPU_DABORT_HANDLER +#endif + + @ + @ IRQs on, then call the main handler + @ + debug_entry r1 + enable_irq + mov r2, sp + adr lr, BSYM(ret_from_exception) + b do_DataAbort + UNWIND(.fnend ) +ENDPROC(__dabt_usr) + + .align 5 +__irq_usr: + usr_entry + kuser_cmpxchg_check + +#ifdef CONFIG_IRQSOFF_TRACER + bl trace_hardirqs_off +#endif + + get_thread_info tsk +#ifdef CONFIG_PREEMPT + ldr r8, [tsk, #TI_PREEMPT] @ get preempt count + add r7, r8, #1 @ increment it + str r7, [tsk, #TI_PREEMPT] +#endif + + irq_handler +#ifdef CONFIG_PREEMPT + ldr r0, [tsk, #TI_PREEMPT] + str r8, [tsk, #TI_PREEMPT] + teq r0, r7 + ARM( strne r0, [r0, -r0] ) + THUMB( movne r0, #0 ) + THUMB( strne r0, [r0] ) +#endif + + mov why, #0 + b ret_to_user_from_irq + UNWIND(.fnend ) +ENDPROC(__irq_usr) + + .ltorg + + .align 5 +__und_usr: + usr_entry + + @ + @ fall through to the emulation code, which returns using r9 if + @ it has emulated the instruction, or the more conventional lr + @ if we are to treat this as a real undefined instruction + @ + @ r0 - instruction + @ + adr r9, BSYM(ret_from_exception) + adr lr, BSYM(__und_usr_unknown) + tst r3, #PSR_T_BIT @ Thumb mode? + itet eq @ explicit IT needed for the 1f label + subeq r4, r2, #4 @ ARM instr at LR - 4 + subne r4, r2, #2 @ Thumb instr at LR - 2 +1: ldreqt r0, [r4] +#ifdef CONFIG_CPU_ENDIAN_BE8 + reveq r0, r0 @ little endian instruction +#endif + beq call_fpe + @ Thumb instruction +#if __LINUX_ARM_ARCH__ >= 7 +2: + ARM( ldrht r5, [r4], #2 ) + THUMB( ldrht r5, [r4] ) + THUMB( add r4, r4, #2 ) + and r0, r5, #0xf800 @ mask bits 111x x... .... .... + cmp r0, #0xe800 @ 32bit instruction if xx != 0 + blo __und_usr_unknown +3: ldrht r0, [r4] + add r2, r2, #2 @ r2 is PC + 2, make it PC + 4 + orr r0, r0, r5, lsl #16 +#else + b __und_usr_unknown +#endif + UNWIND(.fnend ) +ENDPROC(__und_usr) + + @ + @ fallthrough to call_fpe + @ + +/* + * The out of line fixup for the ldrt above. + */ + .pushsection .fixup, "ax" +4: mov pc, r9 + .popsection + .pushsection __ex_table,"a" + .long 1b, 4b +#if __LINUX_ARM_ARCH__ >= 7 + .long 2b, 4b + .long 3b, 4b +#endif + .popsection + +/* + * Check whether the instruction is a co-processor instruction. + * If yes, we need to call the relevant co-processor handler. + * + * Note that we don't do a full check here for the co-processor + * instructions; all instructions with bit 27 set are well + * defined. The only instructions that should fault are the + * co-processor instructions. However, we have to watch out + * for the ARM6/ARM7 SWI bug. + * + * NEON is a special case that has to be handled here. Not all + * NEON instructions are co-processor instructions, so we have + * to make a special case of checking for them. Plus, there's + * five groups of them, so we have a table of mask/opcode pairs + * to check against, and if any match then we branch off into the + * NEON handler code. + * + * Emulators may wish to make use of the following registers: + * r0 = instruction opcode. + * r2 = PC+4 + * r9 = normal "successful" return address + * r10 = this threads thread_info structure. + * lr = unrecognised instruction return address + */ + @ + @ Fall-through from Thumb-2 __und_usr + @ +#ifdef CONFIG_NEON + adr r6, .LCneon_thumb_opcodes + b 2f +#endif +call_fpe: +#ifdef CONFIG_NEON + adr r6, .LCneon_arm_opcodes +2: + ldr r7, [r6], #4 @ mask value + cmp r7, #0 @ end mask? + beq 1f + and r8, r0, r7 + ldr r7, [r6], #4 @ opcode bits matching in mask + cmp r8, r7 @ NEON instruction? + bne 2b + get_thread_info r10 + mov r7, #1 + strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used + strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used + b do_vfp @ let VFP handler handle this +1: +#endif + tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 + tstne r0, #0x04000000 @ bit 26 set on both ARM and Thumb-2 +#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710) + and r8, r0, #0x0f000000 @ mask out op-code bits + teqne r8, #0x0f000000 @ SWI (ARM6/7 bug)? +#endif + moveq pc, lr + get_thread_info r10 @ get current thread + and r8, r0, #0x00000f00 @ mask out CP number + THUMB( lsr r8, r8, #8 ) + mov r7, #1 + add r6, r10, #TI_USED_CP + ARM( strb r7, [r6, r8, lsr #8] ) @ set appropriate used_cp[] + THUMB( strb r7, [r6, r8] ) @ set appropriate used_cp[] +#ifdef CONFIG_IWMMXT + @ Test if we need to give access to iWMMXt coprocessors + ldr r5, [r10, #TI_FLAGS] + rsbs r7, r8, #(1 << 8) @ CP 0 or 1 only + movcss r7, r5, lsr #(TIF_USING_IWMMXT + 1) + bcs iwmmxt_task_enable +#endif + ARM( add pc, pc, r8, lsr #6 ) + THUMB( lsl r8, r8, #2 ) + THUMB( add pc, r8 ) + nop + + movw_pc lr @ CP#0 + W(b) do_fpe @ CP#1 (FPE) + W(b) do_fpe @ CP#2 (FPE) + movw_pc lr @ CP#3 +#ifdef CONFIG_CRUNCH + b crunch_task_enable @ CP#4 (MaverickCrunch) + b crunch_task_enable @ CP#5 (MaverickCrunch) + b crunch_task_enable @ CP#6 (MaverickCrunch) +#else + movw_pc lr @ CP#4 + movw_pc lr @ CP#5 + movw_pc lr @ CP#6 +#endif + movw_pc lr @ CP#7 + movw_pc lr @ CP#8 + movw_pc lr @ CP#9 +#ifdef CONFIG_VFP + W(b) do_vfp @ CP#10 (VFP) + W(b) do_vfp @ CP#11 (VFP) +#else + movw_pc lr @ CP#10 (VFP) + movw_pc lr @ CP#11 (VFP) +#endif + movw_pc lr @ CP#12 + movw_pc lr @ CP#13 + movw_pc lr @ CP#14 (Debug) + movw_pc lr @ CP#15 (Control) + +#ifdef CONFIG_NEON + .align 6 + +.LCneon_arm_opcodes: + .word 0xfe000000 @ mask + .word 0xf2000000 @ opcode + + .word 0xff100000 @ mask + .word 0xf4000000 @ opcode + + .word 0x00000000 @ mask + .word 0x00000000 @ opcode + +.LCneon_thumb_opcodes: + .word 0xef000000 @ mask + .word 0xef000000 @ opcode + + .word 0xff100000 @ mask + .word 0xf9000000 @ opcode + + .word 0x00000000 @ mask + .word 0x00000000 @ opcode +#endif + +do_fpe: + enable_irq + ldr r4, .LCfp + add r10, r10, #TI_FPSTATE @ r10 = workspace + ldr pc, [r4] @ Call FP module USR entry point + +/* + * The FP module is called with these registers set: + * r0 = instruction + * r2 = PC+4 + * r9 = normal "successful" return address + * r10 = FP workspace + * lr = unrecognised FP instruction return address + */ + + .pushsection .data +ENTRY(fp_enter) + .word no_fp + .popsection + +ENTRY(no_fp) + mov pc, lr +ENDPROC(no_fp) + +__und_usr_unknown: + enable_irq + mov r0, sp + adr lr, BSYM(ret_from_exception) + b do_undefinstr +ENDPROC(__und_usr_unknown) + + .align 5 +__pabt_usr: + usr_entry + + mov r0, r2 @ pass address of aborted instruction. +#ifdef MULTI_PABORT + ldr r4, .LCprocfns + mov lr, pc + ldr pc, [r4, #PROCESSOR_PABT_FUNC] +#else + bl CPU_PABORT_HANDLER +#endif + debug_entry r1 + enable_irq @ Enable interrupts + mov r2, sp @ regs + bl do_PrefetchAbort @ call abort handler + UNWIND(.fnend ) + /* fall through */ +/* + * This is the return code to user mode for abort handlers + */ +ENTRY(ret_from_exception) + UNWIND(.fnstart ) + UNWIND(.cantunwind ) + get_thread_info tsk + mov why, #0 + b ret_to_user + UNWIND(.fnend ) +ENDPROC(__pabt_usr) +ENDPROC(ret_from_exception) + +/* + * Register switch for ARMv3 and ARMv4 processors + * r0 = previous task_struct, r1 = previous thread_info, r2 = next thread_info + * previous and next are guaranteed not to be the same. + */ +ENTRY(__switch_to) + UNWIND(.fnstart ) + UNWIND(.cantunwind ) + add ip, r1, #TI_CPU_SAVE + ldr r3, [r2, #TI_TP_VALUE] + ARM( stmia ip!, {r4 - sl, fp, sp, lr} ) @ Store most regs on stack + THUMB( stmia ip!, {r4 - sl, fp} ) @ Store most regs on stack + THUMB( str sp, [ip], #4 ) + THUMB( str lr, [ip], #4 ) +#ifdef CONFIG_CPU_USE_DOMAINS + ldr r6, [r2, #TI_CPU_DOMAIN] +#endif + set_tls r3, r4, r5 +#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) + ldr r7, [r2, #TI_TASK] + ldr r8, =__stack_chk_guard + ldr r7, [r7, #TSK_STACK_CANARY] +#endif +#ifdef CONFIG_CPU_USE_DOMAINS + mcr p15, 0, r6, c3, c0, 0 @ Set domain register +#endif + mov r5, r0 + add r4, r2, #TI_CPU_SAVE + ldr r0, =thread_notify_head + mov r1, #THREAD_NOTIFY_SWITCH + bl atomic_notifier_call_chain +#if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) + str r7, [r8] +#endif + THUMB( mov ip, r4 ) + mov r0, r5 + ARM( ldmia r4, {r4 - sl, fp, sp, pc} ) @ Load all regs saved previously + THUMB( ldmia ip!, {r4 - sl, fp} ) @ Load all regs saved previously + THUMB( ldr sp, [ip], #4 ) + THUMB( ldr pc, [ip] ) + UNWIND(.fnend ) +ENDPROC(__switch_to) + + __INIT + +/* + * User helpers. + * + * These are segment of kernel provided user code reachable from user space + * at a fixed address in kernel memory. This is used to provide user space + * with some operations which require kernel help because of unimplemented + * native feature and/or instructions in many ARM CPUs. The idea is for + * this code to be executed directly in user mode for best efficiency but + * which is too intimate with the kernel counter part to be left to user + * libraries. In fact this code might even differ from one CPU to another + * depending on the available instruction set and restrictions like on + * SMP systems. In other words, the kernel reserves the right to change + * this code as needed without warning. Only the entry points and their + * results are guaranteed to be stable. + * + * Each segment is 32-byte aligned and will be moved to the top of the high + * vector page. New segments (if ever needed) must be added in front of + * existing ones. This mechanism should be used only for things that are + * really small and justified, and not be abused freely. + * + * User space is expected to implement those things inline when optimizing + * for a processor that has the necessary native support, but only if such + * resulting binaries are already to be incompatible with earlier ARM + * processors due to the use of unsupported instructions other than what + * is provided here. In other words don't make binaries unable to run on + * earlier processors just for the sake of not using these kernel helpers + * if your compiled code is not going to use the new instructions for other + * purpose. + */ + THUMB( .arm ) + + .macro usr_ret, reg +#ifdef CONFIG_ARM_THUMB + bx \reg +#else + mov pc, \reg +#endif + .endm + + .align 5 + .globl __kuser_helper_start +__kuser_helper_start: + +/* + * Reference prototype: + * + * void __kernel_memory_barrier(void) + * + * Input: + * + * lr = return address + * + * Output: + * + * none + * + * Clobbered: + * + * none + * + * Definition and user space usage example: + * + * typedef void (__kernel_dmb_t)(void); + * #define __kernel_dmb (*(__kernel_dmb_t *)0xffff0fa0) + * + * Apply any needed memory barrier to preserve consistency with data modified + * manually and __kuser_cmpxchg usage. + * + * This could be used as follows: + * + * #define __kernel_dmb() \ + * asm volatile ( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #95" \ + * : : : "r0", "lr","cc" ) + */ + +__kuser_memory_barrier: @ 0xffff0fa0 + smp_dmb arm + usr_ret lr + + .align 5 + +/* + * Reference prototype: + * + * int __kernel_cmpxchg(int oldval, int newval, int *ptr) + * + * Input: + * + * r0 = oldval + * r1 = newval + * r2 = ptr + * lr = return address + * + * Output: + * + * r0 = returned value (zero or non-zero) + * C flag = set if r0 == 0, clear if r0 != 0 + * + * Clobbered: + * + * r3, ip, flags + * + * Definition and user space usage example: + * + * typedef int (__kernel_cmpxchg_t)(int oldval, int newval, int *ptr); + * #define __kernel_cmpxchg (*(__kernel_cmpxchg_t *)0xffff0fc0) + * + * Atomically store newval in *ptr if *ptr is equal to oldval for user space. + * Return zero if *ptr was changed or non-zero if no exchange happened. + * The C flag is also set if *ptr was changed to allow for assembly + * optimization in the calling code. + * + * Notes: + * + * - This routine already includes memory barriers as needed. + * + * For example, a user space atomic_add implementation could look like this: + * + * #define atomic_add(ptr, val) \ + * ({ register unsigned int *__ptr asm("r2") = (ptr); \ + * register unsigned int __result asm("r1"); \ + * asm volatile ( \ + * "1: @ atomic_add\n\t" \ + * "ldr r0, [r2]\n\t" \ + * "mov r3, #0xffff0fff\n\t" \ + * "add lr, pc, #4\n\t" \ + * "add r1, r0, %2\n\t" \ + * "add pc, r3, #(0xffff0fc0 - 0xffff0fff)\n\t" \ + * "bcc 1b" \ + * : "=&r" (__result) \ + * : "r" (__ptr), "rIL" (val) \ + * : "r0","r3","ip","lr","cc","memory" ); \ + * __result; }) + */ + +__kuser_cmpxchg: @ 0xffff0fc0 + +#if defined(CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG) + + /* + * Poor you. No fast solution possible... + * The kernel itself must perform the operation. + * A special ghost syscall is used for that (see traps.c). + */ + stmfd sp!, {r7, lr} + ldr r7, 1f @ it's 20 bits + swi __ARM_NR_cmpxchg + ldmfd sp!, {r7, pc} +1: .word __ARM_NR_cmpxchg + +#elif __LINUX_ARM_ARCH__ < 6 + +#ifdef CONFIG_MMU + + /* + * The only thing that can break atomicity in this cmpxchg + * implementation is either an IRQ or a data abort exception + * causing another process/thread to be scheduled in the middle + * of the critical sequence. To prevent this, code is added to + * the IRQ and data abort exception handlers to set the pc back + * to the beginning of the critical section if it is found to be + * within that critical section (see kuser_cmpxchg_fixup). + */ +1: ldr r3, [r2] @ load current val + subs r3, r3, r0 @ compare with oldval +2: streq r1, [r2] @ store newval if eq + rsbs r0, r3, #0 @ set return val and C flag + usr_ret lr + + .text +kuser_cmpxchg_fixup: + @ Called from kuser_cmpxchg_check macro. + @ r2 = address of interrupted insn (must be preserved). + @ sp = saved regs. r7 and r8 are clobbered. + @ 1b = first critical insn, 2b = last critical insn. + @ If r2 >= 1b and r2 <= 2b then saved pc_usr is set to 1b. + mov r7, #0xffff0fff + sub r7, r7, #(0xffff0fff - (0xffff0fc0 + (1b - __kuser_cmpxchg))) + subs r8, r2, r7 + rsbcss r8, r8, #(2b - 1b) + strcs r7, [sp, #S_PC] + mov pc, lr + .previous + +#else +#warning "NPTL on non MMU needs fixing" + mov r0, #-1 + adds r0, r0, #0 + usr_ret lr +#endif + +#else + + smp_dmb arm +1: ldrex r3, [r2] + subs r3, r3, r0 + strexeq r3, r1, [r2] + teqeq r3, #1 + beq 1b + rsbs r0, r3, #0 + /* beware -- each __kuser slot must be 8 instructions max */ + ALT_SMP(b __kuser_memory_barrier) + ALT_UP(usr_ret lr) + +#endif + + .align 5 + +/* + * Reference prototype: + * + * int __kernel_get_tls(void) + * + * Input: + * + * lr = return address + * + * Output: + * + * r0 = TLS value + * + * Clobbered: + * + * none + * + * Definition and user space usage example: + * + * typedef int (__kernel_get_tls_t)(void); + * #define __kernel_get_tls (*(__kernel_get_tls_t *)0xffff0fe0) + * + * Get the TLS value as previously set via the __ARM_NR_set_tls syscall. + * + * This could be used as follows: + * + * #define __kernel_get_tls() \ + * ({ register unsigned int __val asm("r0"); \ + * asm( "mov r0, #0xffff0fff; mov lr, pc; sub pc, r0, #31" \ + * : "=r" (__val) : : "lr","cc" ); \ + * __val; }) + */ + +__kuser_get_tls: @ 0xffff0fe0 + ldr r0, [pc, #(16 - 8)] @ read TLS, set in kuser_get_tls_init + usr_ret lr + mrc p15, 0, r0, c13, c0, 3 @ 0xffff0fe8 hardware TLS code + .rep 4 + .word 0 @ 0xffff0ff0 software TLS value, then + .endr @ pad up to __kuser_helper_version + +/* + * Reference declaration: + * + * extern unsigned int __kernel_helper_version; + * + * Definition and user space usage example: + * + * #define __kernel_helper_version (*(unsigned int *)0xffff0ffc) + * + * User space may read this to determine the curent number of helpers + * available. + */ + +__kuser_helper_version: @ 0xffff0ffc + .word ((__kuser_helper_end - __kuser_helper_start) >> 5) + + .globl __kuser_helper_end +__kuser_helper_end: + + THUMB( .thumb ) + +/* + * Vector stubs. + * + * This code is copied to 0xffff0200 so we can use branches in the + * vectors, rather than ldr's. Note that this code must not + * exceed 0x300 bytes. + * + * Common stub entry macro: + * Enter in IRQ mode, spsr = SVC/USR CPSR, lr = SVC/USR PC + * + * SP points to a minimal amount of processor-private memory, the address + * of which is copied into r0 for the mode specific abort handler. + */ + .macro vector_stub, name, mode, correction=0 + .align 5 + +vector_\name: + .if \correction + sub lr, lr, #\correction + .endif + + @ + @ Save r0, lr_<exception> (parent PC) and spsr_<exception> + @ (parent CPSR) + @ + stmia sp, {r0, lr} @ save r0, lr + mrs lr, spsr + str lr, [sp, #8] @ save spsr + + @ + @ Prepare for SVC32 mode. IRQs remain disabled. + @ + mrs r0, cpsr + eor r0, r0, #(\mode ^ SVC_MODE | PSR_ISETSTATE) + msr spsr_cxsf, r0 + + @ + @ the branch table must immediately follow this code + @ + and lr, lr, #0x0f + THUMB( adr r0, 1f ) + THUMB( ldr lr, [r0, lr, lsl #2] ) + mov r0, sp + ARM( ldr lr, [pc, lr, lsl #2] ) + movs pc, lr @ branch to handler in SVC mode +ENDPROC(vector_\name) + + .align 2 + @ handler addresses follow this label +1: + .endm + + .globl __stubs_start +__stubs_start: +/* + * Interrupt dispatcher + */ + vector_stub irq, IRQ_MODE, 4 + + .long __irq_usr @ 0 (USR_26 / USR_32) + .long __irq_invalid @ 1 (FIQ_26 / FIQ_32) + .long __irq_invalid @ 2 (IRQ_26 / IRQ_32) + .long __irq_svc @ 3 (SVC_26 / SVC_32) + .long __irq_invalid @ 4 + .long __irq_invalid @ 5 + .long __irq_invalid @ 6 + .long __irq_invalid @ 7 + .long __irq_invalid @ 8 + .long __irq_invalid @ 9 + .long __irq_invalid @ a + .long __irq_invalid @ b + .long __irq_invalid @ c + .long __irq_invalid @ d + .long __irq_invalid @ e + .long __irq_invalid @ f + +/* + * Data abort dispatcher + * Enter in ABT mode, spsr = USR CPSR, lr = USR PC + */ + vector_stub dabt, ABT_MODE, 8 + + .long __dabt_usr @ 0 (USR_26 / USR_32) + .long __dabt_invalid @ 1 (FIQ_26 / FIQ_32) + .long __dabt_invalid @ 2 (IRQ_26 / IRQ_32) + .long __dabt_svc @ 3 (SVC_26 / SVC_32) + .long __dabt_invalid @ 4 + .long __dabt_invalid @ 5 + .long __dabt_invalid @ 6 + .long __dabt_invalid @ 7 + .long __dabt_invalid @ 8 + .long __dabt_invalid @ 9 + .long __dabt_invalid @ a + .long __dabt_invalid @ b + .long __dabt_invalid @ c + .long __dabt_invalid @ d + .long __dabt_invalid @ e + .long __dabt_invalid @ f + +/* + * Prefetch abort dispatcher + * Enter in ABT mode, spsr = USR CPSR, lr = USR PC + */ + vector_stub pabt, ABT_MODE, 4 + + .long __pabt_usr @ 0 (USR_26 / USR_32) + .long __pabt_invalid @ 1 (FIQ_26 / FIQ_32) + .long __pabt_invalid @ 2 (IRQ_26 / IRQ_32) + .long __pabt_svc @ 3 (SVC_26 / SVC_32) + .long __pabt_invalid @ 4 + .long __pabt_invalid @ 5 + .long __pabt_invalid @ 6 + .long __pabt_invalid @ 7 + .long __pabt_invalid @ 8 + .long __pabt_invalid @ 9 + .long __pabt_invalid @ a + .long __pabt_invalid @ b + .long __pabt_invalid @ c + .long __pabt_invalid @ d + .long __pabt_invalid @ e + .long __pabt_invalid @ f + +/* + * Undef instr entry dispatcher + * Enter in UND mode, spsr = SVC/USR CPSR, lr = SVC/USR PC + */ + vector_stub und, UND_MODE + + .long __und_usr @ 0 (USR_26 / USR_32) + .long __und_invalid @ 1 (FIQ_26 / FIQ_32) + .long __und_invalid @ 2 (IRQ_26 / IRQ_32) + .long __und_svc @ 3 (SVC_26 / SVC_32) + .long __und_invalid @ 4 + .long __und_invalid @ 5 + .long __und_invalid @ 6 + .long __und_invalid @ 7 + .long __und_invalid @ 8 + .long __und_invalid @ 9 + .long __und_invalid @ a + .long __und_invalid @ b + .long __und_invalid @ c + .long __und_invalid @ d + .long __und_invalid @ e + .long __und_invalid @ f + + .align 5 + +/*============================================================================= + * Undefined FIQs + *----------------------------------------------------------------------------- + * Enter in FIQ mode, spsr = ANY CPSR, lr = ANY PC + * MUST PRESERVE SVC SPSR, but need to switch to SVC mode to show our msg. + * Basically to switch modes, we *HAVE* to clobber one register... brain + * damage alert! I don't think that we can execute any code in here in any + * other mode than FIQ... Ok you can switch to another mode, but you can't + * get out of that mode without clobbering one register. + */ +vector_fiq: + disable_fiq + subs pc, lr, #4 + +/*============================================================================= + * Address exception handler + *----------------------------------------------------------------------------- + * These aren't too critical. + * (they're not supposed to happen, and won't happen in 32-bit data mode). + */ + +vector_addrexcptn: + b vector_addrexcptn + +/* + * We group all the following data together to optimise + * for CPUs with separate I & D caches. + */ + .align 5 + +.LCvswi: + .word vector_swi + + .globl __stubs_end +__stubs_end: + + .equ stubs_offset, __vectors_start + 0x200 - __stubs_start + + .globl __vectors_start +__vectors_start: + ARM( swi SYS_ERROR0 ) + THUMB( svc #0 ) + THUMB( nop ) + W(b) vector_und + stubs_offset + W(ldr) pc, .LCvswi + stubs_offset + W(b) vector_pabt + stubs_offset + W(b) vector_dabt + stubs_offset + W(b) vector_addrexcptn + stubs_offset + W(b) vector_irq + stubs_offset + W(b) vector_fiq + stubs_offset + + .globl __vectors_end +__vectors_end: + + .data + + .globl cr_alignment + .globl cr_no_alignment +cr_alignment: + .space 4 +cr_no_alignment: + .space 4 + +#ifdef CONFIG_MULTI_IRQ_HANDLER + .globl handle_arch_irq +handle_arch_irq: + .space 4 +#endif diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S new file mode 100644 index 00000000..b2a27b6b --- /dev/null +++ b/arch/arm/kernel/entry-common.S @@ -0,0 +1,647 @@ +/* + * linux/arch/arm/kernel/entry-common.S + * + * Copyright (C) 2000 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <asm/unistd.h> +#include <asm/ftrace.h> +#include <mach/entry-macro.S> +#include <asm/unwind.h> + +#include "entry-header.S" + + + .align 5 +/* + * This is the fast syscall return path. We do as little as + * possible here, and this includes saving r0 back into the SVC + * stack. + */ +ret_fast_syscall: + UNWIND(.fnstart ) + UNWIND(.cantunwind ) + disable_irq @ disable interrupts + ldr r1, [tsk, #TI_FLAGS] + tst r1, #_TIF_WORK_MASK + bne fast_work_pending +#if defined(CONFIG_IRQSOFF_TRACER) + asm_trace_hardirqs_on +#endif + + /* perform architecture specific actions before user return */ + arch_ret_to_user r1, lr + + restore_user_regs fast = 1, offset = S_OFF + UNWIND(.fnend ) + +/* + * Ok, we need to do extra processing, enter the slow path. + */ +fast_work_pending: + str r0, [sp, #S_R0+S_OFF]! @ returned r0 +work_pending: + tst r1, #_TIF_NEED_RESCHED + bne work_resched + tst r1, #_TIF_SIGPENDING|_TIF_NOTIFY_RESUME + beq no_work_pending + mov r0, sp @ 'regs' + mov r2, why @ 'syscall' + tst r1, #_TIF_SIGPENDING @ delivering a signal? + movne why, #0 @ prevent further restarts + bl do_notify_resume + b ret_slow_syscall @ Check work again + +work_resched: + bl schedule +/* + * "slow" syscall return path. "why" tells us if this was a real syscall. + */ +ENTRY(ret_to_user) +ret_slow_syscall: + disable_irq @ disable interrupts +ENTRY(ret_to_user_from_irq) + ldr r1, [tsk, #TI_FLAGS] + tst r1, #_TIF_WORK_MASK + bne work_pending +no_work_pending: +#if defined(CONFIG_IRQSOFF_TRACER) + asm_trace_hardirqs_on +#endif + /* perform architecture specific actions before user return */ + arch_ret_to_user r1, lr + + restore_user_regs fast = 0, offset = 0 +ENDPROC(ret_to_user_from_irq) +ENDPROC(ret_to_user) + +/* + * This is how we return from a fork. + */ +ENTRY(ret_from_fork) + bl schedule_tail + get_thread_info tsk + ldr r1, [tsk, #TI_FLAGS] @ check for syscall tracing + mov why, #1 + tst r1, #_TIF_SYSCALL_TRACE @ are we tracing syscalls? + beq ret_slow_syscall + mov r1, sp + mov r0, #1 @ trace exit [IP = 1] + bl syscall_trace + b ret_slow_syscall +ENDPROC(ret_from_fork) + + .equ NR_syscalls,0 +#define CALL(x) .equ NR_syscalls,NR_syscalls+1 +#include "calls.S" +#undef CALL +#define CALL(x) .long x + +#ifdef CONFIG_FUNCTION_TRACER +/* + * When compiling with -pg, gcc inserts a call to the mcount routine at the + * start of every function. In mcount, apart from the function's address (in + * lr), we need to get hold of the function's caller's address. + * + * Older GCCs (pre-4.4) inserted a call to a routine called mcount like this: + * + * bl mcount + * + * These versions have the limitation that in order for the mcount routine to + * be able to determine the function's caller's address, an APCS-style frame + * pointer (which is set up with something like the code below) is required. + * + * mov ip, sp + * push {fp, ip, lr, pc} + * sub fp, ip, #4 + * + * With EABI, these frame pointers are not available unless -mapcs-frame is + * specified, and if building as Thumb-2, not even then. + * + * Newer GCCs (4.4+) solve this problem by introducing a new version of mcount, + * with call sites like: + * + * push {lr} + * bl __gnu_mcount_nc + * + * With these compilers, frame pointers are not necessary. + * + * mcount can be thought of as a function called in the middle of a subroutine + * call. As such, it needs to be transparent for both the caller and the + * callee: the original lr needs to be restored when leaving mcount, and no + * registers should be clobbered. (In the __gnu_mcount_nc implementation, we + * clobber the ip register. This is OK because the ARM calling convention + * allows it to be clobbered in subroutines and doesn't use it to hold + * parameters.) + * + * When using dynamic ftrace, we patch out the mcount call by a "mov r0, r0" + * for the mcount case, and a "pop {lr}" for the __gnu_mcount_nc case (see + * arch/arm/kernel/ftrace.c). + */ + +#ifndef CONFIG_OLD_MCOUNT +#if (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 4)) +#error Ftrace requires CONFIG_FRAME_POINTER=y with GCC older than 4.4.0. +#endif +#endif + +.macro __mcount suffix + mcount_enter + ldr r0, =ftrace_trace_function + ldr r2, [r0] + adr r0, .Lftrace_stub + cmp r0, r2 + bne 1f + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + ldr r1, =ftrace_graph_return + ldr r2, [r1] + cmp r0, r2 + bne ftrace_graph_caller\suffix + + ldr r1, =ftrace_graph_entry + ldr r2, [r1] + ldr r0, =ftrace_graph_entry_stub + cmp r0, r2 + bne ftrace_graph_caller\suffix +#endif + + mcount_exit + +1: mcount_get_lr r1 @ lr of instrumented func + mov r0, lr @ instrumented function + sub r0, r0, #MCOUNT_INSN_SIZE + adr lr, BSYM(2f) + mov pc, r2 +2: mcount_exit +.endm + +.macro __ftrace_caller suffix + mcount_enter + + mcount_get_lr r1 @ lr of instrumented func + mov r0, lr @ instrumented function + sub r0, r0, #MCOUNT_INSN_SIZE + + .globl ftrace_call\suffix +ftrace_call\suffix: + bl ftrace_stub + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + .globl ftrace_graph_call\suffix +ftrace_graph_call\suffix: + mov r0, r0 +#endif + + mcount_exit +.endm + +.macro __ftrace_graph_caller + sub r0, fp, #4 @ &lr of instrumented routine (&parent) +#ifdef CONFIG_DYNAMIC_FTRACE + @ called from __ftrace_caller, saved in mcount_enter + ldr r1, [sp, #16] @ instrumented routine (func) +#else + @ called from __mcount, untouched in lr + mov r1, lr @ instrumented routine (func) +#endif + sub r1, r1, #MCOUNT_INSN_SIZE + mov r2, fp @ frame pointer + bl prepare_ftrace_return + mcount_exit +.endm + +#ifdef CONFIG_OLD_MCOUNT +/* + * mcount + */ + +.macro mcount_enter + stmdb sp!, {r0-r3, lr} +.endm + +.macro mcount_get_lr reg + ldr \reg, [fp, #-4] +.endm + +.macro mcount_exit + ldr lr, [fp, #-4] + ldmia sp!, {r0-r3, pc} +.endm + +ENTRY(mcount) +#ifdef CONFIG_DYNAMIC_FTRACE + stmdb sp!, {lr} + ldr lr, [fp, #-4] + ldmia sp!, {pc} +#else + __mcount _old +#endif +ENDPROC(mcount) + +#ifdef CONFIG_DYNAMIC_FTRACE +ENTRY(ftrace_caller_old) + __ftrace_caller _old +ENDPROC(ftrace_caller_old) +#endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +ENTRY(ftrace_graph_caller_old) + __ftrace_graph_caller +ENDPROC(ftrace_graph_caller_old) +#endif + +.purgem mcount_enter +.purgem mcount_get_lr +.purgem mcount_exit +#endif + +/* + * __gnu_mcount_nc + */ + +.macro mcount_enter + stmdb sp!, {r0-r3, lr} +.endm + +.macro mcount_get_lr reg + ldr \reg, [sp, #20] +.endm + +.macro mcount_exit + ldmia sp!, {r0-r3, ip, lr} + mov pc, ip +.endm + +ENTRY(__gnu_mcount_nc) +#ifdef CONFIG_DYNAMIC_FTRACE + mov ip, lr + ldmia sp!, {lr} + mov pc, ip +#else + __mcount +#endif +ENDPROC(__gnu_mcount_nc) + +#ifdef CONFIG_DYNAMIC_FTRACE +ENTRY(ftrace_caller) + __ftrace_caller +ENDPROC(ftrace_caller) +#endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +ENTRY(ftrace_graph_caller) + __ftrace_graph_caller +ENDPROC(ftrace_graph_caller) +#endif + +.purgem mcount_enter +.purgem mcount_get_lr +.purgem mcount_exit + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + .globl return_to_handler +return_to_handler: + stmdb sp!, {r0-r3} + mov r0, fp @ frame pointer + bl ftrace_return_to_handler + mov lr, r0 @ r0 has real ret addr + ldmia sp!, {r0-r3} + mov pc, lr +#endif + +ENTRY(ftrace_stub) +.Lftrace_stub: + mov pc, lr +ENDPROC(ftrace_stub) + +#endif /* CONFIG_FUNCTION_TRACER */ + +/*============================================================================= + * SWI handler + *----------------------------------------------------------------------------- + */ + + /* If we're optimising for StrongARM the resulting code won't + run on an ARM7 and we can save a couple of instructions. + --pb */ +#ifdef CONFIG_CPU_ARM710 +#define A710(code...) code +.Larm710bug: + ldmia sp, {r0 - lr}^ @ Get calling r0 - lr + mov r0, r0 + add sp, sp, #S_FRAME_SIZE + subs pc, lr, #4 +#else +#define A710(code...) +#endif + + .align 5 +ENTRY(vector_swi) + sub sp, sp, #S_FRAME_SIZE + stmia sp, {r0 - r12} @ Calling r0 - r12 + ARM( add r8, sp, #S_PC ) + ARM( stmdb r8, {sp, lr}^ ) @ Calling sp, lr + THUMB( mov r8, sp ) + THUMB( store_user_sp_lr r8, r10, S_SP ) @ calling sp, lr + mrs r8, spsr @ called from non-FIQ mode, so ok. + str lr, [sp, #S_PC] @ Save calling PC + str r8, [sp, #S_PSR] @ Save CPSR + str r0, [sp, #S_OLD_R0] @ Save OLD_R0 + zero_fp + + /* + * Get the system call number. + */ + +#if defined(CONFIG_OABI_COMPAT) + + /* + * If we have CONFIG_OABI_COMPAT then we need to look at the swi + * value to determine if it is an EABI or an old ABI call. + */ +#ifdef CONFIG_ARM_THUMB + tst r8, #PSR_T_BIT + movne r10, #0 @ no thumb OABI emulation + ldreq r10, [lr, #-4] @ get SWI instruction +#else + ldr r10, [lr, #-4] @ get SWI instruction + A710( and ip, r10, #0x0f000000 @ check for SWI ) + A710( teq ip, #0x0f000000 ) + A710( bne .Larm710bug ) +#endif +#ifdef CONFIG_CPU_ENDIAN_BE8 + rev r10, r10 @ little endian instruction +#endif + +#elif defined(CONFIG_AEABI) + + /* + * Pure EABI user space always put syscall number into scno (r7). + */ + A710( ldr ip, [lr, #-4] @ get SWI instruction ) + A710( and ip, ip, #0x0f000000 @ check for SWI ) + A710( teq ip, #0x0f000000 ) + A710( bne .Larm710bug ) + +#elif defined(CONFIG_ARM_THUMB) + + /* Legacy ABI only, possibly thumb mode. */ + tst r8, #PSR_T_BIT @ this is SPSR from save_user_regs + addne scno, r7, #__NR_SYSCALL_BASE @ put OS number in + ldreq scno, [lr, #-4] + +#else + + /* Legacy ABI only. */ + ldr scno, [lr, #-4] @ get SWI instruction + A710( and ip, scno, #0x0f000000 @ check for SWI ) + A710( teq ip, #0x0f000000 ) + A710( bne .Larm710bug ) + +#endif + +#ifdef CONFIG_ALIGNMENT_TRAP + ldr ip, __cr_alignment + ldr ip, [ip] + mcr p15, 0, ip, c1, c0 @ update control register +#endif + enable_irq + + get_thread_info tsk + adr tbl, sys_call_table @ load syscall table pointer + +#if defined(CONFIG_OABI_COMPAT) + /* + * If the swi argument is zero, this is an EABI call and we do nothing. + * + * If this is an old ABI call, get the syscall number into scno and + * get the old ABI syscall table address. + */ + bics r10, r10, #0xff000000 + eorne scno, r10, #__NR_OABI_SYSCALL_BASE + ldrne tbl, =sys_oabi_call_table +#elif !defined(CONFIG_AEABI) + bic scno, scno, #0xff000000 @ mask off SWI op-code + eor scno, scno, #__NR_SYSCALL_BASE @ check OS number +#endif + + ldr r10, [tsk, #TI_FLAGS] @ check for syscall tracing + stmdb sp!, {r4, r5} @ push fifth and sixth args + +#ifdef CONFIG_SECCOMP + tst r10, #_TIF_SECCOMP + beq 1f + mov r0, scno + bl __secure_computing + add r0, sp, #S_R0 + S_OFF @ pointer to regs + ldmia r0, {r0 - r3} @ have to reload r0 - r3 +1: +#endif + + tst r10, #_TIF_SYSCALL_TRACE @ are we tracing syscalls? + bne __sys_trace + + cmp scno, #NR_syscalls @ check upper syscall limit + adr lr, BSYM(ret_fast_syscall) @ return address + ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + + add r1, sp, #S_OFF +2: mov why, #0 @ no longer a real syscall + cmp scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE) + eor r0, scno, #__NR_SYSCALL_BASE @ put OS number back + bcs arm_syscall + b sys_ni_syscall @ not private func +ENDPROC(vector_swi) + + /* + * This is the really slow path. We're going to be doing + * context switches, and waiting for our parent to respond. + */ +__sys_trace: + mov r2, scno + add r1, sp, #S_OFF + mov r0, #0 @ trace entry [IP = 0] + bl syscall_trace + + adr lr, BSYM(__sys_trace_return) @ return address + mov scno, r0 @ syscall number (possibly new) + add r1, sp, #S_R0 + S_OFF @ pointer to regs + cmp scno, #NR_syscalls @ check upper syscall limit + ldmccia r1, {r0 - r3} @ have to reload r0 - r3 + ldrcc pc, [tbl, scno, lsl #2] @ call sys_* routine + b 2b + +__sys_trace_return: + str r0, [sp, #S_R0 + S_OFF]! @ save returned r0 + mov r2, scno + mov r1, sp + mov r0, #1 @ trace exit [IP = 1] + bl syscall_trace + b ret_slow_syscall + + .align 5 +#ifdef CONFIG_ALIGNMENT_TRAP + .type __cr_alignment, #object +__cr_alignment: + .word cr_alignment +#endif + .ltorg + +/* + * This is the syscall table declaration for native ABI syscalls. + * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall. + */ +#define ABI(native, compat) native +#ifdef CONFIG_AEABI +#define OBSOLETE(syscall) sys_ni_syscall +#else +#define OBSOLETE(syscall) syscall +#endif + + .type sys_call_table, #object +ENTRY(sys_call_table) +#include "calls.S" +#undef ABI +#undef OBSOLETE + +/*============================================================================ + * Special system call wrappers + */ +@ r0 = syscall number +@ r8 = syscall table +sys_syscall: + bic scno, r0, #__NR_OABI_SYSCALL_BASE + cmp scno, #__NR_syscall - __NR_SYSCALL_BASE + cmpne scno, #NR_syscalls @ check range + stmloia sp, {r5, r6} @ shuffle args + movlo r0, r1 + movlo r1, r2 + movlo r2, r3 + movlo r3, r4 + ldrlo pc, [tbl, scno, lsl #2] + b sys_ni_syscall +ENDPROC(sys_syscall) + +sys_fork_wrapper: + add r0, sp, #S_OFF + b sys_fork +ENDPROC(sys_fork_wrapper) + +sys_vfork_wrapper: + add r0, sp, #S_OFF + b sys_vfork +ENDPROC(sys_vfork_wrapper) + +sys_execve_wrapper: + add r3, sp, #S_OFF + b sys_execve +ENDPROC(sys_execve_wrapper) + +sys_clone_wrapper: + add ip, sp, #S_OFF + str ip, [sp, #4] + b sys_clone +ENDPROC(sys_clone_wrapper) + +sys_sigreturn_wrapper: + add r0, sp, #S_OFF + mov why, #0 @ prevent syscall restart handling + b sys_sigreturn +ENDPROC(sys_sigreturn_wrapper) + +sys_rt_sigreturn_wrapper: + add r0, sp, #S_OFF + mov why, #0 @ prevent syscall restart handling + b sys_rt_sigreturn +ENDPROC(sys_rt_sigreturn_wrapper) + +sys_sigaltstack_wrapper: + ldr r2, [sp, #S_OFF + S_SP] + b do_sigaltstack +ENDPROC(sys_sigaltstack_wrapper) + +sys_statfs64_wrapper: + teq r1, #88 + moveq r1, #84 + b sys_statfs64 +ENDPROC(sys_statfs64_wrapper) + +sys_fstatfs64_wrapper: + teq r1, #88 + moveq r1, #84 + b sys_fstatfs64 +ENDPROC(sys_fstatfs64_wrapper) + +/* + * Note: off_4k (r5) is always units of 4K. If we can't do the requested + * offset, we return EINVAL. + */ +sys_mmap2: +#if PAGE_SHIFT > 12 + tst r5, #PGOFF_MASK + moveq r5, r5, lsr #PAGE_SHIFT - 12 + streq r5, [sp, #4] + beq sys_mmap_pgoff + mov r0, #-EINVAL + mov pc, lr +#else + str r5, [sp, #4] + b sys_mmap_pgoff +#endif +ENDPROC(sys_mmap2) + +#ifdef CONFIG_OABI_COMPAT + +/* + * These are syscalls with argument register differences + */ + +sys_oabi_pread64: + stmia sp, {r3, r4} + b sys_pread64 +ENDPROC(sys_oabi_pread64) + +sys_oabi_pwrite64: + stmia sp, {r3, r4} + b sys_pwrite64 +ENDPROC(sys_oabi_pwrite64) + +sys_oabi_truncate64: + mov r3, r2 + mov r2, r1 + b sys_truncate64 +ENDPROC(sys_oabi_truncate64) + +sys_oabi_ftruncate64: + mov r3, r2 + mov r2, r1 + b sys_ftruncate64 +ENDPROC(sys_oabi_ftruncate64) + +sys_oabi_readahead: + str r3, [sp] + mov r3, r2 + mov r2, r1 + b sys_readahead +ENDPROC(sys_oabi_readahead) + +/* + * Let's declare a second syscall table for old ABI binaries + * using the compatibility syscall entries. + */ +#define ABI(native, compat) compat +#define OBSOLETE(syscall) syscall + + .type sys_oabi_call_table, #object +ENTRY(sys_oabi_call_table) +#include "calls.S" +#undef ABI +#undef OBSOLETE + +#endif + diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S new file mode 100644 index 00000000..051166c2 --- /dev/null +++ b/arch/arm/kernel/entry-header.S @@ -0,0 +1,200 @@ +#include <linux/init.h> +#include <linux/linkage.h> + +#include <asm/assembler.h> +#include <asm/asm-offsets.h> +#include <asm/errno.h> +#include <asm/thread_info.h> + +@ Bad Abort numbers +@ ----------------- +@ +#define BAD_PREFETCH 0 +#define BAD_DATA 1 +#define BAD_ADDREXCPTN 2 +#define BAD_IRQ 3 +#define BAD_UNDEFINSTR 4 + +@ +@ Most of the stack format comes from struct pt_regs, but with +@ the addition of 8 bytes for storing syscall args 5 and 6. +@ This _must_ remain a multiple of 8 for EABI. +@ +#define S_OFF 8 + +/* + * The SWI code relies on the fact that R0 is at the bottom of the stack + * (due to slow/fast restore user regs). + */ +#if S_R0 != 0 +#error "Please fix" +#endif + + .macro zero_fp +#ifdef CONFIG_FRAME_POINTER + mov fp, #0 +#endif + .endm + + .macro alignment_trap, rtemp +#ifdef CONFIG_ALIGNMENT_TRAP + ldr \rtemp, .LCcralign + ldr \rtemp, [\rtemp] + mcr p15, 0, \rtemp, c1, c0 +#endif + .endm + + @ + @ Store/load the USER SP and LR registers by switching to the SYS + @ mode. Useful in Thumb-2 mode where "stm/ldm rd, {sp, lr}^" is not + @ available. Should only be called from SVC mode + @ + .macro store_user_sp_lr, rd, rtemp, offset = 0 + mrs \rtemp, cpsr + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch to the SYS mode + + str sp, [\rd, #\offset] @ save sp_usr + str lr, [\rd, #\offset + 4] @ save lr_usr + + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch back to the SVC mode + .endm + + .macro load_user_sp_lr, rd, rtemp, offset = 0 + mrs \rtemp, cpsr + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch to the SYS mode + + ldr sp, [\rd, #\offset] @ load sp_usr + ldr lr, [\rd, #\offset + 4] @ load lr_usr + + eor \rtemp, \rtemp, #(SVC_MODE ^ SYSTEM_MODE) + msr cpsr_c, \rtemp @ switch back to the SVC mode + .endm + +#ifndef CONFIG_THUMB2_KERNEL + .macro svc_exit, rpsr + msr spsr_cxsf, \rpsr +#if defined(CONFIG_CPU_V6) + ldr r0, [sp] + strex r1, r2, [sp] @ clear the exclusive monitor + ldmib sp, {r1 - pc}^ @ load r1 - pc, cpsr +#elif defined(CONFIG_CPU_32v6K) + clrex @ clear the exclusive monitor + ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr +#else + ldmia sp, {r0 - pc}^ @ load r0 - pc, cpsr +#endif + .endm + + .macro restore_user_regs, fast = 0, offset = 0 + ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr + ldr lr, [sp, #\offset + S_PC]! @ get pc + msr spsr_cxsf, r1 @ save in spsr_svc +#if defined(CONFIG_CPU_V6) + strex r1, r2, [sp] @ clear the exclusive monitor +#elif defined(CONFIG_CPU_32v6K) + clrex @ clear the exclusive monitor +#endif + .if \fast + ldmdb sp, {r1 - lr}^ @ get calling r1 - lr + .else + ldmdb sp, {r0 - lr}^ @ get calling r0 - lr + .endif + mov r0, r0 @ ARMv5T and earlier require a nop + @ after ldm {}^ + add sp, sp, #S_FRAME_SIZE - S_PC + movs pc, lr @ return & move spsr_svc into cpsr + .endm + + .macro get_thread_info, rd + mov \rd, sp, lsr #13 + mov \rd, \rd, lsl #13 + .endm + + @ + @ 32-bit wide "mov pc, reg" + @ + .macro movw_pc, reg + mov pc, \reg + .endm +#else /* CONFIG_THUMB2_KERNEL */ + .macro svc_exit, rpsr + clrex @ clear the exclusive monitor + ldr r0, [sp, #S_SP] @ top of the stack + ldr r1, [sp, #S_PC] @ return address + tst r0, #4 @ orig stack 8-byte aligned? + stmdb r0, {r1, \rpsr} @ rfe context + ldmia sp, {r0 - r12} + ldr lr, [sp, #S_LR] + addeq sp, sp, #S_FRAME_SIZE - 8 @ aligned + addne sp, sp, #S_FRAME_SIZE - 4 @ not aligned + rfeia sp! + .endm + + .macro restore_user_regs, fast = 0, offset = 0 + clrex @ clear the exclusive monitor + mov r2, sp + load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr + ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr + ldr lr, [sp, #\offset + S_PC] @ get pc + add sp, sp, #\offset + S_SP + msr spsr_cxsf, r1 @ save in spsr_svc + .if \fast + ldmdb sp, {r1 - r12} @ get calling r1 - r12 + .else + ldmdb sp, {r0 - r12} @ get calling r0 - r12 + .endif + add sp, sp, #S_FRAME_SIZE - S_SP + movs pc, lr @ return & move spsr_svc into cpsr + .endm + + .macro get_thread_info, rd + mov \rd, sp + lsr \rd, \rd, #13 + mov \rd, \rd, lsl #13 + .endm + + @ + @ 32-bit wide "mov pc, reg" + @ + .macro movw_pc, reg + mov pc, \reg + nop + .endm +#endif /* !CONFIG_THUMB2_KERNEL */ + + @ + @ Debug exceptions are taken as prefetch or data aborts. + @ We must disable preemption during the handler so that + @ we can access the debug registers safely. + @ + .macro debug_entry, fsr +#if defined(CONFIG_HAVE_HW_BREAKPOINT) && defined(CONFIG_PREEMPT) + ldr r4, =0x40f @ mask out fsr.fs + and r5, r4, \fsr + cmp r5, #2 @ debug exception + bne 1f + get_thread_info r10 + ldr r6, [r10, #TI_PREEMPT] @ get preempt count + add r11, r6, #1 @ increment it + str r11, [r10, #TI_PREEMPT] +1: +#endif + .endm + +/* + * These are the registers used in the syscall handler, and allow us to + * have in theory up to 7 arguments to a function - r0 to r6. + * + * r7 is reserved for the system call number for thumb mode. + * + * Note that tbl == why is intentional. + * + * We must set at least "tsk" and "why" when calling ret_with_reschedule. + */ +scno .req r7 @ syscall number +tbl .req r8 @ syscall table pointer +why .req r8 @ Linux syscall (!= 0) +tsk .req r9 @ current thread_info diff --git a/arch/arm/kernel/etm.c b/arch/arm/kernel/etm.c new file mode 100644 index 00000000..93d06ba0 --- /dev/null +++ b/arch/arm/kernel/etm.c @@ -0,0 +1,1075 @@ +/* + * linux/arch/arm/kernel/etm.c + * + * Driver for ARM's Embedded Trace Macrocell and Embedded Trace Buffer. + * + * Copyright (C) 2009 Nokia Corporation. + * Alexander Shishkin + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/io.h> +#include <linux/slab.h> +#include <linux/sysrq.h> +#include <linux/device.h> +#include <linux/clk.h> +#include <linux/amba/bus.h> +#include <linux/fs.h> +#include <linux/uaccess.h> +#include <linux/miscdevice.h> +#include <linux/vmalloc.h> +#include <linux/mutex.h> +#include <asm/hardware/coresight.h> +#include <asm/sections.h> + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Alexander Shishkin"); + +/* + * ETM tracer state + */ +struct tracectx { + unsigned int etb_bufsz; + void __iomem *etb_regs; + void __iomem **etm_regs; + int etm_regs_count; + unsigned long flags; + int ncmppairs; + int etm_portsz; + int etm_contextid_size; + u32 etb_fc; + unsigned long range_start; + unsigned long range_end; + unsigned long data_range_start; + unsigned long data_range_end; + bool dump_initial_etb; + struct device *dev; + struct clk *emu_clk; + struct mutex mutex; +}; + +static struct tracectx tracer = { + .range_start = (unsigned long)_stext, + .range_end = (unsigned long)_etext, +}; + +static inline bool trace_isrunning(struct tracectx *t) +{ + return !!(t->flags & TRACER_RUNNING); +} + +static int etm_setup_address_range(struct tracectx *t, int id, int n, + unsigned long start, unsigned long end, int exclude, int data) +{ + u32 flags = ETMAAT_ARM | ETMAAT_IGNCONTEXTID | ETMAAT_IGNSECURITY | + ETMAAT_NOVALCMP; + + if (n < 1 || n > t->ncmppairs) + return -EINVAL; + + /* comparators and ranges are numbered starting with 1 as opposed + * to bits in a word */ + n--; + + if (data) + flags |= ETMAAT_DLOADSTORE; + else + flags |= ETMAAT_IEXEC; + + /* first comparator for the range */ + etm_writel(t, id, flags, ETMR_COMP_ACC_TYPE(n * 2)); + etm_writel(t, id, start, ETMR_COMP_VAL(n * 2)); + + /* second comparator is right next to it */ + etm_writel(t, id, flags, ETMR_COMP_ACC_TYPE(n * 2 + 1)); + etm_writel(t, id, end, ETMR_COMP_VAL(n * 2 + 1)); + + if (data) { + flags = exclude ? ETMVDC3_EXCLONLY : 0; + if (exclude) + n += 8; + etm_writel(t, id, flags | BIT(n), ETMR_VIEWDATACTRL3); + } else { + flags = exclude ? ETMTE_INCLEXCL : 0; + etm_writel(t, id, flags | (1 << n), ETMR_TRACEENCTRL); + } + + return 0; +} + +static int trace_start_etm(struct tracectx *t, int id) +{ + u32 v; + unsigned long timeout = TRACER_TIMEOUT; + + v = ETMCTRL_OPTS | ETMCTRL_PROGRAM | ETMCTRL_PORTSIZE(t->etm_portsz); + v |= ETMCTRL_CONTEXTIDSIZE(t->etm_contextid_size); + + if (t->flags & TRACER_CYCLE_ACC) + v |= ETMCTRL_CYCLEACCURATE; + + if (t->flags & TRACER_BRANCHOUTPUT) + v |= ETMCTRL_BRANCH_OUTPUT; + + if (t->flags & TRACER_TRACE_DATA) + v |= ETMCTRL_DATA_DO_ADDR; + + if (t->flags & TRACER_TIMESTAMP) + v |= ETMCTRL_TIMESTAMP_EN; + + if (t->flags & TRACER_RETURN_STACK) + v |= ETMCTRL_RETURN_STACK_EN; + + etm_unlock(t, id); + + etm_writel(t, id, v, ETMR_CTRL); + + while (!(etm_readl(t, id, ETMR_CTRL) & ETMCTRL_PROGRAM) && --timeout) + ; + if (!timeout) { + dev_dbg(t->dev, "Waiting for progbit to assert timed out\n"); + etm_lock(t, id); + return -EFAULT; + } + + if (t->range_start || t->range_end) + etm_setup_address_range(t, id, 1, + t->range_start, t->range_end, 0, 0); + else + etm_writel(t, id, ETMTE_INCLEXCL, ETMR_TRACEENCTRL); + + etm_writel(t, id, 0, ETMR_TRACEENCTRL2); + etm_writel(t, id, 0, ETMR_TRACESSCTRL); + etm_writel(t, id, 0x6f, ETMR_TRACEENEVT); + + etm_writel(t, id, 0, ETMR_VIEWDATACTRL1); + etm_writel(t, id, 0, ETMR_VIEWDATACTRL2); + + if (t->data_range_start || t->data_range_end) + etm_setup_address_range(t, id, 2, t->data_range_start, + t->data_range_end, 0, 1); + else + etm_writel(t, id, ETMVDC3_EXCLONLY, ETMR_VIEWDATACTRL3); + + etm_writel(t, id, 0x6f, ETMR_VIEWDATAEVT); + + v &= ~ETMCTRL_PROGRAM; + v |= ETMCTRL_PORTSEL; + + etm_writel(t, id, v, ETMR_CTRL); + + timeout = TRACER_TIMEOUT; + while (etm_readl(t, id, ETMR_CTRL) & ETMCTRL_PROGRAM && --timeout) + ; + if (!timeout) { + dev_dbg(t->dev, "Waiting for progbit to deassert timed out\n"); + etm_lock(t, id); + return -EFAULT; + } + + etm_lock(t, id); + return 0; +} + +static int trace_start(struct tracectx *t) +{ + int ret; + int id; + u32 etb_fc = t->etb_fc; + + etb_unlock(t); + + t->dump_initial_etb = false; + etb_writel(t, 0, ETBR_WRITEADDR); + etb_writel(t, etb_fc, ETBR_FORMATTERCTRL); + etb_writel(t, 1, ETBR_CTRL); + + etb_lock(t); + + /* configure etm(s) */ + for (id = 0; id < t->etm_regs_count; id++) { + ret = trace_start_etm(t, id); + if (ret) + return ret; + } + + t->flags |= TRACER_RUNNING; + + return 0; +} + +static int trace_stop_etm(struct tracectx *t, int id) +{ + unsigned long timeout = TRACER_TIMEOUT; + + etm_unlock(t, id); + + etm_writel(t, id, 0x440, ETMR_CTRL); + while (!(etm_readl(t, id, ETMR_CTRL) & ETMCTRL_PROGRAM) && --timeout) + ; + if (!timeout) { + dev_err(t->dev, + "etm%d: Waiting for progbit to assert timed out\n", + id); + etm_lock(t, id); + return -EFAULT; + } + + etm_lock(t, id); + return 0; +} + +static int trace_power_down_etm(struct tracectx *t, int id) +{ + unsigned long timeout = TRACER_TIMEOUT; + etm_unlock(t, id); + while (!(etm_readl(t, id, ETMR_STATUS) & ETMST_PROGBIT) && --timeout) + ; + if (!timeout) { + dev_err(t->dev, "etm%d: Waiting for status progbit to assert timed out\n", + id); + etm_lock(t, id); + return -EFAULT; + } + + etm_writel(t, id, 0x441, ETMR_CTRL); + + etm_lock(t, id); + return 0; +} + +static int trace_stop(struct tracectx *t) +{ + int id; + unsigned long timeout = TRACER_TIMEOUT; + u32 etb_fc = t->etb_fc; + + for (id = 0; id < t->etm_regs_count; id++) + trace_stop_etm(t, id); + + for (id = 0; id < t->etm_regs_count; id++) + trace_power_down_etm(t, id); + + etb_unlock(t); + if (etb_fc) { + etb_fc |= ETBFF_STOPFL; + etb_writel(t, t->etb_fc, ETBR_FORMATTERCTRL); + } + etb_writel(t, etb_fc | ETBFF_MANUAL_FLUSH, ETBR_FORMATTERCTRL); + + timeout = TRACER_TIMEOUT; + while (etb_readl(t, ETBR_FORMATTERCTRL) & + ETBFF_MANUAL_FLUSH && --timeout) + ; + if (!timeout) { + dev_dbg(t->dev, "Waiting for formatter flush to commence " + "timed out\n"); + etb_lock(t); + return -EFAULT; + } + + etb_writel(t, 0, ETBR_CTRL); + + etb_lock(t); + + t->flags &= ~TRACER_RUNNING; + + return 0; +} + +static int etb_getdatalen(struct tracectx *t) +{ + u32 v; + int wp; + + v = etb_readl(t, ETBR_STATUS); + + if (v & 1) + return t->etb_bufsz; + + wp = etb_readl(t, ETBR_WRITEADDR); + return wp; +} + +/* sysrq+v will always stop the running trace and leave it at that */ +static void etm_dump(void) +{ + struct tracectx *t = &tracer; + u32 first = 0; + int length; + + if (!t->etb_regs) { + printk(KERN_INFO "No tracing hardware found\n"); + return; + } + + if (trace_isrunning(t)) + trace_stop(t); + + etb_unlock(t); + + length = etb_getdatalen(t); + + if (length == t->etb_bufsz) + first = etb_readl(t, ETBR_WRITEADDR); + + etb_writel(t, first, ETBR_READADDR); + + printk(KERN_INFO "Trace buffer contents length: %d\n", length); + printk(KERN_INFO "--- ETB buffer begin ---\n"); + for (; length; length--) + printk("%08x", cpu_to_be32(etb_readl(t, ETBR_READMEM))); + printk(KERN_INFO "\n--- ETB buffer end ---\n"); + + etb_lock(t); +} + +static void sysrq_etm_dump(int key) +{ + if (!mutex_trylock(&tracer.mutex)) { + printk(KERN_INFO "Tracing hardware busy\n"); + return; + } + dev_dbg(tracer.dev, "Dumping ETB buffer\n"); + etm_dump(); + mutex_unlock(&tracer.mutex); +} + +static struct sysrq_key_op sysrq_etm_op = { + .handler = sysrq_etm_dump, + .help_msg = "ETM buffer dump", + .action_msg = "etm", +}; + +static int etb_open(struct inode *inode, struct file *file) +{ + if (!tracer.etb_regs) + return -ENODEV; + + file->private_data = &tracer; + + return nonseekable_open(inode, file); +} + +static ssize_t etb_read(struct file *file, char __user *data, + size_t len, loff_t *ppos) +{ + int total, i; + long length; + struct tracectx *t = file->private_data; + u32 first = 0; + u32 *buf; + int wpos; + int skip; + long wlength; + loff_t pos = *ppos; + + mutex_lock(&t->mutex); + + if (trace_isrunning(t)) { + length = 0; + goto out; + } + + etb_unlock(t); + + total = etb_getdatalen(t); + if (total == 0 && t->dump_initial_etb) + total = t->etb_bufsz; + if (total == t->etb_bufsz) + first = etb_readl(t, ETBR_WRITEADDR); + + if (pos > total * 4) { + skip = 0; + wpos = total; + } else { + skip = (int)pos % 4; + wpos = (int)pos / 4; + } + total -= wpos; + first = (first + wpos) % t->etb_bufsz; + + etb_writel(t, first, ETBR_READADDR); + + wlength = min(total, DIV_ROUND_UP(skip + (int)len, 4)); + length = min(total * 4 - skip, (int)len); + buf = vmalloc(wlength * 4); + + dev_dbg(t->dev, "ETB read %ld bytes to %lld from %ld words at %d\n", + length, pos, wlength, first); + dev_dbg(t->dev, "ETB buffer length: %d\n", total + wpos); + dev_dbg(t->dev, "ETB status reg: %x\n", etb_readl(t, ETBR_STATUS)); + for (i = 0; i < wlength; i++) + buf[i] = etb_readl(t, ETBR_READMEM); + + etb_lock(t); + + length -= copy_to_user(data, (u8 *)buf + skip, length); + vfree(buf); + *ppos = pos + length; + +out: + mutex_unlock(&t->mutex); + + return length; +} + +static int etb_release(struct inode *inode, struct file *file) +{ + /* there's nothing to do here, actually */ + return 0; +} + +static const struct file_operations etb_fops = { + .owner = THIS_MODULE, + .read = etb_read, + .open = etb_open, + .release = etb_release, + .llseek = no_llseek, +}; + +static struct miscdevice etb_miscdev = { + .name = "tracebuf", + .minor = 0, + .fops = &etb_fops, +}; + +static int __devinit etb_probe(struct amba_device *dev, const struct amba_id *id) +{ + struct tracectx *t = &tracer; + int ret = 0; + + ret = amba_request_regions(dev, NULL); + if (ret) + goto out; + + mutex_lock(&t->mutex); + t->etb_regs = ioremap_nocache(dev->res.start, resource_size(&dev->res)); + if (!t->etb_regs) { + ret = -ENOMEM; + goto out_release; + } + + t->dev = &dev->dev; + t->dump_initial_etb = true; + amba_set_drvdata(dev, t); + + etb_unlock(t); + t->etb_bufsz = etb_readl(t, ETBR_DEPTH); + dev_dbg(&dev->dev, "Size: %x\n", t->etb_bufsz); + + /* make sure trace capture is disabled */ + etb_writel(t, 0, ETBR_CTRL); + etb_writel(t, 0x1000, ETBR_FORMATTERCTRL); + etb_lock(t); + mutex_unlock(&t->mutex); + + etb_miscdev.parent = &dev->dev; + + ret = misc_register(&etb_miscdev); + if (ret) + goto out_unmap; + + /* Get optional clock. Currently used to select clock source on omap3 */ + t->emu_clk = clk_get(&dev->dev, "emu_src_ck"); + if (IS_ERR(t->emu_clk)) + dev_dbg(&dev->dev, "Failed to obtain emu_src_ck.\n"); + else + clk_enable(t->emu_clk); + + dev_dbg(&dev->dev, "ETB AMBA driver initialized.\n"); + +out: + return ret; + +out_unmap: + mutex_lock(&t->mutex); + amba_set_drvdata(dev, NULL); + iounmap(t->etb_regs); + t->etb_regs = NULL; + +out_release: + mutex_unlock(&t->mutex); + amba_release_regions(dev); + + return ret; +} + +static int etb_remove(struct amba_device *dev) +{ + struct tracectx *t = amba_get_drvdata(dev); + + amba_set_drvdata(dev, NULL); + + iounmap(t->etb_regs); + t->etb_regs = NULL; + + if (!IS_ERR(t->emu_clk)) { + clk_disable(t->emu_clk); + clk_put(t->emu_clk); + } + + amba_release_regions(dev); + + return 0; +} + +static struct amba_id etb_ids[] = { + { + .id = 0x0003b907, + .mask = 0x0007ffff, + }, + { 0, 0 }, +}; + +static struct amba_driver etb_driver = { + .drv = { + .name = "etb", + .owner = THIS_MODULE, + }, + .probe = etb_probe, + .remove = etb_remove, + .id_table = etb_ids, +}; + +/* use a sysfs file "trace_running" to start/stop tracing */ +static ssize_t trace_running_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%x\n", trace_isrunning(&tracer)); +} + +static ssize_t trace_running_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned int value; + int ret; + + if (sscanf(buf, "%u", &value) != 1) + return -EINVAL; + + mutex_lock(&tracer.mutex); + if (!tracer.etb_regs) + ret = -ENODEV; + else + ret = value ? trace_start(&tracer) : trace_stop(&tracer); + mutex_unlock(&tracer.mutex); + + return ret ? : n; +} + +static struct kobj_attribute trace_running_attr = + __ATTR(trace_running, 0644, trace_running_show, trace_running_store); + +static ssize_t trace_info_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + u32 etb_wa, etb_ra, etb_st, etb_fc, etm_ctrl, etm_st; + int datalen; + int id; + int ret; + + mutex_lock(&tracer.mutex); + if (tracer.etb_regs) { + etb_unlock(&tracer); + datalen = etb_getdatalen(&tracer); + etb_wa = etb_readl(&tracer, ETBR_WRITEADDR); + etb_ra = etb_readl(&tracer, ETBR_READADDR); + etb_st = etb_readl(&tracer, ETBR_STATUS); + etb_fc = etb_readl(&tracer, ETBR_FORMATTERCTRL); + etb_lock(&tracer); + } else { + etb_wa = etb_ra = etb_st = etb_fc = ~0; + datalen = -1; + } + + ret = sprintf(buf, "Trace buffer len: %d\nComparator pairs: %d\n" + "ETBR_WRITEADDR:\t%08x\n" + "ETBR_READADDR:\t%08x\n" + "ETBR_STATUS:\t%08x\n" + "ETBR_FORMATTERCTRL:\t%08x\n", + datalen, + tracer.ncmppairs, + etb_wa, + etb_ra, + etb_st, + etb_fc + ); + + for (id = 0; id < tracer.etm_regs_count; id++) { + etm_unlock(&tracer, id); + etm_ctrl = etm_readl(&tracer, id, ETMR_CTRL); + etm_st = etm_readl(&tracer, id, ETMR_STATUS); + etm_lock(&tracer, id); + ret += sprintf(buf + ret, "ETMR_CTRL:\t%08x\n" + "ETMR_STATUS:\t%08x\n", + etm_ctrl, + etm_st + ); + } + mutex_unlock(&tracer.mutex); + + return ret; +} + +static struct kobj_attribute trace_info_attr = + __ATTR(trace_info, 0444, trace_info_show, NULL); + +static ssize_t trace_mode_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d %d\n", + !!(tracer.flags & TRACER_CYCLE_ACC), + tracer.etm_portsz); +} + +static ssize_t trace_mode_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned int cycacc, portsz; + + if (sscanf(buf, "%u %u", &cycacc, &portsz) != 2) + return -EINVAL; + + mutex_lock(&tracer.mutex); + if (cycacc) + tracer.flags |= TRACER_CYCLE_ACC; + else + tracer.flags &= ~TRACER_CYCLE_ACC; + + tracer.etm_portsz = portsz & 0x0f; + mutex_unlock(&tracer.mutex); + + return n; +} + +static struct kobj_attribute trace_mode_attr = + __ATTR(trace_mode, 0644, trace_mode_show, trace_mode_store); + +static ssize_t trace_contextid_size_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + /* 0: No context id tracing, 1: One byte, 2: Two bytes, 3: Four bytes */ + return sprintf(buf, "%d\n", (1 << tracer.etm_contextid_size) >> 1); +} + +static ssize_t trace_contextid_size_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned int contextid_size; + + if (sscanf(buf, "%u", &contextid_size) != 1) + return -EINVAL; + + if (contextid_size == 3 || contextid_size > 4) + return -EINVAL; + + mutex_lock(&tracer.mutex); + tracer.etm_contextid_size = fls(contextid_size); + mutex_unlock(&tracer.mutex); + + return n; +} + +static struct kobj_attribute trace_contextid_size_attr = + __ATTR(trace_contextid_size, 0644, + trace_contextid_size_show, trace_contextid_size_store); + +static ssize_t trace_branch_output_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", !!(tracer.flags & TRACER_BRANCHOUTPUT)); +} + +static ssize_t trace_branch_output_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned int branch_output; + + if (sscanf(buf, "%u", &branch_output) != 1) + return -EINVAL; + + mutex_lock(&tracer.mutex); + if (branch_output) { + tracer.flags |= TRACER_BRANCHOUTPUT; + /* Branch broadcasting is incompatible with the return stack */ + tracer.flags &= ~TRACER_RETURN_STACK; + } else { + tracer.flags &= ~TRACER_BRANCHOUTPUT; + } + mutex_unlock(&tracer.mutex); + + return n; +} + +static struct kobj_attribute trace_branch_output_attr = + __ATTR(trace_branch_output, 0644, + trace_branch_output_show, trace_branch_output_store); + +static ssize_t trace_return_stack_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", !!(tracer.flags & TRACER_RETURN_STACK)); +} + +static ssize_t trace_return_stack_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned int return_stack; + + if (sscanf(buf, "%u", &return_stack) != 1) + return -EINVAL; + + mutex_lock(&tracer.mutex); + if (return_stack) { + tracer.flags |= TRACER_RETURN_STACK; + /* Return stack is incompatible with branch broadcasting */ + tracer.flags &= ~TRACER_BRANCHOUTPUT; + } else { + tracer.flags &= ~TRACER_RETURN_STACK; + } + mutex_unlock(&tracer.mutex); + + return n; +} + +static struct kobj_attribute trace_return_stack_attr = + __ATTR(trace_return_stack, 0644, + trace_return_stack_show, trace_return_stack_store); + +static ssize_t trace_timestamp_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%d\n", !!(tracer.flags & TRACER_TIMESTAMP)); +} + +static ssize_t trace_timestamp_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned int timestamp; + + if (sscanf(buf, "%u", ×tamp) != 1) + return -EINVAL; + + mutex_lock(&tracer.mutex); + if (timestamp) + tracer.flags |= TRACER_TIMESTAMP; + else + tracer.flags &= ~TRACER_TIMESTAMP; + mutex_unlock(&tracer.mutex); + + return n; +} + +static struct kobj_attribute trace_timestamp_attr = + __ATTR(trace_timestamp, 0644, + trace_timestamp_show, trace_timestamp_store); + +static ssize_t trace_range_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%08lx %08lx\n", + tracer.range_start, tracer.range_end); +} + +static ssize_t trace_range_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned long range_start, range_end; + + if (sscanf(buf, "%lx %lx", &range_start, &range_end) != 2) + return -EINVAL; + + mutex_lock(&tracer.mutex); + tracer.range_start = range_start; + tracer.range_end = range_end; + mutex_unlock(&tracer.mutex); + + return n; +} + + +static struct kobj_attribute trace_range_attr = + __ATTR(trace_range, 0644, trace_range_show, trace_range_store); + +static ssize_t trace_data_range_show(struct kobject *kobj, + struct kobj_attribute *attr, + char *buf) +{ + unsigned long range_start; + u64 range_end; + mutex_lock(&tracer.mutex); + range_start = tracer.data_range_start; + range_end = tracer.data_range_end; + if (!range_end && (tracer.flags & TRACER_TRACE_DATA)) + range_end = 0x100000000ULL; + mutex_unlock(&tracer.mutex); + return sprintf(buf, "%08lx %08llx\n", range_start, range_end); +} + +static ssize_t trace_data_range_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned long range_start; + u64 range_end; + + if (sscanf(buf, "%lx %llx", &range_start, &range_end) != 2) + return -EINVAL; + + mutex_lock(&tracer.mutex); + tracer.data_range_start = range_start; + tracer.data_range_end = (unsigned long)range_end; + if (range_end) + tracer.flags |= TRACER_TRACE_DATA; + else + tracer.flags &= ~TRACER_TRACE_DATA; + mutex_unlock(&tracer.mutex); + + return n; +} + + +static struct kobj_attribute trace_data_range_attr = + __ATTR(trace_data_range, 0644, + trace_data_range_show, trace_data_range_store); + +static int __devinit etm_probe(struct amba_device *dev, const struct amba_id *id) +{ + struct tracectx *t = &tracer; + int ret = 0; + void __iomem **new_regs; + int new_count; + u32 etmccr; + u32 etmidr; + u32 etmccer = 0; + u8 etm_version = 0; + + mutex_lock(&t->mutex); + new_count = t->etm_regs_count + 1; + new_regs = krealloc(t->etm_regs, + sizeof(t->etm_regs[0]) * new_count, GFP_KERNEL); + + if (!new_regs) { + dev_dbg(&dev->dev, "Failed to allocate ETM register array\n"); + ret = -ENOMEM; + goto out; + } + t->etm_regs = new_regs; + + ret = amba_request_regions(dev, NULL); + if (ret) + goto out; + + t->etm_regs[t->etm_regs_count] = + ioremap_nocache(dev->res.start, resource_size(&dev->res)); + if (!t->etm_regs[t->etm_regs_count]) { + ret = -ENOMEM; + goto out_release; + } + + amba_set_drvdata(dev, t->etm_regs[t->etm_regs_count]); + + t->flags = TRACER_CYCLE_ACC | TRACER_TRACE_DATA | TRACER_BRANCHOUTPUT; + t->etm_portsz = 1; + t->etm_contextid_size = 3; + + etm_unlock(t, t->etm_regs_count); + (void)etm_readl(t, t->etm_regs_count, ETMMR_PDSR); + /* dummy first read */ + (void)etm_readl(&tracer, t->etm_regs_count, ETMMR_OSSRR); + + etmccr = etm_readl(t, t->etm_regs_count, ETMR_CONFCODE); + t->ncmppairs = etmccr & 0xf; + if (etmccr & ETMCCR_ETMIDR_PRESENT) { + etmidr = etm_readl(t, t->etm_regs_count, ETMR_ID); + etm_version = ETMIDR_VERSION(etmidr); + if (etm_version >= ETMIDR_VERSION_3_1) + etmccer = etm_readl(t, t->etm_regs_count, ETMR_CCE); + } + etm_writel(t, t->etm_regs_count, 0x441, ETMR_CTRL); + etm_writel(t, t->etm_regs_count, new_count, ETMR_TRACEIDR); + etm_lock(t, t->etm_regs_count); + + ret = sysfs_create_file(&dev->dev.kobj, + &trace_running_attr.attr); + if (ret) + goto out_unmap; + + /* failing to create any of these two is not fatal */ + ret = sysfs_create_file(&dev->dev.kobj, &trace_info_attr.attr); + if (ret) + dev_dbg(&dev->dev, "Failed to create trace_info in sysfs\n"); + + ret = sysfs_create_file(&dev->dev.kobj, &trace_mode_attr.attr); + if (ret) + dev_dbg(&dev->dev, "Failed to create trace_mode in sysfs\n"); + + ret = sysfs_create_file(&dev->dev.kobj, + &trace_contextid_size_attr.attr); + if (ret) + dev_dbg(&dev->dev, + "Failed to create trace_contextid_size in sysfs\n"); + + ret = sysfs_create_file(&dev->dev.kobj, + &trace_branch_output_attr.attr); + if (ret) + dev_dbg(&dev->dev, + "Failed to create trace_branch_output in sysfs\n"); + + if (etmccer & ETMCCER_RETURN_STACK_IMPLEMENTED) { + ret = sysfs_create_file(&dev->dev.kobj, + &trace_return_stack_attr.attr); + if (ret) + dev_dbg(&dev->dev, + "Failed to create trace_return_stack in sysfs\n"); + } + + if (etmccer & ETMCCER_TIMESTAMPING_IMPLEMENTED) { + ret = sysfs_create_file(&dev->dev.kobj, + &trace_timestamp_attr.attr); + if (ret) + dev_dbg(&dev->dev, + "Failed to create trace_timestamp in sysfs\n"); + } + + ret = sysfs_create_file(&dev->dev.kobj, &trace_range_attr.attr); + if (ret) + dev_dbg(&dev->dev, "Failed to create trace_range in sysfs\n"); + + if (etm_version < ETMIDR_VERSION_PFT_1_0) { + ret = sysfs_create_file(&dev->dev.kobj, + &trace_data_range_attr.attr); + if (ret) + dev_dbg(&dev->dev, + "Failed to create trace_data_range in sysfs\n"); + } else { + tracer.flags &= ~TRACER_TRACE_DATA; + } + + dev_dbg(&dev->dev, "ETM AMBA driver initialized.\n"); + + /* Enable formatter if there are multiple trace sources */ + if (new_count > 1) + t->etb_fc = ETBFF_ENFCONT | ETBFF_ENFTC; + + t->etm_regs_count = new_count; + +out: + mutex_unlock(&t->mutex); + return ret; + +out_unmap: + amba_set_drvdata(dev, NULL); + iounmap(t->etm_regs[t->etm_regs_count]); + +out_release: + amba_release_regions(dev); + + mutex_unlock(&t->mutex); + return ret; +} + +static int etm_remove(struct amba_device *dev) +{ + int i; + struct tracectx *t = &tracer; + void __iomem *etm_regs = amba_get_drvdata(dev); + + sysfs_remove_file(&dev->dev.kobj, &trace_running_attr.attr); + sysfs_remove_file(&dev->dev.kobj, &trace_info_attr.attr); + sysfs_remove_file(&dev->dev.kobj, &trace_mode_attr.attr); + sysfs_remove_file(&dev->dev.kobj, &trace_range_attr.attr); + sysfs_remove_file(&dev->dev.kobj, &trace_data_range_attr.attr); + + amba_set_drvdata(dev, NULL); + + mutex_lock(&t->mutex); + for (i = 0; i < t->etm_regs_count; i++) + if (t->etm_regs[i] == etm_regs) + break; + for (; i < t->etm_regs_count - 1; i++) + t->etm_regs[i] = t->etm_regs[i + 1]; + t->etm_regs_count--; + if (!t->etm_regs_count) { + kfree(t->etm_regs); + t->etm_regs = NULL; + } + mutex_unlock(&t->mutex); + + iounmap(etm_regs); + amba_release_regions(dev); + + return 0; +} + +static struct amba_id etm_ids[] = { + { + .id = 0x0003b921, + .mask = 0x0007ffff, + }, + { + .id = 0x0003b950, + .mask = 0x0007ffff, + }, + { 0, 0 }, +}; + +static struct amba_driver etm_driver = { + .drv = { + .name = "etm", + .owner = THIS_MODULE, + }, + .probe = etm_probe, + .remove = etm_remove, + .id_table = etm_ids, +}; + +static int __init etm_init(void) +{ + int retval; + + mutex_init(&tracer.mutex); + + retval = amba_driver_register(&etb_driver); + if (retval) { + printk(KERN_ERR "Failed to register etb\n"); + return retval; + } + + retval = amba_driver_register(&etm_driver); + if (retval) { + amba_driver_unregister(&etb_driver); + printk(KERN_ERR "Failed to probe etm\n"); + return retval; + } + + /* not being able to install this handler is not fatal */ + (void)register_sysrq_key('v', &sysrq_etm_op); + + return 0; +} + +device_initcall(etm_init); + diff --git a/arch/arm/kernel/fiq.c b/arch/arm/kernel/fiq.c new file mode 100644 index 00000000..4c164ece --- /dev/null +++ b/arch/arm/kernel/fiq.c @@ -0,0 +1,146 @@ +/* + * linux/arch/arm/kernel/fiq.c + * + * Copyright (C) 1998 Russell King + * Copyright (C) 1998, 1999 Phil Blundell + * + * FIQ support written by Philip Blundell <philb@gnu.org>, 1998. + * + * FIQ support re-written by Russell King to be more generic + * + * We now properly support a method by which the FIQ handlers can + * be stacked onto the vector. We still do not support sharing + * the FIQ vector itself. + * + * Operation is as follows: + * 1. Owner A claims FIQ: + * - default_fiq relinquishes control. + * 2. Owner A: + * - inserts code. + * - sets any registers, + * - enables FIQ. + * 3. Owner B claims FIQ: + * - if owner A has a relinquish function. + * - disable FIQs. + * - saves any registers. + * - returns zero. + * 4. Owner B: + * - inserts code. + * - sets any registers, + * - enables FIQ. + * 5. Owner B releases FIQ: + * - Owner A is asked to reacquire FIQ: + * - inserts code. + * - restores saved registers. + * - enables FIQ. + * 6. Goto 3 + */ +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/seq_file.h> + +#include <asm/cacheflush.h> +#include <asm/fiq.h> +#include <asm/irq.h> +#include <asm/system.h> +#include <asm/traps.h> + +static unsigned long no_fiq_insn; + +/* Default reacquire function + * - we always relinquish FIQ control + * - we always reacquire FIQ control + */ +static int fiq_def_op(void *ref, int relinquish) +{ + if (!relinquish) + set_fiq_handler(&no_fiq_insn, sizeof(no_fiq_insn)); + + return 0; +} + +static struct fiq_handler default_owner = { + .name = "default", + .fiq_op = fiq_def_op, +}; + +static struct fiq_handler *current_fiq = &default_owner; + +int show_fiq_list(struct seq_file *p, int prec) +{ + if (current_fiq != &default_owner) + seq_printf(p, "%*s: %s\n", prec, "FIQ", + current_fiq->name); + + return 0; +} + +void set_fiq_handler(void *start, unsigned int length) +{ +#if defined(CONFIG_CPU_USE_DOMAINS) + memcpy((void *)0xffff001c, start, length); +#else + memcpy(vectors_page + 0x1c, start, length); +#endif + flush_icache_range(0xffff001c, 0xffff001c + length); + if (!vectors_high()) + flush_icache_range(0x1c, 0x1c + length); +} + +int claim_fiq(struct fiq_handler *f) +{ + int ret = 0; + + if (current_fiq) { + ret = -EBUSY; + + if (current_fiq->fiq_op != NULL) + ret = current_fiq->fiq_op(current_fiq->dev_id, 1); + } + + if (!ret) { + f->next = current_fiq; + current_fiq = f; + } + + return ret; +} + +void release_fiq(struct fiq_handler *f) +{ + if (current_fiq != f) { + printk(KERN_ERR "%s FIQ trying to release %s FIQ\n", + f->name, current_fiq->name); + dump_stack(); + return; + } + + do + current_fiq = current_fiq->next; + while (current_fiq->fiq_op(current_fiq->dev_id, 0)); +} + +void enable_fiq(int fiq) +{ + enable_irq(fiq + FIQ_START); +} + +void disable_fiq(int fiq) +{ + disable_irq(fiq + FIQ_START); +} + +EXPORT_SYMBOL(set_fiq_handler); +EXPORT_SYMBOL(__set_fiq_regs); /* defined in fiqasm.S */ +EXPORT_SYMBOL(__get_fiq_regs); /* defined in fiqasm.S */ +EXPORT_SYMBOL(claim_fiq); +EXPORT_SYMBOL(release_fiq); +EXPORT_SYMBOL(enable_fiq); +EXPORT_SYMBOL(disable_fiq); + +void __init init_FIQ(void) +{ + no_fiq_insn = *(unsigned long *)0xffff001c; +} diff --git a/arch/arm/kernel/fiqasm.S b/arch/arm/kernel/fiqasm.S new file mode 100644 index 00000000..207f9d65 --- /dev/null +++ b/arch/arm/kernel/fiqasm.S @@ -0,0 +1,49 @@ +/* + * linux/arch/arm/kernel/fiqasm.S + * + * Derived from code originally in linux/arch/arm/kernel/fiq.c: + * + * Copyright (C) 1998 Russell King + * Copyright (C) 1998, 1999 Phil Blundell + * Copyright (C) 2011, Linaro Limited + * + * FIQ support written by Philip Blundell <philb@gnu.org>, 1998. + * + * FIQ support re-written by Russell King to be more generic + * + * v7/Thumb-2 compatibility modifications by Linaro Limited, 2011. + */ + +#include <linux/linkage.h> +#include <asm/assembler.h> + +/* + * Taking an interrupt in FIQ mode is death, so both these functions + * disable irqs for the duration. + */ + +ENTRY(__set_fiq_regs) + mov r2, #PSR_I_BIT | PSR_F_BIT | FIQ_MODE + mrs r1, cpsr + msr cpsr_c, r2 @ select FIQ mode + mov r0, r0 @ avoid hazard prior to ARMv4 + ldmia r0!, {r8 - r12} + ldr sp, [r0], #4 + ldr lr, [r0] + msr cpsr_c, r1 @ return to SVC mode + mov r0, r0 @ avoid hazard prior to ARMv4 + mov pc, lr +ENDPROC(__set_fiq_regs) + +ENTRY(__get_fiq_regs) + mov r2, #PSR_I_BIT | PSR_F_BIT | FIQ_MODE + mrs r1, cpsr + msr cpsr_c, r2 @ select FIQ mode + mov r0, r0 @ avoid hazard prior to ARMv4 + stmia r0!, {r8 - r12} + str sp, [r0], #4 + str lr, [r0] + msr cpsr_c, r1 @ return to SVC mode + mov r0, r0 @ avoid hazard prior to ARMv4 + mov pc, lr +ENDPROC(__get_fiq_regs) diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c new file mode 100644 index 00000000..c0062ad1 --- /dev/null +++ b/arch/arm/kernel/ftrace.c @@ -0,0 +1,288 @@ +/* + * Dynamic function tracing support. + * + * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com> + * Copyright (C) 2010 Rabin Vincent <rabin@rab.in> + * + * For licencing details, see COPYING. + * + * Defines low-level handling of mcount calls when the kernel + * is compiled with the -pg flag. When using dynamic ftrace, the + * mcount call-sites get patched with NOP till they are enabled. + * All code mutation routines here are called under stop_machine(). + */ + +#include <linux/ftrace.h> +#include <linux/uaccess.h> + +#include <asm/cacheflush.h> +#include <asm/ftrace.h> + +#ifdef CONFIG_THUMB2_KERNEL +#define NOP 0xeb04f85d /* pop.w {lr} */ +#else +#define NOP 0xe8bd4000 /* pop {lr} */ +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE +#ifdef CONFIG_OLD_MCOUNT +#define OLD_MCOUNT_ADDR ((unsigned long) mcount) +#define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old) + +#define OLD_NOP 0xe1a00000 /* mov r0, r0 */ + +static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) +{ + return rec->arch.old_mcount ? OLD_NOP : NOP; +} + +static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) +{ + if (!rec->arch.old_mcount) + return addr; + + if (addr == MCOUNT_ADDR) + addr = OLD_MCOUNT_ADDR; + else if (addr == FTRACE_ADDR) + addr = OLD_FTRACE_ADDR; + + return addr; +} +#else +static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec) +{ + return NOP; +} + +static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr) +{ + return addr; +} +#endif + +#ifdef CONFIG_THUMB2_KERNEL +static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr, + bool link) +{ + unsigned long s, j1, j2, i1, i2, imm10, imm11; + unsigned long first, second; + long offset; + + offset = (long)addr - (long)(pc + 4); + if (offset < -16777216 || offset > 16777214) { + WARN_ON_ONCE(1); + return 0; + } + + s = (offset >> 24) & 0x1; + i1 = (offset >> 23) & 0x1; + i2 = (offset >> 22) & 0x1; + imm10 = (offset >> 12) & 0x3ff; + imm11 = (offset >> 1) & 0x7ff; + + j1 = (!i1) ^ s; + j2 = (!i2) ^ s; + + first = 0xf000 | (s << 10) | imm10; + second = 0x9000 | (j1 << 13) | (j2 << 11) | imm11; + if (link) + second |= 1 << 14; + + return (second << 16) | first; +} +#else +static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr, + bool link) +{ + unsigned long opcode = 0xea000000; + long offset; + + if (link) + opcode |= 1 << 24; + + offset = (long)addr - (long)(pc + 8); + if (unlikely(offset < -33554432 || offset > 33554428)) { + /* Can't generate branches that far (from ARM ARM). Ftrace + * doesn't generate branches outside of kernel text. + */ + WARN_ON_ONCE(1); + return 0; + } + + offset = (offset >> 2) & 0x00ffffff; + + return opcode | offset; +} +#endif + +static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr) +{ + return ftrace_gen_branch(pc, addr, true); +} + +static int ftrace_modify_code(unsigned long pc, unsigned long old, + unsigned long new) +{ + unsigned long replaced; + + if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE)) + return -EFAULT; + + if (replaced != old) + return -EINVAL; + + if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE)) + return -EPERM; + + flush_icache_range(pc, pc + MCOUNT_INSN_SIZE); + + return 0; +} + +int ftrace_update_ftrace_func(ftrace_func_t func) +{ + unsigned long pc, old; + unsigned long new; + int ret; + + pc = (unsigned long)&ftrace_call; + memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE); + new = ftrace_call_replace(pc, (unsigned long)func); + + ret = ftrace_modify_code(pc, old, new); + +#ifdef CONFIG_OLD_MCOUNT + if (!ret) { + pc = (unsigned long)&ftrace_call_old; + memcpy(&old, &ftrace_call_old, MCOUNT_INSN_SIZE); + new = ftrace_call_replace(pc, (unsigned long)func); + + ret = ftrace_modify_code(pc, old, new); + } +#endif + + return ret; +} + +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long new, old; + unsigned long ip = rec->ip; + + old = ftrace_nop_replace(rec); + new = ftrace_call_replace(ip, adjust_address(rec, addr)); + + return ftrace_modify_code(rec->ip, old, new); +} + +int ftrace_make_nop(struct module *mod, + struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long ip = rec->ip; + unsigned long old; + unsigned long new; + int ret; + + old = ftrace_call_replace(ip, adjust_address(rec, addr)); + new = ftrace_nop_replace(rec); + ret = ftrace_modify_code(ip, old, new); + +#ifdef CONFIG_OLD_MCOUNT + if (ret == -EINVAL && addr == MCOUNT_ADDR) { + rec->arch.old_mcount = true; + + old = ftrace_call_replace(ip, adjust_address(rec, addr)); + new = ftrace_nop_replace(rec); + ret = ftrace_modify_code(ip, old, new); + } +#endif + + return ret; +} + +int __init ftrace_dyn_arch_init(void *data) +{ + *(unsigned long *)data = 0; + + return 0; +} +#endif /* CONFIG_DYNAMIC_FTRACE */ + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long) &return_to_handler; + struct ftrace_graph_ent trace; + unsigned long old; + int err; + + if (unlikely(atomic_read(¤t->tracing_graph_pause))) + return; + + old = *parent; + *parent = return_hooker; + + err = ftrace_push_return_trace(old, self_addr, &trace.depth, + frame_pointer); + if (err == -EBUSY) { + *parent = old; + return; + } + + trace.func = self_addr; + + /* Only trace if the calling function expects to */ + if (!ftrace_graph_entry(&trace)) { + current->curr_ret_stack--; + *parent = old; + } +} + +#ifdef CONFIG_DYNAMIC_FTRACE +extern unsigned long ftrace_graph_call; +extern unsigned long ftrace_graph_call_old; +extern void ftrace_graph_caller_old(void); + +static int __ftrace_modify_caller(unsigned long *callsite, + void (*func) (void), bool enable) +{ + unsigned long caller_fn = (unsigned long) func; + unsigned long pc = (unsigned long) callsite; + unsigned long branch = ftrace_gen_branch(pc, caller_fn, false); + unsigned long nop = 0xe1a00000; /* mov r0, r0 */ + unsigned long old = enable ? nop : branch; + unsigned long new = enable ? branch : nop; + + return ftrace_modify_code(pc, old, new); +} + +static int ftrace_modify_graph_caller(bool enable) +{ + int ret; + + ret = __ftrace_modify_caller(&ftrace_graph_call, + ftrace_graph_caller, + enable); + +#ifdef CONFIG_OLD_MCOUNT + if (!ret) + ret = __ftrace_modify_caller(&ftrace_graph_call_old, + ftrace_graph_caller_old, + enable); +#endif + + return ret; +} + +int ftrace_enable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(true); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + return ftrace_modify_graph_caller(false); +} +#endif /* CONFIG_DYNAMIC_FTRACE */ +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S new file mode 100644 index 00000000..854bd223 --- /dev/null +++ b/arch/arm/kernel/head-common.S @@ -0,0 +1,204 @@ +/* + * linux/arch/arm/kernel/head-common.S + * + * Copyright (C) 1994-2002 Russell King + * Copyright (c) 2003 ARM Limited + * All Rights Reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define ATAG_CORE 0x54410001 +#define ATAG_CORE_SIZE ((2*4 + 3*4) >> 2) +#define ATAG_CORE_SIZE_EMPTY ((2*4) >> 2) + +#ifdef CONFIG_CPU_BIG_ENDIAN +#define OF_DT_MAGIC 0xd00dfeed +#else +#define OF_DT_MAGIC 0xedfe0dd0 /* 0xd00dfeed in big-endian */ +#endif + +/* + * Exception handling. Something went wrong and we can't proceed. We + * ought to tell the user, but since we don't have any guarantee that + * we're even running on the right architecture, we do virtually nothing. + * + * If CONFIG_DEBUG_LL is set we try to print out something about the error + * and hope for the best (useful if bootloader fails to pass a proper + * machine ID for example). + */ + __HEAD + +/* Determine validity of the r2 atags pointer. The heuristic requires + * that the pointer be aligned, in the first 16k of physical RAM and + * that the ATAG_CORE marker is first and present. If CONFIG_OF_FLATTREE + * is selected, then it will also accept a dtb pointer. Future revisions + * of this function may be more lenient with the physical address and + * may also be able to move the ATAGS block if necessary. + * + * Returns: + * r2 either valid atags pointer, valid dtb pointer, or zero + * r5, r6 corrupted + */ +__vet_atags: + tst r2, #0x3 @ aligned? + bne 1f + + ldr r5, [r2, #0] +#ifdef CONFIG_OF_FLATTREE + ldr r6, =OF_DT_MAGIC @ is it a DTB? + cmp r5, r6 + beq 2f +#endif + cmp r5, #ATAG_CORE_SIZE @ is first tag ATAG_CORE? + cmpne r5, #ATAG_CORE_SIZE_EMPTY + bne 1f + ldr r5, [r2, #4] + ldr r6, =ATAG_CORE + cmp r5, r6 + bne 1f + +2: mov pc, lr @ atag/dtb pointer is ok + +1: mov r2, #0 + mov pc, lr +ENDPROC(__vet_atags) + +/* + * The following fragment of code is executed with the MMU on in MMU mode, + * and uses absolute addresses; this is not position independent. + * + * r0 = cp#15 control register + * r1 = machine ID + * r2 = atags/dtb pointer + * r9 = processor ID + */ + __INIT +__mmap_switched: + adr r3, __mmap_switched_data + + ldmia r3!, {r4, r5, r6, r7} + cmp r4, r5 @ Copy data segment if needed +1: cmpne r5, r6 + ldrne fp, [r4], #4 + strne fp, [r5], #4 + bne 1b + + mov fp, #0 @ Clear BSS (and zero fp) +1: cmp r6, r7 + strcc fp, [r6],#4 + bcc 1b + + ARM( ldmia r3, {r4, r5, r6, r7, sp}) + THUMB( ldmia r3, {r4, r5, r6, r7} ) + THUMB( ldr sp, [r3, #16] ) + str r9, [r4] @ Save processor ID + str r1, [r5] @ Save machine type + str r2, [r6] @ Save atags pointer + bic r4, r0, #CR_A @ Clear 'A' bit + stmia r7, {r0, r4} @ Save control register values + b start_kernel +ENDPROC(__mmap_switched) + + .align 2 + .type __mmap_switched_data, %object +__mmap_switched_data: + .long __data_loc @ r4 + .long _sdata @ r5 + .long __bss_start @ r6 + .long _end @ r7 + .long processor_id @ r4 + .long __machine_arch_type @ r5 + .long __atags_pointer @ r6 + .long cr_alignment @ r7 + .long init_thread_union + THREAD_START_SP @ sp + .size __mmap_switched_data, . - __mmap_switched_data + +/* + * This provides a C-API version of __lookup_processor_type + */ +ENTRY(lookup_processor_type) + stmfd sp!, {r4 - r6, r9, lr} + mov r9, r0 + bl __lookup_processor_type + mov r0, r5 + ldmfd sp!, {r4 - r6, r9, pc} +ENDPROC(lookup_processor_type) + +/* + * Read processor ID register (CP#15, CR0), and look up in the linker-built + * supported processor list. Note that we can't use the absolute addresses + * for the __proc_info lists since we aren't running with the MMU on + * (and therefore, we are not in the correct address space). We have to + * calculate the offset. + * + * r9 = cpuid + * Returns: + * r3, r4, r6 corrupted + * r5 = proc_info pointer in physical address space + * r9 = cpuid (preserved) + */ + __CPUINIT +__lookup_processor_type: + adr r3, __lookup_processor_type_data + ldmia r3, {r4 - r6} + sub r3, r3, r4 @ get offset between virt&phys + add r5, r5, r3 @ convert virt addresses to + add r6, r6, r3 @ physical address space +1: ldmia r5, {r3, r4} @ value, mask + and r4, r4, r9 @ mask wanted bits + teq r3, r4 + beq 2f + add r5, r5, #PROC_INFO_SZ @ sizeof(proc_info_list) + cmp r5, r6 + blo 1b + mov r5, #0 @ unknown processor +2: mov pc, lr +ENDPROC(__lookup_processor_type) + +/* + * Look in <asm/procinfo.h> for information about the __proc_info structure. + */ + .align 2 + .type __lookup_processor_type_data, %object +__lookup_processor_type_data: + .long . + .long __proc_info_begin + .long __proc_info_end + .size __lookup_processor_type_data, . - __lookup_processor_type_data + +__error_p: +#ifdef CONFIG_DEBUG_LL + adr r0, str_p1 + bl printascii + mov r0, r9 + bl printhex8 + adr r0, str_p2 + bl printascii + b __error +str_p1: .asciz "\nError: unrecognized/unsupported processor variant (0x" +str_p2: .asciz ").\n" + .align +#endif +ENDPROC(__error_p) + +__error: +#ifdef CONFIG_ARCH_RPC +/* + * Turn the screen red on a error - RiscPC only. + */ + mov r0, #0x02000000 + mov r3, #0x11 + orr r3, r3, r3, lsl #8 + orr r3, r3, r3, lsl #16 + str r3, [r0], #4 + str r3, [r0], #4 + str r3, [r0], #4 + str r3, [r0], #4 +#endif +1: mov r0, r0 + b 1b +ENDPROC(__error) diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S new file mode 100644 index 00000000..6b1e0ad9 --- /dev/null +++ b/arch/arm/kernel/head-nommu.S @@ -0,0 +1,90 @@ +/* + * linux/arch/arm/kernel/head-nommu.S + * + * Copyright (C) 1994-2002 Russell King + * Copyright (C) 2003-2006 Hyok S. Choi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Common kernel startup code (non-paged MM) + * + */ +#include <linux/linkage.h> +#include <linux/init.h> + +#include <asm/assembler.h> +#include <asm/ptrace.h> +#include <asm/asm-offsets.h> +#include <asm/thread_info.h> +#include <asm/system.h> + +/* + * Kernel startup entry point. + * --------------------------- + * + * This is normally called from the decompressor code. The requirements + * are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0, + * r1 = machine nr. + * + * See linux/arch/arm/tools/mach-types for the complete list of machine + * numbers for r1. + * + */ + __HEAD +ENTRY(stext) + setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode + @ and irqs disabled +#ifndef CONFIG_CPU_CP15 + ldr r9, =CONFIG_PROCESSOR_ID +#else + mrc p15, 0, r9, c0, c0 @ get processor id +#endif + bl __lookup_processor_type @ r5=procinfo r9=cpuid + movs r10, r5 @ invalid processor (r5=0)? + beq __error_p @ yes, error 'p' + + adr lr, BSYM(__after_proc_init) @ return (PIC) address + ARM( add pc, r10, #PROCINFO_INITFUNC ) + THUMB( add r12, r10, #PROCINFO_INITFUNC ) + THUMB( mov pc, r12 ) +ENDPROC(stext) + +/* + * Set the Control Register and Read the process ID. + */ +__after_proc_init: +#ifdef CONFIG_CPU_CP15 + /* + * CP15 system control register value returned in r0 from + * the CPU init function. + */ +#ifdef CONFIG_ALIGNMENT_TRAP + orr r0, r0, #CR_A +#else + bic r0, r0, #CR_A +#endif +#ifdef CONFIG_CPU_DCACHE_DISABLE + bic r0, r0, #CR_C +#endif +#ifdef CONFIG_CPU_BPREDICT_DISABLE + bic r0, r0, #CR_Z +#endif +#ifdef CONFIG_CPU_ICACHE_DISABLE + bic r0, r0, #CR_I +#endif +#ifdef CONFIG_CPU_HIGH_VECTOR + orr r0, r0, #CR_V +#else + bic r0, r0, #CR_V +#endif + mcr p15, 0, r0, c1, c0, 0 @ write control reg +#endif /* CONFIG_CPU_CP15 */ + + b __mmap_switched @ clear the BSS and jump + @ to start_kernel +ENDPROC(__after_proc_init) + .ltorg + +#include "head-common.S" diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S new file mode 100644 index 00000000..673151c7 --- /dev/null +++ b/arch/arm/kernel/head.S @@ -0,0 +1,592 @@ +/* + * linux/arch/arm/kernel/head.S + * + * Copyright (C) 1994-2002 Russell King + * Copyright (c) 2003 ARM Limited + * All Rights Reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Kernel startup code for all 32-bit CPUs + */ +#include <linux/linkage.h> +#include <linux/init.h> + +#include <asm/assembler.h> +#include <asm/domain.h> +#include <asm/ptrace.h> +#include <asm/asm-offsets.h> +#include <asm/memory.h> +#include <asm/thread_info.h> +#include <asm/system.h> + +#ifdef CONFIG_DEBUG_LL +#include <mach/debug-macro.S> +#endif + +/* + * swapper_pg_dir is the virtual address of the initial page table. + * We place the page tables 16K below KERNEL_RAM_VADDR. Therefore, we must + * make sure that KERNEL_RAM_VADDR is correctly set. Currently, we expect + * the least significant 16 bits to be 0x8000, but we could probably + * relax this restriction to KERNEL_RAM_VADDR >= PAGE_OFFSET + 0x4000. + */ +#define KERNEL_RAM_VADDR (PAGE_OFFSET + TEXT_OFFSET) +#if (KERNEL_RAM_VADDR & 0xffff) != 0x8000 +#error KERNEL_RAM_VADDR must start at 0xXXXX8000 +#endif + + .globl swapper_pg_dir + .equ swapper_pg_dir, KERNEL_RAM_VADDR - 0x4000 + + .macro pgtbl, rd, phys + add \rd, \phys, #TEXT_OFFSET - 0x4000 + .endm + +#ifdef CONFIG_XIP_KERNEL +#define KERNEL_START XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR) +#define KERNEL_END _edata_loc +#else +#define KERNEL_START KERNEL_RAM_VADDR +#define KERNEL_END _end +#endif + +/* + * Kernel startup entry point. + * --------------------------- + * + * This is normally called from the decompressor code. The requirements + * are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0, + * r1 = machine nr, r2 = atags or dtb pointer. + * + * This code is mostly position independent, so if you link the kernel at + * 0xc0008000, you call this at __pa(0xc0008000). + * + * See linux/arch/arm/tools/mach-types for the complete list of machine + * numbers for r1. + * + * We're trying to keep crap to a minimum; DO NOT add any machine specific + * crap here - that's what the boot loader (or in extreme, well justified + * circumstances, zImage) is for. + */ + __HEAD +ENTRY(stext) + setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 @ ensure svc mode + @ and irqs disabled + mrc p15, 0, r9, c0, c0 @ get processor id + bl __lookup_processor_type @ r5=procinfo r9=cpuid + movs r10, r5 @ invalid processor (r5=0)? + THUMB( it eq ) @ force fixup-able long branch encoding + beq __error_p @ yes, error 'p' + +#ifndef CONFIG_XIP_KERNEL + adr r3, 2f + ldmia r3, {r4, r8} + sub r4, r3, r4 @ (PHYS_OFFSET - PAGE_OFFSET) + add r8, r8, r4 @ PHYS_OFFSET +#else + ldr r8, =PLAT_PHYS_OFFSET +#endif + + /* + * r1 = machine no, r2 = atags or dtb, + * r8 = phys_offset, r9 = cpuid, r10 = procinfo + */ + bl __vet_atags +#ifdef CONFIG_SMP_ON_UP + bl __fixup_smp +#endif +#ifdef CONFIG_ARM_PATCH_PHYS_VIRT + bl __fixup_pv_table +#endif + bl __create_page_tables + + /* + * The following calls CPU specific code in a position independent + * manner. See arch/arm/mm/proc-*.S for details. r10 = base of + * xxx_proc_info structure selected by __lookup_processor_type + * above. On return, the CPU will be ready for the MMU to be + * turned on, and r0 will hold the CPU control register value. + */ + ldr r13, =__mmap_switched @ address to jump to after + @ mmu has been enabled + adr lr, BSYM(1f) @ return (PIC) address + mov r8, r4 @ set TTBR1 to swapper_pg_dir + ARM( add pc, r10, #PROCINFO_INITFUNC ) + THUMB( add r12, r10, #PROCINFO_INITFUNC ) + THUMB( mov pc, r12 ) +1: b __enable_mmu +ENDPROC(stext) + .ltorg +#ifndef CONFIG_XIP_KERNEL +2: .long . + .long PAGE_OFFSET +#endif + +/* + * Setup the initial page tables. We only setup the barest + * amount which are required to get the kernel running, which + * generally means mapping in the kernel code. + * + * r8 = phys_offset, r9 = cpuid, r10 = procinfo + * + * Returns: + * r0, r3, r5-r7 corrupted + * r4 = physical page table address + */ +__create_page_tables: + pgtbl r4, r8 @ page table address + + /* + * Clear the 16K level 1 swapper page table + */ + mov r0, r4 + mov r3, #0 + add r6, r0, #0x4000 +1: str r3, [r0], #4 + str r3, [r0], #4 + str r3, [r0], #4 + str r3, [r0], #4 + teq r0, r6 + bne 1b + + ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags + + /* + * Create identity mapping to cater for __enable_mmu. + * This identity mapping will be removed by paging_init(). + */ + adr r0, __enable_mmu_loc + ldmia r0, {r3, r5, r6} + sub r0, r0, r3 @ virt->phys offset + add r5, r5, r0 @ phys __enable_mmu + add r6, r6, r0 @ phys __enable_mmu_end + mov r5, r5, lsr #20 + mov r6, r6, lsr #20 + +1: orr r3, r7, r5, lsl #20 @ flags + kernel base + str r3, [r4, r5, lsl #2] @ identity mapping + teq r5, r6 + addne r5, r5, #1 @ next section + bne 1b + + /* + * Now setup the pagetables for our kernel direct + * mapped region. + */ + mov r3, pc + mov r3, r3, lsr #20 + orr r3, r7, r3, lsl #20 + add r0, r4, #(KERNEL_START & 0xff000000) >> 18 + str r3, [r0, #(KERNEL_START & 0x00f00000) >> 18]! + ldr r6, =(KERNEL_END - 1) + add r0, r0, #4 + add r6, r4, r6, lsr #18 +1: cmp r0, r6 + add r3, r3, #1 << 20 + strls r3, [r0], #4 + bls 1b + +#ifdef CONFIG_XIP_KERNEL + /* + * Map some ram to cover our .data and .bss areas. + */ + add r3, r8, #TEXT_OFFSET + orr r3, r3, r7 + add r0, r4, #(KERNEL_RAM_VADDR & 0xff000000) >> 18 + str r3, [r0, #(KERNEL_RAM_VADDR & 0x00f00000) >> 18]! + ldr r6, =(_end - 1) + add r0, r0, #4 + add r6, r4, r6, lsr #18 +1: cmp r0, r6 + add r3, r3, #1 << 20 + strls r3, [r0], #4 + bls 1b +#endif + + /* + * Then map boot params address in r2 or + * the first 1MB of ram if boot params address is not specified. + */ + mov r0, r2, lsr #20 + movs r0, r0, lsl #20 + moveq r0, r8 + sub r3, r0, r8 + add r3, r3, #PAGE_OFFSET + add r3, r4, r3, lsr #18 + orr r6, r7, r0 + str r6, [r3] + +#ifdef CONFIG_DEBUG_LL +#ifndef CONFIG_DEBUG_ICEDCC + /* + * Map in IO space for serial debugging. + * This allows debug messages to be output + * via a serial console before paging_init. + */ + addruart r7, r3 + + mov r3, r3, lsr #20 + mov r3, r3, lsl #2 + + add r0, r4, r3 + rsb r3, r3, #0x4000 @ PTRS_PER_PGD*sizeof(long) + cmp r3, #0x0800 @ limit to 512MB + movhi r3, #0x0800 + add r6, r0, r3 + mov r3, r7, lsr #20 + ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags + orr r3, r7, r3, lsl #20 +1: str r3, [r0], #4 + add r3, r3, #1 << 20 + teq r0, r6 + bne 1b + +#else /* CONFIG_DEBUG_ICEDCC */ + /* we don't need any serial debugging mappings for ICEDCC */ + ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags +#endif /* !CONFIG_DEBUG_ICEDCC */ + +#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS) + /* + * If we're using the NetWinder or CATS, we also need to map + * in the 16550-type serial port for the debug messages + */ + add r0, r4, #0xff000000 >> 18 + orr r3, r7, #0x7c000000 + str r3, [r0] +#endif +#ifdef CONFIG_ARCH_RPC + /* + * Map in screen at 0x02000000 & SCREEN2_BASE + * Similar reasons here - for debug. This is + * only for Acorn RiscPC architectures. + */ + add r0, r4, #0x02000000 >> 18 + orr r3, r7, #0x02000000 + str r3, [r0] + add r0, r4, #0xd8000000 >> 18 + str r3, [r0] +#endif +#endif + mov pc, lr +ENDPROC(__create_page_tables) + .ltorg + .align +__enable_mmu_loc: + .long . + .long __enable_mmu + .long __enable_mmu_end + +#if defined(CONFIG_SMP) + __CPUINIT +ENTRY(secondary_startup) + /* + * Common entry point for secondary CPUs. + * + * Ensure that we're in SVC mode, and IRQs are disabled. Lookup + * the processor type - there is no need to check the machine type + * as it has already been validated by the primary processor. + */ + setmode PSR_F_BIT | PSR_I_BIT | SVC_MODE, r9 + mrc p15, 0, r9, c0, c0 @ get processor id + bl __lookup_processor_type + movs r10, r5 @ invalid processor? + moveq r0, #'p' @ yes, error 'p' + THUMB( it eq ) @ force fixup-able long branch encoding + beq __error_p + + /* + * Use the page tables supplied from __cpu_up. + */ + adr r4, __secondary_data + ldmia r4, {r5, r7, r12} @ address to jump to after + sub lr, r4, r5 @ mmu has been enabled + ldr r4, [r7, lr] @ get secondary_data.pgdir + add r7, r7, #4 + ldr r8, [r7, lr] @ get secondary_data.swapper_pg_dir + adr lr, BSYM(__enable_mmu) @ return address + mov r13, r12 @ __secondary_switched address + ARM( add pc, r10, #PROCINFO_INITFUNC ) @ initialise processor + @ (return control reg) + THUMB( add r12, r10, #PROCINFO_INITFUNC ) + THUMB( mov pc, r12 ) +ENDPROC(secondary_startup) + + /* + * r6 = &secondary_data + */ +ENTRY(__secondary_switched) + ldr sp, [r7, #4] @ get secondary_data.stack + mov fp, #0 + b secondary_start_kernel +ENDPROC(__secondary_switched) + + .align + + .type __secondary_data, %object +__secondary_data: + .long . + .long secondary_data + .long __secondary_switched +#endif /* defined(CONFIG_SMP) */ + + + +/* + * Setup common bits before finally enabling the MMU. Essentially + * this is just loading the page table pointer and domain access + * registers. + * + * r0 = cp#15 control register + * r1 = machine ID + * r2 = atags or dtb pointer + * r4 = page table pointer + * r9 = processor ID + * r13 = *virtual* address to jump to upon completion + */ +__enable_mmu: +#if defined(CONFIG_ALIGNMENT_TRAP) && __LINUX_ARM_ARCH__ < 6 + orr r0, r0, #CR_A +#else + bic r0, r0, #CR_A +#endif +#ifdef CONFIG_CPU_DCACHE_DISABLE + bic r0, r0, #CR_C +#endif +#ifdef CONFIG_CPU_BPREDICT_DISABLE + bic r0, r0, #CR_Z +#endif +#ifdef CONFIG_CPU_ICACHE_DISABLE + bic r0, r0, #CR_I +#endif + mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ + domain_val(DOMAIN_IO, DOMAIN_CLIENT)) + mcr p15, 0, r5, c3, c0, 0 @ load domain access register + mcr p15, 0, r4, c2, c0, 0 @ load page table pointer + b __turn_mmu_on +ENDPROC(__enable_mmu) + +/* + * Enable the MMU. This completely changes the structure of the visible + * memory space. You will not be able to trace execution through this. + * If you have an enquiry about this, *please* check the linux-arm-kernel + * mailing list archives BEFORE sending another post to the list. + * + * r0 = cp#15 control register + * r1 = machine ID + * r2 = atags or dtb pointer + * r9 = processor ID + * r13 = *virtual* address to jump to upon completion + * + * other registers depend on the function called upon completion + */ + .align 5 +__turn_mmu_on: + mov r0, r0 + mcr p15, 0, r0, c1, c0, 0 @ write control reg + mrc p15, 0, r3, c0, c0, 0 @ read id reg + mov r3, r3 + mov r3, r13 + mov pc, r3 +__enable_mmu_end: +ENDPROC(__turn_mmu_on) + + +#ifdef CONFIG_SMP_ON_UP + __INIT +__fixup_smp: + and r3, r9, #0x000f0000 @ architecture version + teq r3, #0x000f0000 @ CPU ID supported? + bne __fixup_smp_on_up @ no, assume UP + + bic r3, r9, #0x00ff0000 + bic r3, r3, #0x0000000f @ mask 0xff00fff0 + mov r4, #0x41000000 + orr r4, r4, #0x0000b000 + orr r4, r4, #0x00000020 @ val 0x4100b020 + teq r3, r4 @ ARM 11MPCore? + moveq pc, lr @ yes, assume SMP + + mrc p15, 0, r0, c0, c0, 5 @ read MPIDR + and r0, r0, #0xc0000000 @ multiprocessing extensions and + teq r0, #0x80000000 @ not part of a uniprocessor system? + moveq pc, lr @ yes, assume SMP + +__fixup_smp_on_up: + adr r0, 1f + ldmia r0, {r3 - r5} + sub r3, r0, r3 + add r4, r4, r3 + add r5, r5, r3 + b __do_fixup_smp_on_up +ENDPROC(__fixup_smp) + + .align +1: .word . + .word __smpalt_begin + .word __smpalt_end + + .pushsection .data + .globl smp_on_up +smp_on_up: + ALT_SMP(.long 1) + ALT_UP(.long 0) + .popsection +#endif + + .text +__do_fixup_smp_on_up: + cmp r4, r5 + movhs pc, lr + ldmia r4!, {r0, r6} + ARM( str r6, [r0, r3] ) + THUMB( add r0, r0, r3 ) +#ifdef __ARMEB__ + THUMB( mov r6, r6, ror #16 ) @ Convert word order for big-endian. +#endif + THUMB( strh r6, [r0], #2 ) @ For Thumb-2, store as two halfwords + THUMB( mov r6, r6, lsr #16 ) @ to be robust against misaligned r3. + THUMB( strh r6, [r0] ) + b __do_fixup_smp_on_up +ENDPROC(__do_fixup_smp_on_up) + +ENTRY(fixup_smp) + stmfd sp!, {r4 - r6, lr} + mov r4, r0 + add r5, r0, r1 + mov r3, #0 + bl __do_fixup_smp_on_up + ldmfd sp!, {r4 - r6, pc} +ENDPROC(fixup_smp) + +#ifdef CONFIG_ARM_PATCH_PHYS_VIRT + +/* __fixup_pv_table - patch the stub instructions with the delta between + * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be 16MiB aligned and + * can be expressed by an immediate shifter operand. The stub instruction + * has a form of '(add|sub) rd, rn, #imm'. + */ + __HEAD +__fixup_pv_table: + adr r0, 1f + ldmia r0, {r3-r5, r7} + sub r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET + add r4, r4, r3 @ adjust table start address + add r5, r5, r3 @ adjust table end address + add r7, r7, r3 @ adjust __pv_phys_offset address + str r8, [r7] @ save computed PHYS_OFFSET to __pv_phys_offset +#ifndef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT + mov r6, r3, lsr #24 @ constant for add/sub instructions + teq r3, r6, lsl #24 @ must be 16MiB aligned +#else + mov r6, r3, lsr #16 @ constant for add/sub instructions + teq r3, r6, lsl #16 @ must be 64kiB aligned +#endif +THUMB( it ne @ cross section branch ) + bne __error + str r6, [r7, #4] @ save to __pv_offset + b __fixup_a_pv_table +ENDPROC(__fixup_pv_table) + + .align +1: .long . + .long __pv_table_begin + .long __pv_table_end +2: .long __pv_phys_offset + + .text +__fixup_a_pv_table: +#ifdef CONFIG_THUMB2_KERNEL +#ifdef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT + lsls r0, r6, #24 + lsr r6, #8 + beq 1f + clz r7, r0 + lsr r0, #24 + lsl r0, r7 + bic r0, 0x0080 + lsrs r7, #1 + orrcs r0, #0x0080 + orr r0, r0, r7, lsl #12 +#endif +1: lsls r6, #24 + beq 4f + clz r7, r6 + lsr r6, #24 + lsl r6, r7 + bic r6, #0x0080 + lsrs r7, #1 + orrcs r6, #0x0080 + orr r6, r6, r7, lsl #12 + orr r6, #0x4000 + b 4f +2: @ at this point the C flag is always clear + add r7, r3 +#ifdef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT + ldrh ip, [r7] + tst ip, 0x0400 @ the i bit tells us LS or MS byte + beq 3f + cmp r0, #0 @ set C flag, and ... + biceq ip, 0x0400 @ immediate zero value has a special encoding + streqh ip, [r7] @ that requires the i bit cleared +#endif +3: ldrh ip, [r7, #2] + and ip, 0x8f00 + orrcc ip, r6 @ mask in offset bits 31-24 + orrcs ip, r0 @ mask in offset bits 23-16 + strh ip, [r7, #2] +4: cmp r4, r5 + ldrcc r7, [r4], #4 @ use branch for delay slot + bcc 2b + bx lr +#else +#ifdef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT + and r0, r6, #255 @ offset bits 23-16 + mov r6, r6, lsr #8 @ offset bits 31-24 +#else + mov r0, #0 @ just in case... +#endif + b 3f +2: ldr ip, [r7, r3] + bic ip, ip, #0x000000ff + tst ip, #0x400 @ rotate shift tells us LS or MS byte + orrne ip, ip, r6 @ mask in offset bits 31-24 + orreq ip, ip, r0 @ mask in offset bits 23-16 + str ip, [r7, r3] +3: cmp r4, r5 + ldrcc r7, [r4], #4 @ use branch for delay slot + bcc 2b + mov pc, lr +#endif +ENDPROC(__fixup_a_pv_table) + +ENTRY(fixup_pv_table) + stmfd sp!, {r4 - r7, lr} + ldr r2, 2f @ get address of __pv_phys_offset + mov r3, #0 @ no offset + mov r4, r0 @ r0 = table start + add r5, r0, r1 @ r1 = table size + ldr r6, [r2, #4] @ get __pv_offset + bl __fixup_a_pv_table + ldmfd sp!, {r4 - r7, pc} +ENDPROC(fixup_pv_table) + + .align +2: .long __pv_phys_offset + + .data + .globl __pv_phys_offset + .type __pv_phys_offset, %object +__pv_phys_offset: + .long 0 + .size __pv_phys_offset, . - __pv_phys_offset +__pv_offset: + .long 0 +#endif + +#include "head-common.S" diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c new file mode 100644 index 00000000..87acc25d --- /dev/null +++ b/arch/arm/kernel/hw_breakpoint.c @@ -0,0 +1,978 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) 2009, 2010 ARM Limited + * + * Author: Will Deacon <will.deacon@arm.com> + */ + +/* + * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility, + * using the CPU's debug registers. + */ +#define pr_fmt(fmt) "hw-breakpoint: " fmt + +#include <linux/errno.h> +#include <linux/hardirq.h> +#include <linux/perf_event.h> +#include <linux/hw_breakpoint.h> +#include <linux/smp.h> + +#include <asm/cacheflush.h> +#include <asm/cputype.h> +#include <asm/current.h> +#include <asm/hw_breakpoint.h> +#include <asm/kdebug.h> +#include <asm/system.h> +#include <asm/traps.h> + +/* Breakpoint currently in use for each BRP. */ +static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]); + +/* Watchpoint currently in use for each WRP. */ +static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]); + +/* Number of BRP/WRP registers on this CPU. */ +static int core_num_brps; +static int core_num_reserved_brps; +static int core_num_wrps; + +/* Debug architecture version. */ +static u8 debug_arch; + +/* Maximum supported watchpoint length. */ +static u8 max_watchpoint_len; + +#define READ_WB_REG_CASE(OP2, M, VAL) \ + case ((OP2 << 4) + M): \ + ARM_DBG_READ(c ## M, OP2, VAL); \ + break + +#define WRITE_WB_REG_CASE(OP2, M, VAL) \ + case ((OP2 << 4) + M): \ + ARM_DBG_WRITE(c ## M, OP2, VAL);\ + break + +#define GEN_READ_WB_REG_CASES(OP2, VAL) \ + READ_WB_REG_CASE(OP2, 0, VAL); \ + READ_WB_REG_CASE(OP2, 1, VAL); \ + READ_WB_REG_CASE(OP2, 2, VAL); \ + READ_WB_REG_CASE(OP2, 3, VAL); \ + READ_WB_REG_CASE(OP2, 4, VAL); \ + READ_WB_REG_CASE(OP2, 5, VAL); \ + READ_WB_REG_CASE(OP2, 6, VAL); \ + READ_WB_REG_CASE(OP2, 7, VAL); \ + READ_WB_REG_CASE(OP2, 8, VAL); \ + READ_WB_REG_CASE(OP2, 9, VAL); \ + READ_WB_REG_CASE(OP2, 10, VAL); \ + READ_WB_REG_CASE(OP2, 11, VAL); \ + READ_WB_REG_CASE(OP2, 12, VAL); \ + READ_WB_REG_CASE(OP2, 13, VAL); \ + READ_WB_REG_CASE(OP2, 14, VAL); \ + READ_WB_REG_CASE(OP2, 15, VAL) + +#define GEN_WRITE_WB_REG_CASES(OP2, VAL) \ + WRITE_WB_REG_CASE(OP2, 0, VAL); \ + WRITE_WB_REG_CASE(OP2, 1, VAL); \ + WRITE_WB_REG_CASE(OP2, 2, VAL); \ + WRITE_WB_REG_CASE(OP2, 3, VAL); \ + WRITE_WB_REG_CASE(OP2, 4, VAL); \ + WRITE_WB_REG_CASE(OP2, 5, VAL); \ + WRITE_WB_REG_CASE(OP2, 6, VAL); \ + WRITE_WB_REG_CASE(OP2, 7, VAL); \ + WRITE_WB_REG_CASE(OP2, 8, VAL); \ + WRITE_WB_REG_CASE(OP2, 9, VAL); \ + WRITE_WB_REG_CASE(OP2, 10, VAL); \ + WRITE_WB_REG_CASE(OP2, 11, VAL); \ + WRITE_WB_REG_CASE(OP2, 12, VAL); \ + WRITE_WB_REG_CASE(OP2, 13, VAL); \ + WRITE_WB_REG_CASE(OP2, 14, VAL); \ + WRITE_WB_REG_CASE(OP2, 15, VAL) + +static u32 read_wb_reg(int n) +{ + u32 val = 0; + + switch (n) { + GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val); + GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val); + GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val); + GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val); + default: + pr_warning("attempt to read from unknown breakpoint " + "register %d\n", n); + } + + return val; +} + +static void write_wb_reg(int n, u32 val) +{ + switch (n) { + GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val); + GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val); + GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val); + GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val); + default: + pr_warning("attempt to write to unknown breakpoint " + "register %d\n", n); + } + isb(); +} + +/* Determine debug architecture. */ +static u8 get_debug_arch(void) +{ + u32 didr; + + /* Do we implement the extended CPUID interface? */ + if (WARN_ONCE((((read_cpuid_id() >> 16) & 0xf) != 0xf), + "CPUID feature registers not supported. " + "Assuming v6 debug is present.\n")) + return ARM_DEBUG_ARCH_V6; + + ARM_DBG_READ(c0, 0, didr); + return (didr >> 16) & 0xf; +} + +u8 arch_get_debug_arch(void) +{ + return debug_arch; +} + +static int debug_arch_supported(void) +{ + u8 arch = get_debug_arch(); + return arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14; +} + +/* Determine number of BRP register available. */ +static int get_num_brp_resources(void) +{ + u32 didr; + ARM_DBG_READ(c0, 0, didr); + return ((didr >> 24) & 0xf) + 1; +} + +/* Does this core support mismatch breakpoints? */ +static int core_has_mismatch_brps(void) +{ + return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 && + get_num_brp_resources() > 1); +} + +/* Determine number of usable WRPs available. */ +static int get_num_wrps(void) +{ + /* + * FIXME: When a watchpoint fires, the only way to work out which + * watchpoint it was is by disassembling the faulting instruction + * and working out the address of the memory access. + * + * Furthermore, we can only do this if the watchpoint was precise + * since imprecise watchpoints prevent us from calculating register + * based addresses. + * + * Providing we have more than 1 breakpoint register, we only report + * a single watchpoint register for the time being. This way, we always + * know which watchpoint fired. In the future we can either add a + * disassembler and address generation emulator, or we can insert a + * check to see if the DFAR is set on watchpoint exception entry + * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows + * that it is set on some implementations]. + */ + +#if 0 + int wrps; + u32 didr; + ARM_DBG_READ(c0, 0, didr); + wrps = ((didr >> 28) & 0xf) + 1; +#endif + int wrps = 1; + + if (core_has_mismatch_brps() && wrps >= get_num_brp_resources()) + wrps = get_num_brp_resources() - 1; + + return wrps; +} + +/* We reserve one breakpoint for each watchpoint. */ +static int get_num_reserved_brps(void) +{ + if (core_has_mismatch_brps()) + return get_num_wrps(); + return 0; +} + +/* Determine number of usable BRPs available. */ +static int get_num_brps(void) +{ + int brps = get_num_brp_resources(); + if (core_has_mismatch_brps()) + brps -= get_num_reserved_brps(); + return brps; +} + +/* + * In order to access the breakpoint/watchpoint control registers, + * we must be running in debug monitor mode. Unfortunately, we can + * be put into halting debug mode at any time by an external debugger + * but there is nothing we can do to prevent that. + */ +static int enable_monitor_mode(void) +{ + u32 dscr; + int ret = 0; + + ARM_DBG_READ(c1, 0, dscr); + + /* Ensure that halting mode is disabled. */ + if (WARN_ONCE(dscr & ARM_DSCR_HDBGEN, + "halting debug mode enabled. Unable to access hardware resources.\n")) { + ret = -EPERM; + goto out; + } + + /* If monitor mode is already enabled, just return. */ + if (dscr & ARM_DSCR_MDBGEN) + goto out; + + /* Write to the corresponding DSCR. */ + switch (get_debug_arch()) { + case ARM_DEBUG_ARCH_V6: + case ARM_DEBUG_ARCH_V6_1: + ARM_DBG_WRITE(c1, 0, (dscr | ARM_DSCR_MDBGEN)); + break; + case ARM_DEBUG_ARCH_V7_ECP14: + ARM_DBG_WRITE(c2, 2, (dscr | ARM_DSCR_MDBGEN)); + break; + default: + ret = -ENODEV; + goto out; + } + + /* Check that the write made it through. */ + ARM_DBG_READ(c1, 0, dscr); + if (!(dscr & ARM_DSCR_MDBGEN)) + ret = -EPERM; + +out: + return ret; +} + +int hw_breakpoint_slots(int type) +{ + if (!debug_arch_supported()) + return 0; + + /* + * We can be called early, so don't rely on + * our static variables being initialised. + */ + switch (type) { + case TYPE_INST: + return get_num_brps(); + case TYPE_DATA: + return get_num_wrps(); + default: + pr_warning("unknown slot type: %d\n", type); + return 0; + } +} + +/* + * Check if 8-bit byte-address select is available. + * This clobbers WRP 0. + */ +static u8 get_max_wp_len(void) +{ + u32 ctrl_reg; + struct arch_hw_breakpoint_ctrl ctrl; + u8 size = 4; + + if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14) + goto out; + + memset(&ctrl, 0, sizeof(ctrl)); + ctrl.len = ARM_BREAKPOINT_LEN_8; + ctrl_reg = encode_ctrl_reg(ctrl); + + write_wb_reg(ARM_BASE_WVR, 0); + write_wb_reg(ARM_BASE_WCR, ctrl_reg); + if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg) + size = 8; + +out: + return size; +} + +u8 arch_get_max_wp_len(void) +{ + return max_watchpoint_len; +} + +/* + * Install a perf counter breakpoint. + */ +int arch_install_hw_breakpoint(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + struct perf_event **slot, **slots; + int i, max_slots, ctrl_base, val_base, ret = 0; + u32 addr, ctrl; + + /* Ensure that we are in monitor mode and halting mode is disabled. */ + ret = enable_monitor_mode(); + if (ret) + goto out; + + addr = info->address; + ctrl = encode_ctrl_reg(info->ctrl) | 0x1; + + if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { + /* Breakpoint */ + ctrl_base = ARM_BASE_BCR; + val_base = ARM_BASE_BVR; + slots = (struct perf_event **)__get_cpu_var(bp_on_reg); + max_slots = core_num_brps; + if (info->step_ctrl.enabled) { + /* Override the breakpoint data with the step data. */ + addr = info->trigger & ~0x3; + ctrl = encode_ctrl_reg(info->step_ctrl); + } + } else { + /* Watchpoint */ + if (info->step_ctrl.enabled) { + /* Install into the reserved breakpoint region. */ + ctrl_base = ARM_BASE_BCR + core_num_brps; + val_base = ARM_BASE_BVR + core_num_brps; + /* Override the watchpoint data with the step data. */ + addr = info->trigger & ~0x3; + ctrl = encode_ctrl_reg(info->step_ctrl); + } else { + ctrl_base = ARM_BASE_WCR; + val_base = ARM_BASE_WVR; + } + slots = (struct perf_event **)__get_cpu_var(wp_on_reg); + max_slots = core_num_wrps; + } + + for (i = 0; i < max_slots; ++i) { + slot = &slots[i]; + + if (!*slot) { + *slot = bp; + break; + } + } + + if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot\n")) { + ret = -EBUSY; + goto out; + } + + /* Setup the address register. */ + write_wb_reg(val_base + i, addr); + + /* Setup the control register. */ + write_wb_reg(ctrl_base + i, ctrl); + +out: + return ret; +} + +void arch_uninstall_hw_breakpoint(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + struct perf_event **slot, **slots; + int i, max_slots, base; + + if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) { + /* Breakpoint */ + base = ARM_BASE_BCR; + slots = (struct perf_event **)__get_cpu_var(bp_on_reg); + max_slots = core_num_brps; + } else { + /* Watchpoint */ + if (info->step_ctrl.enabled) + base = ARM_BASE_BCR + core_num_brps; + else + base = ARM_BASE_WCR; + slots = (struct perf_event **)__get_cpu_var(wp_on_reg); + max_slots = core_num_wrps; + } + + /* Remove the breakpoint. */ + for (i = 0; i < max_slots; ++i) { + slot = &slots[i]; + + if (*slot == bp) { + *slot = NULL; + break; + } + } + + if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot\n")) + return; + + /* Reset the control register. */ + write_wb_reg(base + i, 0); +} + +static int get_hbp_len(u8 hbp_len) +{ + unsigned int len_in_bytes = 0; + + switch (hbp_len) { + case ARM_BREAKPOINT_LEN_1: + len_in_bytes = 1; + break; + case ARM_BREAKPOINT_LEN_2: + len_in_bytes = 2; + break; + case ARM_BREAKPOINT_LEN_4: + len_in_bytes = 4; + break; + case ARM_BREAKPOINT_LEN_8: + len_in_bytes = 8; + break; + } + + return len_in_bytes; +} + +/* + * Check whether bp virtual address is in kernel space. + */ +int arch_check_bp_in_kernelspace(struct perf_event *bp) +{ + unsigned int len; + unsigned long va; + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + + va = info->address; + len = get_hbp_len(info->ctrl.len); + + return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE); +} + +/* + * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl. + * Hopefully this will disappear when ptrace can bypass the conversion + * to generic breakpoint descriptions. + */ +int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl, + int *gen_len, int *gen_type) +{ + /* Type */ + switch (ctrl.type) { + case ARM_BREAKPOINT_EXECUTE: + *gen_type = HW_BREAKPOINT_X; + break; + case ARM_BREAKPOINT_LOAD: + *gen_type = HW_BREAKPOINT_R; + break; + case ARM_BREAKPOINT_STORE: + *gen_type = HW_BREAKPOINT_W; + break; + case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE: + *gen_type = HW_BREAKPOINT_RW; + break; + default: + return -EINVAL; + } + + /* Len */ + switch (ctrl.len) { + case ARM_BREAKPOINT_LEN_1: + *gen_len = HW_BREAKPOINT_LEN_1; + break; + case ARM_BREAKPOINT_LEN_2: + *gen_len = HW_BREAKPOINT_LEN_2; + break; + case ARM_BREAKPOINT_LEN_4: + *gen_len = HW_BREAKPOINT_LEN_4; + break; + case ARM_BREAKPOINT_LEN_8: + *gen_len = HW_BREAKPOINT_LEN_8; + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * Construct an arch_hw_breakpoint from a perf_event. + */ +static int arch_build_bp_info(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + + /* Type */ + switch (bp->attr.bp_type) { + case HW_BREAKPOINT_X: + info->ctrl.type = ARM_BREAKPOINT_EXECUTE; + break; + case HW_BREAKPOINT_R: + info->ctrl.type = ARM_BREAKPOINT_LOAD; + break; + case HW_BREAKPOINT_W: + info->ctrl.type = ARM_BREAKPOINT_STORE; + break; + case HW_BREAKPOINT_RW: + info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE; + break; + default: + return -EINVAL; + } + + /* Len */ + switch (bp->attr.bp_len) { + case HW_BREAKPOINT_LEN_1: + info->ctrl.len = ARM_BREAKPOINT_LEN_1; + break; + case HW_BREAKPOINT_LEN_2: + info->ctrl.len = ARM_BREAKPOINT_LEN_2; + break; + case HW_BREAKPOINT_LEN_4: + info->ctrl.len = ARM_BREAKPOINT_LEN_4; + break; + case HW_BREAKPOINT_LEN_8: + info->ctrl.len = ARM_BREAKPOINT_LEN_8; + if ((info->ctrl.type != ARM_BREAKPOINT_EXECUTE) + && max_watchpoint_len >= 8) + break; + default: + return -EINVAL; + } + + /* + * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes. + * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported + * by the hardware and must be aligned to the appropriate number of + * bytes. + */ + if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE && + info->ctrl.len != ARM_BREAKPOINT_LEN_2 && + info->ctrl.len != ARM_BREAKPOINT_LEN_4) + return -EINVAL; + + /* Address */ + info->address = bp->attr.bp_addr; + + /* Privilege */ + info->ctrl.privilege = ARM_BREAKPOINT_USER; + if (arch_check_bp_in_kernelspace(bp)) + info->ctrl.privilege |= ARM_BREAKPOINT_PRIV; + + /* Enabled? */ + info->ctrl.enabled = !bp->attr.disabled; + + /* Mismatch */ + info->ctrl.mismatch = 0; + + return 0; +} + +/* + * Validate the arch-specific HW Breakpoint register settings. + */ +int arch_validate_hwbkpt_settings(struct perf_event *bp) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + int ret = 0; + u32 offset, alignment_mask = 0x3; + + /* Build the arch_hw_breakpoint. */ + ret = arch_build_bp_info(bp); + if (ret) + goto out; + + /* Check address alignment. */ + if (info->ctrl.len == ARM_BREAKPOINT_LEN_8) + alignment_mask = 0x7; + offset = info->address & alignment_mask; + switch (offset) { + case 0: + /* Aligned */ + break; + case 1: + /* Allow single byte watchpoint. */ + if (info->ctrl.len == ARM_BREAKPOINT_LEN_1) + break; + case 2: + /* Allow halfword watchpoints and breakpoints. */ + if (info->ctrl.len == ARM_BREAKPOINT_LEN_2) + break; + default: + ret = -EINVAL; + goto out; + } + + info->address &= ~alignment_mask; + info->ctrl.len <<= offset; + + /* + * Currently we rely on an overflow handler to take + * care of single-stepping the breakpoint when it fires. + * In the case of userspace breakpoints on a core with V7 debug, + * we can use the mismatch feature as a poor-man's hardware + * single-step, but this only works for per-task breakpoints. + */ + if (WARN_ONCE(!bp->overflow_handler && + (arch_check_bp_in_kernelspace(bp) || !core_has_mismatch_brps() + || !bp->hw.bp_target), + "overflow handler required but none found\n")) { + ret = -EINVAL; + } +out: + return ret; +} + +/* + * Enable/disable single-stepping over the breakpoint bp at address addr. + */ +static void enable_single_step(struct perf_event *bp, u32 addr) +{ + struct arch_hw_breakpoint *info = counter_arch_bp(bp); + + arch_uninstall_hw_breakpoint(bp); + info->step_ctrl.mismatch = 1; + info->step_ctrl.len = ARM_BREAKPOINT_LEN_4; + info->step_ctrl.type = ARM_BREAKPOINT_EXECUTE; + info->step_ctrl.privilege = info->ctrl.privilege; + info->step_ctrl.enabled = 1; + info->trigger = addr; + arch_install_hw_breakpoint(bp); +} + +static void disable_single_step(struct perf_event *bp) +{ + arch_uninstall_hw_breakpoint(bp); + counter_arch_bp(bp)->step_ctrl.enabled = 0; + arch_install_hw_breakpoint(bp); +} + +static void watchpoint_handler(unsigned long unknown, struct pt_regs *regs) +{ + int i; + struct perf_event *wp, **slots; + struct arch_hw_breakpoint *info; + + slots = (struct perf_event **)__get_cpu_var(wp_on_reg); + + /* Without a disassembler, we can only handle 1 watchpoint. */ + BUG_ON(core_num_wrps > 1); + + for (i = 0; i < core_num_wrps; ++i) { + rcu_read_lock(); + + wp = slots[i]; + + if (wp == NULL) { + rcu_read_unlock(); + continue; + } + + /* + * The DFAR is an unknown value. Since we only allow a + * single watchpoint, we can set the trigger to the lowest + * possible faulting address. + */ + info = counter_arch_bp(wp); + info->trigger = wp->attr.bp_addr; + pr_debug("watchpoint fired: address = 0x%x\n", info->trigger); + perf_bp_event(wp, regs); + + /* + * If no overflow handler is present, insert a temporary + * mismatch breakpoint so we can single-step over the + * watchpoint trigger. + */ + if (!wp->overflow_handler) + enable_single_step(wp, instruction_pointer(regs)); + + rcu_read_unlock(); + } +} + +static void watchpoint_single_step_handler(unsigned long pc) +{ + int i; + struct perf_event *wp, **slots; + struct arch_hw_breakpoint *info; + + slots = (struct perf_event **)__get_cpu_var(wp_on_reg); + + for (i = 0; i < core_num_reserved_brps; ++i) { + rcu_read_lock(); + + wp = slots[i]; + + if (wp == NULL) + goto unlock; + + info = counter_arch_bp(wp); + if (!info->step_ctrl.enabled) + goto unlock; + + /* + * Restore the original watchpoint if we've completed the + * single-step. + */ + if (info->trigger != pc) + disable_single_step(wp); + +unlock: + rcu_read_unlock(); + } +} + +static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs) +{ + int i; + u32 ctrl_reg, val, addr; + struct perf_event *bp, **slots; + struct arch_hw_breakpoint *info; + struct arch_hw_breakpoint_ctrl ctrl; + + slots = (struct perf_event **)__get_cpu_var(bp_on_reg); + + /* The exception entry code places the amended lr in the PC. */ + addr = regs->ARM_pc; + + /* Check the currently installed breakpoints first. */ + for (i = 0; i < core_num_brps; ++i) { + rcu_read_lock(); + + bp = slots[i]; + + if (bp == NULL) + goto unlock; + + info = counter_arch_bp(bp); + + /* Check if the breakpoint value matches. */ + val = read_wb_reg(ARM_BASE_BVR + i); + if (val != (addr & ~0x3)) + goto mismatch; + + /* Possible match, check the byte address select to confirm. */ + ctrl_reg = read_wb_reg(ARM_BASE_BCR + i); + decode_ctrl_reg(ctrl_reg, &ctrl); + if ((1 << (addr & 0x3)) & ctrl.len) { + info->trigger = addr; + pr_debug("breakpoint fired: address = 0x%x\n", addr); + perf_bp_event(bp, regs); + if (!bp->overflow_handler) + enable_single_step(bp, addr); + goto unlock; + } + +mismatch: + /* If we're stepping a breakpoint, it can now be restored. */ + if (info->step_ctrl.enabled) + disable_single_step(bp); +unlock: + rcu_read_unlock(); + } + + /* Handle any pending watchpoint single-step breakpoints. */ + watchpoint_single_step_handler(addr); +} + +/* + * Called from either the Data Abort Handler [watchpoint] or the + * Prefetch Abort Handler [breakpoint] with preemption disabled. + */ +static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr, + struct pt_regs *regs) +{ + int ret = 0; + u32 dscr; + + /* We must be called with preemption disabled. */ + WARN_ON(preemptible()); + + /* We only handle watchpoints and hardware breakpoints. */ + ARM_DBG_READ(c1, 0, dscr); + + /* Perform perf callbacks. */ + switch (ARM_DSCR_MOE(dscr)) { + case ARM_ENTRY_BREAKPOINT: + breakpoint_handler(addr, regs); + break; + case ARM_ENTRY_ASYNC_WATCHPOINT: + WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n"); + case ARM_ENTRY_SYNC_WATCHPOINT: + watchpoint_handler(addr, regs); + break; + default: + ret = 1; /* Unhandled fault. */ + } + + /* + * Re-enable preemption after it was disabled in the + * low-level exception handling code. + */ + preempt_enable(); + + return ret; +} + +/* + * One-time initialisation. + */ +static void reset_ctrl_regs(void *info) +{ + int i, cpu = smp_processor_id(); + u32 dbg_power; + cpumask_t *cpumask = info; + + /* + * v7 debug contains save and restore registers so that debug state + * can be maintained across low-power modes without leaving the debug + * logic powered up. It is IMPLEMENTATION DEFINED whether we can access + * the debug registers out of reset, so we must unlock the OS Lock + * Access Register to avoid taking undefined instruction exceptions + * later on. + */ + if (debug_arch >= ARM_DEBUG_ARCH_V7_ECP14) { + /* + * Ensure sticky power-down is clear (i.e. debug logic is + * powered up). + */ + asm volatile("mrc p14, 0, %0, c1, c5, 4" : "=r" (dbg_power)); + if ((dbg_power & 0x1) == 0) { + pr_warning("CPU %d debug is powered down!\n", cpu); + cpumask_or(cpumask, cpumask, cpumask_of(cpu)); + return; + } + + /* + * Unconditionally clear the lock by writing a value + * other than 0xC5ACCE55 to the access register. + */ + asm volatile("mcr p14, 0, %0, c1, c0, 4" : : "r" (0)); + isb(); + + /* + * Clear any configured vector-catch events before + * enabling monitor mode. + */ + asm volatile("mcr p14, 0, %0, c0, c7, 0" : : "r" (0)); + isb(); + } + + if (enable_monitor_mode()) + return; + + /* We must also reset any reserved registers. */ + for (i = 0; i < core_num_brps + core_num_reserved_brps; ++i) { + write_wb_reg(ARM_BASE_BCR + i, 0UL); + write_wb_reg(ARM_BASE_BVR + i, 0UL); + } + + for (i = 0; i < core_num_wrps; ++i) { + write_wb_reg(ARM_BASE_WCR + i, 0UL); + write_wb_reg(ARM_BASE_WVR + i, 0UL); + } +} + +static int __cpuinit dbg_reset_notify(struct notifier_block *self, + unsigned long action, void *cpu) +{ + if (action == CPU_ONLINE) + smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1); + return NOTIFY_OK; +} + +static struct notifier_block __cpuinitdata dbg_reset_nb = { + .notifier_call = dbg_reset_notify, +}; + +static int __init arch_hw_breakpoint_init(void) +{ + u32 dscr; + cpumask_t cpumask = { CPU_BITS_NONE }; + + debug_arch = get_debug_arch(); + + if (!debug_arch_supported()) { + pr_info("debug architecture 0x%x unsupported.\n", debug_arch); + return 0; + } + + /* Determine how many BRPs/WRPs are available. */ + core_num_brps = get_num_brps(); + core_num_reserved_brps = get_num_reserved_brps(); + core_num_wrps = get_num_wrps(); + + pr_info("found %d breakpoint and %d watchpoint registers.\n", + core_num_brps + core_num_reserved_brps, core_num_wrps); + + if (core_num_reserved_brps) + pr_info("%d breakpoint(s) reserved for watchpoint " + "single-step.\n", core_num_reserved_brps); + + /* + * Reset the breakpoint resources. We assume that a halting + * debugger will leave the world in a nice state for us. + */ + on_each_cpu(reset_ctrl_regs, &cpumask, 1); + if (!cpumask_empty(&cpumask)) { + core_num_brps = 0; + core_num_reserved_brps = 0; + core_num_wrps = 0; + return 0; + } + + ARM_DBG_READ(c1, 0, dscr); + if (dscr & ARM_DSCR_HDBGEN) { + max_watchpoint_len = 4; + pr_warning("halting debug mode enabled. Assuming maximum watchpoint size of %u bytes.\n", + max_watchpoint_len); + } else { + /* Work out the maximum supported watchpoint length. */ + max_watchpoint_len = get_max_wp_len(); + pr_info("maximum watchpoint size is %u bytes.\n", + max_watchpoint_len); + } + + /* Register debug fault handler. */ + hook_fault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, + "watchpoint debug exception"); + hook_ifault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, + "breakpoint debug exception"); + + /* Register hotplug notifier. */ + register_cpu_notifier(&dbg_reset_nb); + return 0; +} +arch_initcall(arch_hw_breakpoint_init); + +void hw_breakpoint_pmu_read(struct perf_event *bp) +{ +} + +/* + * Dummy function to register with die_notifier. + */ +int hw_breakpoint_exceptions_notify(struct notifier_block *unused, + unsigned long val, void *data) +{ + return NOTIFY_DONE; +} diff --git a/arch/arm/kernel/init_task.c b/arch/arm/kernel/init_task.c new file mode 100644 index 00000000..e7cbb50d --- /dev/null +++ b/arch/arm/kernel/init_task.c @@ -0,0 +1,37 @@ +/* + * linux/arch/arm/kernel/init_task.c + */ +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/fs.h> +#include <linux/sched.h> +#include <linux/init.h> +#include <linux/init_task.h> +#include <linux/mqueue.h> +#include <linux/uaccess.h> + +#include <asm/pgtable.h> + +static struct signal_struct init_signals = INIT_SIGNALS(init_signals); +static struct sighand_struct init_sighand = INIT_SIGHAND(init_sighand); +/* + * Initial thread structure. + * + * We need to make sure that this is 8192-byte aligned due to the + * way process stacks are handled. This is done by making sure + * the linker maps this in the .text segment right after head.S, + * and making head.S ensure the proper alignment. + * + * The things we do for performance.. + */ +union thread_union init_thread_union __init_task_data = + { INIT_THREAD_INFO(init_task) }; + +/* + * Initial task structure. + * + * All other task structs will be allocated on slabs in fork.c + */ +struct task_struct init_task = INIT_TASK(init_task); + +EXPORT_SYMBOL(init_task); diff --git a/arch/arm/kernel/io.c b/arch/arm/kernel/io.c new file mode 100644 index 00000000..f4470307 --- /dev/null +++ b/arch/arm/kernel/io.c @@ -0,0 +1,50 @@ +#include <linux/module.h> +#include <linux/types.h> +#include <linux/io.h> + +/* + * Copy data from IO memory space to "real" memory space. + * This needs to be optimized. + */ +void _memcpy_fromio(void *to, const volatile void __iomem *from, size_t count) +{ + unsigned char *t = to; + while (count) { + count--; + *t = readb(from); + t++; + from++; + } +} + +/* + * Copy data from "real" memory space to IO memory space. + * This needs to be optimized. + */ +void _memcpy_toio(volatile void __iomem *to, const void *from, size_t count) +{ + const unsigned char *f = from; + while (count) { + count--; + writeb(*f, to); + f++; + to++; + } +} + +/* + * "memset" on IO memory space. + * This needs to be optimized. + */ +void _memset_io(volatile void __iomem *dst, int c, size_t count) +{ + while (count) { + count--; + writeb(c, dst); + dst++; + } +} + +EXPORT_SYMBOL(_memcpy_fromio); +EXPORT_SYMBOL(_memcpy_toio); +EXPORT_SYMBOL(_memset_io); diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c new file mode 100644 index 00000000..26179497 --- /dev/null +++ b/arch/arm/kernel/irq.c @@ -0,0 +1,201 @@ +/* + * linux/arch/arm/kernel/irq.c + * + * Copyright (C) 1992 Linus Torvalds + * Modifications for ARM processor Copyright (C) 1995-2000 Russell King. + * + * Support for Dynamic Tick Timer Copyright (C) 2004-2005 Nokia Corporation. + * Dynamic Tick Timer written by Tony Lindgren <tony@atomide.com> and + * Tuukka Tikkanen <tuukka.tikkanen@elektrobit.com>. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This file contains the code used by various IRQ handling routines: + * asking for different IRQ's should be done through these routines + * instead of just grabbing them. Thus setups with different IRQ numbers + * shouldn't result in any weird surprises, and installing new handlers + * should be easier. + * + * IRQ's are in fact implemented a bit like signal handlers for the kernel. + * Naturally it's not a 1:1 relation, but there are similarities. + */ +#include <linux/kernel_stat.h> +#include <linux/module.h> +#include <linux/signal.h> +#include <linux/ioport.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/random.h> +#include <linux/smp.h> +#include <linux/init.h> +#include <linux/seq_file.h> +#include <linux/errno.h> +#include <linux/list.h> +#include <linux/kallsyms.h> +#include <linux/proc_fs.h> +#include <linux/ftrace.h> + +#include <asm/system.h> +#include <asm/mach/arch.h> +#include <asm/mach/irq.h> +#include <asm/mach/time.h> + +/* + * No architecture-specific irq_finish function defined in arm/arch/irqs.h. + */ +#ifndef irq_finish +#define irq_finish(irq) do { } while (0) +#endif + +unsigned long irq_err_count; + +int arch_show_interrupts(struct seq_file *p, int prec) +{ +#ifdef CONFIG_FIQ + show_fiq_list(p, prec); +#endif +#ifdef CONFIG_SMP + show_ipi_list(p, prec); +#endif +#ifdef CONFIG_LOCAL_TIMERS + show_local_irqs(p, prec); +#endif + seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count); + return 0; +} + +/* + * handle_IRQ handles all hardware IRQ's. Decoded IRQs should + * not come via this function. Instead, they should provide their + * own 'handler'. Used by platform code implementing C-based 1st + * level decoding. + */ +void handle_IRQ(unsigned int irq, struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + + irq_enter(); + + /* + * Some hardware gives randomly wrong interrupts. Rather + * than crashing, do something sensible. + */ + if (unlikely(irq >= nr_irqs)) { + if (printk_ratelimit()) + printk(KERN_WARNING "Bad IRQ%u\n", irq); + ack_bad_irq(irq); + } else { + generic_handle_irq(irq); + } + + /* AT91 specific workaround */ + irq_finish(irq); + + irq_exit(); + set_irq_regs(old_regs); +} + +/* + * asm_do_IRQ is the interface to be used from assembly code. + */ +asmlinkage void __exception_irq_entry +asm_do_IRQ(unsigned int irq, struct pt_regs *regs) +{ + handle_IRQ(irq, regs); +} + +void set_irq_flags(unsigned int irq, unsigned int iflags) +{ + unsigned long clr = 0, set = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; + + if (irq >= nr_irqs) { + printk(KERN_ERR "Trying to set irq flags for IRQ%d\n", irq); + return; + } + + if (iflags & IRQF_VALID) + clr |= IRQ_NOREQUEST; + if (iflags & IRQF_PROBE) + clr |= IRQ_NOPROBE; + if (!(iflags & IRQF_NOAUTOEN)) + clr |= IRQ_NOAUTOEN; + /* Order is clear bits in "clr" then set bits in "set" */ + irq_modify_status(irq, clr, set & ~clr); +} + +void __init init_IRQ(void) +{ + machine_desc->init_irq(); +} + +#ifdef CONFIG_SPARSE_IRQ +int __init arch_probe_nr_irqs(void) +{ + nr_irqs = machine_desc->nr_irqs ? machine_desc->nr_irqs : NR_IRQS; + return nr_irqs; +} +#endif + +#ifdef CONFIG_HOTPLUG_CPU + +static bool migrate_one_irq(struct irq_desc *desc) +{ + struct irq_data *d = irq_desc_get_irq_data(desc); + const struct cpumask *affinity = d->affinity; + struct irq_chip *c; + bool ret = false; + + /* + * If this is a per-CPU interrupt, or the affinity does not + * include this CPU, then we have nothing to do. + */ + if (irqd_is_per_cpu(d) || !cpumask_test_cpu(smp_processor_id(), affinity)) + return false; + + if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) { + affinity = cpu_online_mask; + ret = true; + } + + c = irq_data_get_irq_chip(d); + if (!c->irq_set_affinity) + pr_debug("IRQ%u: unable to set affinity\n", d->irq); + else if (c->irq_set_affinity(d, affinity, true) == IRQ_SET_MASK_OK && ret) + cpumask_copy(d->affinity, affinity); + + return ret; +} + +/* + * The current CPU has been marked offline. Migrate IRQs off this CPU. + * If the affinity settings do not allow other CPUs, force them onto any + * available CPU. + * + * Note: we must iterate over all IRQs, whether they have an attached + * action structure or not, as we need to get chained interrupts too. + */ +void migrate_irqs(void) +{ + unsigned int i; + struct irq_desc *desc; + unsigned long flags; + + local_irq_save(flags); + + for_each_irq_desc(i, desc) { + bool affinity_broken; + + raw_spin_lock(&desc->lock); + affinity_broken = migrate_one_irq(desc); + raw_spin_unlock(&desc->lock); + + if (affinity_broken && printk_ratelimit()) + pr_warning("IRQ%u no longer affine to CPU%u\n", i, + smp_processor_id()); + } + + local_irq_restore(flags); +} +#endif /* CONFIG_HOTPLUG_CPU */ diff --git a/arch/arm/kernel/isa.c b/arch/arm/kernel/isa.c new file mode 100644 index 00000000..34648591 --- /dev/null +++ b/arch/arm/kernel/isa.c @@ -0,0 +1,70 @@ +/* + * linux/arch/arm/kernel/isa.c + * + * Copyright (C) 1999 Phil Blundell + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + * ISA shared memory and I/O port support, and is required to support + * iopl, inb, outb and friends in userspace via glibc emulation. + */ +#include <linux/stddef.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/sysctl.h> +#include <linux/init.h> +#include <linux/io.h> + +static unsigned int isa_membase, isa_portbase, isa_portshift; + +static ctl_table ctl_isa_vars[4] = { + { + .procname = "membase", + .data = &isa_membase, + .maxlen = sizeof(isa_membase), + .mode = 0444, + .proc_handler = proc_dointvec, + }, { + .procname = "portbase", + .data = &isa_portbase, + .maxlen = sizeof(isa_portbase), + .mode = 0444, + .proc_handler = proc_dointvec, + }, { + .procname = "portshift", + .data = &isa_portshift, + .maxlen = sizeof(isa_portshift), + .mode = 0444, + .proc_handler = proc_dointvec, + }, {} +}; + +static struct ctl_table_header *isa_sysctl_header; + +static ctl_table ctl_isa[2] = { + { + .procname = "isa", + .mode = 0555, + .child = ctl_isa_vars, + }, {} +}; + +static ctl_table ctl_bus[2] = { + { + .procname = "bus", + .mode = 0555, + .child = ctl_isa, + }, {} +}; + +void __init +register_isa_ports(unsigned int membase, unsigned int portbase, unsigned int portshift) +{ + isa_membase = membase; + isa_portbase = portbase; + isa_portshift = portshift; + isa_sysctl_header = register_sysctl_table(ctl_bus); +} diff --git a/arch/arm/kernel/iwmmxt.S b/arch/arm/kernel/iwmmxt.S new file mode 100644 index 00000000..7fa3bb0d --- /dev/null +++ b/arch/arm/kernel/iwmmxt.S @@ -0,0 +1,346 @@ +/* + * linux/arch/arm/kernel/iwmmxt.S + * + * XScale iWMMXt (Concan) context switching and handling + * + * Initial code: + * Copyright (c) 2003, Intel Corporation + * + * Full lazy switching support, optimizations and more, by Nicolas Pitre +* Copyright (c) 2003-2004, MontaVista Software, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/linkage.h> +#include <asm/ptrace.h> +#include <asm/thread_info.h> +#include <asm/asm-offsets.h> + +#if defined(CONFIG_CPU_PJ4) +#define PJ4(code...) code +#define XSC(code...) +#else +#define PJ4(code...) +#define XSC(code...) code +#endif + +#define MMX_WR0 (0x00) +#define MMX_WR1 (0x08) +#define MMX_WR2 (0x10) +#define MMX_WR3 (0x18) +#define MMX_WR4 (0x20) +#define MMX_WR5 (0x28) +#define MMX_WR6 (0x30) +#define MMX_WR7 (0x38) +#define MMX_WR8 (0x40) +#define MMX_WR9 (0x48) +#define MMX_WR10 (0x50) +#define MMX_WR11 (0x58) +#define MMX_WR12 (0x60) +#define MMX_WR13 (0x68) +#define MMX_WR14 (0x70) +#define MMX_WR15 (0x78) +#define MMX_WCSSF (0x80) +#define MMX_WCASF (0x84) +#define MMX_WCGR0 (0x88) +#define MMX_WCGR1 (0x8C) +#define MMX_WCGR2 (0x90) +#define MMX_WCGR3 (0x94) + +#define MMX_SIZE (0x98) + + .text + +/* + * Lazy switching of Concan coprocessor context + * + * r10 = struct thread_info pointer + * r9 = ret_from_exception + * lr = undefined instr exit + * + * called from prefetch exception handler with interrupts disabled + */ + +ENTRY(iwmmxt_task_enable) + + XSC(mrc p15, 0, r2, c15, c1, 0) + PJ4(mrc p15, 0, r2, c1, c0, 2) + @ CP0 and CP1 accessible? + XSC(tst r2, #0x3) + PJ4(tst r2, #0xf) + movne pc, lr @ if so no business here + @ enable access to CP0 and CP1 + XSC(orr r2, r2, #0x3) + XSC(mcr p15, 0, r2, c15, c1, 0) + PJ4(orr r2, r2, #0xf) + PJ4(mcr p15, 0, r2, c1, c0, 2) + + ldr r3, =concan_owner + add r0, r10, #TI_IWMMXT_STATE @ get task Concan save area + ldr r2, [sp, #60] @ current task pc value + ldr r1, [r3] @ get current Concan owner + str r0, [r3] @ this task now owns Concan regs + sub r2, r2, #4 @ adjust pc back + str r2, [sp, #60] + + mrc p15, 0, r2, c2, c0, 0 + mov r2, r2 @ cpwait + + teq r1, #0 @ test for last ownership + mov lr, r9 @ normal exit from exception + beq concan_load @ no owner, skip save + +concan_save: + + tmrc r2, wCon + + @ CUP? wCx + tst r2, #0x1 + beq 1f + +concan_dump: + + wstrw wCSSF, [r1, #MMX_WCSSF] + wstrw wCASF, [r1, #MMX_WCASF] + wstrw wCGR0, [r1, #MMX_WCGR0] + wstrw wCGR1, [r1, #MMX_WCGR1] + wstrw wCGR2, [r1, #MMX_WCGR2] + wstrw wCGR3, [r1, #MMX_WCGR3] + +1: @ MUP? wRn + tst r2, #0x2 + beq 2f + + wstrd wR0, [r1, #MMX_WR0] + wstrd wR1, [r1, #MMX_WR1] + wstrd wR2, [r1, #MMX_WR2] + wstrd wR3, [r1, #MMX_WR3] + wstrd wR4, [r1, #MMX_WR4] + wstrd wR5, [r1, #MMX_WR5] + wstrd wR6, [r1, #MMX_WR6] + wstrd wR7, [r1, #MMX_WR7] + wstrd wR8, [r1, #MMX_WR8] + wstrd wR9, [r1, #MMX_WR9] + wstrd wR10, [r1, #MMX_WR10] + wstrd wR11, [r1, #MMX_WR11] + wstrd wR12, [r1, #MMX_WR12] + wstrd wR13, [r1, #MMX_WR13] + wstrd wR14, [r1, #MMX_WR14] + wstrd wR15, [r1, #MMX_WR15] + +2: teq r0, #0 @ anything to load? + moveq pc, lr + +concan_load: + + @ Load wRn + wldrd wR0, [r0, #MMX_WR0] + wldrd wR1, [r0, #MMX_WR1] + wldrd wR2, [r0, #MMX_WR2] + wldrd wR3, [r0, #MMX_WR3] + wldrd wR4, [r0, #MMX_WR4] + wldrd wR5, [r0, #MMX_WR5] + wldrd wR6, [r0, #MMX_WR6] + wldrd wR7, [r0, #MMX_WR7] + wldrd wR8, [r0, #MMX_WR8] + wldrd wR9, [r0, #MMX_WR9] + wldrd wR10, [r0, #MMX_WR10] + wldrd wR11, [r0, #MMX_WR11] + wldrd wR12, [r0, #MMX_WR12] + wldrd wR13, [r0, #MMX_WR13] + wldrd wR14, [r0, #MMX_WR14] + wldrd wR15, [r0, #MMX_WR15] + + @ Load wCx + wldrw wCSSF, [r0, #MMX_WCSSF] + wldrw wCASF, [r0, #MMX_WCASF] + wldrw wCGR0, [r0, #MMX_WCGR0] + wldrw wCGR1, [r0, #MMX_WCGR1] + wldrw wCGR2, [r0, #MMX_WCGR2] + wldrw wCGR3, [r0, #MMX_WCGR3] + + @ clear CUP/MUP (only if r1 != 0) + teq r1, #0 + mov r2, #0 + moveq pc, lr + tmcr wCon, r2 + mov pc, lr + +/* + * Back up Concan regs to save area and disable access to them + * (mainly for gdb or sleep mode usage) + * + * r0 = struct thread_info pointer of target task or NULL for any + */ + +ENTRY(iwmmxt_task_disable) + + stmfd sp!, {r4, lr} + + mrs ip, cpsr + orr r2, ip, #PSR_I_BIT @ disable interrupts + msr cpsr_c, r2 + + ldr r3, =concan_owner + add r2, r0, #TI_IWMMXT_STATE @ get task Concan save area + ldr r1, [r3] @ get current Concan owner + teq r1, #0 @ any current owner? + beq 1f @ no: quit + teq r0, #0 @ any owner? + teqne r1, r2 @ or specified one? + bne 1f @ no: quit + + @ enable access to CP0 and CP1 + XSC(mrc p15, 0, r4, c15, c1, 0) + XSC(orr r4, r4, #0xf) + XSC(mcr p15, 0, r4, c15, c1, 0) + PJ4(mrc p15, 0, r4, c1, c0, 2) + PJ4(orr r4, r4, #0x3) + PJ4(mcr p15, 0, r4, c1, c0, 2) + + mov r0, #0 @ nothing to load + str r0, [r3] @ no more current owner + mrc p15, 0, r2, c2, c0, 0 + mov r2, r2 @ cpwait + bl concan_save + + @ disable access to CP0 and CP1 + XSC(bic r4, r4, #0x3) + XSC(mcr p15, 0, r4, c15, c1, 0) + PJ4(bic r4, r4, #0xf) + PJ4(mcr p15, 0, r4, c1, c0, 2) + + mrc p15, 0, r2, c2, c0, 0 + mov r2, r2 @ cpwait + +1: msr cpsr_c, ip @ restore interrupt mode + ldmfd sp!, {r4, pc} + +/* + * Copy Concan state to given memory address + * + * r0 = struct thread_info pointer of target task + * r1 = memory address where to store Concan state + * + * this is called mainly in the creation of signal stack frames + */ + +ENTRY(iwmmxt_task_copy) + + mrs ip, cpsr + orr r2, ip, #PSR_I_BIT @ disable interrupts + msr cpsr_c, r2 + + ldr r3, =concan_owner + add r2, r0, #TI_IWMMXT_STATE @ get task Concan save area + ldr r3, [r3] @ get current Concan owner + teq r2, r3 @ does this task own it... + beq 1f + + @ current Concan values are in the task save area + msr cpsr_c, ip @ restore interrupt mode + mov r0, r1 + mov r1, r2 + mov r2, #MMX_SIZE + b memcpy + +1: @ this task owns Concan regs -- grab a copy from there + mov r0, #0 @ nothing to load + mov r2, #3 @ save all regs + mov r3, lr @ preserve return address + bl concan_dump + msr cpsr_c, ip @ restore interrupt mode + mov pc, r3 + +/* + * Restore Concan state from given memory address + * + * r0 = struct thread_info pointer of target task + * r1 = memory address where to get Concan state from + * + * this is used to restore Concan state when unwinding a signal stack frame + */ + +ENTRY(iwmmxt_task_restore) + + mrs ip, cpsr + orr r2, ip, #PSR_I_BIT @ disable interrupts + msr cpsr_c, r2 + + ldr r3, =concan_owner + add r2, r0, #TI_IWMMXT_STATE @ get task Concan save area + ldr r3, [r3] @ get current Concan owner + bic r2, r2, #0x7 @ 64-bit alignment + teq r2, r3 @ does this task own it... + beq 1f + + @ this task doesn't own Concan regs -- use its save area + msr cpsr_c, ip @ restore interrupt mode + mov r0, r2 + mov r2, #MMX_SIZE + b memcpy + +1: @ this task owns Concan regs -- load them directly + mov r0, r1 + mov r1, #0 @ don't clear CUP/MUP + mov r3, lr @ preserve return address + bl concan_load + msr cpsr_c, ip @ restore interrupt mode + mov pc, r3 + +/* + * Concan handling on task switch + * + * r0 = next thread_info pointer + * + * Called only from the iwmmxt notifier with task preemption disabled. + */ +ENTRY(iwmmxt_task_switch) + + XSC(mrc p15, 0, r1, c15, c1, 0) + PJ4(mrc p15, 0, r1, c1, c0, 2) + @ CP0 and CP1 accessible? + XSC(tst r1, #0x3) + PJ4(tst r1, #0xf) + bne 1f @ yes: block them for next task + + ldr r2, =concan_owner + add r3, r0, #TI_IWMMXT_STATE @ get next task Concan save area + ldr r2, [r2] @ get current Concan owner + teq r2, r3 @ next task owns it? + movne pc, lr @ no: leave Concan disabled + +1: @ flip Conan access + XSC(eor r1, r1, #0x3) + XSC(mcr p15, 0, r1, c15, c1, 0) + PJ4(eor r1, r1, #0xf) + PJ4(mcr p15, 0, r1, c1, c0, 2) + + mrc p15, 0, r1, c2, c0, 0 + sub pc, lr, r1, lsr #32 @ cpwait and return + +/* + * Remove Concan ownership of given task + * + * r0 = struct thread_info pointer + */ +ENTRY(iwmmxt_task_release) + + mrs r2, cpsr + orr ip, r2, #PSR_I_BIT @ disable interrupts + msr cpsr_c, ip + ldr r3, =concan_owner + add r0, r0, #TI_IWMMXT_STATE @ get task Concan save area + ldr r1, [r3] @ get current Concan owner + eors r0, r0, r1 @ if equal... + streq r0, [r3] @ then clear ownership + msr cpsr_c, r2 @ restore interrupts + mov pc, lr + + .data +concan_owner: + .word 0 + diff --git a/arch/arm/kernel/kgdb.c b/arch/arm/kernel/kgdb.c new file mode 100644 index 00000000..778c2f70 --- /dev/null +++ b/arch/arm/kernel/kgdb.c @@ -0,0 +1,255 @@ +/* + * arch/arm/kernel/kgdb.c + * + * ARM KGDB support + * + * Copyright (c) 2002-2004 MontaVista Software, Inc + * Copyright (c) 2008 Wind River Systems, Inc. + * + * Authors: George Davis <davis_g@mvista.com> + * Deepak Saxena <dsaxena@plexity.net> + */ +#include <linux/irq.h> +#include <linux/kdebug.h> +#include <linux/kgdb.h> +#include <asm/traps.h> + +struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = +{ + { "r0", 4, offsetof(struct pt_regs, ARM_r0)}, + { "r1", 4, offsetof(struct pt_regs, ARM_r1)}, + { "r2", 4, offsetof(struct pt_regs, ARM_r2)}, + { "r3", 4, offsetof(struct pt_regs, ARM_r3)}, + { "r4", 4, offsetof(struct pt_regs, ARM_r4)}, + { "r5", 4, offsetof(struct pt_regs, ARM_r5)}, + { "r6", 4, offsetof(struct pt_regs, ARM_r6)}, + { "r7", 4, offsetof(struct pt_regs, ARM_r7)}, + { "r8", 4, offsetof(struct pt_regs, ARM_r8)}, + { "r9", 4, offsetof(struct pt_regs, ARM_r9)}, + { "r10", 4, offsetof(struct pt_regs, ARM_r10)}, + { "fp", 4, offsetof(struct pt_regs, ARM_fp)}, + { "ip", 4, offsetof(struct pt_regs, ARM_ip)}, + { "sp", 4, offsetof(struct pt_regs, ARM_sp)}, + { "lr", 4, offsetof(struct pt_regs, ARM_lr)}, + { "pc", 4, offsetof(struct pt_regs, ARM_pc)}, + { "f0", 12, -1 }, + { "f1", 12, -1 }, + { "f2", 12, -1 }, + { "f3", 12, -1 }, + { "f4", 12, -1 }, + { "f5", 12, -1 }, + { "f6", 12, -1 }, + { "f7", 12, -1 }, + { "fps", 4, -1 }, + { "cpsr", 4, offsetof(struct pt_regs, ARM_cpsr)}, +}; + +char *dbg_get_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return NULL; + + if (dbg_reg_def[regno].offset != -1) + memcpy(mem, (void *)regs + dbg_reg_def[regno].offset, + dbg_reg_def[regno].size); + else + memset(mem, 0, dbg_reg_def[regno].size); + return dbg_reg_def[regno].name; +} + +int dbg_set_reg(int regno, void *mem, struct pt_regs *regs) +{ + if (regno >= DBG_MAX_REG_NUM || regno < 0) + return -EINVAL; + + if (dbg_reg_def[regno].offset != -1) + memcpy((void *)regs + dbg_reg_def[regno].offset, mem, + dbg_reg_def[regno].size); + return 0; +} + +void +sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task) +{ + struct pt_regs *thread_regs; + int regno; + + /* Just making sure... */ + if (task == NULL) + return; + + /* Initialize to zero */ + for (regno = 0; regno < GDB_MAX_REGS; regno++) + gdb_regs[regno] = 0; + + /* Otherwise, we have only some registers from switch_to() */ + thread_regs = task_pt_regs(task); + gdb_regs[_R0] = thread_regs->ARM_r0; + gdb_regs[_R1] = thread_regs->ARM_r1; + gdb_regs[_R2] = thread_regs->ARM_r2; + gdb_regs[_R3] = thread_regs->ARM_r3; + gdb_regs[_R4] = thread_regs->ARM_r4; + gdb_regs[_R5] = thread_regs->ARM_r5; + gdb_regs[_R6] = thread_regs->ARM_r6; + gdb_regs[_R7] = thread_regs->ARM_r7; + gdb_regs[_R8] = thread_regs->ARM_r8; + gdb_regs[_R9] = thread_regs->ARM_r9; + gdb_regs[_R10] = thread_regs->ARM_r10; + gdb_regs[_FP] = thread_regs->ARM_fp; + gdb_regs[_IP] = thread_regs->ARM_ip; + gdb_regs[_SPT] = thread_regs->ARM_sp; + gdb_regs[_LR] = thread_regs->ARM_lr; + gdb_regs[_PC] = thread_regs->ARM_pc; + gdb_regs[_CPSR] = thread_regs->ARM_cpsr; +} + +void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long pc) +{ + regs->ARM_pc = pc; +} + +static int compiled_break; + +int kgdb_arch_handle_exception(int exception_vector, int signo, + int err_code, char *remcom_in_buffer, + char *remcom_out_buffer, + struct pt_regs *linux_regs) +{ + unsigned long addr; + char *ptr; + + switch (remcom_in_buffer[0]) { + case 'D': + case 'k': + case 'c': + /* + * Try to read optional parameter, pc unchanged if no parm. + * If this was a compiled breakpoint, we need to move + * to the next instruction or we will just breakpoint + * over and over again. + */ + ptr = &remcom_in_buffer[1]; + if (kgdb_hex2long(&ptr, &addr)) + linux_regs->ARM_pc = addr; + else if (compiled_break == 1) + linux_regs->ARM_pc += 4; + + compiled_break = 0; + + return 0; + } + + return -1; +} + +static int kgdb_brk_fn(struct pt_regs *regs, unsigned int instr) +{ + kgdb_handle_exception(1, SIGTRAP, 0, regs); + + return 0; +} + +static int kgdb_compiled_brk_fn(struct pt_regs *regs, unsigned int instr) +{ + compiled_break = 1; + kgdb_handle_exception(1, SIGTRAP, 0, regs); + + return 0; +} + +static struct undef_hook kgdb_brkpt_hook = { + .instr_mask = 0xffffffff, + .instr_val = KGDB_BREAKINST, + .fn = kgdb_brk_fn +}; + +static struct undef_hook kgdb_compiled_brkpt_hook = { + .instr_mask = 0xffffffff, + .instr_val = KGDB_COMPILED_BREAK, + .fn = kgdb_compiled_brk_fn +}; + +static void kgdb_call_nmi_hook(void *ignored) +{ + kgdb_nmicallback(raw_smp_processor_id(), get_irq_regs()); +} + +void kgdb_roundup_cpus(unsigned long flags) +{ + local_irq_enable(); + smp_call_function(kgdb_call_nmi_hook, NULL, 0); + local_irq_disable(); +} + +static int __kgdb_notify(struct die_args *args, unsigned long cmd) +{ + struct pt_regs *regs = args->regs; + + if (kgdb_handle_exception(1, args->signr, cmd, regs)) + return NOTIFY_DONE; + return NOTIFY_STOP; +} +static int +kgdb_notify(struct notifier_block *self, unsigned long cmd, void *ptr) +{ + unsigned long flags; + int ret; + + local_irq_save(flags); + ret = __kgdb_notify(ptr, cmd); + local_irq_restore(flags); + + return ret; +} + +static struct notifier_block kgdb_notifier = { + .notifier_call = kgdb_notify, + .priority = -INT_MAX, +}; + + +/** + * kgdb_arch_init - Perform any architecture specific initalization. + * + * This function will handle the initalization of any architecture + * specific callbacks. + */ +int kgdb_arch_init(void) +{ + int ret = register_die_notifier(&kgdb_notifier); + + if (ret != 0) + return ret; + + register_undef_hook(&kgdb_brkpt_hook); + register_undef_hook(&kgdb_compiled_brkpt_hook); + + return 0; +} + +/** + * kgdb_arch_exit - Perform any architecture specific uninitalization. + * + * This function will handle the uninitalization of any architecture + * specific callbacks, for dynamic registration and unregistration. + */ +void kgdb_arch_exit(void) +{ + unregister_undef_hook(&kgdb_brkpt_hook); + unregister_undef_hook(&kgdb_compiled_brkpt_hook); + unregister_die_notifier(&kgdb_notifier); +} + +/* + * Register our undef instruction hooks with ARM undef core. + * We regsiter a hook specifically looking for the KGB break inst + * and we handle the normal undef case within the do_undefinstr + * handler. + */ +struct kgdb_arch arch_kgdb_ops = { +#ifndef __ARMEB__ + .gdb_bpt_instr = {0xfe, 0xde, 0xff, 0xe7} +#else /* ! __ARMEB__ */ + .gdb_bpt_instr = {0xe7, 0xff, 0xde, 0xfe} +#endif +}; diff --git a/arch/arm/kernel/kprobes-decode.c b/arch/arm/kernel/kprobes-decode.c new file mode 100644 index 00000000..15eeff6a --- /dev/null +++ b/arch/arm/kernel/kprobes-decode.c @@ -0,0 +1,1670 @@ +/* + * arch/arm/kernel/kprobes-decode.c + * + * Copyright (C) 2006, 2007 Motorola Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +/* + * We do not have hardware single-stepping on ARM, This + * effort is further complicated by the ARM not having a + * "next PC" register. Instructions that change the PC + * can't be safely single-stepped in a MP environment, so + * we have a lot of work to do: + * + * In the prepare phase: + * *) If it is an instruction that does anything + * with the CPU mode, we reject it for a kprobe. + * (This is out of laziness rather than need. The + * instructions could be simulated.) + * + * *) Otherwise, decode the instruction rewriting its + * registers to take fixed, ordered registers and + * setting a handler for it to run the instruction. + * + * In the execution phase by an instruction's handler: + * + * *) If the PC is written to by the instruction, the + * instruction must be fully simulated in software. + * + * *) Otherwise, a modified form of the instruction is + * directly executed. Its handler calls the + * instruction in insn[0]. In insn[1] is a + * "mov pc, lr" to return. + * + * Before calling, load up the reordered registers + * from the original instruction's registers. If one + * of the original input registers is the PC, compute + * and adjust the appropriate input register. + * + * After call completes, copy the output registers to + * the original instruction's original registers. + * + * We don't use a real breakpoint instruction since that + * would have us in the kernel go from SVC mode to SVC + * mode losing the link register. Instead we use an + * undefined instruction. To simplify processing, the + * undefined instruction used for kprobes must be reserved + * exclusively for kprobes use. + * + * TODO: ifdef out some instruction decoding based on architecture. + */ + +#include <linux/kernel.h> +#include <linux/kprobes.h> + +#define sign_extend(x, signbit) ((x) | (0 - ((x) & (1 << (signbit))))) + +#define branch_displacement(insn) sign_extend(((insn) & 0xffffff) << 2, 25) + +#define is_r15(insn, bitpos) (((insn) & (0xf << bitpos)) == (0xf << bitpos)) + +/* + * Test if load/store instructions writeback the address register. + * if P (bit 24) == 0 or W (bit 21) == 1 + */ +#define is_writeback(insn) ((insn ^ 0x01000000) & 0x01200000) + +#define PSR_fs (PSR_f|PSR_s) + +#define KPROBE_RETURN_INSTRUCTION 0xe1a0f00e /* mov pc, lr */ + +typedef long (insn_0arg_fn_t)(void); +typedef long (insn_1arg_fn_t)(long); +typedef long (insn_2arg_fn_t)(long, long); +typedef long (insn_3arg_fn_t)(long, long, long); +typedef long (insn_4arg_fn_t)(long, long, long, long); +typedef long long (insn_llret_0arg_fn_t)(void); +typedef long long (insn_llret_3arg_fn_t)(long, long, long); +typedef long long (insn_llret_4arg_fn_t)(long, long, long, long); + +union reg_pair { + long long dr; +#ifdef __LITTLE_ENDIAN + struct { long r0, r1; }; +#else + struct { long r1, r0; }; +#endif +}; + +/* + * For STR and STM instructions, an ARM core may choose to use either + * a +8 or a +12 displacement from the current instruction's address. + * Whichever value is chosen for a given core, it must be the same for + * both instructions and may not change. This function measures it. + */ + +static int str_pc_offset; + +static void __init find_str_pc_offset(void) +{ + int addr, scratch, ret; + + __asm__ ( + "sub %[ret], pc, #4 \n\t" + "str pc, %[addr] \n\t" + "ldr %[scr], %[addr] \n\t" + "sub %[ret], %[scr], %[ret] \n\t" + : [ret] "=r" (ret), [scr] "=r" (scratch), [addr] "+m" (addr)); + + str_pc_offset = ret; +} + +/* + * The insnslot_?arg_r[w]flags() functions below are to keep the + * msr -> *fn -> mrs instruction sequences indivisible so that + * the state of the CPSR flags aren't inadvertently modified + * just before or just after the call. + */ + +static inline long __kprobes +insnslot_0arg_rflags(long cpsr, insn_0arg_fn_t *fn) +{ + register long ret asm("r0"); + + __asm__ __volatile__ ( + "msr cpsr_fs, %[cpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + : "=r" (ret) + : [cpsr] "r" (cpsr), [fn] "r" (fn) + : "lr", "cc" + ); + return ret; +} + +static inline long long __kprobes +insnslot_llret_0arg_rflags(long cpsr, insn_llret_0arg_fn_t *fn) +{ + register long ret0 asm("r0"); + register long ret1 asm("r1"); + union reg_pair fnr; + + __asm__ __volatile__ ( + "msr cpsr_fs, %[cpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + : "=r" (ret0), "=r" (ret1) + : [cpsr] "r" (cpsr), [fn] "r" (fn) + : "lr", "cc" + ); + fnr.r0 = ret0; + fnr.r1 = ret1; + return fnr.dr; +} + +static inline long __kprobes +insnslot_1arg_rflags(long r0, long cpsr, insn_1arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long ret asm("r0"); + + __asm__ __volatile__ ( + "msr cpsr_fs, %[cpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + : "=r" (ret) + : "0" (rr0), [cpsr] "r" (cpsr), [fn] "r" (fn) + : "lr", "cc" + ); + return ret; +} + +static inline long __kprobes +insnslot_2arg_rflags(long r0, long r1, long cpsr, insn_2arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long ret asm("r0"); + + __asm__ __volatile__ ( + "msr cpsr_fs, %[cpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + : "=r" (ret) + : "0" (rr0), "r" (rr1), + [cpsr] "r" (cpsr), [fn] "r" (fn) + : "lr", "cc" + ); + return ret; +} + +static inline long __kprobes +insnslot_3arg_rflags(long r0, long r1, long r2, long cpsr, insn_3arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long rr2 asm("r2") = r2; + register long ret asm("r0"); + + __asm__ __volatile__ ( + "msr cpsr_fs, %[cpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + : "=r" (ret) + : "0" (rr0), "r" (rr1), "r" (rr2), + [cpsr] "r" (cpsr), [fn] "r" (fn) + : "lr", "cc" + ); + return ret; +} + +static inline long long __kprobes +insnslot_llret_3arg_rflags(long r0, long r1, long r2, long cpsr, + insn_llret_3arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long rr2 asm("r2") = r2; + register long ret0 asm("r0"); + register long ret1 asm("r1"); + union reg_pair fnr; + + __asm__ __volatile__ ( + "msr cpsr_fs, %[cpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + : "=r" (ret0), "=r" (ret1) + : "0" (rr0), "r" (rr1), "r" (rr2), + [cpsr] "r" (cpsr), [fn] "r" (fn) + : "lr", "cc" + ); + fnr.r0 = ret0; + fnr.r1 = ret1; + return fnr.dr; +} + +static inline long __kprobes +insnslot_4arg_rflags(long r0, long r1, long r2, long r3, long cpsr, + insn_4arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long rr2 asm("r2") = r2; + register long rr3 asm("r3") = r3; + register long ret asm("r0"); + + __asm__ __volatile__ ( + "msr cpsr_fs, %[cpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + : "=r" (ret) + : "0" (rr0), "r" (rr1), "r" (rr2), "r" (rr3), + [cpsr] "r" (cpsr), [fn] "r" (fn) + : "lr", "cc" + ); + return ret; +} + +static inline long __kprobes +insnslot_1arg_rwflags(long r0, long *cpsr, insn_1arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long ret asm("r0"); + long oldcpsr = *cpsr; + long newcpsr; + + __asm__ __volatile__ ( + "msr cpsr_fs, %[oldcpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + "mrs %[newcpsr], cpsr \n\t" + : "=r" (ret), [newcpsr] "=r" (newcpsr) + : "0" (rr0), [oldcpsr] "r" (oldcpsr), [fn] "r" (fn) + : "lr", "cc" + ); + *cpsr = (oldcpsr & ~PSR_fs) | (newcpsr & PSR_fs); + return ret; +} + +static inline long __kprobes +insnslot_2arg_rwflags(long r0, long r1, long *cpsr, insn_2arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long ret asm("r0"); + long oldcpsr = *cpsr; + long newcpsr; + + __asm__ __volatile__ ( + "msr cpsr_fs, %[oldcpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + "mrs %[newcpsr], cpsr \n\t" + : "=r" (ret), [newcpsr] "=r" (newcpsr) + : "0" (rr0), "r" (rr1), [oldcpsr] "r" (oldcpsr), [fn] "r" (fn) + : "lr", "cc" + ); + *cpsr = (oldcpsr & ~PSR_fs) | (newcpsr & PSR_fs); + return ret; +} + +static inline long __kprobes +insnslot_3arg_rwflags(long r0, long r1, long r2, long *cpsr, + insn_3arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long rr2 asm("r2") = r2; + register long ret asm("r0"); + long oldcpsr = *cpsr; + long newcpsr; + + __asm__ __volatile__ ( + "msr cpsr_fs, %[oldcpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + "mrs %[newcpsr], cpsr \n\t" + : "=r" (ret), [newcpsr] "=r" (newcpsr) + : "0" (rr0), "r" (rr1), "r" (rr2), + [oldcpsr] "r" (oldcpsr), [fn] "r" (fn) + : "lr", "cc" + ); + *cpsr = (oldcpsr & ~PSR_fs) | (newcpsr & PSR_fs); + return ret; +} + +static inline long __kprobes +insnslot_4arg_rwflags(long r0, long r1, long r2, long r3, long *cpsr, + insn_4arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long rr2 asm("r2") = r2; + register long rr3 asm("r3") = r3; + register long ret asm("r0"); + long oldcpsr = *cpsr; + long newcpsr; + + __asm__ __volatile__ ( + "msr cpsr_fs, %[oldcpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + "mrs %[newcpsr], cpsr \n\t" + : "=r" (ret), [newcpsr] "=r" (newcpsr) + : "0" (rr0), "r" (rr1), "r" (rr2), "r" (rr3), + [oldcpsr] "r" (oldcpsr), [fn] "r" (fn) + : "lr", "cc" + ); + *cpsr = (oldcpsr & ~PSR_fs) | (newcpsr & PSR_fs); + return ret; +} + +static inline long long __kprobes +insnslot_llret_4arg_rwflags(long r0, long r1, long r2, long r3, long *cpsr, + insn_llret_4arg_fn_t *fn) +{ + register long rr0 asm("r0") = r0; + register long rr1 asm("r1") = r1; + register long rr2 asm("r2") = r2; + register long rr3 asm("r3") = r3; + register long ret0 asm("r0"); + register long ret1 asm("r1"); + long oldcpsr = *cpsr; + long newcpsr; + union reg_pair fnr; + + __asm__ __volatile__ ( + "msr cpsr_fs, %[oldcpsr] \n\t" + "mov lr, pc \n\t" + "mov pc, %[fn] \n\t" + "mrs %[newcpsr], cpsr \n\t" + : "=r" (ret0), "=r" (ret1), [newcpsr] "=r" (newcpsr) + : "0" (rr0), "r" (rr1), "r" (rr2), "r" (rr3), + [oldcpsr] "r" (oldcpsr), [fn] "r" (fn) + : "lr", "cc" + ); + *cpsr = (oldcpsr & ~PSR_fs) | (newcpsr & PSR_fs); + fnr.r0 = ret0; + fnr.r1 = ret1; + return fnr.dr; +} + +/* + * To avoid the complications of mimicing single-stepping on a + * processor without a Next-PC or a single-step mode, and to + * avoid having to deal with the side-effects of boosting, we + * simulate or emulate (almost) all ARM instructions. + * + * "Simulation" is where the instruction's behavior is duplicated in + * C code. "Emulation" is where the original instruction is rewritten + * and executed, often by altering its registers. + * + * By having all behavior of the kprobe'd instruction completed before + * returning from the kprobe_handler(), all locks (scheduler and + * interrupt) can safely be released. There is no need for secondary + * breakpoints, no race with MP or preemptable kernels, nor having to + * clean up resources counts at a later time impacting overall system + * performance. By rewriting the instruction, only the minimum registers + * need to be loaded and saved back optimizing performance. + * + * Calling the insnslot_*_rwflags version of a function doesn't hurt + * anything even when the CPSR flags aren't updated by the + * instruction. It's just a little slower in return for saving + * a little space by not having a duplicate function that doesn't + * update the flags. (The same optimization can be said for + * instructions that do or don't perform register writeback) + * Also, instructions can either read the flags, only write the + * flags, or read and write the flags. To save combinations + * rather than for sheer performance, flag functions just assume + * read and write of flags. + */ + +static void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + long iaddr = (long)p->addr; + int disp = branch_displacement(insn); + + if (insn & (1 << 24)) + regs->ARM_lr = iaddr + 4; + + regs->ARM_pc = iaddr + 8 + disp; +} + +static void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + long iaddr = (long)p->addr; + int disp = branch_displacement(insn); + + regs->ARM_lr = iaddr + 4; + regs->ARM_pc = iaddr + 8 + disp + ((insn >> 23) & 0x2); + regs->ARM_cpsr |= PSR_T_BIT; +} + +static void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + int rm = insn & 0xf; + long rmv = regs->uregs[rm]; + + if (insn & (1 << 5)) + regs->ARM_lr = (long)p->addr + 4; + + regs->ARM_pc = rmv & ~0x1; + regs->ARM_cpsr &= ~PSR_T_BIT; + if (rmv & 0x1) + regs->ARM_cpsr |= PSR_T_BIT; +} + +static void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + unsigned long mask = 0xf8ff03df; /* Mask out execution state */ + regs->uregs[rd] = regs->ARM_cpsr & mask; +} + +static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs) +{ + kprobe_opcode_t insn = p->opcode; + int rn = (insn >> 16) & 0xf; + int lbit = insn & (1 << 20); + int wbit = insn & (1 << 21); + int ubit = insn & (1 << 23); + int pbit = insn & (1 << 24); + long *addr = (long *)regs->uregs[rn]; + int reg_bit_vector; + int reg_count; + + reg_count = 0; + reg_bit_vector = insn & 0xffff; + while (reg_bit_vector) { + reg_bit_vector &= (reg_bit_vector - 1); + ++reg_count; + } + + if (!ubit) + addr -= reg_count; + addr += (!pbit == !ubit); + + reg_bit_vector = insn & 0xffff; + while (reg_bit_vector) { + int reg = __ffs(reg_bit_vector); + reg_bit_vector &= (reg_bit_vector - 1); + if (lbit) + regs->uregs[reg] = *addr++; + else + *addr++ = regs->uregs[reg]; + } + + if (wbit) { + if (!ubit) + addr -= reg_count; + addr -= (!pbit == !ubit); + regs->uregs[rn] = (long)addr; + } +} + +static void __kprobes simulate_stm1_pc(struct kprobe *p, struct pt_regs *regs) +{ + regs->ARM_pc = (long)p->addr + str_pc_offset; + simulate_ldm1stm1(p, regs); + regs->ARM_pc = (long)p->addr + 4; +} + +static void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs) +{ + regs->uregs[12] = regs->uregs[13]; +} + +static void __kprobes emulate_ldrd(struct kprobe *p, struct pt_regs *regs) +{ + insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + long ppc = (long)p->addr + 8; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + int rm = insn & 0xf; /* rm may be invalid, don't care. */ + long rmv = (rm == 15) ? ppc : regs->uregs[rm]; + long rnv = (rn == 15) ? ppc : regs->uregs[rn]; + + /* Not following the C calling convention here, so need asm(). */ + __asm__ __volatile__ ( + "ldr r0, %[rn] \n\t" + "ldr r1, %[rm] \n\t" + "msr cpsr_fs, %[cpsr]\n\t" + "mov lr, pc \n\t" + "mov pc, %[i_fn] \n\t" + "str r0, %[rn] \n\t" /* in case of writeback */ + "str r2, %[rd0] \n\t" + "str r3, %[rd1] \n\t" + : [rn] "+m" (rnv), + [rd0] "=m" (regs->uregs[rd]), + [rd1] "=m" (regs->uregs[rd+1]) + : [rm] "m" (rmv), + [cpsr] "r" (regs->ARM_cpsr), + [i_fn] "r" (i_fn) + : "r0", "r1", "r2", "r3", "lr", "cc" + ); + if (is_writeback(insn)) + regs->uregs[rn] = rnv; +} + +static void __kprobes emulate_strd(struct kprobe *p, struct pt_regs *regs) +{ + insn_4arg_fn_t *i_fn = (insn_4arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + long ppc = (long)p->addr + 8; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + int rm = insn & 0xf; + long rnv = (rn == 15) ? ppc : regs->uregs[rn]; + /* rm/rmv may be invalid, don't care. */ + long rmv = (rm == 15) ? ppc : regs->uregs[rm]; + long rnv_wb; + + rnv_wb = insnslot_4arg_rflags(rnv, rmv, regs->uregs[rd], + regs->uregs[rd+1], + regs->ARM_cpsr, i_fn); + if (is_writeback(insn)) + regs->uregs[rn] = rnv_wb; +} + +static void __kprobes emulate_ldr(struct kprobe *p, struct pt_regs *regs) +{ + insn_llret_3arg_fn_t *i_fn = (insn_llret_3arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + long ppc = (long)p->addr + 8; + union reg_pair fnr; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + int rm = insn & 0xf; + long rdv; + long rnv = (rn == 15) ? ppc : regs->uregs[rn]; + long rmv = (rm == 15) ? ppc : regs->uregs[rm]; + long cpsr = regs->ARM_cpsr; + + fnr.dr = insnslot_llret_3arg_rflags(rnv, 0, rmv, cpsr, i_fn); + if (rn != 15) + regs->uregs[rn] = fnr.r0; /* Save Rn in case of writeback. */ + rdv = fnr.r1; + + if (rd == 15) { +#if __LINUX_ARM_ARCH__ >= 5 + cpsr &= ~PSR_T_BIT; + if (rdv & 0x1) + cpsr |= PSR_T_BIT; + regs->ARM_cpsr = cpsr; + rdv &= ~0x1; +#else + rdv &= ~0x2; +#endif + } + regs->uregs[rd] = rdv; +} + +static void __kprobes emulate_str(struct kprobe *p, struct pt_regs *regs) +{ + insn_3arg_fn_t *i_fn = (insn_3arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + long iaddr = (long)p->addr; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + int rm = insn & 0xf; + long rdv = (rd == 15) ? iaddr + str_pc_offset : regs->uregs[rd]; + long rnv = (rn == 15) ? iaddr + 8 : regs->uregs[rn]; + long rmv = regs->uregs[rm]; /* rm/rmv may be invalid, don't care. */ + long rnv_wb; + + rnv_wb = insnslot_3arg_rflags(rnv, rdv, rmv, regs->ARM_cpsr, i_fn); + if (rn != 15) + regs->uregs[rn] = rnv_wb; /* Save Rn in case of writeback. */ +} + +static void __kprobes emulate_sat(struct kprobe *p, struct pt_regs *regs) +{ + insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + int rm = insn & 0xf; + long rmv = regs->uregs[rm]; + + /* Writes Q flag */ + regs->uregs[rd] = insnslot_1arg_rwflags(rmv, ®s->ARM_cpsr, i_fn); +} + +static void __kprobes emulate_sel(struct kprobe *p, struct pt_regs *regs) +{ + insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + int rm = insn & 0xf; + long rnv = regs->uregs[rn]; + long rmv = regs->uregs[rm]; + + /* Reads GE bits */ + regs->uregs[rd] = insnslot_2arg_rflags(rnv, rmv, regs->ARM_cpsr, i_fn); +} + +static void __kprobes emulate_none(struct kprobe *p, struct pt_regs *regs) +{ + insn_0arg_fn_t *i_fn = (insn_0arg_fn_t *)&p->ainsn.insn[0]; + + insnslot_0arg_rflags(regs->ARM_cpsr, i_fn); +} + +static void __kprobes emulate_nop(struct kprobe *p, struct pt_regs *regs) +{ +} + +static void __kprobes +emulate_rd12_modify(struct kprobe *p, struct pt_regs *regs) +{ + insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + long rdv = regs->uregs[rd]; + + regs->uregs[rd] = insnslot_1arg_rflags(rdv, regs->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_rd12rn0_modify(struct kprobe *p, struct pt_regs *regs) +{ + insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + int rn = insn & 0xf; + long rdv = regs->uregs[rd]; + long rnv = regs->uregs[rn]; + + regs->uregs[rd] = insnslot_2arg_rflags(rdv, rnv, regs->ARM_cpsr, i_fn); +} + +static void __kprobes emulate_rd12rm0(struct kprobe *p, struct pt_regs *regs) +{ + insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + int rm = insn & 0xf; + long rmv = regs->uregs[rm]; + + regs->uregs[rd] = insnslot_1arg_rflags(rmv, regs->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_rd12rn16rm0_rwflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + int rm = insn & 0xf; + long rnv = regs->uregs[rn]; + long rmv = regs->uregs[rm]; + + regs->uregs[rd] = + insnslot_2arg_rwflags(rnv, rmv, ®s->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_rd16rn12rs8rm0_rwflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_3arg_fn_t *i_fn = (insn_3arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 16) & 0xf; + int rn = (insn >> 12) & 0xf; + int rs = (insn >> 8) & 0xf; + int rm = insn & 0xf; + long rnv = regs->uregs[rn]; + long rsv = regs->uregs[rs]; + long rmv = regs->uregs[rm]; + + regs->uregs[rd] = + insnslot_3arg_rwflags(rnv, rsv, rmv, ®s->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_rd16rs8rm0_rwflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 16) & 0xf; + int rs = (insn >> 8) & 0xf; + int rm = insn & 0xf; + long rsv = regs->uregs[rs]; + long rmv = regs->uregs[rm]; + + regs->uregs[rd] = + insnslot_2arg_rwflags(rsv, rmv, ®s->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_rdhi16rdlo12rs8rm0_rwflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_llret_4arg_fn_t *i_fn = (insn_llret_4arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + union reg_pair fnr; + int rdhi = (insn >> 16) & 0xf; + int rdlo = (insn >> 12) & 0xf; + int rs = (insn >> 8) & 0xf; + int rm = insn & 0xf; + long rsv = regs->uregs[rs]; + long rmv = regs->uregs[rm]; + + fnr.dr = insnslot_llret_4arg_rwflags(regs->uregs[rdhi], + regs->uregs[rdlo], rsv, rmv, + ®s->ARM_cpsr, i_fn); + regs->uregs[rdhi] = fnr.r0; + regs->uregs[rdlo] = fnr.r1; +} + +static void __kprobes +emulate_alu_imm_rflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + long rnv = (rn == 15) ? (long)p->addr + 8 : regs->uregs[rn]; + + regs->uregs[rd] = insnslot_1arg_rflags(rnv, regs->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_alu_imm_rwflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; + long rnv = (rn == 15) ? (long)p->addr + 8 : regs->uregs[rn]; + + regs->uregs[rd] = insnslot_1arg_rwflags(rnv, ®s->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_alu_tests_imm(struct kprobe *p, struct pt_regs *regs) +{ + insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + int rn = (insn >> 16) & 0xf; + long rnv = (rn == 15) ? (long)p->addr + 8 : regs->uregs[rn]; + + insnslot_1arg_rwflags(rnv, ®s->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_alu_rflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_3arg_fn_t *i_fn = (insn_3arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + long ppc = (long)p->addr + 8; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; /* rn/rnv/rs/rsv may be */ + int rs = (insn >> 8) & 0xf; /* invalid, don't care. */ + int rm = insn & 0xf; + long rnv = (rn == 15) ? ppc : regs->uregs[rn]; + long rmv = (rm == 15) ? ppc : regs->uregs[rm]; + long rsv = regs->uregs[rs]; + + regs->uregs[rd] = + insnslot_3arg_rflags(rnv, rmv, rsv, regs->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_alu_rwflags(struct kprobe *p, struct pt_regs *regs) +{ + insn_3arg_fn_t *i_fn = (insn_3arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + long ppc = (long)p->addr + 8; + int rd = (insn >> 12) & 0xf; + int rn = (insn >> 16) & 0xf; /* rn/rnv/rs/rsv may be */ + int rs = (insn >> 8) & 0xf; /* invalid, don't care. */ + int rm = insn & 0xf; + long rnv = (rn == 15) ? ppc : regs->uregs[rn]; + long rmv = (rm == 15) ? ppc : regs->uregs[rm]; + long rsv = regs->uregs[rs]; + + regs->uregs[rd] = + insnslot_3arg_rwflags(rnv, rmv, rsv, ®s->ARM_cpsr, i_fn); +} + +static void __kprobes +emulate_alu_tests(struct kprobe *p, struct pt_regs *regs) +{ + insn_3arg_fn_t *i_fn = (insn_3arg_fn_t *)&p->ainsn.insn[0]; + kprobe_opcode_t insn = p->opcode; + long ppc = (long)p->addr + 8; + int rn = (insn >> 16) & 0xf; + int rs = (insn >> 8) & 0xf; /* rs/rsv may be invalid, don't care. */ + int rm = insn & 0xf; + long rnv = (rn == 15) ? ppc : regs->uregs[rn]; + long rmv = (rm == 15) ? ppc : regs->uregs[rm]; + long rsv = regs->uregs[rs]; + + insnslot_3arg_rwflags(rnv, rmv, rsv, ®s->ARM_cpsr, i_fn); +} + +static enum kprobe_insn __kprobes +prep_emulate_ldr_str(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + int not_imm = (insn & (1 << 26)) ? (insn & (1 << 25)) + : (~insn & (1 << 22)); + + if (is_writeback(insn) && is_r15(insn, 16)) + return INSN_REJECTED; /* Writeback to PC */ + + insn &= 0xfff00fff; + insn |= 0x00001000; /* Rn = r0, Rd = r1 */ + if (not_imm) { + insn &= ~0xf; + insn |= 2; /* Rm = r2 */ + } + asi->insn[0] = insn; + asi->insn_handler = (insn & (1 << 20)) ? emulate_ldr : emulate_str; + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +prep_emulate_rd12_modify(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + + insn &= 0xffff0fff; /* Rd = r0 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_rd12_modify; + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +prep_emulate_rd12rn0_modify(kprobe_opcode_t insn, + struct arch_specific_insn *asi) +{ + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + + insn &= 0xffff0ff0; /* Rd = r0 */ + insn |= 0x00000001; /* Rn = r1 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_rd12rn0_modify; + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +prep_emulate_rd12rm0(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + + insn &= 0xffff0ff0; /* Rd = r0, Rm = r0 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_rd12rm0; + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +prep_emulate_rd12rn16rm0_wflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi) +{ + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + + insn &= 0xfff00ff0; /* Rd = r0, Rn = r0 */ + insn |= 0x00000001; /* Rm = r1 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_rd12rn16rm0_rwflags; + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +prep_emulate_rd16rs8rm0_wflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi) +{ + if (is_r15(insn, 16)) + return INSN_REJECTED; /* Rd is PC */ + + insn &= 0xfff0f0f0; /* Rd = r0, Rs = r0 */ + insn |= 0x00000001; /* Rm = r1 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_rd16rs8rm0_rwflags; + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +prep_emulate_rd16rn12rs8rm0_wflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi) +{ + if (is_r15(insn, 16)) + return INSN_REJECTED; /* Rd is PC */ + + insn &= 0xfff000f0; /* Rd = r0, Rn = r0 */ + insn |= 0x00000102; /* Rs = r1, Rm = r2 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_rd16rn12rs8rm0_rwflags; + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +prep_emulate_rdhi16rdlo12rs8rm0_wflags(kprobe_opcode_t insn, + struct arch_specific_insn *asi) +{ + if (is_r15(insn, 16) || is_r15(insn, 12)) + return INSN_REJECTED; /* RdHi or RdLo is PC */ + + insn &= 0xfff000f0; /* RdHi = r0, RdLo = r1 */ + insn |= 0x00001203; /* Rs = r2, Rm = r3 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_rdhi16rdlo12rs8rm0_rwflags; + return INSN_GOOD; +} + +/* + * For the instruction masking and comparisons in all the "space_*" + * functions below, Do _not_ rearrange the order of tests unless + * you're very, very sure of what you are doing. For the sake of + * efficiency, the masks for some tests sometimes assume other test + * have been done prior to them so the number of patterns to test + * for an instruction set can be as broad as possible to reduce the + * number of tests needed. + */ + +static enum kprobe_insn __kprobes +space_1111(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* memory hint : 1111 0100 x001 xxxx xxxx xxxx xxxx xxxx : */ + /* PLDI : 1111 0100 x101 xxxx xxxx xxxx xxxx xxxx : */ + /* PLDW : 1111 0101 x001 xxxx xxxx xxxx xxxx xxxx : */ + /* PLD : 1111 0101 x101 xxxx xxxx xxxx xxxx xxxx : */ + if ((insn & 0xfe300000) == 0xf4100000) { + asi->insn_handler = emulate_nop; + return INSN_GOOD_NO_SLOT; + } + + /* BLX(1) : 1111 101x xxxx xxxx xxxx xxxx xxxx xxxx : */ + if ((insn & 0xfe000000) == 0xfa000000) { + asi->insn_handler = simulate_blx1; + return INSN_GOOD_NO_SLOT; + } + + /* CPS : 1111 0001 0000 xxx0 xxxx xxxx xx0x xxxx */ + /* SETEND: 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */ + + /* SRS : 1111 100x x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* RFE : 1111 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ + + /* Coprocessor instructions... */ + /* MCRR2 : 1111 1100 0100 xxxx xxxx xxxx xxxx xxxx : (Rd != Rn) */ + /* MRRC2 : 1111 1100 0101 xxxx xxxx xxxx xxxx xxxx : (Rd != Rn) */ + /* LDC2 : 1111 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ + /* STC2 : 1111 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ + /* CDP2 : 1111 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ + /* MCR2 : 1111 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ + /* MRC2 : 1111 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ + + return INSN_REJECTED; +} + +static enum kprobe_insn __kprobes +space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* cccc 0001 0xx0 xxxx xxxx xxxx xxxx xxx0 xxxx */ + if ((insn & 0x0f900010) == 0x01000000) { + + /* MRS cpsr : cccc 0001 0000 xxxx xxxx xxxx 0000 xxxx */ + if ((insn & 0x0ff000f0) == 0x01000000) { + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + asi->insn_handler = simulate_mrs; + return INSN_GOOD_NO_SLOT; + } + + /* SMLALxy : cccc 0001 0100 xxxx xxxx xxxx 1xx0 xxxx */ + if ((insn & 0x0ff00090) == 0x01400080) + return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn, + asi); + + /* SMULWy : cccc 0001 0010 xxxx xxxx xxxx 1x10 xxxx */ + /* SMULxy : cccc 0001 0110 xxxx xxxx xxxx 1xx0 xxxx */ + if ((insn & 0x0ff000b0) == 0x012000a0 || + (insn & 0x0ff00090) == 0x01600080) + return prep_emulate_rd16rs8rm0_wflags(insn, asi); + + /* SMLAxy : cccc 0001 0000 xxxx xxxx xxxx 1xx0 xxxx : Q */ + /* SMLAWy : cccc 0001 0010 xxxx xxxx xxxx 1x00 xxxx : Q */ + if ((insn & 0x0ff00090) == 0x01000080 || + (insn & 0x0ff000b0) == 0x01200080) + return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi); + + /* BXJ : cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */ + /* MSR : cccc 0001 0x10 xxxx xxxx xxxx 0000 xxxx */ + /* MRS spsr : cccc 0001 0100 xxxx xxxx xxxx 0000 xxxx */ + + /* Other instruction encodings aren't yet defined */ + return INSN_REJECTED; + } + + /* cccc 0001 0xx0 xxxx xxxx xxxx xxxx 0xx1 xxxx */ + else if ((insn & 0x0f900090) == 0x01000010) { + + /* BLX(2) : cccc 0001 0010 xxxx xxxx xxxx 0011 xxxx */ + /* BX : cccc 0001 0010 xxxx xxxx xxxx 0001 xxxx */ + if ((insn & 0x0ff000d0) == 0x01200010) { + if ((insn & 0x0ff000ff) == 0x0120003f) + return INSN_REJECTED; /* BLX pc */ + asi->insn_handler = simulate_blx2bx; + return INSN_GOOD_NO_SLOT; + } + + /* CLZ : cccc 0001 0110 xxxx xxxx xxxx 0001 xxxx */ + if ((insn & 0x0ff000f0) == 0x01600010) + return prep_emulate_rd12rm0(insn, asi); + + /* QADD : cccc 0001 0000 xxxx xxxx xxxx 0101 xxxx :Q */ + /* QSUB : cccc 0001 0010 xxxx xxxx xxxx 0101 xxxx :Q */ + /* QDADD : cccc 0001 0100 xxxx xxxx xxxx 0101 xxxx :Q */ + /* QDSUB : cccc 0001 0110 xxxx xxxx xxxx 0101 xxxx :Q */ + if ((insn & 0x0f9000f0) == 0x01000050) + return prep_emulate_rd12rn16rm0_wflags(insn, asi); + + /* BKPT : 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */ + /* SMC : cccc 0001 0110 xxxx xxxx xxxx 0111 xxxx */ + + /* Other instruction encodings aren't yet defined */ + return INSN_REJECTED; + } + + /* cccc 0000 xxxx xxxx xxxx xxxx xxxx 1001 xxxx */ + else if ((insn & 0x0f0000f0) == 0x00000090) { + + /* MUL : cccc 0000 0000 xxxx xxxx xxxx 1001 xxxx : */ + /* MULS : cccc 0000 0001 xxxx xxxx xxxx 1001 xxxx :cc */ + /* MLA : cccc 0000 0010 xxxx xxxx xxxx 1001 xxxx : */ + /* MLAS : cccc 0000 0011 xxxx xxxx xxxx 1001 xxxx :cc */ + /* UMAAL : cccc 0000 0100 xxxx xxxx xxxx 1001 xxxx : */ + /* undef : cccc 0000 0101 xxxx xxxx xxxx 1001 xxxx : */ + /* MLS : cccc 0000 0110 xxxx xxxx xxxx 1001 xxxx : */ + /* undef : cccc 0000 0111 xxxx xxxx xxxx 1001 xxxx : */ + /* UMULL : cccc 0000 1000 xxxx xxxx xxxx 1001 xxxx : */ + /* UMULLS : cccc 0000 1001 xxxx xxxx xxxx 1001 xxxx :cc */ + /* UMLAL : cccc 0000 1010 xxxx xxxx xxxx 1001 xxxx : */ + /* UMLALS : cccc 0000 1011 xxxx xxxx xxxx 1001 xxxx :cc */ + /* SMULL : cccc 0000 1100 xxxx xxxx xxxx 1001 xxxx : */ + /* SMULLS : cccc 0000 1101 xxxx xxxx xxxx 1001 xxxx :cc */ + /* SMLAL : cccc 0000 1110 xxxx xxxx xxxx 1001 xxxx : */ + /* SMLALS : cccc 0000 1111 xxxx xxxx xxxx 1001 xxxx :cc */ + if ((insn & 0x00d00000) == 0x00500000) + return INSN_REJECTED; + else if ((insn & 0x00e00000) == 0x00000000) + return prep_emulate_rd16rs8rm0_wflags(insn, asi); + else if ((insn & 0x00a00000) == 0x00200000) + return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi); + else + return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn, + asi); + } + + /* cccc 000x xxxx xxxx xxxx xxxx xxxx 1xx1 xxxx */ + else if ((insn & 0x0e000090) == 0x00000090) { + + /* SWP : cccc 0001 0000 xxxx xxxx xxxx 1001 xxxx */ + /* SWPB : cccc 0001 0100 xxxx xxxx xxxx 1001 xxxx */ + /* ??? : cccc 0001 0x01 xxxx xxxx xxxx 1001 xxxx */ + /* ??? : cccc 0001 0x10 xxxx xxxx xxxx 1001 xxxx */ + /* ??? : cccc 0001 0x11 xxxx xxxx xxxx 1001 xxxx */ + /* STREX : cccc 0001 1000 xxxx xxxx xxxx 1001 xxxx */ + /* LDREX : cccc 0001 1001 xxxx xxxx xxxx 1001 xxxx */ + /* STREXD: cccc 0001 1010 xxxx xxxx xxxx 1001 xxxx */ + /* LDREXD: cccc 0001 1011 xxxx xxxx xxxx 1001 xxxx */ + /* STREXB: cccc 0001 1100 xxxx xxxx xxxx 1001 xxxx */ + /* LDREXB: cccc 0001 1101 xxxx xxxx xxxx 1001 xxxx */ + /* STREXH: cccc 0001 1110 xxxx xxxx xxxx 1001 xxxx */ + /* LDREXH: cccc 0001 1111 xxxx xxxx xxxx 1001 xxxx */ + + /* LDRD : cccc 000x xxx0 xxxx xxxx xxxx 1101 xxxx */ + /* STRD : cccc 000x xxx0 xxxx xxxx xxxx 1111 xxxx */ + /* LDRH : cccc 000x xxx1 xxxx xxxx xxxx 1011 xxxx */ + /* STRH : cccc 000x xxx0 xxxx xxxx xxxx 1011 xxxx */ + /* LDRSB : cccc 000x xxx1 xxxx xxxx xxxx 1101 xxxx */ + /* LDRSH : cccc 000x xxx1 xxxx xxxx xxxx 1111 xxxx */ + if ((insn & 0x0f0000f0) == 0x01000090) { + if ((insn & 0x0fb000f0) == 0x01000090) { + /* SWP/SWPB */ + return prep_emulate_rd12rn16rm0_wflags(insn, + asi); + } else { + /* STREX/LDREX variants and unallocaed space */ + return INSN_REJECTED; + } + + } else if ((insn & 0x0e1000d0) == 0x00000d0) { + /* STRD/LDRD */ + if ((insn & 0x0000e000) == 0x0000e000) + return INSN_REJECTED; /* Rd is LR or PC */ + if (is_writeback(insn) && is_r15(insn, 16)) + return INSN_REJECTED; /* Writeback to PC */ + + insn &= 0xfff00fff; + insn |= 0x00002000; /* Rn = r0, Rd = r2 */ + if (!(insn & (1 << 22))) { + /* Register index */ + insn &= ~0xf; + insn |= 1; /* Rm = r1 */ + } + asi->insn[0] = insn; + asi->insn_handler = + (insn & (1 << 5)) ? emulate_strd : emulate_ldrd; + return INSN_GOOD; + } + + /* LDRH/STRH/LDRSB/LDRSH */ + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + return prep_emulate_ldr_str(insn, asi); + } + + /* cccc 000x xxxx xxxx xxxx xxxx xxxx xxxx xxxx */ + + /* + * ALU op with S bit and Rd == 15 : + * cccc 000x xxx1 xxxx 1111 xxxx xxxx xxxx + */ + if ((insn & 0x0e10f000) == 0x0010f000) + return INSN_REJECTED; + + /* + * "mov ip, sp" is the most common kprobe'd instruction by far. + * Check and optimize for it explicitly. + */ + if (insn == 0xe1a0c00d) { + asi->insn_handler = simulate_mov_ipsp; + return INSN_GOOD_NO_SLOT; + } + + /* + * Data processing: Immediate-shift / Register-shift + * ALU op : cccc 000x xxxx xxxx xxxx xxxx xxxx xxxx + * CPY : cccc 0001 1010 xxxx xxxx 0000 0000 xxxx + * MOV : cccc 0001 101x xxxx xxxx xxxx xxxx xxxx + * *S (bit 20) updates condition codes + * ADC/SBC/RSC reads the C flag + */ + insn &= 0xfff00ff0; /* Rn = r0, Rd = r0 */ + insn |= 0x00000001; /* Rm = r1 */ + if (insn & 0x010) { + insn &= 0xfffff0ff; /* register shift */ + insn |= 0x00000200; /* Rs = r2 */ + } + asi->insn[0] = insn; + + if ((insn & 0x0f900000) == 0x01100000) { + /* + * TST : cccc 0001 0001 xxxx xxxx xxxx xxxx xxxx + * TEQ : cccc 0001 0011 xxxx xxxx xxxx xxxx xxxx + * CMP : cccc 0001 0101 xxxx xxxx xxxx xxxx xxxx + * CMN : cccc 0001 0111 xxxx xxxx xxxx xxxx xxxx + */ + asi->insn_handler = emulate_alu_tests; + } else { + /* ALU ops which write to Rd */ + asi->insn_handler = (insn & (1 << 20)) ? /* S-bit */ + emulate_alu_rwflags : emulate_alu_rflags; + } + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +space_cccc_001x(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* MOVW : cccc 0011 0000 xxxx xxxx xxxx xxxx xxxx */ + /* MOVT : cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx */ + if ((insn & 0x0fb00000) == 0x03000000) + return prep_emulate_rd12_modify(insn, asi); + + /* hints : cccc 0011 0010 0000 xxxx xxxx xxxx xxxx */ + if ((insn & 0x0fff0000) == 0x03200000) { + unsigned op2 = insn & 0x000000ff; + if (op2 == 0x01 || op2 == 0x04) { + /* YIELD : cccc 0011 0010 0000 xxxx xxxx 0000 0001 */ + /* SEV : cccc 0011 0010 0000 xxxx xxxx 0000 0100 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_none; + return INSN_GOOD; + } else if (op2 <= 0x03) { + /* NOP : cccc 0011 0010 0000 xxxx xxxx 0000 0000 */ + /* WFE : cccc 0011 0010 0000 xxxx xxxx 0000 0010 */ + /* WFI : cccc 0011 0010 0000 xxxx xxxx 0000 0011 */ + /* + * We make WFE and WFI true NOPs to avoid stalls due + * to missing events whilst processing the probe. + */ + asi->insn_handler = emulate_nop; + return INSN_GOOD_NO_SLOT; + } + /* For DBG and unallocated hints it's safest to reject them */ + return INSN_REJECTED; + } + + /* + * MSR : cccc 0011 0x10 xxxx xxxx xxxx xxxx xxxx + * ALU op with S bit and Rd == 15 : + * cccc 001x xxx1 xxxx 1111 xxxx xxxx xxxx + */ + if ((insn & 0x0fb00000) == 0x03200000 || /* MSR */ + (insn & 0x0e10f000) == 0x0210f000) /* ALU s-bit, R15 */ + return INSN_REJECTED; + + /* + * Data processing: 32-bit Immediate + * ALU op : cccc 001x xxxx xxxx xxxx xxxx xxxx xxxx + * MOV : cccc 0011 101x xxxx xxxx xxxx xxxx xxxx + * *S (bit 20) updates condition codes + * ADC/SBC/RSC reads the C flag + */ + insn &= 0xfff00fff; /* Rn = r0 and Rd = r0 */ + asi->insn[0] = insn; + + if ((insn & 0x0f900000) == 0x03100000) { + /* + * TST : cccc 0011 0001 xxxx xxxx xxxx xxxx xxxx + * TEQ : cccc 0011 0011 xxxx xxxx xxxx xxxx xxxx + * CMP : cccc 0011 0101 xxxx xxxx xxxx xxxx xxxx + * CMN : cccc 0011 0111 xxxx xxxx xxxx xxxx xxxx + */ + asi->insn_handler = emulate_alu_tests_imm; + } else { + /* ALU ops which write to Rd */ + asi->insn_handler = (insn & (1 << 20)) ? /* S-bit */ + emulate_alu_imm_rwflags : emulate_alu_imm_rflags; + } + return INSN_GOOD; +} + +static enum kprobe_insn __kprobes +space_cccc_0110__1(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* SEL : cccc 0110 1000 xxxx xxxx xxxx 1011 xxxx GE: !!! */ + if ((insn & 0x0ff000f0) == 0x068000b0) { + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + insn &= 0xfff00ff0; /* Rd = r0, Rn = r0 */ + insn |= 0x00000001; /* Rm = r1 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_sel; + return INSN_GOOD; + } + + /* SSAT : cccc 0110 101x xxxx xxxx xxxx xx01 xxxx :Q */ + /* USAT : cccc 0110 111x xxxx xxxx xxxx xx01 xxxx :Q */ + /* SSAT16 : cccc 0110 1010 xxxx xxxx xxxx 0011 xxxx :Q */ + /* USAT16 : cccc 0110 1110 xxxx xxxx xxxx 0011 xxxx :Q */ + if ((insn & 0x0fa00030) == 0x06a00010 || + (insn & 0x0fb000f0) == 0x06a00030) { + if (is_r15(insn, 12)) + return INSN_REJECTED; /* Rd is PC */ + insn &= 0xffff0ff0; /* Rd = r0, Rm = r0 */ + asi->insn[0] = insn; + asi->insn_handler = emulate_sat; + return INSN_GOOD; + } + + /* REV : cccc 0110 1011 xxxx xxxx xxxx 0011 xxxx */ + /* REV16 : cccc 0110 1011 xxxx xxxx xxxx 1011 xxxx */ + /* RBIT : cccc 0110 1111 xxxx xxxx xxxx 0011 xxxx */ + /* REVSH : cccc 0110 1111 xxxx xxxx xxxx 1011 xxxx */ + if ((insn & 0x0ff00070) == 0x06b00030 || + (insn & 0x0ff00070) == 0x06f00030) + return prep_emulate_rd12rm0(insn, asi); + + /* ??? : cccc 0110 0000 xxxx xxxx xxxx xxx1 xxxx : */ + /* SADD16 : cccc 0110 0001 xxxx xxxx xxxx 0001 xxxx :GE */ + /* SADDSUBX : cccc 0110 0001 xxxx xxxx xxxx 0011 xxxx :GE */ + /* SSUBADDX : cccc 0110 0001 xxxx xxxx xxxx 0101 xxxx :GE */ + /* SSUB16 : cccc 0110 0001 xxxx xxxx xxxx 0111 xxxx :GE */ + /* SADD8 : cccc 0110 0001 xxxx xxxx xxxx 1001 xxxx :GE */ + /* ??? : cccc 0110 0001 xxxx xxxx xxxx 1011 xxxx : */ + /* ??? : cccc 0110 0001 xxxx xxxx xxxx 1101 xxxx : */ + /* SSUB8 : cccc 0110 0001 xxxx xxxx xxxx 1111 xxxx :GE */ + /* QADD16 : cccc 0110 0010 xxxx xxxx xxxx 0001 xxxx : */ + /* QADDSUBX : cccc 0110 0010 xxxx xxxx xxxx 0011 xxxx : */ + /* QSUBADDX : cccc 0110 0010 xxxx xxxx xxxx 0101 xxxx : */ + /* QSUB16 : cccc 0110 0010 xxxx xxxx xxxx 0111 xxxx : */ + /* QADD8 : cccc 0110 0010 xxxx xxxx xxxx 1001 xxxx : */ + /* ??? : cccc 0110 0010 xxxx xxxx xxxx 1011 xxxx : */ + /* ??? : cccc 0110 0010 xxxx xxxx xxxx 1101 xxxx : */ + /* QSUB8 : cccc 0110 0010 xxxx xxxx xxxx 1111 xxxx : */ + /* SHADD16 : cccc 0110 0011 xxxx xxxx xxxx 0001 xxxx : */ + /* SHADDSUBX : cccc 0110 0011 xxxx xxxx xxxx 0011 xxxx : */ + /* SHSUBADDX : cccc 0110 0011 xxxx xxxx xxxx 0101 xxxx : */ + /* SHSUB16 : cccc 0110 0011 xxxx xxxx xxxx 0111 xxxx : */ + /* SHADD8 : cccc 0110 0011 xxxx xxxx xxxx 1001 xxxx : */ + /* ??? : cccc 0110 0011 xxxx xxxx xxxx 1011 xxxx : */ + /* ??? : cccc 0110 0011 xxxx xxxx xxxx 1101 xxxx : */ + /* SHSUB8 : cccc 0110 0011 xxxx xxxx xxxx 1111 xxxx : */ + /* ??? : cccc 0110 0100 xxxx xxxx xxxx xxx1 xxxx : */ + /* UADD16 : cccc 0110 0101 xxxx xxxx xxxx 0001 xxxx :GE */ + /* UADDSUBX : cccc 0110 0101 xxxx xxxx xxxx 0011 xxxx :GE */ + /* USUBADDX : cccc 0110 0101 xxxx xxxx xxxx 0101 xxxx :GE */ + /* USUB16 : cccc 0110 0101 xxxx xxxx xxxx 0111 xxxx :GE */ + /* UADD8 : cccc 0110 0101 xxxx xxxx xxxx 1001 xxxx :GE */ + /* ??? : cccc 0110 0101 xxxx xxxx xxxx 1011 xxxx : */ + /* ??? : cccc 0110 0101 xxxx xxxx xxxx 1101 xxxx : */ + /* USUB8 : cccc 0110 0101 xxxx xxxx xxxx 1111 xxxx :GE */ + /* UQADD16 : cccc 0110 0110 xxxx xxxx xxxx 0001 xxxx : */ + /* UQADDSUBX : cccc 0110 0110 xxxx xxxx xxxx 0011 xxxx : */ + /* UQSUBADDX : cccc 0110 0110 xxxx xxxx xxxx 0101 xxxx : */ + /* UQSUB16 : cccc 0110 0110 xxxx xxxx xxxx 0111 xxxx : */ + /* UQADD8 : cccc 0110 0110 xxxx xxxx xxxx 1001 xxxx : */ + /* ??? : cccc 0110 0110 xxxx xxxx xxxx 1011 xxxx : */ + /* ??? : cccc 0110 0110 xxxx xxxx xxxx 1101 xxxx : */ + /* UQSUB8 : cccc 0110 0110 xxxx xxxx xxxx 1111 xxxx : */ + /* UHADD16 : cccc 0110 0111 xxxx xxxx xxxx 0001 xxxx : */ + /* UHADDSUBX : cccc 0110 0111 xxxx xxxx xxxx 0011 xxxx : */ + /* UHSUBADDX : cccc 0110 0111 xxxx xxxx xxxx 0101 xxxx : */ + /* UHSUB16 : cccc 0110 0111 xxxx xxxx xxxx 0111 xxxx : */ + /* UHADD8 : cccc 0110 0111 xxxx xxxx xxxx 1001 xxxx : */ + /* ??? : cccc 0110 0111 xxxx xxxx xxxx 1011 xxxx : */ + /* ??? : cccc 0110 0111 xxxx xxxx xxxx 1101 xxxx : */ + /* UHSUB8 : cccc 0110 0111 xxxx xxxx xxxx 1111 xxxx : */ + if ((insn & 0x0f800010) == 0x06000010) { + if ((insn & 0x00300000) == 0x00000000 || + (insn & 0x000000e0) == 0x000000a0 || + (insn & 0x000000e0) == 0x000000c0) + return INSN_REJECTED; /* Unallocated space */ + return prep_emulate_rd12rn16rm0_wflags(insn, asi); + } + + /* PKHBT : cccc 0110 1000 xxxx xxxx xxxx x001 xxxx : */ + /* PKHTB : cccc 0110 1000 xxxx xxxx xxxx x101 xxxx : */ + if ((insn & 0x0ff00030) == 0x06800010) + return prep_emulate_rd12rn16rm0_wflags(insn, asi); + + /* SXTAB16 : cccc 0110 1000 xxxx xxxx xxxx 0111 xxxx : */ + /* SXTB16 : cccc 0110 1000 1111 xxxx xxxx 0111 xxxx : */ + /* ??? : cccc 0110 1001 xxxx xxxx xxxx 0111 xxxx : */ + /* SXTAB : cccc 0110 1010 xxxx xxxx xxxx 0111 xxxx : */ + /* SXTB : cccc 0110 1010 1111 xxxx xxxx 0111 xxxx : */ + /* SXTAH : cccc 0110 1011 xxxx xxxx xxxx 0111 xxxx : */ + /* SXTH : cccc 0110 1011 1111 xxxx xxxx 0111 xxxx : */ + /* UXTAB16 : cccc 0110 1100 xxxx xxxx xxxx 0111 xxxx : */ + /* UXTB16 : cccc 0110 1100 1111 xxxx xxxx 0111 xxxx : */ + /* ??? : cccc 0110 1101 xxxx xxxx xxxx 0111 xxxx : */ + /* UXTAB : cccc 0110 1110 xxxx xxxx xxxx 0111 xxxx : */ + /* UXTB : cccc 0110 1110 1111 xxxx xxxx 0111 xxxx : */ + /* UXTAH : cccc 0110 1111 xxxx xxxx xxxx 0111 xxxx : */ + /* UXTH : cccc 0110 1111 1111 xxxx xxxx 0111 xxxx : */ + if ((insn & 0x0f8000f0) == 0x06800070) { + if ((insn & 0x00300000) == 0x00100000) + return INSN_REJECTED; /* Unallocated space */ + + if ((insn & 0x000f0000) == 0x000f0000) + return prep_emulate_rd12rm0(insn, asi); + else + return prep_emulate_rd12rn16rm0_wflags(insn, asi); + } + + /* Other instruction encodings aren't yet defined */ + return INSN_REJECTED; +} + +static enum kprobe_insn __kprobes +space_cccc_0111__1(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* Undef : cccc 0111 1111 xxxx xxxx xxxx 1111 xxxx */ + if ((insn & 0x0ff000f0) == 0x03f000f0) + return INSN_REJECTED; + + /* SMLALD : cccc 0111 0100 xxxx xxxx xxxx 00x1 xxxx */ + /* SMLSLD : cccc 0111 0100 xxxx xxxx xxxx 01x1 xxxx */ + if ((insn & 0x0ff00090) == 0x07400010) + return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn, asi); + + /* SMLAD : cccc 0111 0000 xxxx xxxx xxxx 00x1 xxxx :Q */ + /* SMUAD : cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx :Q */ + /* SMLSD : cccc 0111 0000 xxxx xxxx xxxx 01x1 xxxx :Q */ + /* SMUSD : cccc 0111 0000 xxxx 1111 xxxx 01x1 xxxx : */ + /* SMMLA : cccc 0111 0101 xxxx xxxx xxxx 00x1 xxxx : */ + /* SMMUL : cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx : */ + /* USADA8 : cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx : */ + /* USAD8 : cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx : */ + if ((insn & 0x0ff00090) == 0x07000010 || + (insn & 0x0ff000d0) == 0x07500010 || + (insn & 0x0ff000f0) == 0x07800010) { + + if ((insn & 0x0000f000) == 0x0000f000) + return prep_emulate_rd16rs8rm0_wflags(insn, asi); + else + return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi); + } + + /* SMMLS : cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx : */ + if ((insn & 0x0ff000d0) == 0x075000d0) + return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi); + + /* SBFX : cccc 0111 101x xxxx xxxx xxxx x101 xxxx : */ + /* UBFX : cccc 0111 111x xxxx xxxx xxxx x101 xxxx : */ + if ((insn & 0x0fa00070) == 0x07a00050) + return prep_emulate_rd12rm0(insn, asi); + + /* BFI : cccc 0111 110x xxxx xxxx xxxx x001 xxxx : */ + /* BFC : cccc 0111 110x xxxx xxxx xxxx x001 1111 : */ + if ((insn & 0x0fe00070) == 0x07c00010) { + + if ((insn & 0x0000000f) == 0x0000000f) + return prep_emulate_rd12_modify(insn, asi); + else + return prep_emulate_rd12rn0_modify(insn, asi); + } + + return INSN_REJECTED; +} + +static enum kprobe_insn __kprobes +space_cccc_01xx(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* LDR : cccc 01xx x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* LDRB : cccc 01xx x1x1 xxxx xxxx xxxx xxxx xxxx */ + /* LDRBT : cccc 01x0 x111 xxxx xxxx xxxx xxxx xxxx */ + /* LDRT : cccc 01x0 x011 xxxx xxxx xxxx xxxx xxxx */ + /* STR : cccc 01xx x0x0 xxxx xxxx xxxx xxxx xxxx */ + /* STRB : cccc 01xx x1x0 xxxx xxxx xxxx xxxx xxxx */ + /* STRBT : cccc 01x0 x110 xxxx xxxx xxxx xxxx xxxx */ + /* STRT : cccc 01x0 x010 xxxx xxxx xxxx xxxx xxxx */ + + if ((insn & 0x00500000) == 0x00500000 && is_r15(insn, 12)) + return INSN_REJECTED; /* LDRB into PC */ + + return prep_emulate_ldr_str(insn, asi); +} + +static enum kprobe_insn __kprobes +space_cccc_100x(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* LDM(2) : cccc 100x x101 xxxx 0xxx xxxx xxxx xxxx */ + /* LDM(3) : cccc 100x x1x1 xxxx 1xxx xxxx xxxx xxxx */ + if ((insn & 0x0e708000) == 0x85000000 || + (insn & 0x0e508000) == 0x85010000) + return INSN_REJECTED; + + /* LDM(1) : cccc 100x x0x1 xxxx xxxx xxxx xxxx xxxx */ + /* STM(1) : cccc 100x x0x0 xxxx xxxx xxxx xxxx xxxx */ + asi->insn_handler = ((insn & 0x108000) == 0x008000) ? /* STM & R15 */ + simulate_stm1_pc : simulate_ldm1stm1; + return INSN_GOOD_NO_SLOT; +} + +static enum kprobe_insn __kprobes +space_cccc_101x(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* B : cccc 1010 xxxx xxxx xxxx xxxx xxxx xxxx */ + /* BL : cccc 1011 xxxx xxxx xxxx xxxx xxxx xxxx */ + asi->insn_handler = simulate_bbl; + return INSN_GOOD_NO_SLOT; +} + +static enum kprobe_insn __kprobes +space_cccc_11xx(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + /* Coprocessor instructions... */ + /* MCRR : cccc 1100 0100 xxxx xxxx xxxx xxxx xxxx : (Rd!=Rn) */ + /* MRRC : cccc 1100 0101 xxxx xxxx xxxx xxxx xxxx : (Rd!=Rn) */ + /* LDC : cccc 110x xxx1 xxxx xxxx xxxx xxxx xxxx */ + /* STC : cccc 110x xxx0 xxxx xxxx xxxx xxxx xxxx */ + /* CDP : cccc 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */ + /* MCR : cccc 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */ + /* MRC : cccc 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */ + + /* SVC : cccc 1111 xxxx xxxx xxxx xxxx xxxx xxxx */ + + return INSN_REJECTED; +} + +static unsigned long __kprobes __check_eq(unsigned long cpsr) +{ + return cpsr & PSR_Z_BIT; +} + +static unsigned long __kprobes __check_ne(unsigned long cpsr) +{ + return (~cpsr) & PSR_Z_BIT; +} + +static unsigned long __kprobes __check_cs(unsigned long cpsr) +{ + return cpsr & PSR_C_BIT; +} + +static unsigned long __kprobes __check_cc(unsigned long cpsr) +{ + return (~cpsr) & PSR_C_BIT; +} + +static unsigned long __kprobes __check_mi(unsigned long cpsr) +{ + return cpsr & PSR_N_BIT; +} + +static unsigned long __kprobes __check_pl(unsigned long cpsr) +{ + return (~cpsr) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_vs(unsigned long cpsr) +{ + return cpsr & PSR_V_BIT; +} + +static unsigned long __kprobes __check_vc(unsigned long cpsr) +{ + return (~cpsr) & PSR_V_BIT; +} + +static unsigned long __kprobes __check_hi(unsigned long cpsr) +{ + cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return cpsr & PSR_C_BIT; +} + +static unsigned long __kprobes __check_ls(unsigned long cpsr) +{ + cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (~cpsr) & PSR_C_BIT; +} + +static unsigned long __kprobes __check_ge(unsigned long cpsr) +{ + cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (~cpsr) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_lt(unsigned long cpsr) +{ + cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return cpsr & PSR_N_BIT; +} + +static unsigned long __kprobes __check_gt(unsigned long cpsr) +{ + unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ + return (~temp) & PSR_N_BIT; +} + +static unsigned long __kprobes __check_le(unsigned long cpsr) +{ + unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + temp |= (cpsr << 1); /* PSR_N_BIT |= PSR_Z_BIT */ + return temp & PSR_N_BIT; +} + +static unsigned long __kprobes __check_al(unsigned long cpsr) +{ + return true; +} + +static kprobe_check_cc * const condition_checks[16] = { + &__check_eq, &__check_ne, &__check_cs, &__check_cc, + &__check_mi, &__check_pl, &__check_vs, &__check_vc, + &__check_hi, &__check_ls, &__check_ge, &__check_lt, + &__check_gt, &__check_le, &__check_al, &__check_al +}; + +/* Return: + * INSN_REJECTED If instruction is one not allowed to kprobe, + * INSN_GOOD If instruction is supported and uses instruction slot, + * INSN_GOOD_NO_SLOT If instruction is supported but doesn't use its slot. + * + * For instructions we don't want to kprobe (INSN_REJECTED return result): + * These are generally ones that modify the processor state making + * them "hard" to simulate such as switches processor modes or + * make accesses in alternate modes. Any of these could be simulated + * if the work was put into it, but low return considering they + * should also be very rare. + */ +enum kprobe_insn __kprobes +arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi) +{ + asi->insn_check_cc = condition_checks[insn>>28]; + asi->insn[1] = KPROBE_RETURN_INSTRUCTION; + + if ((insn & 0xf0000000) == 0xf0000000) + + return space_1111(insn, asi); + + else if ((insn & 0x0e000000) == 0x00000000) + + return space_cccc_000x(insn, asi); + + else if ((insn & 0x0e000000) == 0x02000000) + + return space_cccc_001x(insn, asi); + + else if ((insn & 0x0f000010) == 0x06000010) + + return space_cccc_0110__1(insn, asi); + + else if ((insn & 0x0f000010) == 0x07000010) + + return space_cccc_0111__1(insn, asi); + + else if ((insn & 0x0c000000) == 0x04000000) + + return space_cccc_01xx(insn, asi); + + else if ((insn & 0x0e000000) == 0x08000000) + + return space_cccc_100x(insn, asi); + + else if ((insn & 0x0e000000) == 0x0a000000) + + return space_cccc_101x(insn, asi); + + return space_cccc_11xx(insn, asi); +} + +void __init arm_kprobe_decode_init(void) +{ + find_str_pc_offset(); +} diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c new file mode 100644 index 00000000..1656c875 --- /dev/null +++ b/arch/arm/kernel/kprobes.c @@ -0,0 +1,476 @@ +/* + * arch/arm/kernel/kprobes.c + * + * Kprobes on ARM + * + * Abhishek Sagar <sagar.abhishek@gmail.com> + * Copyright (C) 2006, 2007 Motorola Inc. + * + * Nicolas Pitre <nico@marvell.com> + * Copyright (C) 2007 Marvell Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ + +#include <linux/kernel.h> +#include <linux/kprobes.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/stop_machine.h> +#include <linux/stringify.h> +#include <asm/traps.h> +#include <asm/cacheflush.h> + +#define MIN_STACK_SIZE(addr) \ + min((unsigned long)MAX_STACK_SIZE, \ + (unsigned long)current_thread_info() + THREAD_START_SP - (addr)) + +#define flush_insns(addr, cnt) \ + flush_icache_range((unsigned long)(addr), \ + (unsigned long)(addr) + \ + sizeof(kprobe_opcode_t) * (cnt)) + +/* Used as a marker in ARM_pc to note when we're in a jprobe. */ +#define JPROBE_MAGIC_ADDR 0xffffffff + +DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + kprobe_opcode_t insn; + kprobe_opcode_t tmp_insn[MAX_INSN_SIZE]; + unsigned long addr = (unsigned long)p->addr; + int is; + + if (addr & 0x3 || in_exception_text(addr)) + return -EINVAL; + + insn = *p->addr; + p->opcode = insn; + p->ainsn.insn = tmp_insn; + + switch (arm_kprobe_decode_insn(insn, &p->ainsn)) { + case INSN_REJECTED: /* not supported */ + return -EINVAL; + + case INSN_GOOD: /* instruction uses slot */ + p->ainsn.insn = get_insn_slot(); + if (!p->ainsn.insn) + return -ENOMEM; + for (is = 0; is < MAX_INSN_SIZE; ++is) + p->ainsn.insn[is] = tmp_insn[is]; + flush_insns(p->ainsn.insn, MAX_INSN_SIZE); + break; + + case INSN_GOOD_NO_SLOT: /* instruction doesn't need insn slot */ + p->ainsn.insn = NULL; + break; + } + + return 0; +} + +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + *p->addr = KPROBE_BREAKPOINT_INSTRUCTION; + flush_insns(p->addr, 1); +} + +/* + * The actual disarming is done here on each CPU and synchronized using + * stop_machine. This synchronization is necessary on SMP to avoid removing + * a probe between the moment the 'Undefined Instruction' exception is raised + * and the moment the exception handler reads the faulting instruction from + * memory. + */ +int __kprobes __arch_disarm_kprobe(void *p) +{ + struct kprobe *kp = p; + *kp->addr = kp->opcode; + flush_insns(kp->addr, 1); + return 0; +} + +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + stop_machine(__arch_disarm_kprobe, p, &cpu_online_map); +} + +void __kprobes arch_remove_kprobe(struct kprobe *p) +{ + if (p->ainsn.insn) { + free_insn_slot(p->ainsn.insn, 0); + p->ainsn.insn = NULL; + } +} + +static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; +} + +static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __get_cpu_var(current_kprobe) = kcb->prev_kprobe.kp; + kcb->kprobe_status = kcb->prev_kprobe.status; +} + +static void __kprobes set_current_kprobe(struct kprobe *p) +{ + __get_cpu_var(current_kprobe) = p; +} + +static void __kprobes singlestep(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + regs->ARM_pc += 4; + if (p->ainsn.insn_check_cc(regs->ARM_cpsr)) + p->ainsn.insn_handler(p, regs); +} + +/* + * Called with IRQs disabled. IRQs must remain disabled from that point + * all the way until processing this kprobe is complete. The current + * kprobes implementation cannot process more than one nested level of + * kprobe, and that level is reserved for user kprobe handlers, so we can't + * risk encountering a new kprobe in an interrupt handler. + */ +void __kprobes kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *p, *cur; + struct kprobe_ctlblk *kcb; + kprobe_opcode_t *addr = (kprobe_opcode_t *)regs->ARM_pc; + + kcb = get_kprobe_ctlblk(); + cur = kprobe_running(); + p = get_kprobe(addr); + + if (p) { + if (cur) { + /* Kprobe is pending, so we're recursing. */ + switch (kcb->kprobe_status) { + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* A pre- or post-handler probe got us here. */ + kprobes_inc_nmissed_count(p); + save_previous_kprobe(kcb); + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_REENTER; + singlestep(p, regs, kcb); + restore_previous_kprobe(kcb); + break; + default: + /* impossible cases */ + BUG(); + } + } else { + set_current_kprobe(p); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + + /* + * If we have no pre-handler or it returned 0, we + * continue with normal processing. If we have a + * pre-handler and it returned non-zero, it prepped + * for calling the break_handler below on re-entry, + * so get out doing nothing more here. + */ + if (!p->pre_handler || !p->pre_handler(p, regs)) { + kcb->kprobe_status = KPROBE_HIT_SS; + singlestep(p, regs, kcb); + if (p->post_handler) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + p->post_handler(p, regs, 0); + } + reset_current_kprobe(); + } + } + } else if (cur) { + /* We probably hit a jprobe. Call its break handler. */ + if (cur->break_handler && cur->break_handler(cur, regs)) { + kcb->kprobe_status = KPROBE_HIT_SS; + singlestep(cur, regs, kcb); + if (cur->post_handler) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + cur->post_handler(cur, regs, 0); + } + } + reset_current_kprobe(); + } else { + /* + * The probe was removed and a race is in progress. + * There is nothing we can do about it. Let's restart + * the instruction. By the time we can restart, the + * real instruction will be there. + */ + } +} + +static int __kprobes kprobe_trap_handler(struct pt_regs *regs, unsigned int instr) +{ + unsigned long flags; + local_irq_save(flags); + kprobe_handler(regs); + local_irq_restore(flags); + return 0; +} + +int __kprobes kprobe_fault_handler(struct pt_regs *regs, unsigned int fsr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + switch (kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the PC to point back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + regs->ARM_pc = (long)cur->addr; + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + } else { + reset_current_kprobe(); + } + break; + + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* + * We increment the nmissed count for accounting, + * we can also use npre/npostfault count for accounting + * these specific fault cases. + */ + kprobes_inc_nmissed_count(cur); + + /* + * We come here because instructions in the pre/post + * handler caused the page_fault, this could happen + * if handler tries to access user space by + * copy_from_user(), get_user() etc. Let the + * user-specified handler try to fix it. + */ + if (cur->fault_handler && cur->fault_handler(cur, regs, fsr)) + return 1; + break; + + default: + break; + } + + return 0; +} + +int __kprobes kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + /* + * notify_die() is currently never called on ARM, + * so this callback is currently empty. + */ + return NOTIFY_DONE; +} + +/* + * When a retprobed function returns, trampoline_handler() is called, + * calling the kretprobe's handler. We construct a struct pt_regs to + * give a view of registers r0-r11 to the user return-handler. This is + * not a complete pt_regs structure, but that should be plenty sufficient + * for kretprobe handlers which should normally be interested in r0 only + * anyway. + */ +void __naked __kprobes kretprobe_trampoline(void) +{ + __asm__ __volatile__ ( + "stmdb sp!, {r0 - r11} \n\t" + "mov r0, sp \n\t" + "bl trampoline_handler \n\t" + "mov lr, r0 \n\t" + "ldmia sp!, {r0 - r11} \n\t" + "mov pc, lr \n\t" + : : : "memory"); +} + +/* Called from kretprobe_trampoline */ +static __used __kprobes void *trampoline_handler(struct pt_regs *regs) +{ + struct kretprobe_instance *ri = NULL; + struct hlist_head *head, empty_rp; + struct hlist_node *node, *tmp; + unsigned long flags, orig_ret_address = 0; + unsigned long trampoline_address = (unsigned long)&kretprobe_trampoline; + + INIT_HLIST_HEAD(&empty_rp); + kretprobe_hash_lock(current, &head, &flags); + + /* + * It is possible to have multiple instances associated with a given + * task either because multiple functions in the call path have + * a return probe installed on them, and/or more than one return + * probe was registered for a target function. + * + * We can handle this because: + * - instances are always inserted at the head of the list + * - when multiple return probes are registered for the same + * function, the first instance's ret_addr will point to the + * real return address, and all the rest will point to + * kretprobe_trampoline + */ + hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { + if (ri->task != current) + /* another task is sharing our hash bucket */ + continue; + + if (ri->rp && ri->rp->handler) { + __get_cpu_var(current_kprobe) = &ri->rp->kp; + get_kprobe_ctlblk()->kprobe_status = KPROBE_HIT_ACTIVE; + ri->rp->handler(ri, regs); + __get_cpu_var(current_kprobe) = NULL; + } + + orig_ret_address = (unsigned long)ri->ret_addr; + recycle_rp_inst(ri, &empty_rp); + + if (orig_ret_address != trampoline_address) + /* + * This is the real return address. Any other + * instances associated with this task are for + * other calls deeper on the call stack + */ + break; + } + + kretprobe_assert(ri, orig_ret_address, trampoline_address); + kretprobe_hash_unlock(current, &flags); + + hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { + hlist_del(&ri->hlist); + kfree(ri); + } + + return (void *)orig_ret_address; +} + +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *)regs->ARM_lr; + + /* Replace the return addr with trampoline addr. */ + regs->ARM_lr = (unsigned long)&kretprobe_trampoline; +} + +int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct jprobe *jp = container_of(p, struct jprobe, kp); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + long sp_addr = regs->ARM_sp; + + kcb->jprobe_saved_regs = *regs; + memcpy(kcb->jprobes_stack, (void *)sp_addr, MIN_STACK_SIZE(sp_addr)); + regs->ARM_pc = (long)jp->entry; + regs->ARM_cpsr |= PSR_I_BIT; + preempt_disable(); + return 1; +} + +void __kprobes jprobe_return(void) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + __asm__ __volatile__ ( + /* + * Setup an empty pt_regs. Fill SP and PC fields as + * they're needed by longjmp_break_handler. + * + * We allocate some slack between the original SP and start of + * our fabricated regs. To be precise we want to have worst case + * covered which is STMFD with all 16 regs so we allocate 2 * + * sizeof(struct_pt_regs)). + * + * This is to prevent any simulated instruction from writing + * over the regs when they are accessing the stack. + */ + "sub sp, %0, %1 \n\t" + "ldr r0, ="__stringify(JPROBE_MAGIC_ADDR)"\n\t" + "str %0, [sp, %2] \n\t" + "str r0, [sp, %3] \n\t" + "mov r0, sp \n\t" + "bl kprobe_handler \n\t" + + /* + * Return to the context saved by setjmp_pre_handler + * and restored by longjmp_break_handler. + */ + "ldr r0, [sp, %4] \n\t" + "msr cpsr_cxsf, r0 \n\t" + "ldmia sp, {r0 - pc} \n\t" + : + : "r" (kcb->jprobe_saved_regs.ARM_sp), + "I" (sizeof(struct pt_regs) * 2), + "J" (offsetof(struct pt_regs, ARM_sp)), + "J" (offsetof(struct pt_regs, ARM_pc)), + "J" (offsetof(struct pt_regs, ARM_cpsr)) + : "memory", "cc"); +} + +int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) +{ + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + long stack_addr = kcb->jprobe_saved_regs.ARM_sp; + long orig_sp = regs->ARM_sp; + struct jprobe *jp = container_of(p, struct jprobe, kp); + + if (regs->ARM_pc == JPROBE_MAGIC_ADDR) { + if (orig_sp != stack_addr) { + struct pt_regs *saved_regs = + (struct pt_regs *)kcb->jprobe_saved_regs.ARM_sp; + printk("current sp %lx does not match saved sp %lx\n", + orig_sp, stack_addr); + printk("Saved registers for jprobe %p\n", jp); + show_regs(saved_regs); + printk("Current registers\n"); + show_regs(regs); + BUG(); + } + *regs = kcb->jprobe_saved_regs; + memcpy((void *)stack_addr, kcb->jprobes_stack, + MIN_STACK_SIZE(stack_addr)); + preempt_enable_no_resched(); + return 1; + } + return 0; +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + return 0; +} + +static struct undef_hook kprobes_break_hook = { + .instr_mask = 0xffffffff, + .instr_val = KPROBE_BREAKPOINT_INSTRUCTION, + .cpsr_mask = MODE_MASK, + .cpsr_val = SVC_MODE, + .fn = kprobe_trap_handler, +}; + +int __init arch_init_kprobes() +{ + arm_kprobe_decode_init(); + register_undef_hook(&kprobes_break_hook); + return 0; +} diff --git a/arch/arm/kernel/leds.c b/arch/arm/kernel/leds.c new file mode 100644 index 00000000..136e8376 --- /dev/null +++ b/arch/arm/kernel/leds.c @@ -0,0 +1,144 @@ +/* + * LED support code, ripped out of arch/arm/kernel/time.c + * + * Copyright (C) 1994-2001 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/module.h> +#include <linux/init.h> +#include <linux/notifier.h> +#include <linux/cpu.h> +#include <linux/sysdev.h> +#include <linux/syscore_ops.h> + +#include <asm/leds.h> + +static void dummy_leds_event(led_event_t evt) +{ +} + +void (*leds_event)(led_event_t) = dummy_leds_event; + +struct leds_evt_name { + const char name[8]; + int on; + int off; +}; + +static const struct leds_evt_name evt_names[] = { + { "amber", led_amber_on, led_amber_off }, + { "blue", led_blue_on, led_blue_off }, + { "green", led_green_on, led_green_off }, + { "red", led_red_on, led_red_off }, +}; + +static ssize_t leds_store(struct sys_device *dev, + struct sysdev_attribute *attr, + const char *buf, size_t size) +{ + int ret = -EINVAL, len = strcspn(buf, " "); + + if (len > 0 && buf[len] == '\0') + len--; + + if (strncmp(buf, "claim", len) == 0) { + leds_event(led_claim); + ret = size; + } else if (strncmp(buf, "release", len) == 0) { + leds_event(led_release); + ret = size; + } else { + int i; + + for (i = 0; i < ARRAY_SIZE(evt_names); i++) { + if (strlen(evt_names[i].name) != len || + strncmp(buf, evt_names[i].name, len) != 0) + continue; + if (strncmp(buf+len, " on", 3) == 0) { + leds_event(evt_names[i].on); + ret = size; + } else if (strncmp(buf+len, " off", 4) == 0) { + leds_event(evt_names[i].off); + ret = size; + } + break; + } + } + return ret; +} + +static SYSDEV_ATTR(event, 0200, NULL, leds_store); + +static struct sysdev_class leds_sysclass = { + .name = "leds", +}; + +static struct sys_device leds_device = { + .id = 0, + .cls = &leds_sysclass, +}; + +static int leds_suspend(void) +{ + leds_event(led_stop); + return 0; +} + +static void leds_resume(void) +{ + leds_event(led_start); +} + +static void leds_shutdown(void) +{ + leds_event(led_halted); +} + +static struct syscore_ops leds_syscore_ops = { + .shutdown = leds_shutdown, + .suspend = leds_suspend, + .resume = leds_resume, +}; + +static int leds_idle_notifier(struct notifier_block *nb, unsigned long val, + void *data) +{ + switch (val) { + case IDLE_START: + leds_event(led_idle_start); + break; + case IDLE_END: + leds_event(led_idle_end); + break; + } + + return 0; +} + +static struct notifier_block leds_idle_nb = { + .notifier_call = leds_idle_notifier, +}; + +static int __init leds_init(void) +{ + int ret; + ret = sysdev_class_register(&leds_sysclass); + if (ret == 0) + ret = sysdev_register(&leds_device); + if (ret == 0) + ret = sysdev_create_file(&leds_device, &attr_event); + + if (ret == 0) { + register_syscore_ops(&leds_syscore_ops); + idle_notifier_register(&leds_idle_nb); + } + + return ret; +} + +device_initcall(leds_init); + +EXPORT_SYMBOL(leds_event); diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c new file mode 100644 index 00000000..e59bbd49 --- /dev/null +++ b/arch/arm/kernel/machine_kexec.c @@ -0,0 +1,124 @@ +/* + * machine_kexec.c - handle transition of Linux booting another kernel + */ + +#include <linux/mm.h> +#include <linux/kexec.h> +#include <linux/delay.h> +#include <linux/reboot.h> +#include <linux/io.h> +#include <asm/pgtable.h> +#include <asm/pgalloc.h> +#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/mach-types.h> + +extern const unsigned char relocate_new_kernel[]; +extern const unsigned int relocate_new_kernel_size; + +extern void setup_mm_for_reboot(char mode); + +extern unsigned long kexec_start_address; +extern unsigned long kexec_indirection_page; +extern unsigned long kexec_mach_type; +extern unsigned long kexec_boot_atags; + +static atomic_t waiting_for_crash_ipi; + +/* + * Provide a dummy crash_notes definition while crash dump arrives to arm. + * This prevents breakage of crash_notes attribute in kernel/ksysfs.c. + */ + +int machine_kexec_prepare(struct kimage *image) +{ + return 0; +} + +void machine_kexec_cleanup(struct kimage *image) +{ +} + +void machine_crash_nonpanic_core(void *unused) +{ + struct pt_regs regs; + + crash_setup_regs(®s, NULL); + printk(KERN_DEBUG "CPU %u will stop doing anything useful since another CPU has crashed\n", + smp_processor_id()); + crash_save_cpu(®s, smp_processor_id()); + flush_cache_all(); + + atomic_dec(&waiting_for_crash_ipi); + while (1) + cpu_relax(); +} + +void machine_crash_shutdown(struct pt_regs *regs) +{ + unsigned long msecs; + + local_irq_disable(); + + atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1); + smp_call_function(machine_crash_nonpanic_core, NULL, false); + msecs = 1000; /* Wait at most a second for the other cpus to stop */ + while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) { + mdelay(1); + msecs--; + } + if (atomic_read(&waiting_for_crash_ipi) > 0) + printk(KERN_WARNING "Non-crashing CPUs did not react to IPI\n"); + + crash_save_cpu(regs, smp_processor_id()); + + printk(KERN_INFO "Loading crashdump kernel...\n"); +} + +/* + * Function pointer to optional machine-specific reinitialization + */ +void (*kexec_reinit)(void); + +void machine_kexec(struct kimage *image) +{ + unsigned long page_list; + unsigned long reboot_code_buffer_phys; + void *reboot_code_buffer; + + + page_list = image->head & PAGE_MASK; + + /* we need both effective and real address here */ + reboot_code_buffer_phys = + page_to_pfn(image->control_code_page) << PAGE_SHIFT; + reboot_code_buffer = page_address(image->control_code_page); + + /* Prepare parameters for reboot_code_buffer*/ + kexec_start_address = image->start; + kexec_indirection_page = page_list; + kexec_mach_type = machine_arch_type; + kexec_boot_atags = image->start - KEXEC_ARM_ZIMAGE_OFFSET + KEXEC_ARM_ATAGS_OFFSET; + + /* copy our kernel relocation code to the control code page */ + memcpy(reboot_code_buffer, + relocate_new_kernel, relocate_new_kernel_size); + + + flush_icache_range((unsigned long) reboot_code_buffer, + (unsigned long) reboot_code_buffer + KEXEC_CONTROL_PAGE_SIZE); + printk(KERN_INFO "Bye!\n"); + + if (kexec_reinit) + kexec_reinit(); + local_irq_disable(); + local_fiq_disable(); + setup_mm_for_reboot(0); /* mode is not used, so just pass 0*/ + flush_cache_all(); + outer_flush_all(); + outer_disable(); + cpu_proc_fin(); + outer_inv_all(); + flush_cache_all(); + cpu_reset(reboot_code_buffer_phys); +} diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c new file mode 100644 index 00000000..016d6a08 --- /dev/null +++ b/arch/arm/kernel/module.c @@ -0,0 +1,367 @@ +/* + * linux/arch/arm/kernel/module.c + * + * Copyright (C) 2002 Russell King. + * Modified for nommu by Hyok S. Choi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Module allocation method suggested by Andi Kleen. + */ +#include <linux/module.h> +#include <linux/moduleloader.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/elf.h> +#include <linux/vmalloc.h> +#include <linux/fs.h> +#include <linux/string.h> +#include <linux/gfp.h> + +#include <asm/pgtable.h> +#include <asm/sections.h> +#include <asm/smp_plat.h> +#include <asm/unwind.h> + +#ifdef CONFIG_XIP_KERNEL +/* + * The XIP kernel text is mapped in the module area for modules and + * some other stuff to work without any indirect relocations. + * MODULES_VADDR is redefined here and not in asm/memory.h to avoid + * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. + */ +#undef MODULES_VADDR +#define MODULES_VADDR (((unsigned long)_etext + ~PGDIR_MASK) & PGDIR_MASK) +#endif + +#ifdef CONFIG_MMU +void *module_alloc(unsigned long size) +{ + return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, + GFP_KERNEL, PAGE_KERNEL_EXEC, -1, + __builtin_return_address(0)); +} +#else /* CONFIG_MMU */ +void *module_alloc(unsigned long size) +{ + return size == 0 ? NULL : vmalloc(size); +} +#endif /* !CONFIG_MMU */ + +void module_free(struct module *module, void *region) +{ + vfree(region); +} + +int module_frob_arch_sections(Elf_Ehdr *hdr, + Elf_Shdr *sechdrs, + char *secstrings, + struct module *mod) +{ + return 0; +} + +int +apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex, + unsigned int relindex, struct module *module) +{ + Elf32_Shdr *symsec = sechdrs + symindex; + Elf32_Shdr *relsec = sechdrs + relindex; + Elf32_Shdr *dstsec = sechdrs + relsec->sh_info; + Elf32_Rel *rel = (void *)relsec->sh_addr; + unsigned int i; + + for (i = 0; i < relsec->sh_size / sizeof(Elf32_Rel); i++, rel++) { + unsigned long loc; + Elf32_Sym *sym; + const char *symname; + s32 offset; +#ifdef CONFIG_THUMB2_KERNEL + u32 upper, lower, sign, j1, j2; +#endif + + offset = ELF32_R_SYM(rel->r_info); + if (offset < 0 || offset > (symsec->sh_size / sizeof(Elf32_Sym))) { + pr_err("%s: section %u reloc %u: bad relocation sym offset\n", + module->name, relindex, i); + return -ENOEXEC; + } + + sym = ((Elf32_Sym *)symsec->sh_addr) + offset; + symname = strtab + sym->st_name; + + if (rel->r_offset < 0 || rel->r_offset > dstsec->sh_size - sizeof(u32)) { + pr_err("%s: section %u reloc %u sym '%s': out of bounds relocation, offset %d size %u\n", + module->name, relindex, i, symname, + rel->r_offset, dstsec->sh_size); + return -ENOEXEC; + } + + loc = dstsec->sh_addr + rel->r_offset; + + switch (ELF32_R_TYPE(rel->r_info)) { + case R_ARM_NONE: + /* ignore */ + break; + + case R_ARM_ABS32: + *(u32 *)loc += sym->st_value; + break; + + case R_ARM_PC24: + case R_ARM_CALL: + case R_ARM_JUMP24: + offset = (*(u32 *)loc & 0x00ffffff) << 2; + if (offset & 0x02000000) + offset -= 0x04000000; + + offset += sym->st_value - loc; + if (offset & 3 || + offset <= (s32)0xfe000000 || + offset >= (s32)0x02000000) { + pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", + module->name, relindex, i, symname, + ELF32_R_TYPE(rel->r_info), loc, + sym->st_value); + return -ENOEXEC; + } + + offset >>= 2; + + *(u32 *)loc &= 0xff000000; + *(u32 *)loc |= offset & 0x00ffffff; + break; + + case R_ARM_V4BX: + /* Preserve Rm and the condition code. Alter + * other bits to re-code instruction as + * MOV PC,Rm. + */ + *(u32 *)loc &= 0xf000000f; + *(u32 *)loc |= 0x01a0f000; + break; + + case R_ARM_PREL31: + offset = *(u32 *)loc + sym->st_value - loc; + *(u32 *)loc = offset & 0x7fffffff; + break; + + case R_ARM_MOVW_ABS_NC: + case R_ARM_MOVT_ABS: + offset = *(u32 *)loc; + offset = ((offset & 0xf0000) >> 4) | (offset & 0xfff); + offset = (offset ^ 0x8000) - 0x8000; + + offset += sym->st_value; + if (ELF32_R_TYPE(rel->r_info) == R_ARM_MOVT_ABS) + offset >>= 16; + + *(u32 *)loc &= 0xfff0f000; + *(u32 *)loc |= ((offset & 0xf000) << 4) | + (offset & 0x0fff); + break; + +#ifdef CONFIG_THUMB2_KERNEL + case R_ARM_THM_CALL: + case R_ARM_THM_JUMP24: + upper = *(u16 *)loc; + lower = *(u16 *)(loc + 2); + + /* + * 25 bit signed address range (Thumb-2 BL and B.W + * instructions): + * S:I1:I2:imm10:imm11:0 + * where: + * S = upper[10] = offset[24] + * I1 = ~(J1 ^ S) = offset[23] + * I2 = ~(J2 ^ S) = offset[22] + * imm10 = upper[9:0] = offset[21:12] + * imm11 = lower[10:0] = offset[11:1] + * J1 = lower[13] + * J2 = lower[11] + */ + sign = (upper >> 10) & 1; + j1 = (lower >> 13) & 1; + j2 = (lower >> 11) & 1; + offset = (sign << 24) | ((~(j1 ^ sign) & 1) << 23) | + ((~(j2 ^ sign) & 1) << 22) | + ((upper & 0x03ff) << 12) | + ((lower & 0x07ff) << 1); + if (offset & 0x01000000) + offset -= 0x02000000; + offset += sym->st_value - loc; + + /* + * For function symbols, only Thumb addresses are + * allowed (no interworking). + * + * For non-function symbols, the destination + * has no specific ARM/Thumb disposition, so + * the branch is resolved under the assumption + * that interworking is not required. + */ + if ((ELF32_ST_TYPE(sym->st_info) == STT_FUNC && + !(offset & 1)) || + offset <= (s32)0xff000000 || + offset >= (s32)0x01000000) { + pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n", + module->name, relindex, i, symname, + ELF32_R_TYPE(rel->r_info), loc, + sym->st_value); + return -ENOEXEC; + } + + sign = (offset >> 24) & 1; + j1 = sign ^ (~(offset >> 23) & 1); + j2 = sign ^ (~(offset >> 22) & 1); + *(u16 *)loc = (u16)((upper & 0xf800) | (sign << 10) | + ((offset >> 12) & 0x03ff)); + *(u16 *)(loc + 2) = (u16)((lower & 0xd000) | + (j1 << 13) | (j2 << 11) | + ((offset >> 1) & 0x07ff)); + break; + + case R_ARM_THM_MOVW_ABS_NC: + case R_ARM_THM_MOVT_ABS: + upper = *(u16 *)loc; + lower = *(u16 *)(loc + 2); + + /* + * MOVT/MOVW instructions encoding in Thumb-2: + * + * i = upper[10] + * imm4 = upper[3:0] + * imm3 = lower[14:12] + * imm8 = lower[7:0] + * + * imm16 = imm4:i:imm3:imm8 + */ + offset = ((upper & 0x000f) << 12) | + ((upper & 0x0400) << 1) | + ((lower & 0x7000) >> 4) | (lower & 0x00ff); + offset = (offset ^ 0x8000) - 0x8000; + offset += sym->st_value; + + if (ELF32_R_TYPE(rel->r_info) == R_ARM_THM_MOVT_ABS) + offset >>= 16; + + *(u16 *)loc = (u16)((upper & 0xfbf0) | + ((offset & 0xf000) >> 12) | + ((offset & 0x0800) >> 1)); + *(u16 *)(loc + 2) = (u16)((lower & 0x8f00) | + ((offset & 0x0700) << 4) | + (offset & 0x00ff)); + break; +#endif + + default: + printk(KERN_ERR "%s: unknown relocation: %u\n", + module->name, ELF32_R_TYPE(rel->r_info)); + return -ENOEXEC; + } + } + return 0; +} + +int +apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab, + unsigned int symindex, unsigned int relsec, struct module *module) +{ + printk(KERN_ERR "module %s: ADD RELOCATION unsupported\n", + module->name); + return -ENOEXEC; +} + +struct mod_unwind_map { + const Elf_Shdr *unw_sec; + const Elf_Shdr *txt_sec; +}; + +static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr, + const Elf_Shdr *sechdrs, const char *name) +{ + const Elf_Shdr *s, *se; + const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + + for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) + if (strcmp(name, secstrs + s->sh_name) == 0) + return s; + + return NULL; +} + +extern void fixup_pv_table(const void *, unsigned long); +extern void fixup_smp(const void *, unsigned long); + +int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs, + struct module *mod) +{ + const Elf_Shdr *s = NULL; +#ifdef CONFIG_ARM_UNWIND + const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + const Elf_Shdr *sechdrs_end = sechdrs + hdr->e_shnum; + struct mod_unwind_map maps[ARM_SEC_MAX]; + int i; + + memset(maps, 0, sizeof(maps)); + + for (s = sechdrs; s < sechdrs_end; s++) { + const char *secname = secstrs + s->sh_name; + + if (!(s->sh_flags & SHF_ALLOC)) + continue; + + if (strcmp(".ARM.exidx.init.text", secname) == 0) + maps[ARM_SEC_INIT].unw_sec = s; + else if (strcmp(".ARM.exidx.devinit.text", secname) == 0) + maps[ARM_SEC_DEVINIT].unw_sec = s; + else if (strcmp(".ARM.exidx", secname) == 0) + maps[ARM_SEC_CORE].unw_sec = s; + else if (strcmp(".ARM.exidx.exit.text", secname) == 0) + maps[ARM_SEC_EXIT].unw_sec = s; + else if (strcmp(".ARM.exidx.devexit.text", secname) == 0) + maps[ARM_SEC_DEVEXIT].unw_sec = s; + else if (strcmp(".init.text", secname) == 0) + maps[ARM_SEC_INIT].txt_sec = s; + else if (strcmp(".devinit.text", secname) == 0) + maps[ARM_SEC_DEVINIT].txt_sec = s; + else if (strcmp(".text", secname) == 0) + maps[ARM_SEC_CORE].txt_sec = s; + else if (strcmp(".exit.text", secname) == 0) + maps[ARM_SEC_EXIT].txt_sec = s; + else if (strcmp(".devexit.text", secname) == 0) + maps[ARM_SEC_DEVEXIT].txt_sec = s; + } + + for (i = 0; i < ARM_SEC_MAX; i++) + if (maps[i].unw_sec && maps[i].txt_sec) + mod->arch.unwind[i] = + unwind_table_add(maps[i].unw_sec->sh_addr, + maps[i].unw_sec->sh_size, + maps[i].txt_sec->sh_addr, + maps[i].txt_sec->sh_size); +#endif +#ifdef CONFIG_ARM_PATCH_PHYS_VIRT + s = find_mod_section(hdr, sechdrs, ".pv_table"); + if (s) + fixup_pv_table((void *)s->sh_addr, s->sh_size); +#endif + s = find_mod_section(hdr, sechdrs, ".alt.smp.init"); + if (s && !is_smp()) + fixup_smp((void *)s->sh_addr, s->sh_size); + return 0; +} + +void +module_arch_cleanup(struct module *mod) +{ +#ifdef CONFIG_ARM_UNWIND + int i; + + for (i = 0; i < ARM_SEC_MAX; i++) + if (mod->arch.unwind[i]) + unwind_table_del(mod->arch.unwind[i]); +#endif +} diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c new file mode 100644 index 00000000..2b5b1421 --- /dev/null +++ b/arch/arm/kernel/perf_event.c @@ -0,0 +1,775 @@ +#undef DEBUG + +/* + * ARM performance counter support. + * + * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles + * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com> + * + * This code is based on the sparc64 perf event code, which is in turn based + * on the x86 code. Callchain code is based on the ARM OProfile backtrace + * code. + */ +#define pr_fmt(fmt) "hw perfevents: " fmt + +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/perf_event.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> +#include <linux/uaccess.h> + +#include <asm/cputype.h> +#include <asm/irq.h> +#include <asm/irq_regs.h> +#include <asm/pmu.h> +#include <asm/stacktrace.h> + +static struct platform_device *pmu_device; + +/* + * Hardware lock to serialize accesses to PMU registers. Needed for the + * read/modify/write sequences. + */ +static DEFINE_RAW_SPINLOCK(pmu_lock); + +/* + * ARMv6 supports a maximum of 3 events, starting from index 1. If we add + * another platform that supports more, we need to increase this to be the + * largest of all platforms. + * + * ARMv7 supports up to 32 events: + * cycle counter CCNT + 31 events counters CNT0..30. + * Cortex-A8 has 1+4 counters, Cortex-A9 has 1+6 counters. + */ +#define ARMPMU_MAX_HWEVENTS 33 + +/* The events for a given CPU. */ +struct cpu_hw_events { + /* + * The events that are active on the CPU for the given index. Index 0 + * is reserved. + */ + struct perf_event *events[ARMPMU_MAX_HWEVENTS]; + + /* + * A 1 bit for an index indicates that the counter is being used for + * an event. A 0 means that the counter can be used. + */ + unsigned long used_mask[BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)]; + + /* + * A 1 bit for an index indicates that the counter is actively being + * used. + */ + unsigned long active_mask[BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)]; +}; +static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events); + +struct arm_pmu { + enum arm_perf_pmu_ids id; + const char *name; + irqreturn_t (*handle_irq)(int irq_num, void *dev); + void (*enable)(struct hw_perf_event *evt, int idx); + void (*disable)(struct hw_perf_event *evt, int idx); + int (*get_event_idx)(struct cpu_hw_events *cpuc, + struct hw_perf_event *hwc); + u32 (*read_counter)(int idx); + void (*write_counter)(int idx, u32 val); + void (*start)(void); + void (*stop)(void); + void (*reset)(void *); + const unsigned (*cache_map)[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; + const unsigned (*event_map)[PERF_COUNT_HW_MAX]; + u32 raw_event_mask; + int num_events; + u64 max_period; +}; + +/* Set at runtime when we know what CPU type we are. */ +static const struct arm_pmu *armpmu; + +enum arm_perf_pmu_ids +armpmu_get_pmu_id(void) +{ + int id = -ENODEV; + + if (armpmu != NULL) + id = armpmu->id; + + return id; +} +EXPORT_SYMBOL_GPL(armpmu_get_pmu_id); + +int +armpmu_get_max_events(void) +{ + int max_events = 0; + + if (armpmu != NULL) + max_events = armpmu->num_events; + + return max_events; +} +EXPORT_SYMBOL_GPL(armpmu_get_max_events); + +int perf_num_counters(void) +{ + return armpmu_get_max_events(); +} +EXPORT_SYMBOL_GPL(perf_num_counters); + +#define HW_OP_UNSUPPORTED 0xFFFF + +#define C(_x) \ + PERF_COUNT_HW_CACHE_##_x + +#define CACHE_OP_UNSUPPORTED 0xFFFF + +static int +armpmu_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result, ret; + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return -EINVAL; + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return -EINVAL; + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return -EINVAL; + + ret = (int)(*armpmu->cache_map)[cache_type][cache_op][cache_result]; + + if (ret == CACHE_OP_UNSUPPORTED) + return -ENOENT; + + return ret; +} + +static int +armpmu_map_event(u64 config) +{ + int mapping = (*armpmu->event_map)[config]; + return mapping == HW_OP_UNSUPPORTED ? -EOPNOTSUPP : mapping; +} + +static int +armpmu_map_raw_event(u64 config) +{ + return (int)(config & armpmu->raw_event_mask); +} + +static int +armpmu_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, + int idx) +{ + s64 left = local64_read(&hwc->period_left); + s64 period = hwc->sample_period; + int ret = 0; + + if (unlikely(left <= -period)) { + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (unlikely(left <= 0)) { + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (left > (s64)armpmu->max_period) + left = armpmu->max_period; + + local64_set(&hwc->prev_count, (u64)-left); + + armpmu->write_counter(idx, (u64)(-left) & 0xffffffff); + + perf_event_update_userpage(event); + + return ret; +} + +static u64 +armpmu_event_update(struct perf_event *event, + struct hw_perf_event *hwc, + int idx, int overflow) +{ + u64 delta, prev_raw_count, new_raw_count; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = armpmu->read_counter(idx); + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + new_raw_count &= armpmu->max_period; + prev_raw_count &= armpmu->max_period; + + if (overflow) + delta = armpmu->max_period - prev_raw_count + new_raw_count + 1; + else + delta = new_raw_count - prev_raw_count; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); + + return new_raw_count; +} + +static void +armpmu_read(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + + /* Don't read disabled counters! */ + if (hwc->idx < 0) + return; + + armpmu_event_update(event, hwc, hwc->idx, 0); +} + +static void +armpmu_stop(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!armpmu) + return; + + /* + * ARM pmu always has to update the counter, so ignore + * PERF_EF_UPDATE, see comments in armpmu_start(). + */ + if (!(hwc->state & PERF_HES_STOPPED)) { + armpmu->disable(hwc, hwc->idx); + barrier(); /* why? */ + armpmu_event_update(event, hwc, hwc->idx, 0); + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; + } +} + +static void +armpmu_start(struct perf_event *event, int flags) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!armpmu) + return; + + /* + * ARM pmu always has to reprogram the period, so ignore + * PERF_EF_RELOAD, see the comment below. + */ + if (flags & PERF_EF_RELOAD) + WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); + + hwc->state = 0; + /* + * Set the period again. Some counters can't be stopped, so when we + * were stopped we simply disabled the IRQ source and the counter + * may have been left counting. If we don't do this step then we may + * get an interrupt too soon or *way* too late if the overflow has + * happened since disabling. + */ + armpmu_event_set_period(event, hwc, hwc->idx); + armpmu->enable(hwc, hwc->idx); +} + +static void +armpmu_del(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx = hwc->idx; + + WARN_ON(idx < 0); + + clear_bit(idx, cpuc->active_mask); + armpmu_stop(event, PERF_EF_UPDATE); + cpuc->events[idx] = NULL; + clear_bit(idx, cpuc->used_mask); + + perf_event_update_userpage(event); +} + +static int +armpmu_add(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + struct hw_perf_event *hwc = &event->hw; + int idx; + int err = 0; + + perf_pmu_disable(event->pmu); + + /* If we don't have a space for the counter then finish early. */ + idx = armpmu->get_event_idx(cpuc, hwc); + if (idx < 0) { + err = idx; + goto out; + } + + /* + * If there is an event in the counter we are going to use then make + * sure it is disabled. + */ + event->hw.idx = idx; + armpmu->disable(hwc, idx); + cpuc->events[idx] = event; + set_bit(idx, cpuc->active_mask); + + hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; + if (flags & PERF_EF_START) + armpmu_start(event, PERF_EF_RELOAD); + + /* Propagate our changes to the userspace mapping. */ + perf_event_update_userpage(event); + +out: + perf_pmu_enable(event->pmu); + return err; +} + +static struct pmu pmu; + +static int +validate_event(struct cpu_hw_events *cpuc, + struct perf_event *event) +{ + struct hw_perf_event fake_event = event->hw; + + if (event->pmu != &pmu || event->state <= PERF_EVENT_STATE_OFF) + return 1; + + return armpmu->get_event_idx(cpuc, &fake_event) >= 0; +} + +static int +validate_group(struct perf_event *event) +{ + struct perf_event *sibling, *leader = event->group_leader; + struct cpu_hw_events fake_pmu; + + memset(&fake_pmu, 0, sizeof(fake_pmu)); + + if (!validate_event(&fake_pmu, leader)) + return -ENOSPC; + + list_for_each_entry(sibling, &leader->sibling_list, group_entry) { + if (!validate_event(&fake_pmu, sibling)) + return -ENOSPC; + } + + if (!validate_event(&fake_pmu, event)) + return -ENOSPC; + + return 0; +} + +static irqreturn_t armpmu_platform_irq(int irq, void *dev) +{ + struct arm_pmu_platdata *plat = dev_get_platdata(&pmu_device->dev); + + return plat->handle_irq(irq, dev, armpmu->handle_irq); +} + +static int +armpmu_reserve_hardware(void) +{ + struct arm_pmu_platdata *plat; + irq_handler_t handle_irq; + int i, err = -ENODEV, irq; + + pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU); + if (IS_ERR(pmu_device)) { + pr_warning("unable to reserve pmu\n"); + return PTR_ERR(pmu_device); + } + + init_pmu(ARM_PMU_DEVICE_CPU); + + plat = dev_get_platdata(&pmu_device->dev); + if (plat && plat->handle_irq) + handle_irq = armpmu_platform_irq; + else + handle_irq = armpmu->handle_irq; + + if (pmu_device->num_resources < 1) { + pr_err("no irqs for PMUs defined\n"); + return -ENODEV; + } + + for (i = 0; i < pmu_device->num_resources; ++i) { + irq = platform_get_irq(pmu_device, i); + if (irq < 0) + continue; + + err = request_irq(irq, handle_irq, + IRQF_DISABLED | IRQF_NOBALANCING, + "armpmu", NULL); + if (err) { + pr_warning("unable to request IRQ%d for ARM perf " + "counters\n", irq); + break; + } + } + + if (err) { + for (i = i - 1; i >= 0; --i) { + irq = platform_get_irq(pmu_device, i); + if (irq >= 0) + free_irq(irq, NULL); + } + release_pmu(pmu_device); + pmu_device = NULL; + } + + return err; +} + +static void +armpmu_release_hardware(void) +{ + int i, irq; + + for (i = pmu_device->num_resources - 1; i >= 0; --i) { + irq = platform_get_irq(pmu_device, i); + if (irq >= 0) + free_irq(irq, NULL); + } + armpmu->stop(); + + release_pmu(pmu_device); + pmu_device = NULL; +} + +static atomic_t active_events = ATOMIC_INIT(0); +static DEFINE_MUTEX(pmu_reserve_mutex); + +static void +hw_perf_event_destroy(struct perf_event *event) +{ + if (atomic_dec_and_mutex_lock(&active_events, &pmu_reserve_mutex)) { + armpmu_release_hardware(); + mutex_unlock(&pmu_reserve_mutex); + } +} + +static int +__hw_perf_event_init(struct perf_event *event) +{ + struct hw_perf_event *hwc = &event->hw; + int mapping, err; + + /* Decode the generic type into an ARM event identifier. */ + if (PERF_TYPE_HARDWARE == event->attr.type) { + mapping = armpmu_map_event(event->attr.config); + } else if (PERF_TYPE_HW_CACHE == event->attr.type) { + mapping = armpmu_map_cache_event(event->attr.config); + } else if (PERF_TYPE_RAW == event->attr.type) { + mapping = armpmu_map_raw_event(event->attr.config); + } else { + pr_debug("event type %x not supported\n", event->attr.type); + return -EOPNOTSUPP; + } + + if (mapping < 0) { + pr_debug("event %x:%llx not supported\n", event->attr.type, + event->attr.config); + return mapping; + } + + /* + * Check whether we need to exclude the counter from certain modes. + * The ARM performance counters are on all of the time so if someone + * has asked us for some excludes then we have to fail. + */ + if (event->attr.exclude_kernel || event->attr.exclude_user || + event->attr.exclude_hv || event->attr.exclude_idle) { + pr_debug("ARM performance counters do not support " + "mode exclusion\n"); + return -EPERM; + } + + /* + * We don't assign an index until we actually place the event onto + * hardware. Use -1 to signify that we haven't decided where to put it + * yet. For SMP systems, each core has it's own PMU so we can't do any + * clever allocation or constraints checking at this point. + */ + hwc->idx = -1; + + /* + * Store the event encoding into the config_base field. config and + * event_base are unused as the only 2 things we need to know are + * the event mapping and the counter to use. The counter to use is + * also the indx and the config_base is the event type. + */ + hwc->config_base = (unsigned long)mapping; + hwc->config = 0; + hwc->event_base = 0; + + if (!hwc->sample_period) { + hwc->sample_period = armpmu->max_period; + hwc->last_period = hwc->sample_period; + local64_set(&hwc->period_left, hwc->sample_period); + } + + err = 0; + if (event->group_leader != event) { + err = validate_group(event); + if (err) + return -EINVAL; + } + + return err; +} + +static int armpmu_event_init(struct perf_event *event) +{ + int err = 0; + + switch (event->attr.type) { + case PERF_TYPE_RAW: + case PERF_TYPE_HARDWARE: + case PERF_TYPE_HW_CACHE: + break; + + default: + return -ENOENT; + } + + if (!armpmu) + return -ENODEV; + + event->destroy = hw_perf_event_destroy; + + if (!atomic_inc_not_zero(&active_events)) { + mutex_lock(&pmu_reserve_mutex); + if (atomic_read(&active_events) == 0) { + err = armpmu_reserve_hardware(); + } + + if (!err) + atomic_inc(&active_events); + mutex_unlock(&pmu_reserve_mutex); + } + + if (err) + return err; + + err = __hw_perf_event_init(event); + if (err) + hw_perf_event_destroy(event); + + return err; +} + +static void armpmu_enable(struct pmu *pmu) +{ + /* Enable all of the perf events on hardware. */ + int idx, enabled = 0; + struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); + + if (!armpmu) + return; + + for (idx = 0; idx <= armpmu->num_events; ++idx) { + struct perf_event *event = cpuc->events[idx]; + + if (!event) + continue; + + armpmu->enable(&event->hw, idx); + enabled = 1; + } + + if (enabled) + armpmu->start(); +} + +static void armpmu_disable(struct pmu *pmu) +{ + if (armpmu) + armpmu->stop(); +} + +static struct pmu pmu = { + .pmu_enable = armpmu_enable, + .pmu_disable = armpmu_disable, + .event_init = armpmu_event_init, + .add = armpmu_add, + .del = armpmu_del, + .start = armpmu_start, + .stop = armpmu_stop, + .read = armpmu_read, +}; + +/* Include the PMU-specific implementations. */ +#include "perf_event_xscale.c" +#include "perf_event_v6.c" +#include "perf_event_v7.c" + +/* + * Ensure the PMU has sane values out of reset. + * This requires SMP to be available, so exists as a separate initcall. + */ +static int __init +armpmu_reset(void) +{ + if (armpmu && armpmu->reset) + return on_each_cpu(armpmu->reset, NULL, 1); + return 0; +} +arch_initcall(armpmu_reset); + +static int __init +init_hw_perf_events(void) +{ + unsigned long cpuid = read_cpuid_id(); + unsigned long implementor = (cpuid & 0xFF000000) >> 24; + unsigned long part_number = (cpuid & 0xFFF0); + + /* ARM Ltd CPUs. */ + if (0x41 == implementor) { + switch (part_number) { + case 0xB360: /* ARM1136 */ + case 0xB560: /* ARM1156 */ + case 0xB760: /* ARM1176 */ + armpmu = armv6pmu_init(); + break; + case 0xB020: /* ARM11mpcore */ + armpmu = armv6mpcore_pmu_init(); + break; + case 0xC080: /* Cortex-A8 */ + armpmu = armv7_a8_pmu_init(); + break; + case 0xC090: /* Cortex-A9 */ + armpmu = armv7_a9_pmu_init(); + break; + } + /* Intel CPUs [xscale]. */ + } else if (0x69 == implementor) { + part_number = (cpuid >> 13) & 0x7; + switch (part_number) { + case 1: + armpmu = xscale1pmu_init(); + break; + case 2: + armpmu = xscale2pmu_init(); + break; + } + } + + if (armpmu) { + pr_info("enabled with %s PMU driver, %d counters available\n", + armpmu->name, armpmu->num_events); + } else { + pr_info("no hardware support available\n"); + } + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); + + return 0; +} +early_initcall(init_hw_perf_events); + +/* + * Callchain handling code. + */ + +/* + * The registers we're interested in are at the end of the variable + * length saved register structure. The fp points at the end of this + * structure so the address of this struct is: + * (struct frame_tail *)(xxx->fp)-1 + * + * This code has been adapted from the ARM OProfile support. + */ +struct frame_tail { + struct frame_tail __user *fp; + unsigned long sp; + unsigned long lr; +} __attribute__((packed)); + +/* + * Get the return address for a single stackframe and return a pointer to the + * next frame tail. + */ +static struct frame_tail __user * +user_backtrace(struct frame_tail __user *tail, + struct perf_callchain_entry *entry) +{ + struct frame_tail buftail; + + /* Also check accessibility of one struct frame_tail beyond */ + if (!access_ok(VERIFY_READ, tail, sizeof(buftail))) + return NULL; + if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail))) + return NULL; + + perf_callchain_store(entry, buftail.lr); + + /* + * Frame pointers should strictly progress back up the stack + * (towards higher addresses). + */ + if (tail + 1 >= buftail.fp) + return NULL; + + return buftail.fp - 1; +} + +void +perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) +{ + struct frame_tail __user *tail; + + + tail = (struct frame_tail __user *)regs->ARM_fp - 1; + + while ((entry->nr < PERF_MAX_STACK_DEPTH) && + tail && !((unsigned long)tail & 0x3)) + tail = user_backtrace(tail, entry); +} + +/* + * Gets called by walk_stackframe() for every stackframe. This will be called + * whist unwinding the stackframe and is like a subroutine return so we use + * the PC. + */ +static int +callchain_trace(struct stackframe *fr, + void *data) +{ + struct perf_callchain_entry *entry = data; + perf_callchain_store(entry, fr->pc); + return 0; +} + +void +perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs) +{ + struct stackframe fr; + + fr.fp = regs->ARM_fp; + fr.sp = regs->ARM_sp; + fr.lr = regs->ARM_lr; + fr.pc = regs->ARM_pc; + walk_stackframe(&fr, callchain_trace, entry); +} diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c new file mode 100644 index 00000000..f1e8dd94 --- /dev/null +++ b/arch/arm/kernel/perf_event_v6.c @@ -0,0 +1,672 @@ +/* + * ARMv6 Performance counter handling code. + * + * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles + * + * ARMv6 has 2 configurable performance counters and a single cycle counter. + * They all share a single reset bit but can be written to zero so we can use + * that for a reset. + * + * The counters can't be individually enabled or disabled so when we remove + * one event and replace it with another we could get spurious counts from the + * wrong event. However, we can take advantage of the fact that the + * performance counters can export events to the event bus, and the event bus + * itself can be monitored. This requires that we *don't* export the events to + * the event bus. The procedure for disabling a configurable counter is: + * - change the counter to count the ETMEXTOUT[0] signal (0x20). This + * effectively stops the counter from counting. + * - disable the counter's interrupt generation (each counter has it's + * own interrupt enable bit). + * Once stopped, the counter value can be written as 0 to reset. + * + * To enable a counter: + * - enable the counter's interrupt generation. + * - set the new event type. + * + * Note: the dedicated cycle counter only counts cycles and can't be + * enabled/disabled independently of the others. When we want to disable the + * cycle counter, we have to just disable the interrupt reporting and start + * ignoring that counter. When re-enabling, we have to reset the value and + * enable the interrupt. + */ + +#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) +enum armv6_perf_types { + ARMV6_PERFCTR_ICACHE_MISS = 0x0, + ARMV6_PERFCTR_IBUF_STALL = 0x1, + ARMV6_PERFCTR_DDEP_STALL = 0x2, + ARMV6_PERFCTR_ITLB_MISS = 0x3, + ARMV6_PERFCTR_DTLB_MISS = 0x4, + ARMV6_PERFCTR_BR_EXEC = 0x5, + ARMV6_PERFCTR_BR_MISPREDICT = 0x6, + ARMV6_PERFCTR_INSTR_EXEC = 0x7, + ARMV6_PERFCTR_DCACHE_HIT = 0x9, + ARMV6_PERFCTR_DCACHE_ACCESS = 0xA, + ARMV6_PERFCTR_DCACHE_MISS = 0xB, + ARMV6_PERFCTR_DCACHE_WBACK = 0xC, + ARMV6_PERFCTR_SW_PC_CHANGE = 0xD, + ARMV6_PERFCTR_MAIN_TLB_MISS = 0xF, + ARMV6_PERFCTR_EXPL_D_ACCESS = 0x10, + ARMV6_PERFCTR_LSU_FULL_STALL = 0x11, + ARMV6_PERFCTR_WBUF_DRAINED = 0x12, + ARMV6_PERFCTR_CPU_CYCLES = 0xFF, + ARMV6_PERFCTR_NOP = 0x20, +}; + +enum armv6_counters { + ARMV6_CYCLE_COUNTER = 1, + ARMV6_COUNTER0, + ARMV6_COUNTER1, +}; + +/* + * The hardware events that we support. We do support cache operations but + * we have harvard caches and no way to combine instruction and data + * accesses/misses in hardware. + */ +static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV6_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6_PERFCTR_INSTR_EXEC, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6_PERFCTR_BR_MISPREDICT, + [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED, +}; + +static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + /* + * The performance counters don't differentiate between read + * and write accesses/misses so this isn't strictly correct, + * but it's the best we can do. Writes and reads get + * combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV6_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV6_PERFCTR_DCACHE_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6_PERFCTR_ICACHE_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + /* + * The ARM performance counters can count micro DTLB misses, + * micro ITLB misses and main TLB misses. There isn't an event + * for TLB misses, so use the micro misses here and if users + * want the main TLB misses they can use a raw counter. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6_PERFCTR_DTLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6_PERFCTR_ITLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + +enum armv6mpcore_perf_types { + ARMV6MPCORE_PERFCTR_ICACHE_MISS = 0x0, + ARMV6MPCORE_PERFCTR_IBUF_STALL = 0x1, + ARMV6MPCORE_PERFCTR_DDEP_STALL = 0x2, + ARMV6MPCORE_PERFCTR_ITLB_MISS = 0x3, + ARMV6MPCORE_PERFCTR_DTLB_MISS = 0x4, + ARMV6MPCORE_PERFCTR_BR_EXEC = 0x5, + ARMV6MPCORE_PERFCTR_BR_NOTPREDICT = 0x6, + ARMV6MPCORE_PERFCTR_BR_MISPREDICT = 0x7, + ARMV6MPCORE_PERFCTR_INSTR_EXEC = 0x8, + ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS = 0xA, + ARMV6MPCORE_PERFCTR_DCACHE_RDMISS = 0xB, + ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS = 0xC, + ARMV6MPCORE_PERFCTR_DCACHE_WRMISS = 0xD, + ARMV6MPCORE_PERFCTR_DCACHE_EVICTION = 0xE, + ARMV6MPCORE_PERFCTR_SW_PC_CHANGE = 0xF, + ARMV6MPCORE_PERFCTR_MAIN_TLB_MISS = 0x10, + ARMV6MPCORE_PERFCTR_EXPL_MEM_ACCESS = 0x11, + ARMV6MPCORE_PERFCTR_LSU_FULL_STALL = 0x12, + ARMV6MPCORE_PERFCTR_WBUF_DRAINED = 0x13, + ARMV6MPCORE_PERFCTR_CPU_CYCLES = 0xFF, +}; + +/* + * The hardware events that we support. We do support cache operations but + * we have harvard caches and no way to combine instruction and data + * accesses/misses in hardware. + */ +static const unsigned armv6mpcore_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV6MPCORE_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_INSTR_EXEC, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_BR_EXEC, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV6MPCORE_PERFCTR_BR_MISPREDICT, + [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED, +}; + +static const unsigned armv6mpcore_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = + ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS, + [C(RESULT_MISS)] = + ARMV6MPCORE_PERFCTR_DCACHE_RDMISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = + ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS, + [C(RESULT_MISS)] = + ARMV6MPCORE_PERFCTR_DCACHE_WRMISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ICACHE_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ICACHE_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + /* + * The ARM performance counters can count micro DTLB misses, + * micro ITLB misses and main TLB misses. There isn't an event + * for TLB misses, so use the micro misses here and if users + * want the main TLB misses they can use a raw counter. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_DTLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV6MPCORE_PERFCTR_ITLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + +static inline unsigned long +armv6_pmcr_read(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c15, c12, 0" : "=r"(val)); + return val; +} + +static inline void +armv6_pmcr_write(unsigned long val) +{ + asm volatile("mcr p15, 0, %0, c15, c12, 0" : : "r"(val)); +} + +#define ARMV6_PMCR_ENABLE (1 << 0) +#define ARMV6_PMCR_CTR01_RESET (1 << 1) +#define ARMV6_PMCR_CCOUNT_RESET (1 << 2) +#define ARMV6_PMCR_CCOUNT_DIV (1 << 3) +#define ARMV6_PMCR_COUNT0_IEN (1 << 4) +#define ARMV6_PMCR_COUNT1_IEN (1 << 5) +#define ARMV6_PMCR_CCOUNT_IEN (1 << 6) +#define ARMV6_PMCR_COUNT0_OVERFLOW (1 << 8) +#define ARMV6_PMCR_COUNT1_OVERFLOW (1 << 9) +#define ARMV6_PMCR_CCOUNT_OVERFLOW (1 << 10) +#define ARMV6_PMCR_EVT_COUNT0_SHIFT 20 +#define ARMV6_PMCR_EVT_COUNT0_MASK (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT) +#define ARMV6_PMCR_EVT_COUNT1_SHIFT 12 +#define ARMV6_PMCR_EVT_COUNT1_MASK (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT) + +#define ARMV6_PMCR_OVERFLOWED_MASK \ + (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \ + ARMV6_PMCR_CCOUNT_OVERFLOW) + +static inline int +armv6_pmcr_has_overflowed(unsigned long pmcr) +{ + return pmcr & ARMV6_PMCR_OVERFLOWED_MASK; +} + +static inline int +armv6_pmcr_counter_has_overflowed(unsigned long pmcr, + enum armv6_counters counter) +{ + int ret = 0; + + if (ARMV6_CYCLE_COUNTER == counter) + ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW; + else if (ARMV6_COUNTER0 == counter) + ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW; + else if (ARMV6_COUNTER1 == counter) + ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW; + else + WARN_ONCE(1, "invalid counter number (%d)\n", counter); + + return ret; +} + +static inline u32 +armv6pmu_read_counter(int counter) +{ + unsigned long value = 0; + + if (ARMV6_CYCLE_COUNTER == counter) + asm volatile("mrc p15, 0, %0, c15, c12, 1" : "=r"(value)); + else if (ARMV6_COUNTER0 == counter) + asm volatile("mrc p15, 0, %0, c15, c12, 2" : "=r"(value)); + else if (ARMV6_COUNTER1 == counter) + asm volatile("mrc p15, 0, %0, c15, c12, 3" : "=r"(value)); + else + WARN_ONCE(1, "invalid counter number (%d)\n", counter); + + return value; +} + +static inline void +armv6pmu_write_counter(int counter, + u32 value) +{ + if (ARMV6_CYCLE_COUNTER == counter) + asm volatile("mcr p15, 0, %0, c15, c12, 1" : : "r"(value)); + else if (ARMV6_COUNTER0 == counter) + asm volatile("mcr p15, 0, %0, c15, c12, 2" : : "r"(value)); + else if (ARMV6_COUNTER1 == counter) + asm volatile("mcr p15, 0, %0, c15, c12, 3" : : "r"(value)); + else + WARN_ONCE(1, "invalid counter number (%d)\n", counter); +} + +static void +armv6pmu_enable_event(struct hw_perf_event *hwc, + int idx) +{ + unsigned long val, mask, evt, flags; + + if (ARMV6_CYCLE_COUNTER == idx) { + mask = 0; + evt = ARMV6_PMCR_CCOUNT_IEN; + } else if (ARMV6_COUNTER0 == idx) { + mask = ARMV6_PMCR_EVT_COUNT0_MASK; + evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) | + ARMV6_PMCR_COUNT0_IEN; + } else if (ARMV6_COUNTER1 == idx) { + mask = ARMV6_PMCR_EVT_COUNT1_MASK; + evt = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) | + ARMV6_PMCR_COUNT1_IEN; + } else { + WARN_ONCE(1, "invalid counter number (%d)\n", idx); + return; + } + + /* + * Mask out the current event and set the counter to count the event + * that we're interested in. + */ + raw_spin_lock_irqsave(&pmu_lock, flags); + val = armv6_pmcr_read(); + val &= ~mask; + val |= evt; + armv6_pmcr_write(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static irqreturn_t +armv6pmu_handle_irq(int irq_num, + void *dev) +{ + unsigned long pmcr = armv6_pmcr_read(); + struct perf_sample_data data; + struct cpu_hw_events *cpuc; + struct pt_regs *regs; + int idx; + + if (!armv6_pmcr_has_overflowed(pmcr)) + return IRQ_NONE; + + regs = get_irq_regs(); + + /* + * The interrupts are cleared by writing the overflow flags back to + * the control register. All of the other bits don't have any effect + * if they are rewritten, so write the whole value back. + */ + armv6_pmcr_write(pmcr); + + perf_sample_data_init(&data, 0); + + cpuc = &__get_cpu_var(cpu_hw_events); + for (idx = 0; idx <= armpmu->num_events; ++idx) { + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc; + + if (!test_bit(idx, cpuc->active_mask)) + continue; + + /* + * We have a single interrupt for all counters. Check that + * each counter has overflowed before we process it. + */ + if (!armv6_pmcr_counter_has_overflowed(pmcr, idx)) + continue; + + hwc = &event->hw; + armpmu_event_update(event, hwc, idx, 1); + data.period = event->hw.last_period; + if (!armpmu_event_set_period(event, hwc, idx)) + continue; + + if (perf_event_overflow(event, 0, &data, regs)) + armpmu->disable(hwc, idx); + } + + /* + * Handle the pending perf events. + * + * Note: this call *must* be run with interrupts disabled. For + * platforms that can have the PMU interrupts raised as an NMI, this + * will not work. + */ + irq_work_run(); + + return IRQ_HANDLED; +} + +static void +armv6pmu_start(void) +{ + unsigned long flags, val; + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = armv6_pmcr_read(); + val |= ARMV6_PMCR_ENABLE; + armv6_pmcr_write(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void +armv6pmu_stop(void) +{ + unsigned long flags, val; + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = armv6_pmcr_read(); + val &= ~ARMV6_PMCR_ENABLE; + armv6_pmcr_write(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static int +armv6pmu_get_event_idx(struct cpu_hw_events *cpuc, + struct hw_perf_event *event) +{ + /* Always place a cycle counter into the cycle counter. */ + if (ARMV6_PERFCTR_CPU_CYCLES == event->config_base) { + if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask)) + return -EAGAIN; + + return ARMV6_CYCLE_COUNTER; + } else { + /* + * For anything other than a cycle counter, try and use + * counter0 and counter1. + */ + if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask)) + return ARMV6_COUNTER1; + + if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask)) + return ARMV6_COUNTER0; + + /* The counters are all in use. */ + return -EAGAIN; + } +} + +static void +armv6pmu_disable_event(struct hw_perf_event *hwc, + int idx) +{ + unsigned long val, mask, evt, flags; + + if (ARMV6_CYCLE_COUNTER == idx) { + mask = ARMV6_PMCR_CCOUNT_IEN; + evt = 0; + } else if (ARMV6_COUNTER0 == idx) { + mask = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK; + evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT; + } else if (ARMV6_COUNTER1 == idx) { + mask = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK; + evt = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT; + } else { + WARN_ONCE(1, "invalid counter number (%d)\n", idx); + return; + } + + /* + * Mask out the current event and set the counter to count the number + * of ETM bus signal assertion cycles. The external reporting should + * be disabled and so this should never increment. + */ + raw_spin_lock_irqsave(&pmu_lock, flags); + val = armv6_pmcr_read(); + val &= ~mask; + val |= evt; + armv6_pmcr_write(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void +armv6mpcore_pmu_disable_event(struct hw_perf_event *hwc, + int idx) +{ + unsigned long val, mask, flags, evt = 0; + + if (ARMV6_CYCLE_COUNTER == idx) { + mask = ARMV6_PMCR_CCOUNT_IEN; + } else if (ARMV6_COUNTER0 == idx) { + mask = ARMV6_PMCR_COUNT0_IEN; + } else if (ARMV6_COUNTER1 == idx) { + mask = ARMV6_PMCR_COUNT1_IEN; + } else { + WARN_ONCE(1, "invalid counter number (%d)\n", idx); + return; + } + + /* + * Unlike UP ARMv6, we don't have a way of stopping the counters. We + * simply disable the interrupt reporting. + */ + raw_spin_lock_irqsave(&pmu_lock, flags); + val = armv6_pmcr_read(); + val &= ~mask; + val |= evt; + armv6_pmcr_write(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static const struct arm_pmu armv6pmu = { + .id = ARM_PERF_PMU_ID_V6, + .name = "v6", + .handle_irq = armv6pmu_handle_irq, + .enable = armv6pmu_enable_event, + .disable = armv6pmu_disable_event, + .read_counter = armv6pmu_read_counter, + .write_counter = armv6pmu_write_counter, + .get_event_idx = armv6pmu_get_event_idx, + .start = armv6pmu_start, + .stop = armv6pmu_stop, + .cache_map = &armv6_perf_cache_map, + .event_map = &armv6_perf_map, + .raw_event_mask = 0xFF, + .num_events = 3, + .max_period = (1LLU << 32) - 1, +}; + +static const struct arm_pmu *__init armv6pmu_init(void) +{ + return &armv6pmu; +} + +/* + * ARMv6mpcore is almost identical to single core ARMv6 with the exception + * that some of the events have different enumerations and that there is no + * *hack* to stop the programmable counters. To stop the counters we simply + * disable the interrupt reporting and update the event. When unthrottling we + * reset the period and enable the interrupt reporting. + */ +static const struct arm_pmu armv6mpcore_pmu = { + .id = ARM_PERF_PMU_ID_V6MP, + .name = "v6mpcore", + .handle_irq = armv6pmu_handle_irq, + .enable = armv6pmu_enable_event, + .disable = armv6mpcore_pmu_disable_event, + .read_counter = armv6pmu_read_counter, + .write_counter = armv6pmu_write_counter, + .get_event_idx = armv6pmu_get_event_idx, + .start = armv6pmu_start, + .stop = armv6pmu_stop, + .cache_map = &armv6mpcore_perf_cache_map, + .event_map = &armv6mpcore_perf_map, + .raw_event_mask = 0xFF, + .num_events = 3, + .max_period = (1LLU << 32) - 1, +}; + +static const struct arm_pmu *__init armv6mpcore_pmu_init(void) +{ + return &armv6mpcore_pmu; +} +#else +static const struct arm_pmu *__init armv6pmu_init(void) +{ + return NULL; +} + +static const struct arm_pmu *__init armv6mpcore_pmu_init(void) +{ + return NULL; +} +#endif /* CONFIG_CPU_V6 || CONFIG_CPU_V6K */ diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c new file mode 100644 index 00000000..43727630 --- /dev/null +++ b/arch/arm/kernel/perf_event_v7.c @@ -0,0 +1,918 @@ +/* + * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code. + * + * ARMv7 support: Jean Pihet <jpihet@mvista.com> + * 2010 (c) MontaVista Software, LLC. + * + * Copied from ARMv6 code, with the low level code inspired + * by the ARMv7 Oprofile code. + * + * Cortex-A8 has up to 4 configurable performance counters and + * a single cycle counter. + * Cortex-A9 has up to 31 configurable performance counters and + * a single cycle counter. + * + * All counters can be enabled/disabled and IRQ masked separately. The cycle + * counter and all 4 performance counters together can be reset separately. + */ + +#ifdef CONFIG_CPU_V7 +/* Common ARMv7 event types */ +enum armv7_perf_types { + ARMV7_PERFCTR_PMNC_SW_INCR = 0x00, + ARMV7_PERFCTR_IFETCH_MISS = 0x01, + ARMV7_PERFCTR_ITLB_MISS = 0x02, + ARMV7_PERFCTR_DCACHE_REFILL = 0x03, + ARMV7_PERFCTR_DCACHE_ACCESS = 0x04, + ARMV7_PERFCTR_DTLB_REFILL = 0x05, + ARMV7_PERFCTR_DREAD = 0x06, + ARMV7_PERFCTR_DWRITE = 0x07, + + ARMV7_PERFCTR_EXC_TAKEN = 0x09, + ARMV7_PERFCTR_EXC_EXECUTED = 0x0A, + ARMV7_PERFCTR_CID_WRITE = 0x0B, + /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS. + * It counts: + * - all branch instructions, + * - instructions that explicitly write the PC, + * - exception generating instructions. + */ + ARMV7_PERFCTR_PC_WRITE = 0x0C, + ARMV7_PERFCTR_PC_IMM_BRANCH = 0x0D, + ARMV7_PERFCTR_UNALIGNED_ACCESS = 0x0F, + ARMV7_PERFCTR_PC_BRANCH_MIS_PRED = 0x10, + ARMV7_PERFCTR_CLOCK_CYCLES = 0x11, + + ARMV7_PERFCTR_PC_BRANCH_MIS_USED = 0x12, + + ARMV7_PERFCTR_CPU_CYCLES = 0xFF +}; + +/* ARMv7 Cortex-A8 specific event types */ +enum armv7_a8_perf_types { + ARMV7_PERFCTR_INSTR_EXECUTED = 0x08, + + ARMV7_PERFCTR_PC_PROC_RETURN = 0x0E, + + ARMV7_PERFCTR_WRITE_BUFFER_FULL = 0x40, + ARMV7_PERFCTR_L2_STORE_MERGED = 0x41, + ARMV7_PERFCTR_L2_STORE_BUFF = 0x42, + ARMV7_PERFCTR_L2_ACCESS = 0x43, + ARMV7_PERFCTR_L2_CACH_MISS = 0x44, + ARMV7_PERFCTR_AXI_READ_CYCLES = 0x45, + ARMV7_PERFCTR_AXI_WRITE_CYCLES = 0x46, + ARMV7_PERFCTR_MEMORY_REPLAY = 0x47, + ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY = 0x48, + ARMV7_PERFCTR_L1_DATA_MISS = 0x49, + ARMV7_PERFCTR_L1_INST_MISS = 0x4A, + ARMV7_PERFCTR_L1_DATA_COLORING = 0x4B, + ARMV7_PERFCTR_L1_NEON_DATA = 0x4C, + ARMV7_PERFCTR_L1_NEON_CACH_DATA = 0x4D, + ARMV7_PERFCTR_L2_NEON = 0x4E, + ARMV7_PERFCTR_L2_NEON_HIT = 0x4F, + ARMV7_PERFCTR_L1_INST = 0x50, + ARMV7_PERFCTR_PC_RETURN_MIS_PRED = 0x51, + ARMV7_PERFCTR_PC_BRANCH_FAILED = 0x52, + ARMV7_PERFCTR_PC_BRANCH_TAKEN = 0x53, + ARMV7_PERFCTR_PC_BRANCH_EXECUTED = 0x54, + ARMV7_PERFCTR_OP_EXECUTED = 0x55, + ARMV7_PERFCTR_CYCLES_INST_STALL = 0x56, + ARMV7_PERFCTR_CYCLES_INST = 0x57, + ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL = 0x58, + ARMV7_PERFCTR_CYCLES_NEON_INST_STALL = 0x59, + ARMV7_PERFCTR_NEON_CYCLES = 0x5A, + + ARMV7_PERFCTR_PMU0_EVENTS = 0x70, + ARMV7_PERFCTR_PMU1_EVENTS = 0x71, + ARMV7_PERFCTR_PMU_EVENTS = 0x72, +}; + +/* ARMv7 Cortex-A9 specific event types */ +enum armv7_a9_perf_types { + ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC = 0x40, + ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC = 0x41, + ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC = 0x42, + + ARMV7_PERFCTR_COHERENT_LINE_MISS = 0x50, + ARMV7_PERFCTR_COHERENT_LINE_HIT = 0x51, + + ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES = 0x60, + ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES = 0x61, + ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62, + ARMV7_PERFCTR_STREX_EXECUTED_PASSED = 0x63, + ARMV7_PERFCTR_STREX_EXECUTED_FAILED = 0x64, + ARMV7_PERFCTR_DATA_EVICTION = 0x65, + ARMV7_PERFCTR_ISSUE_STAGE_NO_INST = 0x66, + ARMV7_PERFCTR_ISSUE_STAGE_EMPTY = 0x67, + ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE = 0x68, + + ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E, + + ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST = 0x70, + ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71, + ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST = 0x72, + ARMV7_PERFCTR_FP_EXECUTED_INST = 0x73, + ARMV7_PERFCTR_NEON_EXECUTED_INST = 0x74, + + ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80, + ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES = 0x81, + ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES = 0x82, + ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES = 0x83, + ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES = 0x84, + ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES = 0x85, + ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES = 0x86, + + ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES = 0x8A, + ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B, + + ARMV7_PERFCTR_ISB_INST = 0x90, + ARMV7_PERFCTR_DSB_INST = 0x91, + ARMV7_PERFCTR_DMB_INST = 0x92, + ARMV7_PERFCTR_EXT_INTERRUPTS = 0x93, + + ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED = 0xA0, + ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED = 0xA1, + ARMV7_PERFCTR_PLE_FIFO_FLUSH = 0xA2, + ARMV7_PERFCTR_PLE_RQST_COMPLETED = 0xA3, + ARMV7_PERFCTR_PLE_FIFO_OVERFLOW = 0xA4, + ARMV7_PERFCTR_PLE_RQST_PROG = 0xA5 +}; + +/* + * Cortex-A8 HW events mapping + * + * The hardware events that we support. We do support cache operations but + * we have harvard caches and no way to combine instruction and data + * accesses/misses in hardware. + */ +static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = ARMV7_PERFCTR_INSTR_EXECUTED, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, +}; + +static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + /* + * The performance counters don't differentiate between read + * and write accesses/misses so this isn't strictly correct, + * but it's the best we can do. Writes and reads get + * combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L1_INST, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L1_INST_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_L2_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_L2_CACH_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + /* + * Only ITLB misses and DTLB refills are supported. + * If users want the DTLB refills misses a raw counter + * must be used. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, + [C(RESULT_MISS)] + = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, + [C(RESULT_MISS)] + = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + +/* + * Cortex-A9 HW events mapping + */ +static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = ARMV7_PERFCTR_CPU_CYCLES, + [PERF_COUNT_HW_INSTRUCTIONS] = + ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE, + [PERF_COUNT_HW_CACHE_REFERENCES] = ARMV7_PERFCTR_DCACHE_ACCESS, + [PERF_COUNT_HW_CACHE_MISSES] = ARMV7_PERFCTR_DCACHE_REFILL, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE, + [PERF_COUNT_HW_BRANCH_MISSES] = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + [PERF_COUNT_HW_BUS_CYCLES] = ARMV7_PERFCTR_CLOCK_CYCLES, +}; + +static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + /* + * The performance counters don't differentiate between read + * and write accesses/misses so this isn't strictly correct, + * but it's the best we can do. Writes and reads get + * combined. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DCACHE_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_IFETCH_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + /* + * Only ITLB misses and DTLB refills are supported. + * If users want the DTLB refills misses a raw counter + * must be used. + */ + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_DTLB_REFILL, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = ARMV7_PERFCTR_ITLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, + [C(RESULT_MISS)] + = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = ARMV7_PERFCTR_PC_WRITE, + [C(RESULT_MISS)] + = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + +/* + * Perf Events counters + */ +enum armv7_counters { + ARMV7_CYCLE_COUNTER = 1, /* Cycle counter */ + ARMV7_COUNTER0 = 2, /* First event counter */ +}; + +/* + * The cycle counter is ARMV7_CYCLE_COUNTER. + * The first event counter is ARMV7_COUNTER0. + * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1). + */ +#define ARMV7_COUNTER_LAST (ARMV7_COUNTER0 + armpmu->num_events - 1) + +/* + * ARMv7 low level PMNC access + */ + +/* + * Per-CPU PMNC: config reg + */ +#define ARMV7_PMNC_E (1 << 0) /* Enable all counters */ +#define ARMV7_PMNC_P (1 << 1) /* Reset all counters */ +#define ARMV7_PMNC_C (1 << 2) /* Cycle counter reset */ +#define ARMV7_PMNC_D (1 << 3) /* CCNT counts every 64th cpu cycle */ +#define ARMV7_PMNC_X (1 << 4) /* Export to ETM */ +#define ARMV7_PMNC_DP (1 << 5) /* Disable CCNT if non-invasive debug*/ +#define ARMV7_PMNC_N_SHIFT 11 /* Number of counters supported */ +#define ARMV7_PMNC_N_MASK 0x1f +#define ARMV7_PMNC_MASK 0x3f /* Mask for writable bits */ + +/* + * Available counters + */ +#define ARMV7_CNT0 0 /* First event counter */ +#define ARMV7_CCNT 31 /* Cycle counter */ + +/* Perf Event to low level counters mapping */ +#define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0) + +/* + * CNTENS: counters enable reg + */ +#define ARMV7_CNTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) +#define ARMV7_CNTENS_C (1 << ARMV7_CCNT) + +/* + * CNTENC: counters disable reg + */ +#define ARMV7_CNTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) +#define ARMV7_CNTENC_C (1 << ARMV7_CCNT) + +/* + * INTENS: counters overflow interrupt enable reg + */ +#define ARMV7_INTENS_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) +#define ARMV7_INTENS_C (1 << ARMV7_CCNT) + +/* + * INTENC: counters overflow interrupt disable reg + */ +#define ARMV7_INTENC_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) +#define ARMV7_INTENC_C (1 << ARMV7_CCNT) + +/* + * EVTSEL: Event selection reg + */ +#define ARMV7_EVTSEL_MASK 0xff /* Mask for writable bits */ + +/* + * SELECT: Counter selection reg + */ +#define ARMV7_SELECT_MASK 0x1f /* Mask for writable bits */ + +/* + * FLAG: counters overflow flag status reg + */ +#define ARMV7_FLAG_P(idx) (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx)) +#define ARMV7_FLAG_C (1 << ARMV7_CCNT) +#define ARMV7_FLAG_MASK 0xffffffff /* Mask for writable bits */ +#define ARMV7_OVERFLOWED_MASK ARMV7_FLAG_MASK + +static inline unsigned long armv7_pmnc_read(void) +{ + u32 val; + asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val)); + return val; +} + +static inline void armv7_pmnc_write(unsigned long val) +{ + val &= ARMV7_PMNC_MASK; + isb(); + asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val)); +} + +static inline int armv7_pmnc_has_overflowed(unsigned long pmnc) +{ + return pmnc & ARMV7_OVERFLOWED_MASK; +} + +static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc, + enum armv7_counters counter) +{ + int ret = 0; + + if (counter == ARMV7_CYCLE_COUNTER) + ret = pmnc & ARMV7_FLAG_C; + else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST)) + ret = pmnc & ARMV7_FLAG_P(counter); + else + pr_err("CPU%u checking wrong counter %d overflow status\n", + smp_processor_id(), counter); + + return ret; +} + +static inline int armv7_pmnc_select_counter(unsigned int idx) +{ + u32 val; + + if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) { + pr_err("CPU%u selecting wrong PMNC counter" + " %d\n", smp_processor_id(), idx); + return -1; + } + + val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK; + asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val)); + isb(); + + return idx; +} + +static inline u32 armv7pmu_read_counter(int idx) +{ + unsigned long value = 0; + + if (idx == ARMV7_CYCLE_COUNTER) + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value)); + else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { + if (armv7_pmnc_select_counter(idx) == idx) + asm volatile("mrc p15, 0, %0, c9, c13, 2" + : "=r" (value)); + } else + pr_err("CPU%u reading wrong counter %d\n", + smp_processor_id(), idx); + + return value; +} + +static inline void armv7pmu_write_counter(int idx, u32 value) +{ + if (idx == ARMV7_CYCLE_COUNTER) + asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); + else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) { + if (armv7_pmnc_select_counter(idx) == idx) + asm volatile("mcr p15, 0, %0, c9, c13, 2" + : : "r" (value)); + } else + pr_err("CPU%u writing wrong counter %d\n", + smp_processor_id(), idx); +} + +static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val) +{ + if (armv7_pmnc_select_counter(idx) == idx) { + val &= ARMV7_EVTSEL_MASK; + asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val)); + } +} + +static inline u32 armv7_pmnc_enable_counter(unsigned int idx) +{ + u32 val; + + if ((idx != ARMV7_CYCLE_COUNTER) && + ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { + pr_err("CPU%u enabling wrong PMNC counter" + " %d\n", smp_processor_id(), idx); + return -1; + } + + if (idx == ARMV7_CYCLE_COUNTER) + val = ARMV7_CNTENS_C; + else + val = ARMV7_CNTENS_P(idx); + + asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val)); + + return idx; +} + +static inline u32 armv7_pmnc_disable_counter(unsigned int idx) +{ + u32 val; + + + if ((idx != ARMV7_CYCLE_COUNTER) && + ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { + pr_err("CPU%u disabling wrong PMNC counter" + " %d\n", smp_processor_id(), idx); + return -1; + } + + if (idx == ARMV7_CYCLE_COUNTER) + val = ARMV7_CNTENC_C; + else + val = ARMV7_CNTENC_P(idx); + + asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val)); + + return idx; +} + +static inline u32 armv7_pmnc_enable_intens(unsigned int idx) +{ + u32 val; + + if ((idx != ARMV7_CYCLE_COUNTER) && + ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { + pr_err("CPU%u enabling wrong PMNC counter" + " interrupt enable %d\n", smp_processor_id(), idx); + return -1; + } + + if (idx == ARMV7_CYCLE_COUNTER) + val = ARMV7_INTENS_C; + else + val = ARMV7_INTENS_P(idx); + + asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val)); + + return idx; +} + +static inline u32 armv7_pmnc_disable_intens(unsigned int idx) +{ + u32 val; + + if ((idx != ARMV7_CYCLE_COUNTER) && + ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) { + pr_err("CPU%u disabling wrong PMNC counter" + " interrupt enable %d\n", smp_processor_id(), idx); + return -1; + } + + if (idx == ARMV7_CYCLE_COUNTER) + val = ARMV7_INTENC_C; + else + val = ARMV7_INTENC_P(idx); + + asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val)); + + return idx; +} + +static inline u32 armv7_pmnc_getreset_flags(void) +{ + u32 val; + + /* Read */ + asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); + + /* Write to clear flags */ + val &= ARMV7_FLAG_MASK; + asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val)); + + return val; +} + +#ifdef DEBUG +static void armv7_pmnc_dump_regs(void) +{ + u32 val; + unsigned int cnt; + + printk(KERN_INFO "PMNC registers dump:\n"); + + asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val)); + printk(KERN_INFO "PMNC =0x%08x\n", val); + + asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val)); + printk(KERN_INFO "CNTENS=0x%08x\n", val); + + asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val)); + printk(KERN_INFO "INTENS=0x%08x\n", val); + + asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val)); + printk(KERN_INFO "FLAGS =0x%08x\n", val); + + asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val)); + printk(KERN_INFO "SELECT=0x%08x\n", val); + + asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val)); + printk(KERN_INFO "CCNT =0x%08x\n", val); + + for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) { + armv7_pmnc_select_counter(cnt); + asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val)); + printk(KERN_INFO "CNT[%d] count =0x%08x\n", + cnt-ARMV7_EVENT_CNT_TO_CNTx, val); + asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val)); + printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n", + cnt-ARMV7_EVENT_CNT_TO_CNTx, val); + } +} +#endif + +static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx) +{ + unsigned long flags; + + /* + * Enable counter and interrupt, and set the counter to count + * the event that we're interested in. + */ + raw_spin_lock_irqsave(&pmu_lock, flags); + + /* + * Disable counter + */ + armv7_pmnc_disable_counter(idx); + + /* + * Set event (if destined for PMNx counters) + * We don't need to set the event if it's a cycle count + */ + if (idx != ARMV7_CYCLE_COUNTER) + armv7_pmnc_write_evtsel(idx, hwc->config_base); + + /* + * Enable interrupt for this counter + */ + armv7_pmnc_enable_intens(idx); + + /* + * Enable counter + */ + armv7_pmnc_enable_counter(idx); + + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx) +{ + unsigned long flags; + + /* + * Disable counter and interrupt + */ + raw_spin_lock_irqsave(&pmu_lock, flags); + + /* + * Disable counter + */ + armv7_pmnc_disable_counter(idx); + + /* + * Disable interrupt for this counter + */ + armv7_pmnc_disable_intens(idx); + + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev) +{ + unsigned long pmnc; + struct perf_sample_data data; + struct cpu_hw_events *cpuc; + struct pt_regs *regs; + int idx; + + /* + * Get and reset the IRQ flags + */ + pmnc = armv7_pmnc_getreset_flags(); + + /* + * Did an overflow occur? + */ + if (!armv7_pmnc_has_overflowed(pmnc)) + return IRQ_NONE; + + /* + * Handle the counter(s) overflow(s) + */ + regs = get_irq_regs(); + + perf_sample_data_init(&data, 0); + + cpuc = &__get_cpu_var(cpu_hw_events); + for (idx = 0; idx <= armpmu->num_events; ++idx) { + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc; + + if (!test_bit(idx, cpuc->active_mask)) + continue; + + /* + * We have a single interrupt for all counters. Check that + * each counter has overflowed before we process it. + */ + if (!armv7_pmnc_counter_has_overflowed(pmnc, idx)) + continue; + + hwc = &event->hw; + armpmu_event_update(event, hwc, idx, 1); + data.period = event->hw.last_period; + if (!armpmu_event_set_period(event, hwc, idx)) + continue; + + if (perf_event_overflow(event, 0, &data, regs)) + armpmu->disable(hwc, idx); + } + + /* + * Handle the pending perf events. + * + * Note: this call *must* be run with interrupts disabled. For + * platforms that can have the PMU interrupts raised as an NMI, this + * will not work. + */ + irq_work_run(); + + return IRQ_HANDLED; +} + +static void armv7pmu_start(void) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&pmu_lock, flags); + /* Enable all counters */ + armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void armv7pmu_stop(void) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&pmu_lock, flags); + /* Disable all counters */ + armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc, + struct hw_perf_event *event) +{ + int idx; + + /* Always place a cycle counter into the cycle counter. */ + if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) { + if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask)) + return -EAGAIN; + + return ARMV7_CYCLE_COUNTER; + } else { + /* + * For anything other than a cycle counter, try and use + * the events counters + */ + for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) { + if (!test_and_set_bit(idx, cpuc->used_mask)) + return idx; + } + + /* The counters are all in use. */ + return -EAGAIN; + } +} + +static void armv7pmu_reset(void *info) +{ + u32 idx, nb_cnt = armpmu->num_events; + + /* The counter and interrupt enable registers are unknown at reset. */ + for (idx = 1; idx < nb_cnt; ++idx) + armv7pmu_disable_event(NULL, idx); + + /* Initialize & Reset PMNC: C and P bits */ + armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C); +} + +static struct arm_pmu armv7pmu = { + .handle_irq = armv7pmu_handle_irq, + .enable = armv7pmu_enable_event, + .disable = armv7pmu_disable_event, + .read_counter = armv7pmu_read_counter, + .write_counter = armv7pmu_write_counter, + .get_event_idx = armv7pmu_get_event_idx, + .start = armv7pmu_start, + .stop = armv7pmu_stop, + .reset = armv7pmu_reset, + .raw_event_mask = 0xFF, + .max_period = (1LLU << 32) - 1, +}; + +static u32 __init armv7_read_num_pmnc_events(void) +{ + u32 nb_cnt; + + /* Read the nb of CNTx counters supported from PMNC */ + nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK; + + /* Add the CPU cycles counter and return */ + return nb_cnt + 1; +} + +static const struct arm_pmu *__init armv7_a8_pmu_init(void) +{ + armv7pmu.id = ARM_PERF_PMU_ID_CA8; + armv7pmu.name = "ARMv7 Cortex-A8"; + armv7pmu.cache_map = &armv7_a8_perf_cache_map; + armv7pmu.event_map = &armv7_a8_perf_map; + armv7pmu.num_events = armv7_read_num_pmnc_events(); + return &armv7pmu; +} + +static const struct arm_pmu *__init armv7_a9_pmu_init(void) +{ + armv7pmu.id = ARM_PERF_PMU_ID_CA9; + armv7pmu.name = "ARMv7 Cortex-A9"; + armv7pmu.cache_map = &armv7_a9_perf_cache_map; + armv7pmu.event_map = &armv7_a9_perf_map; + armv7pmu.num_events = armv7_read_num_pmnc_events(); + return &armv7pmu; +} +#else +static const struct arm_pmu *__init armv7_a8_pmu_init(void) +{ + return NULL; +} + +static const struct arm_pmu *__init armv7_a9_pmu_init(void) +{ + return NULL; +} +#endif /* CONFIG_CPU_V7 */ diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c new file mode 100644 index 00000000..39affbe4 --- /dev/null +++ b/arch/arm/kernel/perf_event_xscale.c @@ -0,0 +1,807 @@ +/* + * ARMv5 [xscale] Performance counter handling code. + * + * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com> + * + * Based on the previous xscale OProfile code. + * + * There are two variants of the xscale PMU that we support: + * - xscale1pmu: 2 event counters and a cycle counter + * - xscale2pmu: 4 event counters and a cycle counter + * The two variants share event definitions, but have different + * PMU structures. + */ + +#ifdef CONFIG_CPU_XSCALE +enum xscale_perf_types { + XSCALE_PERFCTR_ICACHE_MISS = 0x00, + XSCALE_PERFCTR_ICACHE_NO_DELIVER = 0x01, + XSCALE_PERFCTR_DATA_STALL = 0x02, + XSCALE_PERFCTR_ITLB_MISS = 0x03, + XSCALE_PERFCTR_DTLB_MISS = 0x04, + XSCALE_PERFCTR_BRANCH = 0x05, + XSCALE_PERFCTR_BRANCH_MISS = 0x06, + XSCALE_PERFCTR_INSTRUCTION = 0x07, + XSCALE_PERFCTR_DCACHE_FULL_STALL = 0x08, + XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09, + XSCALE_PERFCTR_DCACHE_ACCESS = 0x0A, + XSCALE_PERFCTR_DCACHE_MISS = 0x0B, + XSCALE_PERFCTR_DCACHE_WRITE_BACK = 0x0C, + XSCALE_PERFCTR_PC_CHANGED = 0x0D, + XSCALE_PERFCTR_BCU_REQUEST = 0x10, + XSCALE_PERFCTR_BCU_FULL = 0x11, + XSCALE_PERFCTR_BCU_DRAIN = 0x12, + XSCALE_PERFCTR_BCU_ECC_NO_ELOG = 0x14, + XSCALE_PERFCTR_BCU_1_BIT_ERR = 0x15, + XSCALE_PERFCTR_RMW = 0x16, + /* XSCALE_PERFCTR_CCNT is not hardware defined */ + XSCALE_PERFCTR_CCNT = 0xFE, + XSCALE_PERFCTR_UNUSED = 0xFF, +}; + +enum xscale_counters { + XSCALE_CYCLE_COUNTER = 1, + XSCALE_COUNTER0, + XSCALE_COUNTER1, + XSCALE_COUNTER2, + XSCALE_COUNTER3, +}; + +static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = { + [PERF_COUNT_HW_CPU_CYCLES] = XSCALE_PERFCTR_CCNT, + [PERF_COUNT_HW_INSTRUCTIONS] = XSCALE_PERFCTR_INSTRUCTION, + [PERF_COUNT_HW_CACHE_REFERENCES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_CACHE_MISSES] = HW_OP_UNSUPPORTED, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH, + [PERF_COUNT_HW_BRANCH_MISSES] = XSCALE_PERFCTR_BRANCH_MISS, + [PERF_COUNT_HW_BUS_CYCLES] = HW_OP_UNSUPPORTED, +}; + +static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX] = { + [C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = XSCALE_PERFCTR_DCACHE_ACCESS, + [C(RESULT_MISS)] = XSCALE_PERFCTR_DCACHE_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = XSCALE_PERFCTR_ICACHE_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = XSCALE_PERFCTR_ICACHE_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = XSCALE_PERFCTR_DTLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = XSCALE_PERFCTR_ITLB_MISS, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, + [C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = CACHE_OP_UNSUPPORTED, + [C(RESULT_MISS)] = CACHE_OP_UNSUPPORTED, + }, + }, +}; + +#define XSCALE_PMU_ENABLE 0x001 +#define XSCALE_PMN_RESET 0x002 +#define XSCALE_CCNT_RESET 0x004 +#define XSCALE_PMU_RESET (CCNT_RESET | PMN_RESET) +#define XSCALE_PMU_CNT64 0x008 + +#define XSCALE1_OVERFLOWED_MASK 0x700 +#define XSCALE1_CCOUNT_OVERFLOW 0x400 +#define XSCALE1_COUNT0_OVERFLOW 0x100 +#define XSCALE1_COUNT1_OVERFLOW 0x200 +#define XSCALE1_CCOUNT_INT_EN 0x040 +#define XSCALE1_COUNT0_INT_EN 0x010 +#define XSCALE1_COUNT1_INT_EN 0x020 +#define XSCALE1_COUNT0_EVT_SHFT 12 +#define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT) +#define XSCALE1_COUNT1_EVT_SHFT 20 +#define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT) + +static inline u32 +xscale1pmu_read_pmnc(void) +{ + u32 val; + asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val)); + return val; +} + +static inline void +xscale1pmu_write_pmnc(u32 val) +{ + /* upper 4bits and 7, 11 are write-as-0 */ + val &= 0xffff77f; + asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val)); +} + +static inline int +xscale1_pmnc_counter_has_overflowed(unsigned long pmnc, + enum xscale_counters counter) +{ + int ret = 0; + + switch (counter) { + case XSCALE_CYCLE_COUNTER: + ret = pmnc & XSCALE1_CCOUNT_OVERFLOW; + break; + case XSCALE_COUNTER0: + ret = pmnc & XSCALE1_COUNT0_OVERFLOW; + break; + case XSCALE_COUNTER1: + ret = pmnc & XSCALE1_COUNT1_OVERFLOW; + break; + default: + WARN_ONCE(1, "invalid counter number (%d)\n", counter); + } + + return ret; +} + +static irqreturn_t +xscale1pmu_handle_irq(int irq_num, void *dev) +{ + unsigned long pmnc; + struct perf_sample_data data; + struct cpu_hw_events *cpuc; + struct pt_regs *regs; + int idx; + + /* + * NOTE: there's an A stepping erratum that states if an overflow + * bit already exists and another occurs, the previous + * Overflow bit gets cleared. There's no workaround. + * Fixed in B stepping or later. + */ + pmnc = xscale1pmu_read_pmnc(); + + /* + * Write the value back to clear the overflow flags. Overflow + * flags remain in pmnc for use below. We also disable the PMU + * while we process the interrupt. + */ + xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE); + + if (!(pmnc & XSCALE1_OVERFLOWED_MASK)) + return IRQ_NONE; + + regs = get_irq_regs(); + + perf_sample_data_init(&data, 0); + + cpuc = &__get_cpu_var(cpu_hw_events); + for (idx = 0; idx <= armpmu->num_events; ++idx) { + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc; + + if (!test_bit(idx, cpuc->active_mask)) + continue; + + if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx)) + continue; + + hwc = &event->hw; + armpmu_event_update(event, hwc, idx, 1); + data.period = event->hw.last_period; + if (!armpmu_event_set_period(event, hwc, idx)) + continue; + + if (perf_event_overflow(event, 0, &data, regs)) + armpmu->disable(hwc, idx); + } + + irq_work_run(); + + /* + * Re-enable the PMU. + */ + pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE; + xscale1pmu_write_pmnc(pmnc); + + return IRQ_HANDLED; +} + +static void +xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx) +{ + unsigned long val, mask, evt, flags; + + switch (idx) { + case XSCALE_CYCLE_COUNTER: + mask = 0; + evt = XSCALE1_CCOUNT_INT_EN; + break; + case XSCALE_COUNTER0: + mask = XSCALE1_COUNT0_EVT_MASK; + evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) | + XSCALE1_COUNT0_INT_EN; + break; + case XSCALE_COUNTER1: + mask = XSCALE1_COUNT1_EVT_MASK; + evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) | + XSCALE1_COUNT1_INT_EN; + break; + default: + WARN_ONCE(1, "invalid counter number (%d)\n", idx); + return; + } + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = xscale1pmu_read_pmnc(); + val &= ~mask; + val |= evt; + xscale1pmu_write_pmnc(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void +xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx) +{ + unsigned long val, mask, evt, flags; + + switch (idx) { + case XSCALE_CYCLE_COUNTER: + mask = XSCALE1_CCOUNT_INT_EN; + evt = 0; + break; + case XSCALE_COUNTER0: + mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK; + evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT; + break; + case XSCALE_COUNTER1: + mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK; + evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT; + break; + default: + WARN_ONCE(1, "invalid counter number (%d)\n", idx); + return; + } + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = xscale1pmu_read_pmnc(); + val &= ~mask; + val |= evt; + xscale1pmu_write_pmnc(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static int +xscale1pmu_get_event_idx(struct cpu_hw_events *cpuc, + struct hw_perf_event *event) +{ + if (XSCALE_PERFCTR_CCNT == event->config_base) { + if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask)) + return -EAGAIN; + + return XSCALE_CYCLE_COUNTER; + } else { + if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask)) + return XSCALE_COUNTER1; + + if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask)) + return XSCALE_COUNTER0; + + return -EAGAIN; + } +} + +static void +xscale1pmu_start(void) +{ + unsigned long flags, val; + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = xscale1pmu_read_pmnc(); + val |= XSCALE_PMU_ENABLE; + xscale1pmu_write_pmnc(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void +xscale1pmu_stop(void) +{ + unsigned long flags, val; + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = xscale1pmu_read_pmnc(); + val &= ~XSCALE_PMU_ENABLE; + xscale1pmu_write_pmnc(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static inline u32 +xscale1pmu_read_counter(int counter) +{ + u32 val = 0; + + switch (counter) { + case XSCALE_CYCLE_COUNTER: + asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val)); + break; + case XSCALE_COUNTER0: + asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val)); + break; + case XSCALE_COUNTER1: + asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val)); + break; + } + + return val; +} + +static inline void +xscale1pmu_write_counter(int counter, u32 val) +{ + switch (counter) { + case XSCALE_CYCLE_COUNTER: + asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val)); + break; + case XSCALE_COUNTER0: + asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val)); + break; + case XSCALE_COUNTER1: + asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val)); + break; + } +} + +static const struct arm_pmu xscale1pmu = { + .id = ARM_PERF_PMU_ID_XSCALE1, + .name = "xscale1", + .handle_irq = xscale1pmu_handle_irq, + .enable = xscale1pmu_enable_event, + .disable = xscale1pmu_disable_event, + .read_counter = xscale1pmu_read_counter, + .write_counter = xscale1pmu_write_counter, + .get_event_idx = xscale1pmu_get_event_idx, + .start = xscale1pmu_start, + .stop = xscale1pmu_stop, + .cache_map = &xscale_perf_cache_map, + .event_map = &xscale_perf_map, + .raw_event_mask = 0xFF, + .num_events = 3, + .max_period = (1LLU << 32) - 1, +}; + +static const struct arm_pmu *__init xscale1pmu_init(void) +{ + return &xscale1pmu; +} + +#define XSCALE2_OVERFLOWED_MASK 0x01f +#define XSCALE2_CCOUNT_OVERFLOW 0x001 +#define XSCALE2_COUNT0_OVERFLOW 0x002 +#define XSCALE2_COUNT1_OVERFLOW 0x004 +#define XSCALE2_COUNT2_OVERFLOW 0x008 +#define XSCALE2_COUNT3_OVERFLOW 0x010 +#define XSCALE2_CCOUNT_INT_EN 0x001 +#define XSCALE2_COUNT0_INT_EN 0x002 +#define XSCALE2_COUNT1_INT_EN 0x004 +#define XSCALE2_COUNT2_INT_EN 0x008 +#define XSCALE2_COUNT3_INT_EN 0x010 +#define XSCALE2_COUNT0_EVT_SHFT 0 +#define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT) +#define XSCALE2_COUNT1_EVT_SHFT 8 +#define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT) +#define XSCALE2_COUNT2_EVT_SHFT 16 +#define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT) +#define XSCALE2_COUNT3_EVT_SHFT 24 +#define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT) + +static inline u32 +xscale2pmu_read_pmnc(void) +{ + u32 val; + asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val)); + /* bits 1-2 and 4-23 are read-unpredictable */ + return val & 0xff000009; +} + +static inline void +xscale2pmu_write_pmnc(u32 val) +{ + /* bits 4-23 are write-as-0, 24-31 are write ignored */ + val &= 0xf; + asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val)); +} + +static inline u32 +xscale2pmu_read_overflow_flags(void) +{ + u32 val; + asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val)); + return val; +} + +static inline void +xscale2pmu_write_overflow_flags(u32 val) +{ + asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val)); +} + +static inline u32 +xscale2pmu_read_event_select(void) +{ + u32 val; + asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val)); + return val; +} + +static inline void +xscale2pmu_write_event_select(u32 val) +{ + asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val)); +} + +static inline u32 +xscale2pmu_read_int_enable(void) +{ + u32 val; + asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val)); + return val; +} + +static void +xscale2pmu_write_int_enable(u32 val) +{ + asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val)); +} + +static inline int +xscale2_pmnc_counter_has_overflowed(unsigned long of_flags, + enum xscale_counters counter) +{ + int ret = 0; + + switch (counter) { + case XSCALE_CYCLE_COUNTER: + ret = of_flags & XSCALE2_CCOUNT_OVERFLOW; + break; + case XSCALE_COUNTER0: + ret = of_flags & XSCALE2_COUNT0_OVERFLOW; + break; + case XSCALE_COUNTER1: + ret = of_flags & XSCALE2_COUNT1_OVERFLOW; + break; + case XSCALE_COUNTER2: + ret = of_flags & XSCALE2_COUNT2_OVERFLOW; + break; + case XSCALE_COUNTER3: + ret = of_flags & XSCALE2_COUNT3_OVERFLOW; + break; + default: + WARN_ONCE(1, "invalid counter number (%d)\n", counter); + } + + return ret; +} + +static irqreturn_t +xscale2pmu_handle_irq(int irq_num, void *dev) +{ + unsigned long pmnc, of_flags; + struct perf_sample_data data; + struct cpu_hw_events *cpuc; + struct pt_regs *regs; + int idx; + + /* Disable the PMU. */ + pmnc = xscale2pmu_read_pmnc(); + xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE); + + /* Check the overflow flag register. */ + of_flags = xscale2pmu_read_overflow_flags(); + if (!(of_flags & XSCALE2_OVERFLOWED_MASK)) + return IRQ_NONE; + + /* Clear the overflow bits. */ + xscale2pmu_write_overflow_flags(of_flags); + + regs = get_irq_regs(); + + perf_sample_data_init(&data, 0); + + cpuc = &__get_cpu_var(cpu_hw_events); + for (idx = 0; idx <= armpmu->num_events; ++idx) { + struct perf_event *event = cpuc->events[idx]; + struct hw_perf_event *hwc; + + if (!test_bit(idx, cpuc->active_mask)) + continue; + + if (!xscale2_pmnc_counter_has_overflowed(pmnc, idx)) + continue; + + hwc = &event->hw; + armpmu_event_update(event, hwc, idx, 1); + data.period = event->hw.last_period; + if (!armpmu_event_set_period(event, hwc, idx)) + continue; + + if (perf_event_overflow(event, 0, &data, regs)) + armpmu->disable(hwc, idx); + } + + irq_work_run(); + + /* + * Re-enable the PMU. + */ + pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE; + xscale2pmu_write_pmnc(pmnc); + + return IRQ_HANDLED; +} + +static void +xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx) +{ + unsigned long flags, ien, evtsel; + + ien = xscale2pmu_read_int_enable(); + evtsel = xscale2pmu_read_event_select(); + + switch (idx) { + case XSCALE_CYCLE_COUNTER: + ien |= XSCALE2_CCOUNT_INT_EN; + break; + case XSCALE_COUNTER0: + ien |= XSCALE2_COUNT0_INT_EN; + evtsel &= ~XSCALE2_COUNT0_EVT_MASK; + evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT; + break; + case XSCALE_COUNTER1: + ien |= XSCALE2_COUNT1_INT_EN; + evtsel &= ~XSCALE2_COUNT1_EVT_MASK; + evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT; + break; + case XSCALE_COUNTER2: + ien |= XSCALE2_COUNT2_INT_EN; + evtsel &= ~XSCALE2_COUNT2_EVT_MASK; + evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT; + break; + case XSCALE_COUNTER3: + ien |= XSCALE2_COUNT3_INT_EN; + evtsel &= ~XSCALE2_COUNT3_EVT_MASK; + evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT; + break; + default: + WARN_ONCE(1, "invalid counter number (%d)\n", idx); + return; + } + + raw_spin_lock_irqsave(&pmu_lock, flags); + xscale2pmu_write_event_select(evtsel); + xscale2pmu_write_int_enable(ien); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void +xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx) +{ + unsigned long flags, ien, evtsel; + + ien = xscale2pmu_read_int_enable(); + evtsel = xscale2pmu_read_event_select(); + + switch (idx) { + case XSCALE_CYCLE_COUNTER: + ien &= ~XSCALE2_CCOUNT_INT_EN; + break; + case XSCALE_COUNTER0: + ien &= ~XSCALE2_COUNT0_INT_EN; + evtsel &= ~XSCALE2_COUNT0_EVT_MASK; + evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT; + break; + case XSCALE_COUNTER1: + ien &= ~XSCALE2_COUNT1_INT_EN; + evtsel &= ~XSCALE2_COUNT1_EVT_MASK; + evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT; + break; + case XSCALE_COUNTER2: + ien &= ~XSCALE2_COUNT2_INT_EN; + evtsel &= ~XSCALE2_COUNT2_EVT_MASK; + evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT; + break; + case XSCALE_COUNTER3: + ien &= ~XSCALE2_COUNT3_INT_EN; + evtsel &= ~XSCALE2_COUNT3_EVT_MASK; + evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT; + break; + default: + WARN_ONCE(1, "invalid counter number (%d)\n", idx); + return; + } + + raw_spin_lock_irqsave(&pmu_lock, flags); + xscale2pmu_write_event_select(evtsel); + xscale2pmu_write_int_enable(ien); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static int +xscale2pmu_get_event_idx(struct cpu_hw_events *cpuc, + struct hw_perf_event *event) +{ + int idx = xscale1pmu_get_event_idx(cpuc, event); + if (idx >= 0) + goto out; + + if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask)) + idx = XSCALE_COUNTER3; + else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask)) + idx = XSCALE_COUNTER2; +out: + return idx; +} + +static void +xscale2pmu_start(void) +{ + unsigned long flags, val; + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64; + val |= XSCALE_PMU_ENABLE; + xscale2pmu_write_pmnc(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static void +xscale2pmu_stop(void) +{ + unsigned long flags, val; + + raw_spin_lock_irqsave(&pmu_lock, flags); + val = xscale2pmu_read_pmnc(); + val &= ~XSCALE_PMU_ENABLE; + xscale2pmu_write_pmnc(val); + raw_spin_unlock_irqrestore(&pmu_lock, flags); +} + +static inline u32 +xscale2pmu_read_counter(int counter) +{ + u32 val = 0; + + switch (counter) { + case XSCALE_CYCLE_COUNTER: + asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val)); + break; + case XSCALE_COUNTER0: + asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val)); + break; + case XSCALE_COUNTER1: + asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val)); + break; + case XSCALE_COUNTER2: + asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val)); + break; + case XSCALE_COUNTER3: + asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val)); + break; + } + + return val; +} + +static inline void +xscale2pmu_write_counter(int counter, u32 val) +{ + switch (counter) { + case XSCALE_CYCLE_COUNTER: + asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val)); + break; + case XSCALE_COUNTER0: + asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val)); + break; + case XSCALE_COUNTER1: + asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val)); + break; + case XSCALE_COUNTER2: + asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val)); + break; + case XSCALE_COUNTER3: + asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val)); + break; + } +} + +static const struct arm_pmu xscale2pmu = { + .id = ARM_PERF_PMU_ID_XSCALE2, + .name = "xscale2", + .handle_irq = xscale2pmu_handle_irq, + .enable = xscale2pmu_enable_event, + .disable = xscale2pmu_disable_event, + .read_counter = xscale2pmu_read_counter, + .write_counter = xscale2pmu_write_counter, + .get_event_idx = xscale2pmu_get_event_idx, + .start = xscale2pmu_start, + .stop = xscale2pmu_stop, + .cache_map = &xscale_perf_cache_map, + .event_map = &xscale_perf_map, + .raw_event_mask = 0xFF, + .num_events = 5, + .max_period = (1LLU << 32) - 1, +}; + +static const struct arm_pmu *__init xscale2pmu_init(void) +{ + return &xscale2pmu; +} +#else +static const struct arm_pmu *__init xscale1pmu_init(void) +{ + return NULL; +} + +static const struct arm_pmu *__init xscale2pmu_init(void) +{ + return NULL; +} +#endif /* CONFIG_CPU_XSCALE */ diff --git a/arch/arm/kernel/pj4-cp0.c b/arch/arm/kernel/pj4-cp0.c new file mode 100644 index 00000000..a4b1b074 --- /dev/null +++ b/arch/arm/kernel/pj4-cp0.c @@ -0,0 +1,94 @@ +/* + * linux/arch/arm/kernel/pj4-cp0.c + * + * PJ4 iWMMXt coprocessor context switching and handling + * + * Copyright (c) 2010 Marvell International Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/signal.h> +#include <linux/sched.h> +#include <linux/init.h> +#include <linux/io.h> +#include <asm/thread_notify.h> + +static int iwmmxt_do(struct notifier_block *self, unsigned long cmd, void *t) +{ + struct thread_info *thread = t; + + switch (cmd) { + case THREAD_NOTIFY_FLUSH: + /* + * flush_thread() zeroes thread->fpstate, so no need + * to do anything here. + * + * FALLTHROUGH: Ensure we don't try to overwrite our newly + * initialised state information on the first fault. + */ + + case THREAD_NOTIFY_EXIT: + iwmmxt_task_release(thread); + break; + + case THREAD_NOTIFY_SWITCH: + iwmmxt_task_switch(thread); + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block iwmmxt_notifier_block = { + .notifier_call = iwmmxt_do, +}; + + +static u32 __init pj4_cp_access_read(void) +{ + u32 value; + + __asm__ __volatile__ ( + "mrc p15, 0, %0, c1, c0, 2\n\t" + : "=r" (value)); + return value; +} + +static void __init pj4_cp_access_write(u32 value) +{ + u32 temp; + + __asm__ __volatile__ ( + "mcr p15, 0, %1, c1, c0, 2\n\t" + "mrc p15, 0, %0, c1, c0, 2\n\t" + "mov %0, %0\n\t" + "sub pc, pc, #4\n\t" + : "=r" (temp) : "r" (value)); +} + + +/* + * Disable CP0/CP1 on boot, and let call_fpe() and the iWMMXt lazy + * switch code handle iWMMXt context switching. + */ +static int __init pj4_cp0_init(void) +{ + u32 cp_access; + + cp_access = pj4_cp_access_read() & ~0xf; + pj4_cp_access_write(cp_access); + + printk(KERN_INFO "PJ4 iWMMXt coprocessor enabled.\n"); + elf_hwcap |= HWCAP_IWMMXT; + thread_register_notifier(&iwmmxt_notifier_block); + + return 0; +} + +late_initcall(pj4_cp0_init); diff --git a/arch/arm/kernel/pmu.c b/arch/arm/kernel/pmu.c new file mode 100644 index 00000000..2c79eec1 --- /dev/null +++ b/arch/arm/kernel/pmu.c @@ -0,0 +1,148 @@ +/* + * linux/arch/arm/kernel/pmu.c + * + * Copyright (C) 2009 picoChip Designs Ltd, Jamie Iles + * Copyright (C) 2010 ARM Ltd, Will Deacon + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + */ + +#define pr_fmt(fmt) "PMU: " fmt + +#include <linux/cpumask.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +#include <asm/pmu.h> + +static volatile long pmu_lock; + +static struct platform_device *pmu_devices[ARM_NUM_PMU_DEVICES]; + +static int __devinit pmu_device_probe(struct platform_device *pdev) +{ + + if (pdev->id < 0 || pdev->id >= ARM_NUM_PMU_DEVICES) { + pr_warning("received registration request for unknown " + "device %d\n", pdev->id); + return -EINVAL; + } + + if (pmu_devices[pdev->id]) + pr_warning("registering new PMU device type %d overwrites " + "previous registration!\n", pdev->id); + else + pr_info("registered new PMU device of type %d\n", + pdev->id); + + pmu_devices[pdev->id] = pdev; + return 0; +} + +static struct platform_driver pmu_driver = { + .driver = { + .name = "arm-pmu", + }, + .probe = pmu_device_probe, +}; + +static int __init register_pmu_driver(void) +{ + return platform_driver_register(&pmu_driver); +} +device_initcall(register_pmu_driver); + +struct platform_device * +reserve_pmu(enum arm_pmu_type device) +{ + struct platform_device *pdev; + + if (test_and_set_bit_lock(device, &pmu_lock)) { + pdev = ERR_PTR(-EBUSY); + } else if (pmu_devices[device] == NULL) { + clear_bit_unlock(device, &pmu_lock); + pdev = ERR_PTR(-ENODEV); + } else { + pdev = pmu_devices[device]; + } + + return pdev; +} +EXPORT_SYMBOL_GPL(reserve_pmu); + +int +release_pmu(struct platform_device *pdev) +{ + if (WARN_ON(pdev != pmu_devices[pdev->id])) + return -EINVAL; + clear_bit_unlock(pdev->id, &pmu_lock); + return 0; +} +EXPORT_SYMBOL_GPL(release_pmu); + +static int +set_irq_affinity(int irq, + unsigned int cpu) +{ +#ifdef CONFIG_SMP + int err = irq_set_affinity(irq, cpumask_of(cpu)); + if (err) + pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n", + irq, cpu); + return err; +#else + return -EINVAL; +#endif +} + +static int +init_cpu_pmu(void) +{ + int i, irqs, err = 0; + struct platform_device *pdev = pmu_devices[ARM_PMU_DEVICE_CPU]; + + if (!pdev) + return -ENODEV; + + irqs = pdev->num_resources; + + /* + * If we have a single PMU interrupt that we can't shift, assume that + * we're running on a uniprocessor machine and continue. + */ + if (irqs == 1 && !irq_can_set_affinity(platform_get_irq(pdev, 0))) + return 0; + + for (i = 0; i < irqs; ++i) { + err = set_irq_affinity(platform_get_irq(pdev, i), i); + if (err) + break; + } + + return err; +} + +int +init_pmu(enum arm_pmu_type device) +{ + int err = 0; + + switch (device) { + case ARM_PMU_DEVICE_CPU: + err = init_cpu_pmu(); + break; + default: + pr_warning("attempt to initialise unknown device %d\n", + device); + err = -EINVAL; + } + + return err; +} +EXPORT_SYMBOL_GPL(init_pmu); diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c new file mode 100644 index 00000000..300649c7 --- /dev/null +++ b/arch/arm/kernel/process.c @@ -0,0 +1,626 @@ +/* + * linux/arch/arm/kernel/process.c + * + * Copyright (C) 1996-2000 Russell King - Converted to ARM. + * Original Copyright (C) 1995 Linus Torvalds + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <stdarg.h> + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/stddef.h> +#include <linux/unistd.h> +#include <linux/user.h> +#include <linux/delay.h> +#include <linux/reboot.h> +#include <linux/interrupt.h> +#include <linux/kallsyms.h> +#include <linux/init.h> +#include <linux/cpu.h> +#include <linux/elfcore.h> +#include <linux/pm.h> +#include <linux/tick.h> +#include <linux/utsname.h> +#include <linux/uaccess.h> +#include <linux/random.h> +#include <linux/hw_breakpoint.h> +#include <linux/console.h> + +#include <asm/cacheflush.h> +#include <asm/processor.h> +#include <asm/system.h> +#include <asm/thread_notify.h> +#include <asm/stacktrace.h> +#include <asm/mach/time.h> + +#ifdef CONFIG_CC_STACKPROTECTOR +#include <linux/stackprotector.h> +unsigned long __stack_chk_guard __read_mostly; +EXPORT_SYMBOL(__stack_chk_guard); +#endif + +static const char *processor_modes[] = { + "USER_26", "FIQ_26" , "IRQ_26" , "SVC_26" , "UK4_26" , "UK5_26" , "UK6_26" , "UK7_26" , + "UK8_26" , "UK9_26" , "UK10_26", "UK11_26", "UK12_26", "UK13_26", "UK14_26", "UK15_26", + "USER_32", "FIQ_32" , "IRQ_32" , "SVC_32" , "UK4_32" , "UK5_32" , "UK6_32" , "ABT_32" , + "UK8_32" , "UK9_32" , "UK10_32", "UND_32" , "UK12_32", "UK13_32", "UK14_32", "SYS_32" +}; + +static const char *isa_modes[] = { + "ARM" , "Thumb" , "Jazelle", "ThumbEE" +}; + +extern void setup_mm_for_reboot(char mode); + +static volatile int hlt_counter; + +#include <mach/system.h> + +#ifdef CONFIG_SMP +void arch_trigger_all_cpu_backtrace(void) +{ + smp_send_all_cpu_backtrace(); +} +#else +void arch_trigger_all_cpu_backtrace(void) +{ + dump_stack(); +} +#endif + +void disable_hlt(void) +{ + hlt_counter++; +} + +EXPORT_SYMBOL(disable_hlt); + +void enable_hlt(void) +{ + hlt_counter--; +} + +EXPORT_SYMBOL(enable_hlt); + +static int __init nohlt_setup(char *__unused) +{ + hlt_counter = 1; + return 1; +} + +static int __init hlt_setup(char *__unused) +{ + hlt_counter = 0; + return 1; +} + +__setup("nohlt", nohlt_setup); +__setup("hlt", hlt_setup); + +#ifdef CONFIG_ARM_FLUSH_CONSOLE_ON_RESTART +void arm_machine_flush_console(void) +{ + printk("\n"); + pr_emerg("Restarting %s\n", linux_banner); + if (console_trylock()) { + console_unlock(); + return; + } + + mdelay(50); + + local_irq_disable(); + if (!console_trylock()) + pr_emerg("arm_restart: Console was locked! Busting\n"); + else + pr_emerg("arm_restart: Console was locked!\n"); + console_unlock(); +} +#else +void arm_machine_flush_console(void) +{ +} +#endif + +void arm_machine_restart(char mode, const char *cmd) +{ + /* Flush the console to make sure all the relevant messages make it + * out to the console drivers */ + arm_machine_flush_console(); + + /* Disable interrupts first */ + local_irq_disable(); + local_fiq_disable(); + + /* + * Tell the mm system that we are going to reboot - + * we may need it to insert some 1:1 mappings so that + * soft boot works. + */ + setup_mm_for_reboot(mode); + + /* Clean and invalidate caches */ + flush_cache_all(); + + /* Turn off caching */ + cpu_proc_fin(); + + /* Push out any further dirty data, and ensure cache is empty */ + flush_cache_all(); + + /* + * Now call the architecture specific reboot code. + */ + arch_reset(mode, cmd); + + /* + * Whoops - the architecture was unable to reboot. + * Tell the user! + */ + mdelay(1000); + printk("Reboot failed -- System halted\n"); + while (1); +} + +/* + * Function pointers to optional machine specific functions + */ +void (*pm_power_off)(void); +EXPORT_SYMBOL(pm_power_off); + +void (*arm_pm_restart)(char str, const char *cmd) = arm_machine_restart; +EXPORT_SYMBOL_GPL(arm_pm_restart); + +static void do_nothing(void *unused) +{ +} + +/* + * cpu_idle_wait - Used to ensure that all the CPUs discard old value of + * pm_idle and update to new pm_idle value. Required while changing pm_idle + * handler on SMP systems. + * + * Caller must have changed pm_idle to the new value before the call. Old + * pm_idle value will not be used by any CPU after the return of this function. + */ +void cpu_idle_wait(void) +{ + smp_mb(); + /* kick all the CPUs so that they exit out of pm_idle */ + smp_call_function(do_nothing, NULL, 1); +} +EXPORT_SYMBOL_GPL(cpu_idle_wait); + +/* + * This is our default idle handler. We need to disable + * interrupts here to ensure we don't miss a wakeup call. + */ +static void default_idle(void) +{ + if (!need_resched()) + arch_idle(); + local_irq_enable(); +} + +void (*pm_idle)(void) = default_idle; +EXPORT_SYMBOL(pm_idle); + +/* + * The idle thread, has rather strange semantics for calling pm_idle, + * but this is what x86 does and we need to do the same, so that + * things like cpuidle get called in the same way. The only difference + * is that we always respect 'hlt_counter' to prevent low power idle. + */ +void cpu_idle(void) +{ + local_fiq_enable(); + + /* endless idle loop with no priority at all */ + while (1) { + idle_notifier_call_chain(IDLE_START); + tick_nohz_stop_sched_tick(1); + while (!need_resched()) { +#ifdef CONFIG_HOTPLUG_CPU + if (cpu_is_offline(smp_processor_id())) + cpu_die(); +#endif + + local_irq_disable(); +#ifdef CONFIG_PL310_ERRATA_769419 + wmb(); +#endif + if (hlt_counter || tick_check_broadcast_pending()) { + local_irq_enable(); + cpu_relax(); + } else { + stop_critical_timings(); + pm_idle(); + start_critical_timings(); + /* + * This will eventually be removed - pm_idle + * functions should always return with IRQs + * enabled. + */ + WARN_ON(irqs_disabled()); + local_irq_enable(); + } + } + tick_nohz_restart_sched_tick(); + idle_notifier_call_chain(IDLE_END); + preempt_enable_no_resched(); + schedule(); + preempt_disable(); + } +} + +static char reboot_mode = 'h'; + +int __init reboot_setup(char *str) +{ + reboot_mode = str[0]; + return 1; +} + +__setup("reboot=", reboot_setup); + +void machine_shutdown(void) +{ +#ifdef CONFIG_SMP + smp_send_stop(); +#endif +} + +void machine_halt(void) +{ + machine_shutdown(); + while (1); +} + +void machine_power_off(void) +{ + machine_shutdown(); + if (pm_power_off) + pm_power_off(); +} + +void machine_restart(char *cmd) +{ + machine_shutdown(); + arm_pm_restart(reboot_mode, cmd); +} + +/* + * dump a block of kernel memory from around the given address + */ +static void show_data(unsigned long addr, int nbytes, const char *name) +{ + int i, j; + int nlines; + u32 *p; + + /* + * don't attempt to dump non-kernel addresses or + * values that are probably just small negative numbers + */ + if (addr < PAGE_OFFSET || addr > -256UL) + return; + + printk("\n%s: %#lx:\n", name, addr); + + /* + * round address down to a 32 bit boundary + * and always dump a multiple of 32 bytes + */ + p = (u32 *)(addr & ~(sizeof(u32) - 1)); + nbytes += (addr & (sizeof(u32) - 1)); + nlines = (nbytes + 31) / 32; + + + for (i = 0; i < nlines; i++) { + /* + * just display low 16 bits of address to keep + * each line of the dump < 80 characters + */ + printk("%04lx ", (unsigned long)p & 0xffff); + for (j = 0; j < 8; j++) { + u32 data; + if (probe_kernel_address(p, data)) { + printk(" ********"); + } else { + printk(" %08x", data); + } + ++p; + } + printk("\n"); + } +} + +static void show_extra_register_data(struct pt_regs *regs, int nbytes) +{ + mm_segment_t fs; + + fs = get_fs(); + set_fs(KERNEL_DS); + show_data(regs->ARM_pc - nbytes, nbytes * 2, "PC"); + show_data(regs->ARM_lr - nbytes, nbytes * 2, "LR"); + show_data(regs->ARM_sp - nbytes, nbytes * 2, "SP"); + show_data(regs->ARM_ip - nbytes, nbytes * 2, "IP"); + show_data(regs->ARM_fp - nbytes, nbytes * 2, "FP"); + show_data(regs->ARM_r0 - nbytes, nbytes * 2, "R0"); + show_data(regs->ARM_r1 - nbytes, nbytes * 2, "R1"); + show_data(regs->ARM_r2 - nbytes, nbytes * 2, "R2"); + show_data(regs->ARM_r3 - nbytes, nbytes * 2, "R3"); + show_data(regs->ARM_r4 - nbytes, nbytes * 2, "R4"); + show_data(regs->ARM_r5 - nbytes, nbytes * 2, "R5"); + show_data(regs->ARM_r6 - nbytes, nbytes * 2, "R6"); + show_data(regs->ARM_r7 - nbytes, nbytes * 2, "R7"); + show_data(regs->ARM_r8 - nbytes, nbytes * 2, "R8"); + show_data(regs->ARM_r9 - nbytes, nbytes * 2, "R9"); + show_data(regs->ARM_r10 - nbytes, nbytes * 2, "R10"); + set_fs(fs); +} + +void __show_regs(struct pt_regs *regs) +{ + unsigned long flags; + char buf[64]; + + printk("CPU: %d %s (%s %.*s)\n", + raw_smp_processor_id(), print_tainted(), + init_utsname()->release, + (int)strcspn(init_utsname()->version, " "), + init_utsname()->version); + print_symbol("PC is at %s\n", instruction_pointer(regs)); + print_symbol("LR is at %s\n", regs->ARM_lr); + printk("pc : [<%08lx>] lr : [<%08lx>] psr: %08lx\n" + "sp : %08lx ip : %08lx fp : %08lx\n", + regs->ARM_pc, regs->ARM_lr, regs->ARM_cpsr, + regs->ARM_sp, regs->ARM_ip, regs->ARM_fp); + printk("r10: %08lx r9 : %08lx r8 : %08lx\n", + regs->ARM_r10, regs->ARM_r9, + regs->ARM_r8); + printk("r7 : %08lx r6 : %08lx r5 : %08lx r4 : %08lx\n", + regs->ARM_r7, regs->ARM_r6, + regs->ARM_r5, regs->ARM_r4); + printk("r3 : %08lx r2 : %08lx r1 : %08lx r0 : %08lx\n", + regs->ARM_r3, regs->ARM_r2, + regs->ARM_r1, regs->ARM_r0); + + flags = regs->ARM_cpsr; + buf[0] = flags & PSR_N_BIT ? 'N' : 'n'; + buf[1] = flags & PSR_Z_BIT ? 'Z' : 'z'; + buf[2] = flags & PSR_C_BIT ? 'C' : 'c'; + buf[3] = flags & PSR_V_BIT ? 'V' : 'v'; + buf[4] = '\0'; + + printk("Flags: %s IRQs o%s FIQs o%s Mode %s ISA %s Segment %s\n", + buf, interrupts_enabled(regs) ? "n" : "ff", + fast_interrupts_enabled(regs) ? "n" : "ff", + processor_modes[processor_mode(regs)], + isa_modes[isa_mode(regs)], + get_fs() == get_ds() ? "kernel" : "user"); +#ifdef CONFIG_CPU_CP15 + { + unsigned int ctrl; + + buf[0] = '\0'; +#ifdef CONFIG_CPU_CP15_MMU + { + unsigned int transbase, dac; + asm("mrc p15, 0, %0, c2, c0\n\t" + "mrc p15, 0, %1, c3, c0\n" + : "=r" (transbase), "=r" (dac)); + snprintf(buf, sizeof(buf), " Table: %08x DAC: %08x", + transbase, dac); + } +#endif + asm("mrc p15, 0, %0, c1, c0\n" : "=r" (ctrl)); + + printk("Control: %08x%s\n", ctrl, buf); + } +#endif + + show_extra_register_data(regs, 128); +} + +void show_regs(struct pt_regs * regs) +{ + printk("\n"); + printk("Pid: %d, comm: %20s\n", task_pid_nr(current), current->comm); + __show_regs(regs); + __backtrace(); +} + +ATOMIC_NOTIFIER_HEAD(thread_notify_head); + +EXPORT_SYMBOL_GPL(thread_notify_head); + +/* + * Free current thread data structures etc.. + */ +void exit_thread(void) +{ + thread_notify(THREAD_NOTIFY_EXIT, current_thread_info()); +} + +void flush_thread(void) +{ + struct thread_info *thread = current_thread_info(); + struct task_struct *tsk = current; + + flush_ptrace_hw_breakpoint(tsk); + + memset(thread->used_cp, 0, sizeof(thread->used_cp)); + memset(&tsk->thread.debug, 0, sizeof(struct debug_info)); + memset(&thread->fpstate, 0, sizeof(union fp_state)); + + thread_notify(THREAD_NOTIFY_FLUSH, thread); +} + +void release_thread(struct task_struct *dead_task) +{ +} + +asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); + +int +copy_thread(unsigned long clone_flags, unsigned long stack_start, + unsigned long stk_sz, struct task_struct *p, struct pt_regs *regs) +{ + struct thread_info *thread = task_thread_info(p); + struct pt_regs *childregs = task_pt_regs(p); + + *childregs = *regs; + childregs->ARM_r0 = 0; + childregs->ARM_sp = stack_start; + + memset(&thread->cpu_context, 0, sizeof(struct cpu_context_save)); + thread->cpu_context.sp = (unsigned long)childregs; + thread->cpu_context.pc = (unsigned long)ret_from_fork; + + clear_ptrace_hw_breakpoint(p); + + if (clone_flags & CLONE_SETTLS) + thread->tp_value = regs->ARM_r3; + + thread_notify(THREAD_NOTIFY_COPY, thread); + + return 0; +} + +/* + * Fill in the task's elfregs structure for a core dump. + */ +int dump_task_regs(struct task_struct *t, elf_gregset_t *elfregs) +{ + elf_core_copy_regs(elfregs, task_pt_regs(t)); + return 1; +} + +/* + * fill in the fpe structure for a core dump... + */ +int dump_fpu (struct pt_regs *regs, struct user_fp *fp) +{ + struct thread_info *thread = current_thread_info(); + int used_math = thread->used_cp[1] | thread->used_cp[2]; + + if (used_math) + memcpy(fp, &thread->fpstate.soft, sizeof (*fp)); + + return used_math != 0; +} +EXPORT_SYMBOL(dump_fpu); + +/* + * Shuffle the argument into the correct register before calling the + * thread function. r4 is the thread argument, r5 is the pointer to + * the thread function, and r6 points to the exit function. + */ +extern void kernel_thread_helper(void); +asm( ".pushsection .text\n" +" .align\n" +" .type kernel_thread_helper, #function\n" +"kernel_thread_helper:\n" +#ifdef CONFIG_TRACE_IRQFLAGS +" bl trace_hardirqs_on\n" +#endif +" msr cpsr_c, r7\n" +" mov r0, r4\n" +" mov lr, r6\n" +" mov pc, r5\n" +" .size kernel_thread_helper, . - kernel_thread_helper\n" +" .popsection"); + +#ifdef CONFIG_ARM_UNWIND +extern void kernel_thread_exit(long code); +asm( ".pushsection .text\n" +" .align\n" +" .type kernel_thread_exit, #function\n" +"kernel_thread_exit:\n" +" .fnstart\n" +" .cantunwind\n" +" bl do_exit\n" +" nop\n" +" .fnend\n" +" .size kernel_thread_exit, . - kernel_thread_exit\n" +" .popsection"); +#else +#define kernel_thread_exit do_exit +#endif + +/* + * Create a kernel thread. + */ +pid_t kernel_thread(int (*fn)(void *), void *arg, unsigned long flags) +{ + struct pt_regs regs; + + memset(®s, 0, sizeof(regs)); + + regs.ARM_r4 = (unsigned long)arg; + regs.ARM_r5 = (unsigned long)fn; + regs.ARM_r6 = (unsigned long)kernel_thread_exit; + regs.ARM_r7 = SVC_MODE | PSR_ENDSTATE | PSR_ISETSTATE; + regs.ARM_pc = (unsigned long)kernel_thread_helper; + regs.ARM_cpsr = regs.ARM_r7 | PSR_I_BIT; + + return do_fork(flags|CLONE_VM|CLONE_UNTRACED, 0, ®s, 0, NULL, NULL); +} +EXPORT_SYMBOL(kernel_thread); + +unsigned long get_wchan(struct task_struct *p) +{ + struct stackframe frame; + int count = 0; + if (!p || p == current || p->state == TASK_RUNNING) + return 0; + + frame.fp = thread_saved_fp(p); + frame.sp = thread_saved_sp(p); + frame.lr = 0; /* recovered from the stack */ + frame.pc = thread_saved_pc(p); + do { + int ret = unwind_frame(&frame); + if (ret < 0) + return 0; + if (!in_sched_functions(frame.pc)) + return frame.pc; + } while (count ++ < 16); + return 0; +} + +unsigned long arch_randomize_brk(struct mm_struct *mm) +{ + unsigned long range_end = mm->brk + 0x02000000; + return randomize_range(mm->brk, range_end, 0) ? : mm->brk; +} + +#ifdef CONFIG_MMU +/* + * The vectors page is always readable from user space for the + * atomic helpers and the signal restart code. Let's declare a mapping + * for it so it is visible through ptrace and /proc/<pid>/mem. + */ + +int vectors_user_mapping(void) +{ + struct mm_struct *mm = current->mm; + return install_special_mapping(mm, 0xffff0000, PAGE_SIZE, + VM_READ | VM_EXEC | + VM_MAYREAD | VM_MAYEXEC | + VM_ALWAYSDUMP | VM_RESERVED, + NULL); +} + +const char *arch_vma_name(struct vm_area_struct *vma) +{ + return (vma->vm_start == 0xffff0000) ? "[vectors]" : NULL; +} +#endif diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c new file mode 100644 index 00000000..172ae01c --- /dev/null +++ b/arch/arm/kernel/ptrace.c @@ -0,0 +1,961 @@ +/* + * linux/arch/arm/kernel/ptrace.c + * + * By Ross Biro 1/23/92 + * edited by Linus Torvalds + * ARM modifications Copyright (C) 2000 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/smp.h> +#include <linux/ptrace.h> +#include <linux/user.h> +#include <linux/security.h> +#include <linux/init.h> +#include <linux/signal.h> +#include <linux/uaccess.h> +#include <linux/perf_event.h> +#include <linux/hw_breakpoint.h> +#include <linux/regset.h> + +#include <asm/pgtable.h> +#include <asm/system.h> +#include <asm/traps.h> + +#define REG_PC 15 +#define REG_PSR 16 +/* + * does not yet catch signals sent when the child dies. + * in exit.c or in signal.c. + */ + +#if 0 +/* + * Breakpoint SWI instruction: SWI &9F0001 + */ +#define BREAKINST_ARM 0xef9f0001 +#define BREAKINST_THUMB 0xdf00 /* fill this in later */ +#else +/* + * New breakpoints - use an undefined instruction. The ARM architecture + * reference manual guarantees that the following instruction space + * will produce an undefined instruction exception on all CPUs: + * + * ARM: xxxx 0111 1111 xxxx xxxx xxxx 1111 xxxx + * Thumb: 1101 1110 xxxx xxxx + */ +#define BREAKINST_ARM 0xe7f001f0 +#define BREAKINST_THUMB 0xde01 +#endif + +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(r) \ + {.name = #r, .offset = offsetof(struct pt_regs, ARM_##r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME(r0), + REG_OFFSET_NAME(r1), + REG_OFFSET_NAME(r2), + REG_OFFSET_NAME(r3), + REG_OFFSET_NAME(r4), + REG_OFFSET_NAME(r5), + REG_OFFSET_NAME(r6), + REG_OFFSET_NAME(r7), + REG_OFFSET_NAME(r8), + REG_OFFSET_NAME(r9), + REG_OFFSET_NAME(r10), + REG_OFFSET_NAME(fp), + REG_OFFSET_NAME(ip), + REG_OFFSET_NAME(sp), + REG_OFFSET_NAME(lr), + REG_OFFSET_NAME(pc), + REG_OFFSET_NAME(cpsr), + REG_OFFSET_NAME(ORIG_r0), + REG_OFFSET_END, +}; + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_query_register_name() - query register name from its offset + * @offset: the offset of a register in struct pt_regs. + * + * regs_query_register_name() returns the name of a register from its + * offset in struct pt_regs. If the @offset is invalid, this returns NULL; + */ +const char *regs_query_register_name(unsigned int offset) +{ + const struct pt_regs_offset *roff; + for (roff = regoffset_table; roff->name != NULL; roff++) + if (roff->offset == offset) + return roff->name; + return NULL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) +{ + return ((addr & ~(THREAD_SIZE - 1)) == + (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} + +/* + * this routine will get a word off of the processes privileged stack. + * the offset is how far from the base addr as stored in the THREAD. + * this routine assumes that all the privileged stacks are in our + * data space. + */ +static inline long get_user_reg(struct task_struct *task, int offset) +{ + return task_pt_regs(task)->uregs[offset]; +} + +/* + * this routine will put a word on the processes privileged stack. + * the offset is how far from the base addr as stored in the THREAD. + * this routine assumes that all the privileged stacks are in our + * data space. + */ +static inline int +put_user_reg(struct task_struct *task, int offset, long data) +{ + struct pt_regs newregs, *regs = task_pt_regs(task); + int ret = -EINVAL; + + newregs = *regs; + newregs.uregs[offset] = data; + + if (valid_user_regs(&newregs)) { + regs->uregs[offset] = data; + ret = 0; + } + + return ret; +} + +/* + * Called by kernel/ptrace.c when detaching.. + */ +void ptrace_disable(struct task_struct *child) +{ + /* Nothing to do. */ +} + +/* + * Handle hitting a breakpoint. + */ +void ptrace_break(struct task_struct *tsk, struct pt_regs *regs) +{ + siginfo_t info; + + info.si_signo = SIGTRAP; + info.si_errno = 0; + info.si_code = TRAP_BRKPT; + info.si_addr = (void __user *)instruction_pointer(regs); + + force_sig_info(SIGTRAP, &info, tsk); +} + +static int break_trap(struct pt_regs *regs, unsigned int instr) +{ + ptrace_break(current, regs); + return 0; +} + +static struct undef_hook arm_break_hook = { + .instr_mask = 0x0fffffff, + .instr_val = 0x07f001f0, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = 0, + .fn = break_trap, +}; + +static struct undef_hook thumb_break_hook = { + .instr_mask = 0xffff, + .instr_val = 0xde01, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = PSR_T_BIT, + .fn = break_trap, +}; + +static int thumb2_break_trap(struct pt_regs *regs, unsigned int instr) +{ + unsigned int instr2; + void __user *pc; + + /* Check the second half of the instruction. */ + pc = (void __user *)(instruction_pointer(regs) + 2); + + if (processor_mode(regs) == SVC_MODE) { + instr2 = *(u16 *) pc; + } else { + get_user(instr2, (u16 __user *)pc); + } + + if (instr2 == 0xa000) { + ptrace_break(current, regs); + return 0; + } else { + return 1; + } +} + +static struct undef_hook thumb2_break_hook = { + .instr_mask = 0xffff, + .instr_val = 0xf7f0, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = PSR_T_BIT, + .fn = thumb2_break_trap, +}; + +static int __init ptrace_break_init(void) +{ + register_undef_hook(&arm_break_hook); + register_undef_hook(&thumb_break_hook); + register_undef_hook(&thumb2_break_hook); + return 0; +} + +core_initcall(ptrace_break_init); + +/* + * Read the word at offset "off" into the "struct user". We + * actually access the pt_regs stored on the kernel stack. + */ +static int ptrace_read_user(struct task_struct *tsk, unsigned long off, + unsigned long __user *ret) +{ + unsigned long tmp; + + if (off & 3 || off >= sizeof(struct user)) + return -EIO; + + tmp = 0; + if (off == PT_TEXT_ADDR) + tmp = tsk->mm->start_code; + else if (off == PT_DATA_ADDR) + tmp = tsk->mm->start_data; + else if (off == PT_TEXT_END_ADDR) + tmp = tsk->mm->end_code; + else if (off < sizeof(struct pt_regs)) + tmp = get_user_reg(tsk, off >> 2); + + return put_user(tmp, ret); +} + +/* + * Write the word at offset "off" into "struct user". We + * actually access the pt_regs stored on the kernel stack. + */ +static int ptrace_write_user(struct task_struct *tsk, unsigned long off, + unsigned long val) +{ + if (off & 3 || off >= sizeof(struct user)) + return -EIO; + + if (off >= sizeof(struct pt_regs)) + return 0; + + return put_user_reg(tsk, off >> 2, val); +} + +#ifdef CONFIG_IWMMXT + +/* + * Get the child iWMMXt state. + */ +static int ptrace_getwmmxregs(struct task_struct *tsk, void __user *ufp) +{ + struct thread_info *thread = task_thread_info(tsk); + + if (!test_ti_thread_flag(thread, TIF_USING_IWMMXT)) + return -ENODATA; + iwmmxt_task_disable(thread); /* force it to ram */ + return copy_to_user(ufp, &thread->fpstate.iwmmxt, IWMMXT_SIZE) + ? -EFAULT : 0; +} + +/* + * Set the child iWMMXt state. + */ +static int ptrace_setwmmxregs(struct task_struct *tsk, void __user *ufp) +{ + struct thread_info *thread = task_thread_info(tsk); + + if (!test_ti_thread_flag(thread, TIF_USING_IWMMXT)) + return -EACCES; + iwmmxt_task_release(thread); /* force a reload */ + return copy_from_user(&thread->fpstate.iwmmxt, ufp, IWMMXT_SIZE) + ? -EFAULT : 0; +} + +#endif + +#ifdef CONFIG_CRUNCH +/* + * Get the child Crunch state. + */ +static int ptrace_getcrunchregs(struct task_struct *tsk, void __user *ufp) +{ + struct thread_info *thread = task_thread_info(tsk); + + crunch_task_disable(thread); /* force it to ram */ + return copy_to_user(ufp, &thread->crunchstate, CRUNCH_SIZE) + ? -EFAULT : 0; +} + +/* + * Set the child Crunch state. + */ +static int ptrace_setcrunchregs(struct task_struct *tsk, void __user *ufp) +{ + struct thread_info *thread = task_thread_info(tsk); + + crunch_task_release(thread); /* force a reload */ + return copy_from_user(&thread->crunchstate, ufp, CRUNCH_SIZE) + ? -EFAULT : 0; +} +#endif + +#ifdef CONFIG_HAVE_HW_BREAKPOINT +/* + * Convert a virtual register number into an index for a thread_info + * breakpoint array. Breakpoints are identified using positive numbers + * whilst watchpoints are negative. The registers are laid out as pairs + * of (address, control), each pair mapping to a unique hw_breakpoint struct. + * Register 0 is reserved for describing resource information. + */ +static int ptrace_hbp_num_to_idx(long num) +{ + if (num < 0) + num = (ARM_MAX_BRP << 1) - num; + return (num - 1) >> 1; +} + +/* + * Returns the virtual register number for the address of the + * breakpoint at index idx. + */ +static long ptrace_hbp_idx_to_num(int idx) +{ + long mid = ARM_MAX_BRP << 1; + long num = (idx << 1) + 1; + return num > mid ? mid - num : num; +} + +/* + * Handle hitting a HW-breakpoint. + */ +static void ptrace_hbptriggered(struct perf_event *bp, int unused, + struct perf_sample_data *data, + struct pt_regs *regs) +{ + struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp); + long num; + int i; + siginfo_t info; + + for (i = 0; i < ARM_MAX_HBP_SLOTS; ++i) + if (current->thread.debug.hbp[i] == bp) + break; + + num = (i == ARM_MAX_HBP_SLOTS) ? 0 : ptrace_hbp_idx_to_num(i); + + info.si_signo = SIGTRAP; + info.si_errno = (int)num; + info.si_code = TRAP_HWBKPT; + info.si_addr = (void __user *)(bkpt->trigger); + + force_sig_info(SIGTRAP, &info, current); +} + +/* + * Set ptrace breakpoint pointers to zero for this task. + * This is required in order to prevent child processes from unregistering + * breakpoints held by their parent. + */ +void clear_ptrace_hw_breakpoint(struct task_struct *tsk) +{ + memset(tsk->thread.debug.hbp, 0, sizeof(tsk->thread.debug.hbp)); +} + +/* + * Unregister breakpoints from this task and reset the pointers in + * the thread_struct. + */ +void flush_ptrace_hw_breakpoint(struct task_struct *tsk) +{ + int i; + struct thread_struct *t = &tsk->thread; + + for (i = 0; i < ARM_MAX_HBP_SLOTS; i++) { + if (t->debug.hbp[i]) { + unregister_hw_breakpoint(t->debug.hbp[i]); + t->debug.hbp[i] = NULL; + } + } +} + +static u32 ptrace_get_hbp_resource_info(void) +{ + u8 num_brps, num_wrps, debug_arch, wp_len; + u32 reg = 0; + + num_brps = hw_breakpoint_slots(TYPE_INST); + num_wrps = hw_breakpoint_slots(TYPE_DATA); + debug_arch = arch_get_debug_arch(); + wp_len = arch_get_max_wp_len(); + + reg |= debug_arch; + reg <<= 8; + reg |= wp_len; + reg <<= 8; + reg |= num_wrps; + reg <<= 8; + reg |= num_brps; + + return reg; +} + +static struct perf_event *ptrace_hbp_create(struct task_struct *tsk, int type) +{ + struct perf_event_attr attr; + + ptrace_breakpoint_init(&attr); + + /* Initialise fields to sane defaults. */ + attr.bp_addr = 0; + attr.bp_len = HW_BREAKPOINT_LEN_4; + attr.bp_type = type; + attr.disabled = 1; + + return register_user_hw_breakpoint(&attr, ptrace_hbptriggered, tsk); +} + +static int ptrace_gethbpregs(struct task_struct *tsk, long num, + unsigned long __user *data) +{ + u32 reg; + int idx, ret = 0; + struct perf_event *bp; + struct arch_hw_breakpoint_ctrl arch_ctrl; + + if (num == 0) { + reg = ptrace_get_hbp_resource_info(); + } else { + idx = ptrace_hbp_num_to_idx(num); + if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) { + ret = -EINVAL; + goto out; + } + + bp = tsk->thread.debug.hbp[idx]; + if (!bp) { + reg = 0; + goto put; + } + + arch_ctrl = counter_arch_bp(bp)->ctrl; + + /* + * Fix up the len because we may have adjusted it + * to compensate for an unaligned address. + */ + while (!(arch_ctrl.len & 0x1)) + arch_ctrl.len >>= 1; + + if (num & 0x1) + reg = bp->attr.bp_addr; + else + reg = encode_ctrl_reg(arch_ctrl); + } + +put: + if (put_user(reg, data)) + ret = -EFAULT; + +out: + return ret; +} + +static int ptrace_sethbpregs(struct task_struct *tsk, long num, + unsigned long __user *data) +{ + int idx, gen_len, gen_type, implied_type, ret = 0; + u32 user_val; + struct perf_event *bp; + struct arch_hw_breakpoint_ctrl ctrl; + struct perf_event_attr attr; + + if (num == 0) + goto out; + else if (num < 0) + implied_type = HW_BREAKPOINT_RW; + else + implied_type = HW_BREAKPOINT_X; + + idx = ptrace_hbp_num_to_idx(num); + if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) { + ret = -EINVAL; + goto out; + } + + if (get_user(user_val, data)) { + ret = -EFAULT; + goto out; + } + + bp = tsk->thread.debug.hbp[idx]; + if (!bp) { + bp = ptrace_hbp_create(tsk, implied_type); + if (IS_ERR(bp)) { + ret = PTR_ERR(bp); + goto out; + } + tsk->thread.debug.hbp[idx] = bp; + } + + attr = bp->attr; + + if (num & 0x1) { + /* Address */ + attr.bp_addr = user_val; + } else { + /* Control */ + decode_ctrl_reg(user_val, &ctrl); + ret = arch_bp_generic_fields(ctrl, &gen_len, &gen_type); + if (ret) + goto out; + + if ((gen_type & implied_type) != gen_type) { + ret = -EINVAL; + goto out; + } + + attr.bp_len = gen_len; + attr.bp_type = gen_type; + attr.disabled = !ctrl.enabled; + } + + ret = modify_user_hw_breakpoint(bp, &attr); +out: + return ret; +} +#endif + +/* regset get/set implementations */ + +static int gpr_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, + regs, + 0, sizeof(*regs)); +} + +static int gpr_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + struct pt_regs newregs; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &newregs, + 0, sizeof(newregs)); + if (ret) + return ret; + + if (!valid_user_regs(&newregs)) + return -EINVAL; + + *task_pt_regs(target) = newregs; + return 0; +} + +static int fpa_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, + &task_thread_info(target)->fpstate, + 0, sizeof(struct user_fp)); +} + +static int fpa_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct thread_info *thread = task_thread_info(target); + + thread->used_cp[1] = thread->used_cp[2] = 1; + + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &thread->fpstate, + 0, sizeof(struct user_fp)); +} + +#ifdef CONFIG_VFP +/* + * VFP register get/set implementations. + * + * With respect to the kernel, struct user_fp is divided into three chunks: + * 16 or 32 real VFP registers (d0-d15 or d0-31) + * These are transferred to/from the real registers in the task's + * vfp_hard_struct. The number of registers depends on the kernel + * configuration. + * + * 16 or 0 fake VFP registers (d16-d31 or empty) + * i.e., the user_vfp structure has space for 32 registers even if + * the kernel doesn't have them all. + * + * vfp_get() reads this chunk as zero where applicable + * vfp_set() ignores this chunk + * + * 1 word for the FPSCR + * + * The bounds-checking logic built into user_regset_copyout and friends + * means that we can make a simple sequence of calls to map the relevant data + * to/from the specified slice of the user regset structure. + */ +static int vfp_get(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + void *kbuf, void __user *ubuf) +{ + int ret; + struct thread_info *thread = task_thread_info(target); + struct vfp_hard_struct const *vfp = &thread->vfpstate.hard; + const size_t user_fpregs_offset = offsetof(struct user_vfp, fpregs); + const size_t user_fpscr_offset = offsetof(struct user_vfp, fpscr); + + vfp_sync_hwstate(thread); + + ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf, + &vfp->fpregs, + user_fpregs_offset, + user_fpregs_offset + sizeof(vfp->fpregs)); + if (ret) + return ret; + + ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf, + user_fpregs_offset + sizeof(vfp->fpregs), + user_fpscr_offset); + if (ret) + return ret; + + return user_regset_copyout(&pos, &count, &kbuf, &ubuf, + &vfp->fpscr, + user_fpscr_offset, + user_fpscr_offset + sizeof(vfp->fpscr)); +} + +/* + * For vfp_set() a read-modify-write is done on the VFP registers, + * in order to avoid writing back a half-modified set of registers on + * failure. + */ +static int vfp_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + int ret; + struct thread_info *thread = task_thread_info(target); + struct vfp_hard_struct new_vfp; + const size_t user_fpregs_offset = offsetof(struct user_vfp, fpregs); + const size_t user_fpscr_offset = offsetof(struct user_vfp, fpscr); + + vfp_sync_hwstate(thread); + new_vfp = thread->vfpstate.hard; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &new_vfp.fpregs, + user_fpregs_offset, + user_fpregs_offset + sizeof(new_vfp.fpregs)); + if (ret) + return ret; + + ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + user_fpregs_offset + sizeof(new_vfp.fpregs), + user_fpscr_offset); + if (ret) + return ret; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &new_vfp.fpscr, + user_fpscr_offset, + user_fpscr_offset + sizeof(new_vfp.fpscr)); + if (ret) + return ret; + + vfp_flush_hwstate(thread); + thread->vfpstate.hard = new_vfp; + + return 0; +} +#endif /* CONFIG_VFP */ + +enum arm_regset { + REGSET_GPR, + REGSET_FPR, +#ifdef CONFIG_VFP + REGSET_VFP, +#endif +}; + +static const struct user_regset arm_regsets[] = { + [REGSET_GPR] = { + .core_note_type = NT_PRSTATUS, + .n = ELF_NGREG, + .size = sizeof(u32), + .align = sizeof(u32), + .get = gpr_get, + .set = gpr_set + }, + [REGSET_FPR] = { + /* + * For the FPA regs in fpstate, the real fields are a mixture + * of sizes, so pretend that the registers are word-sized: + */ + .core_note_type = NT_PRFPREG, + .n = sizeof(struct user_fp) / sizeof(u32), + .size = sizeof(u32), + .align = sizeof(u32), + .get = fpa_get, + .set = fpa_set + }, +#ifdef CONFIG_VFP + [REGSET_VFP] = { + /* + * Pretend that the VFP regs are word-sized, since the FPSCR is + * a single word dangling at the end of struct user_vfp: + */ + .core_note_type = NT_ARM_VFP, + .n = ARM_VFPREGS_SIZE / sizeof(u32), + .size = sizeof(u32), + .align = sizeof(u32), + .get = vfp_get, + .set = vfp_set + }, +#endif /* CONFIG_VFP */ +}; + +static const struct user_regset_view user_arm_view = { + .name = "arm", .e_machine = ELF_ARCH, .ei_osabi = ELF_OSABI, + .regsets = arm_regsets, .n = ARRAY_SIZE(arm_regsets) +}; + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ + return &user_arm_view; +} + +long arch_ptrace(struct task_struct *child, long request, + unsigned long addr, unsigned long data) +{ + int ret; + unsigned long __user *datap = (unsigned long __user *) data; + + switch (request) { + case PTRACE_PEEKUSR: + ret = ptrace_read_user(child, addr, datap); + break; + + case PTRACE_POKEUSR: + ret = ptrace_write_user(child, addr, data); + break; + + case PTRACE_GETREGS: + ret = copy_regset_to_user(child, + &user_arm_view, REGSET_GPR, + 0, sizeof(struct pt_regs), + datap); + break; + + case PTRACE_SETREGS: + ret = copy_regset_from_user(child, + &user_arm_view, REGSET_GPR, + 0, sizeof(struct pt_regs), + datap); + break; + + case PTRACE_GETFPREGS: + ret = copy_regset_to_user(child, + &user_arm_view, REGSET_FPR, + 0, sizeof(union fp_state), + datap); + break; + + case PTRACE_SETFPREGS: + ret = copy_regset_from_user(child, + &user_arm_view, REGSET_FPR, + 0, sizeof(union fp_state), + datap); + break; + +#ifdef CONFIG_IWMMXT + case PTRACE_GETWMMXREGS: + ret = ptrace_getwmmxregs(child, datap); + break; + + case PTRACE_SETWMMXREGS: + ret = ptrace_setwmmxregs(child, datap); + break; +#endif + + case PTRACE_GET_THREAD_AREA: + ret = put_user(task_thread_info(child)->tp_value, + datap); + break; + + case PTRACE_SET_SYSCALL: + task_thread_info(child)->syscall = data; + ret = 0; + break; + +#ifdef CONFIG_CRUNCH + case PTRACE_GETCRUNCHREGS: + ret = ptrace_getcrunchregs(child, datap); + break; + + case PTRACE_SETCRUNCHREGS: + ret = ptrace_setcrunchregs(child, datap); + break; +#endif + +#ifdef CONFIG_VFP + case PTRACE_GETVFPREGS: + ret = copy_regset_to_user(child, + &user_arm_view, REGSET_VFP, + 0, ARM_VFPREGS_SIZE, + datap); + break; + + case PTRACE_SETVFPREGS: + ret = copy_regset_from_user(child, + &user_arm_view, REGSET_VFP, + 0, ARM_VFPREGS_SIZE, + datap); + break; +#endif + +#ifdef CONFIG_HAVE_HW_BREAKPOINT + case PTRACE_GETHBPREGS: + if (ptrace_get_breakpoints(child) < 0) + return -ESRCH; + + ret = ptrace_gethbpregs(child, addr, + (unsigned long __user *)data); + ptrace_put_breakpoints(child); + break; + case PTRACE_SETHBPREGS: + if (ptrace_get_breakpoints(child) < 0) + return -ESRCH; + + ret = ptrace_sethbpregs(child, addr, + (unsigned long __user *)data); + ptrace_put_breakpoints(child); + break; +#endif + + default: + ret = ptrace_request(child, request, addr, data); + break; + } + + return ret; +} + +asmlinkage int syscall_trace(int why, struct pt_regs *regs, int scno) +{ + unsigned long ip; + + if (!test_thread_flag(TIF_SYSCALL_TRACE)) + return scno; + if (!(current->ptrace & PT_PTRACED)) + return scno; + + /* + * Save IP. IP is used to denote syscall entry/exit: + * IP = 0 -> entry, = 1 -> exit + */ + ip = regs->ARM_ip; + regs->ARM_ip = why; + + current_thread_info()->syscall = scno; + + /* the 0x80 provides a way for the tracing parent to distinguish + between a syscall stop and SIGTRAP delivery */ + ptrace_notify(SIGTRAP | ((current->ptrace & PT_TRACESYSGOOD) + ? 0x80 : 0)); + /* + * this isn't the same as continuing with a signal, but it will do + * for normal use. strace only continues with a signal if the + * stopping signal is not SIGTRAP. -brl + */ + if (current->exit_code) { + send_sig(current->exit_code, current, 1); + current->exit_code = 0; + } + regs->ARM_ip = ip; + + return current_thread_info()->syscall; +} diff --git a/arch/arm/kernel/relocate_kernel.S b/arch/arm/kernel/relocate_kernel.S new file mode 100644 index 00000000..9cf4cbf8 --- /dev/null +++ b/arch/arm/kernel/relocate_kernel.S @@ -0,0 +1,87 @@ +/* + * relocate_kernel.S - put the kernel image in place to boot + */ + +#include <asm/kexec.h> + + .globl relocate_new_kernel +relocate_new_kernel: + + ldr r0,kexec_indirection_page + ldr r1,kexec_start_address + + /* + * If there is no indirection page (we are doing crashdumps) + * skip any relocation. + */ + cmp r0, #0 + beq 2f + +0: /* top, read another word for the indirection page */ + ldr r3, [r0],#4 + + /* Is it a destination page. Put destination address to r4 */ + tst r3,#1,0 + beq 1f + bic r4,r3,#1 + b 0b +1: + /* Is it an indirection page */ + tst r3,#2,0 + beq 1f + bic r0,r3,#2 + b 0b +1: + + /* are we done ? */ + tst r3,#4,0 + beq 1f + b 2f + +1: + /* is it source ? */ + tst r3,#8,0 + beq 0b + bic r3,r3,#8 + mov r6,#1024 +9: + ldr r5,[r3],#4 + str r5,[r4],#4 + subs r6,r6,#1 + bne 9b + b 0b + +2: + /* Jump to relocated kernel */ + mov lr,r1 + mov r0,#0 + ldr r1,kexec_mach_type + ldr r2,kexec_boot_atags + mov pc,lr + + .align + + .globl kexec_start_address +kexec_start_address: + .long 0x0 + + .globl kexec_indirection_page +kexec_indirection_page: + .long 0x0 + + .globl kexec_mach_type +kexec_mach_type: + .long 0x0 + + /* phy addr of the atags for the new kernel */ + .globl kexec_boot_atags +kexec_boot_atags: + .long 0x0 + +relocate_new_kernel_end: + + .globl relocate_new_kernel_size +relocate_new_kernel_size: + .long relocate_new_kernel_end - relocate_new_kernel + + diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c new file mode 100644 index 00000000..0b13a72f --- /dev/null +++ b/arch/arm/kernel/return_address.c @@ -0,0 +1,72 @@ +/* + * arch/arm/kernel/return_address.c + * + * Copyright (C) 2009 Uwe Kleine-Koenig <u.kleine-koenig@pengutronix.de> + * for Pengutronix + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + */ +#include <linux/module.h> +#include <linux/ftrace.h> + +#if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) +#include <linux/sched.h> + +#include <asm/stacktrace.h> + +struct return_address_data { + unsigned int level; + void *addr; +}; + +static int save_return_addr(struct stackframe *frame, void *d) +{ + struct return_address_data *data = d; + + if (!data->level) { + data->addr = (void *)frame->lr; + + return 1; + } else { + --data->level; + return 0; + } +} + +void *return_address(unsigned int level) +{ + struct return_address_data data; + struct stackframe frame; + register unsigned long current_sp asm ("sp"); + + data.level = level + 1; + + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_sp; + frame.lr = (unsigned long)__builtin_return_address(0); + frame.pc = (unsigned long)return_address; + + walk_stackframe(&frame, save_return_addr, &data); + + if (!data.level) + return data.addr; + else + return NULL; +} + +#else /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) */ + +#if defined(CONFIG_ARM_UNWIND) +#warning "TODO: return_address should use unwind tables" +#endif + +void *return_address(unsigned int level) +{ + return NULL; +} + +#endif /* if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) / else */ + +EXPORT_SYMBOL_GPL(return_address); diff --git a/arch/arm/kernel/sched_clock.c b/arch/arm/kernel/sched_clock.c new file mode 100644 index 00000000..9a46370f --- /dev/null +++ b/arch/arm/kernel/sched_clock.c @@ -0,0 +1,74 @@ +/* + * sched_clock.c: support for extending counters to full 64-bit ns counter + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/clocksource.h> +#include <linux/init.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/timer.h> + +#include <asm/sched_clock.h> + +static void sched_clock_poll(unsigned long wrap_ticks); +static DEFINE_TIMER(sched_clock_timer, sched_clock_poll, 0, 0); +static void (*sched_clock_update_fn)(void); + +static void sched_clock_poll(unsigned long wrap_ticks) +{ + mod_timer(&sched_clock_timer, round_jiffies(jiffies + wrap_ticks)); + sched_clock_update_fn(); +} + +void __init init_sched_clock(struct clock_data *cd, void (*update)(void), + unsigned int clock_bits, unsigned long rate) +{ + unsigned long r, w; + u64 res, wrap; + char r_unit; + + sched_clock_update_fn = update; + + /* calculate the mult/shift to convert counter ticks to ns. */ + clocks_calc_mult_shift(&cd->mult, &cd->shift, rate, NSEC_PER_SEC, 0); + + r = rate; + if (r >= 4000000) { + r /= 1000000; + r_unit = 'M'; + } else { + r /= 1000; + r_unit = 'k'; + } + + /* calculate how many ns until we wrap */ + wrap = cyc_to_ns((1ULL << clock_bits) - 1, cd->mult, cd->shift); + do_div(wrap, NSEC_PER_MSEC); + w = wrap; + + /* calculate the ns resolution of this counter */ + res = cyc_to_ns(1ULL, cd->mult, cd->shift); + pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n", + clock_bits, r, r_unit, res, w); + + /* + * Start the timer to keep sched_clock() properly updated and + * sets the initial epoch. + */ + sched_clock_timer.data = msecs_to_jiffies(w - (w / 10)); + update(); + + /* + * Ensure that sched_clock() starts off at 0ns + */ + cd->epoch_ns = 0; +} + +void __init sched_clock_postinit(void) +{ + sched_clock_poll(sched_clock_timer.data); +} diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c new file mode 100644 index 00000000..acbb447a --- /dev/null +++ b/arch/arm/kernel/setup.c @@ -0,0 +1,1068 @@ +/* + * linux/arch/arm/kernel/setup.c + * + * Copyright (C) 1995-2001 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/stddef.h> +#include <linux/ioport.h> +#include <linux/delay.h> +#include <linux/utsname.h> +#include <linux/initrd.h> +#include <linux/console.h> +#include <linux/bootmem.h> +#include <linux/seq_file.h> +#include <linux/screen_info.h> +#include <linux/init.h> +#include <linux/kexec.h> +#include <linux/of_fdt.h> +#include <linux/crash_dump.h> +#include <linux/root_dev.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/smp.h> +#include <linux/fs.h> +#include <linux/proc_fs.h> +#include <linux/memblock.h> + +#include <asm/unified.h> +#include <asm/cpu.h> +#include <asm/cputype.h> +#include <asm/elf.h> +#include <asm/procinfo.h> +#include <asm/sections.h> +#include <asm/setup.h> +#include <asm/smp_plat.h> +#include <asm/mach-types.h> +#include <asm/cacheflush.h> +#include <asm/cachetype.h> +#include <asm/tlbflush.h> + +#include <asm/prom.h> +#include <asm/mach/arch.h> +#include <asm/mach/irq.h> +#include <asm/mach/time.h> +#include <asm/traps.h> +#include <asm/unwind.h> + +#if defined(CONFIG_DEPRECATED_PARAM_STRUCT) +#include "compat.h" +#endif +#include "atags.h" +#include "tcm.h" + +#ifndef MEM_SIZE +#define MEM_SIZE (16*1024*1024) +#endif + +#if defined(CONFIG_FPE_NWFPE) || defined(CONFIG_FPE_FASTFPE) +char fpe_type[8]; + +static int __init fpe_setup(char *line) +{ + memcpy(fpe_type, line, 8); + return 1; +} + +__setup("fpe=", fpe_setup); +#endif + +extern void paging_init(struct machine_desc *desc); +extern void sanity_check_meminfo(void); +extern void reboot_setup(char *str); + +unsigned int processor_id; +EXPORT_SYMBOL(processor_id); +unsigned int __machine_arch_type __read_mostly; +EXPORT_SYMBOL(__machine_arch_type); +unsigned int cacheid __read_mostly; +EXPORT_SYMBOL(cacheid); + +unsigned int __atags_pointer __initdata; + +unsigned int system_rev; +EXPORT_SYMBOL(system_rev); + +unsigned int system_serial_low; +EXPORT_SYMBOL(system_serial_low); + +unsigned int system_serial_high; +EXPORT_SYMBOL(system_serial_high); + +unsigned int elf_hwcap __read_mostly; +EXPORT_SYMBOL(elf_hwcap); + + +#ifdef MULTI_CPU +struct processor processor __read_mostly; +#endif +#ifdef MULTI_TLB +struct cpu_tlb_fns cpu_tlb __read_mostly; +#endif +#ifdef MULTI_USER +struct cpu_user_fns cpu_user __read_mostly; +#endif +#ifdef MULTI_CACHE +struct cpu_cache_fns cpu_cache __read_mostly; +#endif +#ifdef CONFIG_OUTER_CACHE +struct outer_cache_fns outer_cache __read_mostly; +EXPORT_SYMBOL(outer_cache); +#endif + +struct stack { + u32 irq[3]; + u32 abt[3]; + u32 und[3]; +} ____cacheline_aligned; + +static struct stack stacks[NR_CPUS]; + +char elf_platform[ELF_PLATFORM_SIZE]; +EXPORT_SYMBOL(elf_platform); + +static const char *cpu_name; +static const char *machine_name; +static char __initdata cmd_line[COMMAND_LINE_SIZE]; +struct machine_desc *machine_desc __initdata; + +static char default_command_line[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE; +static union { char c[4]; unsigned long l; } endian_test __initdata = { { 'l', '?', '?', 'b' } }; +#define ENDIANNESS ((char)endian_test.l) + +DEFINE_PER_CPU(struct cpuinfo_arm, cpu_data); + +/* + * Standard memory resources + */ +static struct resource mem_res[] = { + { + .name = "Video RAM", + .start = 0, + .end = 0, + .flags = IORESOURCE_MEM + }, + { + .name = "Kernel text", + .start = 0, + .end = 0, + .flags = IORESOURCE_MEM + }, + { + .name = "Kernel data", + .start = 0, + .end = 0, + .flags = IORESOURCE_MEM + } +}; + +#define video_ram mem_res[0] +#define kernel_code mem_res[1] +#define kernel_data mem_res[2] + +static struct resource io_res[] = { + { + .name = "reserved", + .start = 0x3bc, + .end = 0x3be, + .flags = IORESOURCE_IO | IORESOURCE_BUSY + }, + { + .name = "reserved", + .start = 0x378, + .end = 0x37f, + .flags = IORESOURCE_IO | IORESOURCE_BUSY + }, + { + .name = "reserved", + .start = 0x278, + .end = 0x27f, + .flags = IORESOURCE_IO | IORESOURCE_BUSY + } +}; + +#define lp0 io_res[0] +#define lp1 io_res[1] +#define lp2 io_res[2] + +static const char *proc_arch[] = { + "undefined/unknown", + "3", + "4", + "4T", + "5", + "5T", + "5TE", + "5TEJ", + "6TEJ", + "7", + "?(11)", + "?(12)", + "?(13)", + "?(14)", + "?(15)", + "?(16)", + "?(17)", +}; + +int cpu_architecture(void) +{ + int cpu_arch; + + if ((read_cpuid_id() & 0x0008f000) == 0) { + cpu_arch = CPU_ARCH_UNKNOWN; + } else if ((read_cpuid_id() & 0x0008f000) == 0x00007000) { + cpu_arch = (read_cpuid_id() & (1 << 23)) ? CPU_ARCH_ARMv4T : CPU_ARCH_ARMv3; + } else if ((read_cpuid_id() & 0x00080000) == 0x00000000) { + cpu_arch = (read_cpuid_id() >> 16) & 7; + if (cpu_arch) + cpu_arch += CPU_ARCH_ARMv3; + } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) { + unsigned int mmfr0; + + /* Revised CPUID format. Read the Memory Model Feature + * Register 0 and check for VMSAv7 or PMSAv7 */ + asm("mrc p15, 0, %0, c0, c1, 4" + : "=r" (mmfr0)); + if ((mmfr0 & 0x0000000f) >= 0x00000003 || + (mmfr0 & 0x000000f0) >= 0x00000030) + cpu_arch = CPU_ARCH_ARMv7; + else if ((mmfr0 & 0x0000000f) == 0x00000002 || + (mmfr0 & 0x000000f0) == 0x00000020) + cpu_arch = CPU_ARCH_ARMv6; + else + cpu_arch = CPU_ARCH_UNKNOWN; + } else + cpu_arch = CPU_ARCH_UNKNOWN; + + return cpu_arch; +} + +static int cpu_has_aliasing_icache(unsigned int arch) +{ + int aliasing_icache; + unsigned int id_reg, num_sets, line_size; + + /* arch specifies the register format */ + switch (arch) { + case CPU_ARCH_ARMv7: + asm("mcr p15, 2, %0, c0, c0, 0 @ set CSSELR" + : /* No output operands */ + : "r" (1)); + isb(); + asm("mrc p15, 1, %0, c0, c0, 0 @ read CCSIDR" + : "=r" (id_reg)); + line_size = 4 << ((id_reg & 0x7) + 2); + num_sets = ((id_reg >> 13) & 0x7fff) + 1; + aliasing_icache = (line_size * num_sets) > PAGE_SIZE; + break; + case CPU_ARCH_ARMv6: + aliasing_icache = read_cpuid_cachetype() & (1 << 11); + break; + default: + /* I-cache aliases will be handled by D-cache aliasing code */ + aliasing_icache = 0; + } + + return aliasing_icache; +} + +static void __init cacheid_init(void) +{ + unsigned int cachetype = read_cpuid_cachetype(); + unsigned int arch = cpu_architecture(); + + if (arch >= CPU_ARCH_ARMv6) { + if ((cachetype & (7 << 29)) == 4 << 29) { + /* ARMv7 register format */ + cacheid = CACHEID_VIPT_NONALIASING; + if ((cachetype & (3 << 14)) == 1 << 14) + cacheid |= CACHEID_ASID_TAGGED; + else if (cpu_has_aliasing_icache(CPU_ARCH_ARMv7)) + cacheid |= CACHEID_VIPT_I_ALIASING; + } else if (cachetype & (1 << 23)) { + cacheid = CACHEID_VIPT_ALIASING; + } else { + cacheid = CACHEID_VIPT_NONALIASING; + if (cpu_has_aliasing_icache(CPU_ARCH_ARMv6)) + cacheid |= CACHEID_VIPT_I_ALIASING; + } + } else { + cacheid = CACHEID_VIVT; + } + + printk("CPU: %s data cache, %s instruction cache\n", + cache_is_vivt() ? "VIVT" : + cache_is_vipt_aliasing() ? "VIPT aliasing" : + cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown", + cache_is_vivt() ? "VIVT" : + icache_is_vivt_asid_tagged() ? "VIVT ASID tagged" : + icache_is_vipt_aliasing() ? "VIPT aliasing" : + cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown"); +} + +/* + * These functions re-use the assembly code in head.S, which + * already provide the required functionality. + */ +extern struct proc_info_list *lookup_processor_type(unsigned int); + +void __init early_print(const char *str, ...) +{ + extern void printascii(const char *); + char buf[256]; + va_list ap; + + va_start(ap, str); + vsnprintf(buf, sizeof(buf), str, ap); + va_end(ap); + +#ifdef CONFIG_DEBUG_LL + printascii(buf); +#endif + printk("%s", buf); +} + +static void __init feat_v6_fixup(void) +{ + int id = read_cpuid_id(); + + if ((id & 0xff0f0000) != 0x41070000) + return; + + /* + * HWCAP_TLS is available only on 1136 r1p0 and later, + * see also kuser_get_tls_init. + */ + if ((((id >> 4) & 0xfff) == 0xb36) && (((id >> 20) & 3) == 0)) + elf_hwcap &= ~HWCAP_TLS; +} + +static void __init setup_processor(void) +{ + struct proc_info_list *list; + + /* + * locate processor in the list of supported processor + * types. The linker builds this table for us from the + * entries in arch/arm/mm/proc-*.S + */ + list = lookup_processor_type(read_cpuid_id()); + if (!list) { + printk("CPU configuration botched (ID %08x), unable " + "to continue.\n", read_cpuid_id()); + while (1); + } + + cpu_name = list->cpu_name; + +#ifdef MULTI_CPU + processor = *list->proc; +#endif +#ifdef MULTI_TLB + cpu_tlb = *list->tlb; +#endif +#ifdef MULTI_USER + cpu_user = *list->user; +#endif +#ifdef MULTI_CACHE + cpu_cache = *list->cache; +#endif + + printk("CPU: %s [%08x] revision %d (ARMv%s), cr=%08lx\n", + cpu_name, read_cpuid_id(), read_cpuid_id() & 15, + proc_arch[cpu_architecture()], cr_alignment); + + sprintf(init_utsname()->machine, "%s%c", list->arch_name, ENDIANNESS); + sprintf(elf_platform, "%s%c", list->elf_name, ENDIANNESS); + elf_hwcap = list->elf_hwcap; +#ifndef CONFIG_ARM_THUMB + elf_hwcap &= ~HWCAP_THUMB; +#endif + + feat_v6_fixup(); + + cacheid_init(); + cpu_proc_init(); +} + +/* + * cpu_init - initialise one CPU. + * + * cpu_init sets up the per-CPU stacks. + */ +void cpu_init(void) +{ + unsigned int cpu = smp_processor_id(); + struct stack *stk = &stacks[cpu]; + + if (cpu >= NR_CPUS) { + printk(KERN_CRIT "CPU%u: bad primary CPU number\n", cpu); + BUG(); + } + + /* + * Define the placement constraint for the inline asm directive below. + * In Thumb-2, msr with an immediate value is not allowed. + */ +#ifdef CONFIG_THUMB2_KERNEL +#define PLC "r" +#else +#define PLC "I" +#endif + + /* + * setup stacks for re-entrant exception handlers + */ + __asm__ ( + "msr cpsr_c, %1\n\t" + "add r14, %0, %2\n\t" + "mov sp, r14\n\t" + "msr cpsr_c, %3\n\t" + "add r14, %0, %4\n\t" + "mov sp, r14\n\t" + "msr cpsr_c, %5\n\t" + "add r14, %0, %6\n\t" + "mov sp, r14\n\t" + "msr cpsr_c, %7" + : + : "r" (stk), + PLC (PSR_F_BIT | PSR_I_BIT | IRQ_MODE), + "I" (offsetof(struct stack, irq[0])), + PLC (PSR_F_BIT | PSR_I_BIT | ABT_MODE), + "I" (offsetof(struct stack, abt[0])), + PLC (PSR_F_BIT | PSR_I_BIT | UND_MODE), + "I" (offsetof(struct stack, und[0])), + PLC (PSR_F_BIT | PSR_I_BIT | SVC_MODE) + : "r14"); +} + +void __init dump_machine_table(void) +{ + struct machine_desc *p; + + early_print("Available machine support:\n\nID (hex)\tNAME\n"); + for_each_machine_desc(p) + early_print("%08x\t%s\n", p->nr, p->name); + + early_print("\nPlease check your kernel config and/or bootloader.\n"); + + while (true) + /* can't use cpu_relax() here as it may require MMU setup */; +} + +int __init arm_add_memory(phys_addr_t start, unsigned long size) +{ + struct membank *bank = &meminfo.bank[meminfo.nr_banks]; + + if (meminfo.nr_banks >= NR_BANKS) { + printk(KERN_CRIT "NR_BANKS too low, " + "ignoring memory at 0x%08llx\n", (long long)start); + return -EINVAL; + } + + /* + * Ensure that start/size are aligned to a page boundary. + * Size is appropriately rounded down, start is rounded up. + */ + size -= start & ~PAGE_MASK; + bank->start = PAGE_ALIGN(start); + bank->size = size & PAGE_MASK; + + /* + * Check whether this memory region has non-zero size or + * invalid node number. + */ + if (bank->size == 0) + return -EINVAL; + + meminfo.nr_banks++; + return 0; +} + +/* + * Pick out the memory size. We look for mem=size@start, + * where start and size are "size[KkMm]" + */ +static int __init early_mem(char *p) +{ + static int usermem __initdata = 0; + unsigned long size; + phys_addr_t start; + char *endp; + + /* + * If the user specifies memory size, we + * blow away any automatically generated + * size. + */ + if (usermem == 0) { + usermem = 1; + meminfo.nr_banks = 0; + } + + start = PHYS_OFFSET; + size = memparse(p, &endp); + if (*endp == '@') + start = memparse(endp + 1, NULL); + + arm_add_memory(start, size); + + return 0; +} +early_param("mem", early_mem); + +static void __init +setup_ramdisk(int doload, int prompt, int image_start, unsigned int rd_sz) +{ +#ifdef CONFIG_BLK_DEV_RAM + extern int rd_size, rd_image_start, rd_prompt, rd_doload; + + rd_image_start = image_start; + rd_prompt = prompt; + rd_doload = doload; + + if (rd_sz) + rd_size = rd_sz; +#endif +} + +static void __init request_standard_resources(struct machine_desc *mdesc) +{ + struct memblock_region *region; + struct resource *res; + + kernel_code.start = virt_to_phys(_text); + kernel_code.end = virt_to_phys(_etext - 1); + kernel_data.start = virt_to_phys(_sdata); + kernel_data.end = virt_to_phys(_end - 1); + + for_each_memblock(memory, region) { + res = alloc_bootmem_low(sizeof(*res)); + res->name = "System RAM"; + res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region)); + res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1; + res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; + + request_resource(&iomem_resource, res); + + if (kernel_code.start >= res->start && + kernel_code.end <= res->end) + request_resource(res, &kernel_code); + if (kernel_data.start >= res->start && + kernel_data.end <= res->end) + request_resource(res, &kernel_data); + } + + if (mdesc->video_start) { + video_ram.start = mdesc->video_start; + video_ram.end = mdesc->video_end; + request_resource(&iomem_resource, &video_ram); + } + + /* + * Some machines don't have the possibility of ever + * possessing lp0, lp1 or lp2 + */ + if (mdesc->reserve_lp0) + request_resource(&ioport_resource, &lp0); + if (mdesc->reserve_lp1) + request_resource(&ioport_resource, &lp1); + if (mdesc->reserve_lp2) + request_resource(&ioport_resource, &lp2); +} + +/* + * Tag parsing. + * + * This is the new way of passing data to the kernel at boot time. Rather + * than passing a fixed inflexible structure to the kernel, we pass a list + * of variable-sized tags to the kernel. The first tag must be a ATAG_CORE + * tag for the list to be recognised (to distinguish the tagged list from + * a param_struct). The list is terminated with a zero-length tag (this tag + * is not parsed in any way). + */ +static int __init parse_tag_core(const struct tag *tag) +{ + if (tag->hdr.size > 2) { + if ((tag->u.core.flags & 1) == 0) + root_mountflags &= ~MS_RDONLY; + ROOT_DEV = old_decode_dev(tag->u.core.rootdev); + } + return 0; +} + +__tagtable(ATAG_CORE, parse_tag_core); + +static int __init parse_tag_mem32(const struct tag *tag) +{ + return arm_add_memory(tag->u.mem.start, tag->u.mem.size); +} + +__tagtable(ATAG_MEM, parse_tag_mem32); + +#if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) +struct screen_info screen_info = { + .orig_video_lines = 30, + .orig_video_cols = 80, + .orig_video_mode = 0, + .orig_video_ega_bx = 0, + .orig_video_isVGA = 1, + .orig_video_points = 8 +}; + +static int __init parse_tag_videotext(const struct tag *tag) +{ + screen_info.orig_x = tag->u.videotext.x; + screen_info.orig_y = tag->u.videotext.y; + screen_info.orig_video_page = tag->u.videotext.video_page; + screen_info.orig_video_mode = tag->u.videotext.video_mode; + screen_info.orig_video_cols = tag->u.videotext.video_cols; + screen_info.orig_video_ega_bx = tag->u.videotext.video_ega_bx; + screen_info.orig_video_lines = tag->u.videotext.video_lines; + screen_info.orig_video_isVGA = tag->u.videotext.video_isvga; + screen_info.orig_video_points = tag->u.videotext.video_points; + return 0; +} + +__tagtable(ATAG_VIDEOTEXT, parse_tag_videotext); +#endif + +static int __init parse_tag_ramdisk(const struct tag *tag) +{ + setup_ramdisk((tag->u.ramdisk.flags & 1) == 0, + (tag->u.ramdisk.flags & 2) == 0, + tag->u.ramdisk.start, tag->u.ramdisk.size); + return 0; +} + +__tagtable(ATAG_RAMDISK, parse_tag_ramdisk); + +static int __init parse_tag_serialnr(const struct tag *tag) +{ + system_serial_low = tag->u.serialnr.low; + system_serial_high = tag->u.serialnr.high; + return 0; +} + +__tagtable(ATAG_SERIAL, parse_tag_serialnr); + +static int __init parse_tag_revision(const struct tag *tag) +{ + system_rev = tag->u.revision.rev; + return 0; +} + +__tagtable(ATAG_REVISION, parse_tag_revision); + +static int __init parse_tag_cmdline(const struct tag *tag) +{ +#if defined(CONFIG_CMDLINE_EXTEND) + strlcat(default_command_line, " ", COMMAND_LINE_SIZE); + strlcat(default_command_line, tag->u.cmdline.cmdline, + COMMAND_LINE_SIZE); +#elif defined(CONFIG_CMDLINE_FORCE) + pr_warning("Ignoring tag cmdline (using the default kernel command line)\n"); +#else + strlcpy(default_command_line, tag->u.cmdline.cmdline, + COMMAND_LINE_SIZE); +#endif + return 0; +} + +__tagtable(ATAG_CMDLINE, parse_tag_cmdline); + +/* + * Scan the tag table for this tag, and call its parse function. + * The tag table is built by the linker from all the __tagtable + * declarations. + */ +static int __init parse_tag(const struct tag *tag) +{ + extern struct tagtable __tagtable_begin, __tagtable_end; + struct tagtable *t; + + for (t = &__tagtable_begin; t < &__tagtable_end; t++) + if (tag->hdr.tag == t->tag) { + t->parse(tag); + break; + } + + return t < &__tagtable_end; +} + +/* + * Parse all tags in the list, checking both the global and architecture + * specific tag tables. + */ +static void __init parse_tags(const struct tag *t) +{ + for (; t->hdr.size; t = tag_next(t)) + if (!parse_tag(t)) + printk(KERN_WARNING + "Ignoring unrecognised tag 0x%08x\n", + t->hdr.tag); +} + +/* + * This holds our defaults. + */ +static struct init_tags { + struct tag_header hdr1; + struct tag_core core; + struct tag_header hdr2; + struct tag_mem32 mem; + struct tag_header hdr3; +} init_tags __initdata = { + { tag_size(tag_core), ATAG_CORE }, + { 1, PAGE_SIZE, 0xff }, + { tag_size(tag_mem32), ATAG_MEM }, + { MEM_SIZE }, + { 0, ATAG_NONE } +}; + +static int __init customize_machine(void) +{ + /* customizes platform devices, or adds new ones */ + if (machine_desc->init_machine) + machine_desc->init_machine(); + return 0; +} +arch_initcall(customize_machine); + +#ifdef CONFIG_KEXEC +static inline unsigned long long get_total_mem(void) +{ + unsigned long total; + + total = max_low_pfn - min_low_pfn; + return total << PAGE_SHIFT; +} + +/** + * reserve_crashkernel() - reserves memory are for crash kernel + * + * This function reserves memory area given in "crashkernel=" kernel command + * line parameter. The memory reserved is used by a dump capture kernel when + * primary kernel is crashing. + */ +static void __init reserve_crashkernel(void) +{ + unsigned long long crash_size, crash_base; + unsigned long long total_mem; + int ret; + + total_mem = get_total_mem(); + ret = parse_crashkernel(boot_command_line, total_mem, + &crash_size, &crash_base); + if (ret) + return; + + ret = reserve_bootmem(crash_base, crash_size, BOOTMEM_EXCLUSIVE); + if (ret < 0) { + printk(KERN_WARNING "crashkernel reservation failed - " + "memory is in use (0x%lx)\n", (unsigned long)crash_base); + return; + } + + printk(KERN_INFO "Reserving %ldMB of memory at %ldMB " + "for crashkernel (System RAM: %ldMB)\n", + (unsigned long)(crash_size >> 20), + (unsigned long)(crash_base >> 20), + (unsigned long)(total_mem >> 20)); + + crashk_res.start = crash_base; + crashk_res.end = crash_base + crash_size - 1; + insert_resource(&iomem_resource, &crashk_res); +} +#else +static inline void reserve_crashkernel(void) {} +#endif /* CONFIG_KEXEC */ + +static void __init squash_mem_tags(struct tag *tag) +{ + for (; tag->hdr.size; tag = tag_next(tag)) + if (tag->hdr.tag == ATAG_MEM) + tag->hdr.tag = ATAG_NONE; +} + +static struct machine_desc * __init setup_machine_tags(unsigned int nr) +{ + struct tag *tags = (struct tag *)&init_tags; + struct machine_desc *mdesc = NULL, *p; + char *from = default_command_line; + + init_tags.mem.start = PHYS_OFFSET; + + /* + * locate machine in the list of supported machines. + */ + for_each_machine_desc(p) + if (nr == p->nr) { + printk("Machine: %s\n", p->name); + mdesc = p; + break; + } + + if (!mdesc) { + early_print("\nError: unrecognized/unsupported machine ID" + " (r1 = 0x%08x).\n\n", nr); + dump_machine_table(); /* does not return */ + } + + if (__atags_pointer) + tags = phys_to_virt(__atags_pointer); + else if (mdesc->boot_params) { +#ifdef CONFIG_MMU + /* + * We still are executing with a minimal MMU mapping created + * with the presumption that the machine default for this + * is located in the first MB of RAM. Anything else will + * fault and silently hang the kernel at this point. + */ + if (mdesc->boot_params < PHYS_OFFSET || + mdesc->boot_params >= PHYS_OFFSET + SZ_1M) { + printk(KERN_WARNING + "Default boot params at physical 0x%08lx out of reach\n", + mdesc->boot_params); + } else +#endif + { + tags = phys_to_virt(mdesc->boot_params); + } + } + +#if defined(CONFIG_DEPRECATED_PARAM_STRUCT) + /* + * If we have the old style parameters, convert them to + * a tag list. + */ + if (tags->hdr.tag != ATAG_CORE) + convert_to_tag_list(tags); +#endif + + if (tags->hdr.tag != ATAG_CORE) { +#if defined(CONFIG_OF) + /* + * If CONFIG_OF is set, then assume this is a reasonably + * modern system that should pass boot parameters + */ + early_print("Warning: Neither atags nor dtb found\n"); +#endif + tags = (struct tag *)&init_tags; + } + + if (mdesc->fixup) + mdesc->fixup(mdesc, tags, &from, &meminfo); + + if (tags->hdr.tag == ATAG_CORE) { + if (meminfo.nr_banks != 0) + squash_mem_tags(tags); + save_atags(tags); + parse_tags(tags); + } + + /* parse_early_param needs a boot_command_line */ + strlcpy(boot_command_line, from, COMMAND_LINE_SIZE); + + return mdesc; +} + + +void __init setup_arch(char **cmdline_p) +{ + struct machine_desc *mdesc; + + unwind_init(); + + setup_processor(); + mdesc = setup_machine_fdt(__atags_pointer); + if (!mdesc) + mdesc = setup_machine_tags(machine_arch_type); + machine_desc = mdesc; + machine_name = mdesc->name; + + if (mdesc->soft_reboot) + reboot_setup("s"); + + init_mm.start_code = (unsigned long) _text; + init_mm.end_code = (unsigned long) _etext; + init_mm.end_data = (unsigned long) _edata; + init_mm.brk = (unsigned long) _end; + + /* populate cmd_line too for later use, preserving boot_command_line */ + strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE); + *cmdline_p = cmd_line; + + parse_early_param(); + + sanity_check_meminfo(); + arm_memblock_init(&meminfo, mdesc); + + paging_init(mdesc); + request_standard_resources(mdesc); + + unflatten_device_tree(); + +#ifdef CONFIG_SMP + if (is_smp()) + smp_init_cpus(); +#endif + reserve_crashkernel(); + + cpu_init(); + tcm_init(); + +#ifdef CONFIG_MULTI_IRQ_HANDLER + handle_arch_irq = mdesc->handle_irq; +#endif + +#ifdef CONFIG_VT +#if defined(CONFIG_VGA_CONSOLE) + conswitchp = &vga_con; +#elif defined(CONFIG_DUMMY_CONSOLE) + conswitchp = &dummy_con; +#endif +#endif + early_trap_init(); + + if (mdesc->init_early) + mdesc->init_early(); +} + + +static int __init topology_init(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct cpuinfo_arm *cpuinfo = &per_cpu(cpu_data, cpu); + cpuinfo->cpu.hotpluggable = 1; + register_cpu(&cpuinfo->cpu, cpu); + } + + return 0; +} +subsys_initcall(topology_init); + +#ifdef CONFIG_HAVE_PROC_CPU +static int __init proc_cpu_init(void) +{ + struct proc_dir_entry *res; + + res = proc_mkdir("cpu", NULL); + if (!res) + return -ENOMEM; + return 0; +} +fs_initcall(proc_cpu_init); +#endif + +static const char *hwcap_str[] = { + "swp", + "half", + "thumb", + "26bit", + "fastmult", + "fpa", + "vfp", + "edsp", + "java", + "iwmmxt", + "crunch", + "thumbee", + "neon", + "vfpv3", + "vfpv3d16", + NULL +}; + +static int c_show(struct seq_file *m, void *v) +{ + int i; + + seq_printf(m, "Processor\t: %s rev %d (%s)\n", + cpu_name, read_cpuid_id() & 15, elf_platform); + +#if defined(CONFIG_SMP) + for_each_online_cpu(i) { + /* + * glibc reads /proc/cpuinfo to determine the number of + * online processors, looking for lines beginning with + * "processor". Give glibc what it expects. + */ + seq_printf(m, "processor\t: %d\n", i); + seq_printf(m, "BogoMIPS\t: %lu.%02lu\n\n", + per_cpu(cpu_data, i).loops_per_jiffy / (500000UL/HZ), + (per_cpu(cpu_data, i).loops_per_jiffy / (5000UL/HZ)) % 100); + } +#else /* CONFIG_SMP */ + seq_printf(m, "BogoMIPS\t: %lu.%02lu\n", + loops_per_jiffy / (500000/HZ), + (loops_per_jiffy / (5000/HZ)) % 100); +#endif + + /* dump out the processor features */ + seq_puts(m, "Features\t: "); + + for (i = 0; hwcap_str[i]; i++) + if (elf_hwcap & (1 << i)) + seq_printf(m, "%s ", hwcap_str[i]); + + seq_printf(m, "\nCPU implementer\t: 0x%02x\n", read_cpuid_id() >> 24); + seq_printf(m, "CPU architecture: %s\n", proc_arch[cpu_architecture()]); + + if ((read_cpuid_id() & 0x0008f000) == 0x00000000) { + /* pre-ARM7 */ + seq_printf(m, "CPU part\t: %07x\n", read_cpuid_id() >> 4); + } else { + if ((read_cpuid_id() & 0x0008f000) == 0x00007000) { + /* ARM7 */ + seq_printf(m, "CPU variant\t: 0x%02x\n", + (read_cpuid_id() >> 16) & 127); + } else { + /* post-ARM7 */ + seq_printf(m, "CPU variant\t: 0x%x\n", + (read_cpuid_id() >> 20) & 15); + } + seq_printf(m, "CPU part\t: 0x%03x\n", + (read_cpuid_id() >> 4) & 0xfff); + } + seq_printf(m, "CPU revision\t: %d\n", read_cpuid_id() & 15); + + seq_puts(m, "\n"); + + seq_printf(m, "Hardware\t: %s\n", machine_name); + seq_printf(m, "Revision\t: %04x\n", system_rev); + seq_printf(m, "Serial\t\t: %08x%08x\n", + system_serial_high, system_serial_low); + + return 0; +} + +static void *c_start(struct seq_file *m, loff_t *pos) +{ + return *pos < 1 ? (void *)1 : NULL; +} + +static void *c_next(struct seq_file *m, void *v, loff_t *pos) +{ + ++*pos; + return NULL; +} + +static void c_stop(struct seq_file *m, void *v) +{ +} + +const struct seq_operations cpuinfo_op = { + .start = c_start, + .next = c_next, + .stop = c_stop, + .show = c_show +}; diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c new file mode 100644 index 00000000..9e617bd4 --- /dev/null +++ b/arch/arm/kernel/signal.c @@ -0,0 +1,796 @@ +/* + * linux/arch/arm/kernel/signal.c + * + * Copyright (C) 1995-2009 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/errno.h> +#include <linux/signal.h> +#include <linux/personality.h> +#include <linux/freezer.h> +#include <linux/uaccess.h> +#include <linux/tracehook.h> + +#include <asm/elf.h> +#include <asm/cacheflush.h> +#include <asm/ucontext.h> +#include <asm/unistd.h> +#include <asm/vfp.h> + +#include "signal.h" + +#define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP))) + +/* + * For ARM syscalls, we encode the syscall number into the instruction. + */ +#define SWI_SYS_SIGRETURN (0xef000000|(__NR_sigreturn)|(__NR_OABI_SYSCALL_BASE)) +#define SWI_SYS_RT_SIGRETURN (0xef000000|(__NR_rt_sigreturn)|(__NR_OABI_SYSCALL_BASE)) +#define SWI_SYS_RESTART (0xef000000|__NR_restart_syscall|__NR_OABI_SYSCALL_BASE) + +/* + * With EABI, the syscall number has to be loaded into r7. + */ +#define MOV_R7_NR_SIGRETURN (0xe3a07000 | (__NR_sigreturn - __NR_SYSCALL_BASE)) +#define MOV_R7_NR_RT_SIGRETURN (0xe3a07000 | (__NR_rt_sigreturn - __NR_SYSCALL_BASE)) + +/* + * For Thumb syscalls, we pass the syscall number via r7. We therefore + * need two 16-bit instructions. + */ +#define SWI_THUMB_SIGRETURN (0xdf00 << 16 | 0x2700 | (__NR_sigreturn - __NR_SYSCALL_BASE)) +#define SWI_THUMB_RT_SIGRETURN (0xdf00 << 16 | 0x2700 | (__NR_rt_sigreturn - __NR_SYSCALL_BASE)) + +const unsigned long sigreturn_codes[7] = { + MOV_R7_NR_SIGRETURN, SWI_SYS_SIGRETURN, SWI_THUMB_SIGRETURN, + MOV_R7_NR_RT_SIGRETURN, SWI_SYS_RT_SIGRETURN, SWI_THUMB_RT_SIGRETURN, +}; + +/* + * Either we support OABI only, or we have EABI with the OABI + * compat layer enabled. In the later case we don't know if + * user space is EABI or not, and if not we must not clobber r7. + * Always using the OABI syscall solves that issue and works for + * all those cases. + */ +const unsigned long syscall_restart_code[2] = { + SWI_SYS_RESTART, /* swi __NR_restart_syscall */ + 0xe49df004, /* ldr pc, [sp], #4 */ +}; + +/* + * atomically swap in the new signal mask, and wait for a signal. + */ +asmlinkage int sys_sigsuspend(int restart, unsigned long oldmask, old_sigset_t mask) +{ + mask &= _BLOCKABLE; + spin_lock_irq(¤t->sighand->siglock); + current->saved_sigmask = current->blocked; + siginitset(¤t->blocked, mask); + recalc_sigpending(); + spin_unlock_irq(¤t->sighand->siglock); + + current->state = TASK_INTERRUPTIBLE; + schedule(); + set_restore_sigmask(); + return -ERESTARTNOHAND; +} + +asmlinkage int +sys_sigaction(int sig, const struct old_sigaction __user *act, + struct old_sigaction __user *oact) +{ + struct k_sigaction new_ka, old_ka; + int ret; + + if (act) { + old_sigset_t mask; + if (!access_ok(VERIFY_READ, act, sizeof(*act)) || + __get_user(new_ka.sa.sa_handler, &act->sa_handler) || + __get_user(new_ka.sa.sa_restorer, &act->sa_restorer)) + return -EFAULT; + __get_user(new_ka.sa.sa_flags, &act->sa_flags); + __get_user(mask, &act->sa_mask); + siginitset(&new_ka.sa.sa_mask, mask); + } + + ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); + + if (!ret && oact) { + if (!access_ok(VERIFY_WRITE, oact, sizeof(*oact)) || + __put_user(old_ka.sa.sa_handler, &oact->sa_handler) || + __put_user(old_ka.sa.sa_restorer, &oact->sa_restorer)) + return -EFAULT; + __put_user(old_ka.sa.sa_flags, &oact->sa_flags); + __put_user(old_ka.sa.sa_mask.sig[0], &oact->sa_mask); + } + + return ret; +} + +#ifdef CONFIG_CRUNCH +static int preserve_crunch_context(struct crunch_sigframe __user *frame) +{ + char kbuf[sizeof(*frame) + 8]; + struct crunch_sigframe *kframe; + + /* the crunch context must be 64 bit aligned */ + kframe = (struct crunch_sigframe *)((unsigned long)(kbuf + 8) & ~7); + kframe->magic = CRUNCH_MAGIC; + kframe->size = CRUNCH_STORAGE_SIZE; + crunch_task_copy(current_thread_info(), &kframe->storage); + return __copy_to_user(frame, kframe, sizeof(*frame)); +} + +static int restore_crunch_context(struct crunch_sigframe __user *frame) +{ + char kbuf[sizeof(*frame) + 8]; + struct crunch_sigframe *kframe; + + /* the crunch context must be 64 bit aligned */ + kframe = (struct crunch_sigframe *)((unsigned long)(kbuf + 8) & ~7); + if (__copy_from_user(kframe, frame, sizeof(*frame))) + return -1; + if (kframe->magic != CRUNCH_MAGIC || + kframe->size != CRUNCH_STORAGE_SIZE) + return -1; + crunch_task_restore(current_thread_info(), &kframe->storage); + return 0; +} +#endif + +#ifdef CONFIG_IWMMXT + +static int preserve_iwmmxt_context(struct iwmmxt_sigframe *frame) +{ + char kbuf[sizeof(*frame) + 8]; + struct iwmmxt_sigframe *kframe; + + /* the iWMMXt context must be 64 bit aligned */ + kframe = (struct iwmmxt_sigframe *)((unsigned long)(kbuf + 8) & ~7); + kframe->magic = IWMMXT_MAGIC; + kframe->size = IWMMXT_STORAGE_SIZE; + iwmmxt_task_copy(current_thread_info(), &kframe->storage); + return __copy_to_user(frame, kframe, sizeof(*frame)); +} + +static int restore_iwmmxt_context(struct iwmmxt_sigframe *frame) +{ + char kbuf[sizeof(*frame) + 8]; + struct iwmmxt_sigframe *kframe; + + /* the iWMMXt context must be 64 bit aligned */ + kframe = (struct iwmmxt_sigframe *)((unsigned long)(kbuf + 8) & ~7); + if (__copy_from_user(kframe, frame, sizeof(*frame))) + return -1; + if (kframe->magic != IWMMXT_MAGIC || + kframe->size != IWMMXT_STORAGE_SIZE) + return -1; + iwmmxt_task_restore(current_thread_info(), &kframe->storage); + return 0; +} + +#endif + +#ifdef CONFIG_VFP + +static int preserve_vfp_context(struct vfp_sigframe __user *frame) +{ + struct thread_info *thread = current_thread_info(); + struct vfp_hard_struct *h = &thread->vfpstate.hard; + const unsigned long magic = VFP_MAGIC; + const unsigned long size = VFP_STORAGE_SIZE; + int err = 0; + + vfp_sync_hwstate(thread); + __put_user_error(magic, &frame->magic, err); + __put_user_error(size, &frame->size, err); + + /* + * Copy the floating point registers. There can be unused + * registers see asm/hwcap.h for details. + */ + err |= __copy_to_user(&frame->ufp.fpregs, &h->fpregs, + sizeof(h->fpregs)); + /* + * Copy the status and control register. + */ + __put_user_error(h->fpscr, &frame->ufp.fpscr, err); + + /* + * Copy the exception registers. + */ + __put_user_error(h->fpexc, &frame->ufp_exc.fpexc, err); + __put_user_error(h->fpinst, &frame->ufp_exc.fpinst, err); + __put_user_error(h->fpinst2, &frame->ufp_exc.fpinst2, err); + + return err ? -EFAULT : 0; +} + +static int restore_vfp_context(struct vfp_sigframe __user *frame) +{ + struct thread_info *thread = current_thread_info(); + struct vfp_hard_struct *h = &thread->vfpstate.hard; + unsigned long magic; + unsigned long size; + unsigned long fpexc; + int err = 0; + + __get_user_error(magic, &frame->magic, err); + __get_user_error(size, &frame->size, err); + + if (err) + return -EFAULT; + if (magic != VFP_MAGIC || size != VFP_STORAGE_SIZE) + return -EINVAL; + + vfp_flush_hwstate(thread); + + /* + * Copy the floating point registers. There can be unused + * registers see asm/hwcap.h for details. + */ + err |= __copy_from_user(&h->fpregs, &frame->ufp.fpregs, + sizeof(h->fpregs)); + /* + * Copy the status and control register. + */ + __get_user_error(h->fpscr, &frame->ufp.fpscr, err); + + /* + * Sanitise and restore the exception registers. + */ + __get_user_error(fpexc, &frame->ufp_exc.fpexc, err); + /* Ensure the VFP is enabled. */ + fpexc |= FPEXC_EN; + /* Ensure FPINST2 is invalid and the exception flag is cleared. */ + fpexc &= ~(FPEXC_EX | FPEXC_FP2V); + h->fpexc = fpexc; + + __get_user_error(h->fpinst, &frame->ufp_exc.fpinst, err); + __get_user_error(h->fpinst2, &frame->ufp_exc.fpinst2, err); + + return err ? -EFAULT : 0; +} + +#endif + +/* + * Do a signal return; undo the signal stack. These are aligned to 64-bit. + */ +struct sigframe { + struct ucontext uc; + unsigned long retcode[2]; +}; + +struct rt_sigframe { + struct siginfo info; + struct sigframe sig; +}; + +static int restore_sigframe(struct pt_regs *regs, struct sigframe __user *sf) +{ + struct aux_sigframe __user *aux; + sigset_t set; + int err; + + err = __copy_from_user(&set, &sf->uc.uc_sigmask, sizeof(set)); + if (err == 0) { + sigdelsetmask(&set, ~_BLOCKABLE); + spin_lock_irq(¤t->sighand->siglock); + current->blocked = set; + recalc_sigpending(); + spin_unlock_irq(¤t->sighand->siglock); + } + + __get_user_error(regs->ARM_r0, &sf->uc.uc_mcontext.arm_r0, err); + __get_user_error(regs->ARM_r1, &sf->uc.uc_mcontext.arm_r1, err); + __get_user_error(regs->ARM_r2, &sf->uc.uc_mcontext.arm_r2, err); + __get_user_error(regs->ARM_r3, &sf->uc.uc_mcontext.arm_r3, err); + __get_user_error(regs->ARM_r4, &sf->uc.uc_mcontext.arm_r4, err); + __get_user_error(regs->ARM_r5, &sf->uc.uc_mcontext.arm_r5, err); + __get_user_error(regs->ARM_r6, &sf->uc.uc_mcontext.arm_r6, err); + __get_user_error(regs->ARM_r7, &sf->uc.uc_mcontext.arm_r7, err); + __get_user_error(regs->ARM_r8, &sf->uc.uc_mcontext.arm_r8, err); + __get_user_error(regs->ARM_r9, &sf->uc.uc_mcontext.arm_r9, err); + __get_user_error(regs->ARM_r10, &sf->uc.uc_mcontext.arm_r10, err); + __get_user_error(regs->ARM_fp, &sf->uc.uc_mcontext.arm_fp, err); + __get_user_error(regs->ARM_ip, &sf->uc.uc_mcontext.arm_ip, err); + __get_user_error(regs->ARM_sp, &sf->uc.uc_mcontext.arm_sp, err); + __get_user_error(regs->ARM_lr, &sf->uc.uc_mcontext.arm_lr, err); + __get_user_error(regs->ARM_pc, &sf->uc.uc_mcontext.arm_pc, err); + __get_user_error(regs->ARM_cpsr, &sf->uc.uc_mcontext.arm_cpsr, err); + + err |= !valid_user_regs(regs); + + aux = (struct aux_sigframe __user *) sf->uc.uc_regspace; +#ifdef CONFIG_CRUNCH + if (err == 0) + err |= restore_crunch_context(&aux->crunch); +#endif +#ifdef CONFIG_IWMMXT + if (err == 0 && test_thread_flag(TIF_USING_IWMMXT)) + err |= restore_iwmmxt_context(&aux->iwmmxt); +#endif +#ifdef CONFIG_VFP + if (err == 0) + err |= restore_vfp_context(&aux->vfp); +#endif + + return err; +} + +asmlinkage int sys_sigreturn(struct pt_regs *regs) +{ + struct sigframe __user *frame; + + /* Always make any pending restarted system calls return -EINTR */ + current_thread_info()->restart_block.fn = do_no_restart_syscall; + + /* + * Since we stacked the signal on a 64-bit boundary, + * then 'sp' should be word aligned here. If it's + * not, then the user is trying to mess with us. + */ + if (regs->ARM_sp & 7) + goto badframe; + + frame = (struct sigframe __user *)regs->ARM_sp; + + if (!access_ok(VERIFY_READ, frame, sizeof (*frame))) + goto badframe; + + if (restore_sigframe(regs, frame)) + goto badframe; + + return regs->ARM_r0; + +badframe: + force_sig(SIGSEGV, current); + return 0; +} + +asmlinkage int sys_rt_sigreturn(struct pt_regs *regs) +{ + struct rt_sigframe __user *frame; + + /* Always make any pending restarted system calls return -EINTR */ + current_thread_info()->restart_block.fn = do_no_restart_syscall; + + /* + * Since we stacked the signal on a 64-bit boundary, + * then 'sp' should be word aligned here. If it's + * not, then the user is trying to mess with us. + */ + if (regs->ARM_sp & 7) + goto badframe; + + frame = (struct rt_sigframe __user *)regs->ARM_sp; + + if (!access_ok(VERIFY_READ, frame, sizeof (*frame))) + goto badframe; + + if (restore_sigframe(regs, &frame->sig)) + goto badframe; + + if (do_sigaltstack(&frame->sig.uc.uc_stack, NULL, regs->ARM_sp) == -EFAULT) + goto badframe; + + return regs->ARM_r0; + +badframe: + force_sig(SIGSEGV, current); + return 0; +} + +static int +setup_sigframe(struct sigframe __user *sf, struct pt_regs *regs, sigset_t *set) +{ + struct aux_sigframe __user *aux; + int err = 0; + + __put_user_error(regs->ARM_r0, &sf->uc.uc_mcontext.arm_r0, err); + __put_user_error(regs->ARM_r1, &sf->uc.uc_mcontext.arm_r1, err); + __put_user_error(regs->ARM_r2, &sf->uc.uc_mcontext.arm_r2, err); + __put_user_error(regs->ARM_r3, &sf->uc.uc_mcontext.arm_r3, err); + __put_user_error(regs->ARM_r4, &sf->uc.uc_mcontext.arm_r4, err); + __put_user_error(regs->ARM_r5, &sf->uc.uc_mcontext.arm_r5, err); + __put_user_error(regs->ARM_r6, &sf->uc.uc_mcontext.arm_r6, err); + __put_user_error(regs->ARM_r7, &sf->uc.uc_mcontext.arm_r7, err); + __put_user_error(regs->ARM_r8, &sf->uc.uc_mcontext.arm_r8, err); + __put_user_error(regs->ARM_r9, &sf->uc.uc_mcontext.arm_r9, err); + __put_user_error(regs->ARM_r10, &sf->uc.uc_mcontext.arm_r10, err); + __put_user_error(regs->ARM_fp, &sf->uc.uc_mcontext.arm_fp, err); + __put_user_error(regs->ARM_ip, &sf->uc.uc_mcontext.arm_ip, err); + __put_user_error(regs->ARM_sp, &sf->uc.uc_mcontext.arm_sp, err); + __put_user_error(regs->ARM_lr, &sf->uc.uc_mcontext.arm_lr, err); + __put_user_error(regs->ARM_pc, &sf->uc.uc_mcontext.arm_pc, err); + __put_user_error(regs->ARM_cpsr, &sf->uc.uc_mcontext.arm_cpsr, err); + + __put_user_error(current->thread.trap_no, &sf->uc.uc_mcontext.trap_no, err); + __put_user_error(current->thread.error_code, &sf->uc.uc_mcontext.error_code, err); + __put_user_error(current->thread.address, &sf->uc.uc_mcontext.fault_address, err); + __put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err); + + err |= __copy_to_user(&sf->uc.uc_sigmask, set, sizeof(*set)); + + aux = (struct aux_sigframe __user *) sf->uc.uc_regspace; +#ifdef CONFIG_CRUNCH + if (err == 0) + err |= preserve_crunch_context(&aux->crunch); +#endif +#ifdef CONFIG_IWMMXT + if (err == 0 && test_thread_flag(TIF_USING_IWMMXT)) + err |= preserve_iwmmxt_context(&aux->iwmmxt); +#endif +#ifdef CONFIG_VFP + if (err == 0) + err |= preserve_vfp_context(&aux->vfp); +#endif + __put_user_error(0, &aux->end_magic, err); + + return err; +} + +static inline void __user * +get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, int framesize) +{ + unsigned long sp = regs->ARM_sp; + void __user *frame; + + /* + * This is the X/Open sanctioned signal stack switching. + */ + if ((ka->sa.sa_flags & SA_ONSTACK) && !sas_ss_flags(sp)) + sp = current->sas_ss_sp + current->sas_ss_size; + + /* + * ATPCS B01 mandates 8-byte alignment + */ + frame = (void __user *)((sp - framesize) & ~7); + + /* + * Check that we can actually write to the signal frame. + */ + if (!access_ok(VERIFY_WRITE, frame, framesize)) + frame = NULL; + + return frame; +} + +static int +setup_return(struct pt_regs *regs, struct k_sigaction *ka, + unsigned long __user *rc, void __user *frame, int usig) +{ + unsigned long handler = (unsigned long)ka->sa.sa_handler; + unsigned long retcode; + int thumb = 0; + unsigned long cpsr = regs->ARM_cpsr & ~(PSR_f | PSR_E_BIT); + + cpsr |= PSR_ENDSTATE; + + /* + * Maybe we need to deliver a 32-bit signal to a 26-bit task. + */ + if (ka->sa.sa_flags & SA_THIRTYTWO) + cpsr = (cpsr & ~MODE_MASK) | USR_MODE; + +#ifdef CONFIG_ARM_THUMB + if (elf_hwcap & HWCAP_THUMB) { + /* + * The LSB of the handler determines if we're going to + * be using THUMB or ARM mode for this signal handler. + */ + thumb = handler & 1; + + if (thumb) { + cpsr |= PSR_T_BIT; +#if __LINUX_ARM_ARCH__ >= 7 + /* clear the If-Then Thumb-2 execution state */ + cpsr &= ~PSR_IT_MASK; +#endif + } else + cpsr &= ~PSR_T_BIT; + } +#endif + + if (ka->sa.sa_flags & SA_RESTORER) { + retcode = (unsigned long)ka->sa.sa_restorer; + } else { + unsigned int idx = thumb << 1; + + if (ka->sa.sa_flags & SA_SIGINFO) + idx += 3; + + if (__put_user(sigreturn_codes[idx], rc) || + __put_user(sigreturn_codes[idx+1], rc+1)) + return 1; + + if (cpsr & MODE32_BIT) { + /* + * 32-bit code can use the new high-page + * signal return code support. + */ + retcode = KERN_SIGRETURN_CODE + (idx << 2) + thumb; + } else { + /* + * Ensure that the instruction cache sees + * the return code written onto the stack. + */ + flush_icache_range((unsigned long)rc, + (unsigned long)(rc + 2)); + + retcode = ((unsigned long)rc) + thumb; + } + } + + regs->ARM_r0 = usig; + regs->ARM_sp = (unsigned long)frame; + regs->ARM_lr = retcode; + regs->ARM_pc = handler; + regs->ARM_cpsr = cpsr; + + return 0; +} + +static int +setup_frame(int usig, struct k_sigaction *ka, sigset_t *set, struct pt_regs *regs) +{ + struct sigframe __user *frame = get_sigframe(ka, regs, sizeof(*frame)); + int err = 0; + + if (!frame) + return 1; + + /* + * Set uc.uc_flags to a value which sc.trap_no would never have. + */ + __put_user_error(0x5ac3c35a, &frame->uc.uc_flags, err); + + err |= setup_sigframe(frame, regs, set); + if (err == 0) + err = setup_return(regs, ka, frame->retcode, frame, usig); + + return err; +} + +static int +setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info, + sigset_t *set, struct pt_regs *regs) +{ + struct rt_sigframe __user *frame = get_sigframe(ka, regs, sizeof(*frame)); + stack_t stack; + int err = 0; + + if (!frame) + return 1; + + err |= copy_siginfo_to_user(&frame->info, info); + + __put_user_error(0, &frame->sig.uc.uc_flags, err); + __put_user_error(NULL, &frame->sig.uc.uc_link, err); + + memset(&stack, 0, sizeof(stack)); + stack.ss_sp = (void __user *)current->sas_ss_sp; + stack.ss_flags = sas_ss_flags(regs->ARM_sp); + stack.ss_size = current->sas_ss_size; + err |= __copy_to_user(&frame->sig.uc.uc_stack, &stack, sizeof(stack)); + + err |= setup_sigframe(&frame->sig, regs, set); + if (err == 0) + err = setup_return(regs, ka, frame->sig.retcode, frame, usig); + + if (err == 0) { + /* + * For realtime signals we must also set the second and third + * arguments for the signal handler. + * -- Peter Maydell <pmaydell@chiark.greenend.org.uk> 2000-12-06 + */ + regs->ARM_r1 = (unsigned long)&frame->info; + regs->ARM_r2 = (unsigned long)&frame->sig.uc; + } + + return err; +} + +/* + * OK, we're invoking a handler + */ +static int +handle_signal(unsigned long sig, struct k_sigaction *ka, + siginfo_t *info, sigset_t *oldset, + struct pt_regs * regs) +{ + struct thread_info *thread = current_thread_info(); + struct task_struct *tsk = current; + int usig = sig; + int ret; + + /* + * translate the signal + */ + if (usig < 32 && thread->exec_domain && thread->exec_domain->signal_invmap) + usig = thread->exec_domain->signal_invmap[usig]; + + /* + * Set up the stack frame + */ + if (ka->sa.sa_flags & SA_SIGINFO) + ret = setup_rt_frame(usig, ka, info, oldset, regs); + else + ret = setup_frame(usig, ka, oldset, regs); + + /* + * Check that the resulting registers are actually sane. + */ + ret |= !valid_user_regs(regs); + + if (ret != 0) { + force_sigsegv(sig, tsk); + return ret; + } + + /* + * Block the signal if we were successful. + */ + spin_lock_irq(&tsk->sighand->siglock); + sigorsets(&tsk->blocked, &tsk->blocked, + &ka->sa.sa_mask); + if (!(ka->sa.sa_flags & SA_NODEFER)) + sigaddset(&tsk->blocked, sig); + recalc_sigpending(); + spin_unlock_irq(&tsk->sighand->siglock); + + return 0; +} + +/* + * Note that 'init' is a special process: it doesn't get signals it doesn't + * want to handle. Thus you cannot kill init even with a SIGKILL even by + * mistake. + * + * Note that we go through the signals twice: once to check the signals that + * the kernel can handle, and then we build all the user-level signal handling + * stack-frames in one go after that. + */ +static void do_signal(struct pt_regs *regs, int syscall) +{ + unsigned int retval = 0, continue_addr = 0, restart_addr = 0; + struct k_sigaction ka; + siginfo_t info; + int signr; + + /* + * We want the common case to go fast, which + * is why we may in certain cases get here from + * kernel mode. Just return without doing anything + * if so. + */ + if (!user_mode(regs)) + return; + + /* + * If we were from a system call, check for system call restarting... + */ + if (syscall) { + continue_addr = regs->ARM_pc; + restart_addr = continue_addr - (thumb_mode(regs) ? 2 : 4); + retval = regs->ARM_r0; + + /* + * Prepare for system call restart. We do this here so that a + * debugger will see the already changed PSW. + */ + switch (retval) { + case -ERESTARTNOHAND: + case -ERESTARTSYS: + case -ERESTARTNOINTR: + regs->ARM_r0 = regs->ARM_ORIG_r0; + regs->ARM_pc = restart_addr; + break; + case -ERESTART_RESTARTBLOCK: + regs->ARM_r0 = -EINTR; + break; + } + } + + if (try_to_freeze()) + goto no_signal; + + /* + * Get the signal to deliver. When running under ptrace, at this + * point the debugger may change all our registers ... + */ + signr = get_signal_to_deliver(&info, &ka, regs, NULL); + if (signr > 0) { + sigset_t *oldset; + + /* + * Depending on the signal settings we may need to revert the + * decision to restart the system call. But skip this if a + * debugger has chosen to restart at a different PC. + */ + if (regs->ARM_pc == restart_addr) { + if (retval == -ERESTARTNOHAND + || (retval == -ERESTARTSYS + && !(ka.sa.sa_flags & SA_RESTART))) { + regs->ARM_r0 = -EINTR; + regs->ARM_pc = continue_addr; + } + } + + if (test_thread_flag(TIF_RESTORE_SIGMASK)) + oldset = ¤t->saved_sigmask; + else + oldset = ¤t->blocked; + if (handle_signal(signr, &ka, &info, oldset, regs) == 0) { + /* + * A signal was successfully delivered; the saved + * sigmask will have been stored in the signal frame, + * and will be restored by sigreturn, so we can simply + * clear the TIF_RESTORE_SIGMASK flag. + */ + if (test_thread_flag(TIF_RESTORE_SIGMASK)) + clear_thread_flag(TIF_RESTORE_SIGMASK); + } + return; + } + + no_signal: + if (syscall) { + /* + * Handle restarting a different system call. As above, + * if a debugger has chosen to restart at a different PC, + * ignore the restart. + */ + if (retval == -ERESTART_RESTARTBLOCK + && regs->ARM_pc == continue_addr) { + if (thumb_mode(regs)) { + regs->ARM_r7 = __NR_restart_syscall - __NR_SYSCALL_BASE; + regs->ARM_pc -= 2; + } else { +#if defined(CONFIG_AEABI) && !defined(CONFIG_OABI_COMPAT) + regs->ARM_r7 = __NR_restart_syscall; + regs->ARM_pc -= 4; +#else + u32 __user *usp; + + regs->ARM_sp -= 4; + usp = (u32 __user *)regs->ARM_sp; + + if (put_user(regs->ARM_pc, usp) == 0) { + regs->ARM_pc = KERN_RESTART_CODE; + } else { + regs->ARM_sp += 4; + force_sigsegv(0, current); + } +#endif + } + } + + /* If there's no signal to deliver, we just put the saved sigmask + * back. + */ + if (test_thread_flag(TIF_RESTORE_SIGMASK)) { + clear_thread_flag(TIF_RESTORE_SIGMASK); + sigprocmask(SIG_SETMASK, ¤t->saved_sigmask, NULL); + } + } +} + +asmlinkage void +do_notify_resume(struct pt_regs *regs, unsigned int thread_flags, int syscall) +{ + if (thread_flags & _TIF_SIGPENDING) + do_signal(regs, syscall); + + if (thread_flags & _TIF_NOTIFY_RESUME) { + clear_thread_flag(TIF_NOTIFY_RESUME); + tracehook_notify_resume(regs); + if (current->replacement_session_keyring) + key_replace_session_keyring(); + } +} diff --git a/arch/arm/kernel/signal.h b/arch/arm/kernel/signal.h new file mode 100644 index 00000000..6fcfe839 --- /dev/null +++ b/arch/arm/kernel/signal.h @@ -0,0 +1,14 @@ +/* + * linux/arch/arm/kernel/signal.h + * + * Copyright (C) 2005-2009 Russell King. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#define KERN_SIGRETURN_CODE (CONFIG_VECTORS_BASE + 0x00000500) +#define KERN_RESTART_CODE (KERN_SIGRETURN_CODE + sizeof(sigreturn_codes)) + +extern const unsigned long sigreturn_codes[7]; +extern const unsigned long syscall_restart_code[2]; diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S new file mode 100644 index 00000000..6398ead9 --- /dev/null +++ b/arch/arm/kernel/sleep.S @@ -0,0 +1,142 @@ +#include <linux/linkage.h> +#include <linux/threads.h> +#include <asm/asm-offsets.h> +#include <asm/assembler.h> +#include <asm/glue-cache.h> +#include <asm/glue-proc.h> +#include <asm/system.h> + .text + +/* + * Save CPU state for a suspend + * r1 = v:p offset + * r3 = virtual return function + * Note: sp is decremented to allocate space for CPU state on stack + * r0-r3,r9,r10,lr corrupted + */ +ENTRY(cpu_suspend) + mov r9, lr +#ifdef MULTI_CPU + ldr r10, =processor + mov r2, sp @ current virtual SP + ldr r0, [r10, #CPU_SLEEP_SIZE] @ size of CPU sleep state + ldr ip, [r10, #CPU_DO_RESUME] @ virtual resume function + sub sp, sp, r0 @ allocate CPU state on stack + mov r0, sp @ save pointer + add ip, ip, r1 @ convert resume fn to phys + stmfd sp!, {r1, r2, r3, ip} @ save v:p, virt SP, retfn, phys resume fn + ldr r3, =sleep_save_sp + add r2, sp, r1 @ convert SP to phys +#ifdef CONFIG_SMP + ALT_SMP(mrc p15, 0, lr, c0, c0, 5) + ALT_UP(mov lr, #0) + and lr, lr, #15 + str r2, [r3, lr, lsl #2] @ save phys SP +#else + str r2, [r3] @ save phys SP +#endif + mov lr, pc + ldr pc, [r10, #CPU_DO_SUSPEND] @ save CPU state +#else + mov r2, sp @ current virtual SP + ldr r0, =cpu_suspend_size + sub sp, sp, r0 @ allocate CPU state on stack + mov r0, sp @ save pointer + stmfd sp!, {r1, r2, r3} @ save v:p, virt SP, return fn + ldr r3, =sleep_save_sp + add r2, sp, r1 @ convert SP to phys +#ifdef CONFIG_SMP + ALT_SMP(mrc p15, 0, lr, c0, c0, 5) + ALT_UP(mov lr, #0) + and lr, lr, #15 + str r2, [r3, lr, lsl #2] @ save phys SP +#else + str r2, [r3] @ save phys SP +#endif + bl cpu_do_suspend +#endif + + @ flush data cache +#ifdef MULTI_CACHE + ldr r10, =cpu_cache + mov lr, r9 + ldr pc, [r10, #CACHE_FLUSH_KERN_ALL] +#else + mov lr, r9 + b __cpuc_flush_kern_all +#endif +ENDPROC(cpu_suspend) + .ltorg + +/* + * r0 = control register value + * r1 = v:p offset (preserved by cpu_do_resume) + * r2 = phys page table base + * r3 = L1 section flags + */ +ENTRY(cpu_resume_mmu) + adr r4, cpu_resume_turn_mmu_on + mov r4, r4, lsr #20 + orr r3, r3, r4, lsl #20 + ldr r5, [r2, r4, lsl #2] @ save old mapping + str r3, [r2, r4, lsl #2] @ setup 1:1 mapping for mmu code + sub r2, r2, r1 + ldr r3, =cpu_resume_after_mmu + bic r1, r0, #CR_C @ ensure D-cache is disabled + b cpu_resume_turn_mmu_on +ENDPROC(cpu_resume_mmu) + .ltorg + .align 5 +cpu_resume_turn_mmu_on: + mcr p15, 0, r1, c1, c0, 0 @ turn on MMU, I-cache, etc + mrc p15, 0, r1, c0, c0, 0 @ read id reg + mov r1, r1 + mov r1, r1 + mov pc, r3 @ jump to virtual address +ENDPROC(cpu_resume_turn_mmu_on) +cpu_resume_after_mmu: + str r5, [r2, r4, lsl #2] @ restore old mapping + mcr p15, 0, r0, c1, c0, 0 @ turn on D-cache + mov pc, lr +ENDPROC(cpu_resume_after_mmu) + +/* + * Note: Yes, part of the following code is located into the .data section. + * This is to allow sleep_save_sp to be accessed with a relative load + * while we can't rely on any MMU translation. We could have put + * sleep_save_sp in the .text section as well, but some setups might + * insist on it to be truly read-only. + */ + .data + .align +ENTRY(cpu_resume) +#ifdef CONFIG_SMP + adr r0, sleep_save_sp + ALT_SMP(mrc p15, 0, r1, c0, c0, 5) + ALT_UP(mov r1, #0) + and r1, r1, #15 + ldr r0, [r0, r1, lsl #2] @ stack phys addr +#else + ldr r0, sleep_save_sp @ stack phys addr +#endif + setmode PSR_I_BIT | PSR_F_BIT | SVC_MODE, r1 @ set SVC, irqs off +#ifdef MULTI_CPU + @ load v:p, stack, return fn, resume fn + ARM( ldmia r0!, {r1, sp, lr, pc} ) +THUMB( ldmia r0!, {r1, r2, r3, r4} ) +THUMB( mov sp, r2 ) +THUMB( mov lr, r3 ) +THUMB( bx r4 ) +#else + @ load v:p, stack, return fn + ARM( ldmia r0!, {r1, sp, lr} ) +THUMB( ldmia r0!, {r1, r2, lr} ) +THUMB( mov sp, r2 ) + b cpu_do_resume +#endif +ENDPROC(cpu_resume) + +sleep_save_sp: + .rept CONFIG_NR_CPUS + .long 0 @ preserve stack phys ptr here + .endr diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c new file mode 100644 index 00000000..24cdba4e --- /dev/null +++ b/arch/arm/kernel/smp.c @@ -0,0 +1,686 @@ +/* + * linux/arch/arm/kernel/smp.c + * + * Copyright (C) 2002 ARM Limited, All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/spinlock.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/cache.h> +#include <linux/profile.h> +#include <linux/errno.h> +#include <linux/ftrace.h> +#include <linux/mm.h> +#include <linux/err.h> +#include <linux/cpu.h> +#include <linux/smp.h> +#include <linux/seq_file.h> +#include <linux/irq.h> +#include <linux/percpu.h> +#include <linux/clockchips.h> +#include <linux/completion.h> + +#include <asm/atomic.h> +#include <asm/cacheflush.h> +#include <asm/cpu.h> +#include <asm/cputype.h> +#include <asm/mmu_context.h> +#include <asm/pgtable.h> +#include <asm/pgalloc.h> +#include <asm/processor.h> +#include <asm/sections.h> +#include <asm/tlbflush.h> +#include <asm/ptrace.h> +#include <asm/localtimer.h> + +/* + * as from 2.5, kernels no longer have an init_tasks structure + * so we need some other way of telling a new secondary core + * where to place its SVC stack + */ +struct secondary_data secondary_data; + +enum ipi_msg_type { + IPI_TIMER = 2, + IPI_RESCHEDULE, + IPI_CALL_FUNC, + IPI_CALL_FUNC_SINGLE, + IPI_CPU_STOP, + IPI_CPU_BACKTRACE, +}; + +static DECLARE_COMPLETION(cpu_running); + +int __cpuinit __cpu_up(unsigned int cpu) +{ + struct cpuinfo_arm *ci = &per_cpu(cpu_data, cpu); + struct task_struct *idle = ci->idle; + pgd_t *pgd; + int ret; + + /* + * Spawn a new process manually, if not already done. + * Grab a pointer to its task struct so we can mess with it + */ + if (!idle) { + idle = fork_idle(cpu); + if (IS_ERR(idle)) { + printk(KERN_ERR "CPU%u: fork() failed\n", cpu); + return PTR_ERR(idle); + } + ci->idle = idle; + } else { + /* + * Since this idle thread is being re-used, call + * init_idle() to reinitialize the thread structure. + */ + init_idle(idle, cpu); + } + + /* + * Allocate initial page tables to allow the new CPU to + * enable the MMU safely. This essentially means a set + * of our "standard" page tables, with the addition of + * a 1:1 mapping for the physical address of the kernel. + */ + pgd = pgd_alloc(&init_mm); + if (!pgd) + return -ENOMEM; + + if (PHYS_OFFSET != PAGE_OFFSET) { +#ifndef CONFIG_HOTPLUG_CPU + identity_mapping_add(pgd, __pa(__init_begin), __pa(__init_end)); +#endif + identity_mapping_add(pgd, __pa(_stext), __pa(_etext)); + identity_mapping_add(pgd, __pa(_sdata), __pa(_edata)); + } + + /* + * We need to tell the secondary core where to find + * its stack and the page tables. + */ + secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; + secondary_data.pgdir = virt_to_phys(pgd); + secondary_data.swapper_pg_dir = virt_to_phys(swapper_pg_dir); + __cpuc_flush_dcache_area(&secondary_data, sizeof(secondary_data)); + outer_clean_range(__pa(&secondary_data), __pa(&secondary_data + 1)); + + /* + * Now bring the CPU into our world. + */ + ret = boot_secondary(cpu, idle); + if (ret == 0) { + /* + * CPU was successfully started, wait for it + * to come online or time out. + */ + wait_for_completion_timeout(&cpu_running, + msecs_to_jiffies(1000)); + + if (!cpu_online(cpu)) { + pr_crit("CPU%u: failed to come online\n", cpu); + ret = -EIO; + } + } else { + pr_err("CPU%u: failed to boot: %d\n", cpu, ret); + } + + secondary_data.stack = NULL; + secondary_data.pgdir = 0; + + if (PHYS_OFFSET != PAGE_OFFSET) { +#ifndef CONFIG_HOTPLUG_CPU + identity_mapping_del(pgd, __pa(__init_begin), __pa(__init_end)); +#endif + identity_mapping_del(pgd, __pa(_stext), __pa(_etext)); + identity_mapping_del(pgd, __pa(_sdata), __pa(_edata)); + } + + pgd_free(&init_mm, pgd); + + return ret; +} + +#ifdef CONFIG_HOTPLUG_CPU +static void percpu_timer_stop(void); + +/* + * __cpu_disable runs on the processor to be shutdown. + */ +int __cpu_disable(void) +{ + unsigned int cpu = smp_processor_id(); + struct task_struct *p; + int ret; + + ret = platform_cpu_disable(cpu); + if (ret) + return ret; + + /* + * Take this CPU offline. Once we clear this, we can't return, + * and we must not schedule until we're ready to give up the cpu. + */ + set_cpu_online(cpu, false); + + /* + * OK - migrate IRQs away from this CPU + */ + migrate_irqs(); + + /* + * Stop the local timer for this CPU. + */ + percpu_timer_stop(); + + /* + * Flush user cache and TLB mappings, and then remove this CPU + * from the vm mask set of all processes. + */ + flush_cache_all(); + local_flush_tlb_all(); + + read_lock(&tasklist_lock); + for_each_process(p) { + if (p->mm) + cpumask_clear_cpu(cpu, mm_cpumask(p->mm)); + } + read_unlock(&tasklist_lock); + + return 0; +} + +static DECLARE_COMPLETION(cpu_died); + +/* + * called on the thread which is asking for a CPU to be shutdown - + * waits until shutdown has completed, or it is timed out. + */ +void __cpu_die(unsigned int cpu) +{ + if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) { + pr_err("CPU%u: cpu didn't die\n", cpu); + return; + } + printk(KERN_NOTICE "CPU%u: shutdown\n", cpu); + + if (!platform_cpu_kill(cpu)) + printk("CPU%u: unable to kill\n", cpu); +} + +/* + * Called from the idle thread for the CPU which has been shutdown. + * + * Note that we disable IRQs here, but do not re-enable them + * before returning to the caller. This is also the behaviour + * of the other hotplug-cpu capable cores, so presumably coming + * out of idle fixes this. + */ +void __ref cpu_die(void) +{ + unsigned int cpu = smp_processor_id(); + + idle_task_exit(); + + local_irq_disable(); + mb(); + + /* Tell __cpu_die() that this CPU is now safe to dispose of */ + complete(&cpu_died); + + /* + * actual CPU shutdown procedure is at least platform (if not + * CPU) specific. + */ + platform_cpu_die(cpu); + + /* + * Do not return to the idle loop - jump back to the secondary + * cpu initialisation. There's some initialisation which needs + * to be repeated to undo the effects of taking the CPU offline. + */ + __asm__("mov sp, %0\n" + " mov fp, #0\n" + " b secondary_start_kernel" + : + : "r" (task_stack_page(current) + THREAD_SIZE - 8)); +} +#endif /* CONFIG_HOTPLUG_CPU */ + +/* + * Called by both boot and secondaries to move global data into + * per-processor storage. + */ +static void __cpuinit smp_store_cpu_info(unsigned int cpuid) +{ + struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid); + + cpu_info->loops_per_jiffy = loops_per_jiffy; +} + +/* + * This is the secondary CPU boot entry. We're using this CPUs + * idle thread stack, but a set of temporary page tables. + */ +asmlinkage void __cpuinit secondary_start_kernel(void) +{ + struct mm_struct *mm = &init_mm; + unsigned int cpu = smp_processor_id(); + + /* + * All kernel threads share the same mm context; grab a + * reference and switch to it. + */ + atomic_inc(&mm->mm_count); + current->active_mm = mm; + cpumask_set_cpu(cpu, mm_cpumask(mm)); + cpu_switch_mm(mm->pgd, mm); + enter_lazy_tlb(mm, current); + local_flush_tlb_all(); + + printk("CPU%u: Booted secondary processor\n", cpu); + + cpu_init(); + preempt_disable(); + trace_hardirqs_off(); + + /* + * Give the platform a chance to do its own initialisation. + */ + platform_secondary_init(cpu); + + notify_cpu_starting(cpu); + + calibrate_delay(); + + smp_store_cpu_info(cpu); + + /* + * OK, now it's safe to let the boot CPU continue. Wait for + * the CPU migration code to notice that the CPU is online + * before we continue - which happens after __cpu_up returns. + */ + set_cpu_online(cpu, true); + complete(&cpu_running); + + /* + * Setup the percpu timer for this CPU. + */ + percpu_timer_setup(); + + local_irq_enable(); + local_fiq_enable(); + + /* + * OK, it's off to the idle thread for us + */ + cpu_idle(); +} + +void __init smp_cpus_done(unsigned int max_cpus) +{ + int cpu; + unsigned long bogosum = 0; + + for_each_online_cpu(cpu) + bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy; + + printk(KERN_INFO "SMP: Total of %d processors activated " + "(%lu.%02lu BogoMIPS).\n", + num_online_cpus(), + bogosum / (500000/HZ), + (bogosum / (5000/HZ)) % 100); +} + +void __init smp_prepare_boot_cpu(void) +{ + unsigned int cpu = smp_processor_id(); + + per_cpu(cpu_data, cpu).idle = current; +} + +void __init smp_prepare_cpus(unsigned int max_cpus) +{ + unsigned int ncores = num_possible_cpus(); + + smp_store_cpu_info(smp_processor_id()); + + /* + * are we trying to boot more cores than exist? + */ + if (max_cpus > ncores) + max_cpus = ncores; + + if (max_cpus > 1) { + /* + * Enable the local timer or broadcast device for the + * boot CPU, but only if we have more than one CPU. + */ + percpu_timer_setup(); + + /* + * Initialise the SCU if there are more than one CPU + * and let them know where to start. + */ + platform_smp_prepare_cpus(max_cpus); + } +} + +static void (*smp_cross_call)(const struct cpumask *, unsigned int); + +void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int)) +{ + smp_cross_call = fn; +} + +void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + smp_cross_call(mask, IPI_CALL_FUNC); +} + +void arch_send_call_function_single_ipi(int cpu) +{ + smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE); +} + +static const char *ipi_types[NR_IPI] = { +#define S(x,s) [x - IPI_TIMER] = s + S(IPI_TIMER, "Timer broadcast interrupts"), + S(IPI_RESCHEDULE, "Rescheduling interrupts"), + S(IPI_CALL_FUNC, "Function call interrupts"), + S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"), + S(IPI_CPU_STOP, "CPU stop interrupts"), + S(IPI_CPU_BACKTRACE, "CPU backtrace"), +}; + +void show_ipi_list(struct seq_file *p, int prec) +{ + unsigned int cpu, i; + + for (i = 0; i < NR_IPI; i++) { + seq_printf(p, "%*s%u: ", prec - 1, "IPI", i); + + for_each_present_cpu(cpu) + seq_printf(p, "%10u ", + __get_irq_stat(cpu, ipi_irqs[i])); + + seq_printf(p, " %s\n", ipi_types[i]); + } +} + +u64 smp_irq_stat_cpu(unsigned int cpu) +{ + u64 sum = 0; + int i; + + for (i = 0; i < NR_IPI; i++) + sum += __get_irq_stat(cpu, ipi_irqs[i]); + +#ifdef CONFIG_LOCAL_TIMERS + sum += __get_irq_stat(cpu, local_timer_irqs); +#endif + + return sum; +} + +/* + * Timer (local or broadcast) support + */ +static DEFINE_PER_CPU(struct clock_event_device, percpu_clockevent); + +static void ipi_timer(void) +{ + struct clock_event_device *evt = &__get_cpu_var(percpu_clockevent); + evt->event_handler(evt); +} + +#ifdef CONFIG_LOCAL_TIMERS +asmlinkage void __exception_irq_entry do_local_timer(struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + int cpu = smp_processor_id(); + + if (local_timer_ack()) { + __inc_irq_stat(cpu, local_timer_irqs); + irq_enter(); + ipi_timer(); + irq_exit(); + } + + set_irq_regs(old_regs); +} + +void show_local_irqs(struct seq_file *p, int prec) +{ + unsigned int cpu; + + seq_printf(p, "%*s: ", prec, "LOC"); + + for_each_present_cpu(cpu) + seq_printf(p, "%10u ", __get_irq_stat(cpu, local_timer_irqs)); + + seq_printf(p, " Local timer interrupts\n"); +} +#endif + +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST +static void smp_timer_broadcast(const struct cpumask *mask) +{ + smp_cross_call(mask, IPI_TIMER); +} +#else +#define smp_timer_broadcast NULL +#endif + +static void broadcast_timer_set_mode(enum clock_event_mode mode, + struct clock_event_device *evt) +{ +} + +static void __cpuinit broadcast_timer_setup(struct clock_event_device *evt) +{ + evt->name = "dummy_timer"; + evt->features = CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_PERIODIC | + CLOCK_EVT_FEAT_DUMMY; + evt->rating = 400; + evt->mult = 1; + evt->set_mode = broadcast_timer_set_mode; + + clockevents_register_device(evt); +} + +void __cpuinit percpu_timer_setup(void) +{ + unsigned int cpu = smp_processor_id(); + struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu); + + evt->cpumask = cpumask_of(cpu); + evt->broadcast = smp_timer_broadcast; + + if (local_timer_setup(evt)) + broadcast_timer_setup(evt); +} + +#ifdef CONFIG_HOTPLUG_CPU +/* + * The generic clock events code purposely does not stop the local timer + * on CPU_DEAD/CPU_DEAD_FROZEN hotplug events, so we have to do it + * manually here. + */ +static void percpu_timer_stop(void) +{ + unsigned int cpu = smp_processor_id(); + struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu); + + evt->set_mode(CLOCK_EVT_MODE_UNUSED, evt); +} +#endif + +static DEFINE_SPINLOCK(stop_lock); + +/* + * ipi_cpu_stop - handle IPI from smp_send_stop() + */ +static void ipi_cpu_stop(unsigned int cpu) +{ + if (system_state == SYSTEM_BOOTING || + system_state == SYSTEM_RUNNING) { + spin_lock(&stop_lock); + printk(KERN_CRIT "CPU%u: stopping\n", cpu); + dump_stack(); + spin_unlock(&stop_lock); + } + + set_cpu_online(cpu, false); + + local_fiq_disable(); + local_irq_disable(); + + while (1) + cpu_relax(); +} + +static cpumask_t backtrace_mask; +static DEFINE_RAW_SPINLOCK(backtrace_lock); + +/* "in progress" flag of arch_trigger_all_cpu_backtrace */ +static unsigned long backtrace_flag; + +void smp_send_all_cpu_backtrace(void) +{ + unsigned int this_cpu = smp_processor_id(); + int i; + + if (test_and_set_bit(0, &backtrace_flag)) + /* + * If there is already a trigger_all_cpu_backtrace() in progress + * (backtrace_flag == 1), don't output double cpu dump infos. + */ + return; + + cpumask_copy(&backtrace_mask, cpu_online_mask); + cpu_clear(this_cpu, backtrace_mask); + + pr_info("Backtrace for cpu %d (current):\n", this_cpu); + dump_stack(); + + pr_info("\nsending IPI to all other CPUs:\n"); + smp_cross_call(&backtrace_mask, IPI_CPU_BACKTRACE); + + /* Wait for up to 10 seconds for all other CPUs to do the backtrace */ + for (i = 0; i < 10 * 1000; i++) { + if (cpumask_empty(&backtrace_mask)) + break; + mdelay(1); + } + + clear_bit(0, &backtrace_flag); + smp_mb__after_clear_bit(); +} + +/* + * ipi_cpu_backtrace - handle IPI from smp_send_all_cpu_backtrace() + */ +static void ipi_cpu_backtrace(unsigned int cpu, struct pt_regs *regs) +{ + if (cpu_isset(cpu, backtrace_mask)) { + raw_spin_lock(&backtrace_lock); + pr_warning("IPI backtrace for cpu %d\n", cpu); + show_regs(regs); + raw_spin_unlock(&backtrace_lock); + cpu_clear(cpu, backtrace_mask); + } +} + +/* + * Main handler for inter-processor interrupts + */ +asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs) +{ + unsigned int cpu = smp_processor_id(); + struct pt_regs *old_regs = set_irq_regs(regs); + + if (ipinr >= IPI_TIMER && ipinr < IPI_TIMER + NR_IPI) + __inc_irq_stat(cpu, ipi_irqs[ipinr - IPI_TIMER]); + + switch (ipinr) { + case IPI_TIMER: + irq_enter(); + ipi_timer(); + irq_exit(); + break; + + case IPI_RESCHEDULE: + scheduler_ipi(); + break; + + case IPI_CALL_FUNC: + irq_enter(); + generic_smp_call_function_interrupt(); + irq_exit(); + break; + + case IPI_CALL_FUNC_SINGLE: + irq_enter(); + generic_smp_call_function_single_interrupt(); + irq_exit(); + break; + + case IPI_CPU_STOP: + irq_enter(); + ipi_cpu_stop(cpu); + irq_exit(); + break; + + case IPI_CPU_BACKTRACE: + ipi_cpu_backtrace(cpu, regs); + break; + + default: + printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n", + cpu, ipinr); + break; + } + set_irq_regs(old_regs); +} + +void smp_send_reschedule(int cpu) +{ + smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE); +} + +void smp_send_stop(void) +{ + unsigned long timeout; + + if (num_online_cpus() > 1) { + cpumask_t mask = cpu_online_map; + cpu_clear(smp_processor_id(), mask); + + smp_cross_call(&mask, IPI_CPU_STOP); + } + + /* Wait up to one second for other CPUs to stop */ + timeout = USEC_PER_SEC; + while (num_online_cpus() > 1 && timeout--) + udelay(1); + + if (num_online_cpus() > 1) + pr_warning("SMP: failed to stop secondary CPUs\n"); +} + +/* + * not supported here + */ +int setup_profiling_timer(unsigned int multiplier) +{ + return -EINVAL; +} diff --git a/arch/arm/kernel/smp_scu.c b/arch/arm/kernel/smp_scu.c new file mode 100644 index 00000000..cb7dd40e --- /dev/null +++ b/arch/arm/kernel/smp_scu.c @@ -0,0 +1,85 @@ +/* + * linux/arch/arm/kernel/smp_scu.c + * + * Copyright (C) 2002 ARM Ltd. + * All Rights Reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/init.h> +#include <linux/io.h> + +#include <asm/smp_scu.h> +#include <asm/cacheflush.h> +#include <asm/cputype.h> + +#define SCU_CTRL 0x00 +#define SCU_CONFIG 0x04 +#define SCU_CPU_STATUS 0x08 +#define SCU_INVALIDATE 0x0c +#define SCU_FPGA_REVISION 0x10 + +/* + * Get the number of CPU cores from the SCU configuration + */ +unsigned int __init scu_get_core_count(void __iomem *scu_base) +{ + unsigned int ncores = __raw_readl(scu_base + SCU_CONFIG); + return (ncores & 0x03) + 1; +} + +/* + * Enable the SCU + */ +void __init scu_enable(void __iomem *scu_base) +{ + u32 scu_ctrl; + +#ifdef CONFIG_ARM_ERRATA_764369 + /* Cortex-A9 only */ + if ((read_cpuid(CPUID_ID) & 0xff0ffff0) == 0x410fc090) { + scu_ctrl = __raw_readl(scu_base + 0x30); + if (!(scu_ctrl & 1)) + __raw_writel(scu_ctrl | 0x1, scu_base + 0x30); + } +#endif + + scu_ctrl = __raw_readl(scu_base + SCU_CTRL); + /* already enabled? */ + if (scu_ctrl & 1) + return; + + scu_ctrl |= 1; + __raw_writel(scu_ctrl, scu_base + SCU_CTRL); + + /* + * Ensure that the data accessed by CPU0 before the SCU was + * initialised is visible to the other CPUs. + */ + flush_cache_all(); +} + +/* + * Set the executing CPUs power mode as defined. This will be in + * preparation for it executing a WFI instruction. + * + * This function must be called with preemption disabled, and as it + * has the side effect of disabling coherency, caches must have been + * flushed. Interrupts must also have been disabled. + */ +int scu_power_mode(void __iomem *scu_base, unsigned int mode) +{ + unsigned int val; + int cpu = smp_processor_id(); + + if (mode > 3 || mode == 1 || cpu > 3) + return -EINVAL; + + val = __raw_readb(scu_base + SCU_CPU_STATUS + cpu) & ~0x03; + val |= mode; + __raw_writeb(val, scu_base + SCU_CPU_STATUS + cpu); + + return 0; +} diff --git a/arch/arm/kernel/smp_tlb.c b/arch/arm/kernel/smp_tlb.c new file mode 100644 index 00000000..7dcb3528 --- /dev/null +++ b/arch/arm/kernel/smp_tlb.c @@ -0,0 +1,139 @@ +/* + * linux/arch/arm/kernel/smp_tlb.c + * + * Copyright (C) 2002 ARM Limited, All Rights Reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/preempt.h> +#include <linux/smp.h> + +#include <asm/smp_plat.h> +#include <asm/tlbflush.h> + +static void on_each_cpu_mask(void (*func)(void *), void *info, int wait, + const struct cpumask *mask) +{ + preempt_disable(); + + smp_call_function_many(mask, func, info, wait); + if (cpumask_test_cpu(smp_processor_id(), mask)) + func(info); + + preempt_enable(); +} + +/**********************************************************************/ + +/* + * TLB operations + */ +struct tlb_args { + struct vm_area_struct *ta_vma; + unsigned long ta_start; + unsigned long ta_end; +}; + +static inline void ipi_flush_tlb_all(void *ignored) +{ + local_flush_tlb_all(); +} + +static inline void ipi_flush_tlb_mm(void *arg) +{ + struct mm_struct *mm = (struct mm_struct *)arg; + + local_flush_tlb_mm(mm); +} + +static inline void ipi_flush_tlb_page(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_page(ta->ta_vma, ta->ta_start); +} + +static inline void ipi_flush_tlb_kernel_page(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_kernel_page(ta->ta_start); +} + +static inline void ipi_flush_tlb_range(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end); +} + +static inline void ipi_flush_tlb_kernel_range(void *arg) +{ + struct tlb_args *ta = (struct tlb_args *)arg; + + local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end); +} + +void flush_tlb_all(void) +{ + if (tlb_ops_need_broadcast()) + on_each_cpu(ipi_flush_tlb_all, NULL, 1); + else + local_flush_tlb_all(); +} + +void flush_tlb_mm(struct mm_struct *mm) +{ + if (tlb_ops_need_broadcast()) + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mm_cpumask(mm)); + else + local_flush_tlb_mm(mm); +} + +void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) +{ + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_vma = vma; + ta.ta_start = uaddr; + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mm_cpumask(vma->vm_mm)); + } else + local_flush_tlb_page(vma, uaddr); +} + +void flush_tlb_kernel_page(unsigned long kaddr) +{ + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_start = kaddr; + on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1); + } else + local_flush_tlb_kernel_page(kaddr); +} + +void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_vma = vma; + ta.ta_start = start; + ta.ta_end = end; + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mm_cpumask(vma->vm_mm)); + } else + local_flush_tlb_range(vma, start, end); +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + if (tlb_ops_need_broadcast()) { + struct tlb_args ta; + ta.ta_start = start; + ta.ta_end = end; + on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1); + } else + local_flush_tlb_kernel_range(start, end); +} + diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c new file mode 100644 index 00000000..31deb5ae --- /dev/null +++ b/arch/arm/kernel/smp_twd.c @@ -0,0 +1,223 @@ +/* + * linux/arch/arm/kernel/smp_twd.c + * + * Copyright (C) 2002 ARM Ltd. + * All Rights Reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/smp.h> +#include <linux/jiffies.h> +#include <linux/clockchips.h> +#include <linux/irq.h> +#include <linux/io.h> +#include <linux/clk.h> +#include <linux/cpufreq.h> +#include <linux/err.h> + +#include <asm/smp_twd.h> +#include <asm/hardware/gic.h> + +/* set up by the platform code */ +void __iomem *twd_base; +static struct clk *twd_clk; + +static unsigned long twd_timer_rate; + +static struct clock_event_device __percpu **twd_evt; + +static void twd_set_mode(enum clock_event_mode mode, + struct clock_event_device *clk) +{ + unsigned long ctrl; + + switch (mode) { + case CLOCK_EVT_MODE_PERIODIC: + /* timer load already set up */ + ctrl = TWD_TIMER_CONTROL_ENABLE | TWD_TIMER_CONTROL_IT_ENABLE + | TWD_TIMER_CONTROL_PERIODIC; + __raw_writel(twd_timer_rate / HZ, twd_base + TWD_TIMER_LOAD); + gic_enable_ppi(clk->irq); + break; + case CLOCK_EVT_MODE_ONESHOT: + /* period set, and timer enabled in 'next_event' hook */ + ctrl = TWD_TIMER_CONTROL_IT_ENABLE | TWD_TIMER_CONTROL_ONESHOT; + gic_enable_ppi(clk->irq); + break; + case CLOCK_EVT_MODE_UNUSED: + case CLOCK_EVT_MODE_SHUTDOWN: + default: + ctrl = 0; + gic_disable_ppi(clk->irq); + } + + __raw_writel(ctrl, twd_base + TWD_TIMER_CONTROL); +} + +static int twd_set_next_event(unsigned long evt, + struct clock_event_device *unused) +{ + unsigned long ctrl = __raw_readl(twd_base + TWD_TIMER_CONTROL); + + ctrl |= TWD_TIMER_CONTROL_ENABLE; + + __raw_writel(evt, twd_base + TWD_TIMER_COUNTER); + __raw_writel(ctrl, twd_base + TWD_TIMER_CONTROL); + + return 0; +} + +/* + * local_timer_ack: checks for a local timer interrupt. + * + * If a local timer interrupt has occurred, acknowledge and return 1. + * Otherwise, return 0. + */ +int twd_timer_ack(void) +{ + if (__raw_readl(twd_base + TWD_TIMER_INTSTAT)) { + __raw_writel(1, twd_base + TWD_TIMER_INTSTAT); + return 1; + } + + return 0; +} + +static struct clk *twd_get_clock(void) +{ + return clk_get_sys("smp_twd", NULL); +} + +#ifdef CONFIG_CPU_FREQ +/* + * Updates clockevent frequency when the cpu frequency changes. + * Called on the cpu that is changing frequency with interrupts disabled. + */ +static void twd_update_frequency(void *data) +{ + twd_timer_rate = clk_get_rate(twd_clk); + + clockevents_update_freq(*__this_cpu_ptr(twd_evt), twd_timer_rate); +} + +static int twd_cpufreq_transition(struct notifier_block *nb, + unsigned long state, void *data) +{ + struct cpufreq_freqs *freqs = data; + + /* + * The twd clock events must be reprogrammed to account for the new + * frequency. The timer is local to a cpu, so cross-call to the + * changing cpu. + * + * Only wait for it to finish, if the cpu is active to avoid + * deadlock when cpu1 is spinning on while(!cpu_active(cpu1)) during + * booting of that cpu. + */ + if (state == CPUFREQ_POSTCHANGE || state == CPUFREQ_RESUMECHANGE) + smp_call_function_single(freqs->cpu, twd_update_frequency, + NULL, cpu_active(freqs->cpu)); + + return NOTIFY_OK; +} + +static struct notifier_block twd_cpufreq_nb = { + .notifier_call = twd_cpufreq_transition, +}; + +static int twd_cpufreq_init(void) +{ + if (twd_evt && *__this_cpu_ptr(twd_evt) && !IS_ERR(twd_clk)) + return cpufreq_register_notifier(&twd_cpufreq_nb, + CPUFREQ_TRANSITION_NOTIFIER); + + return 0; +} +core_initcall(twd_cpufreq_init); + +#endif +static void __cpuinit twd_calibrate_rate(void) +{ + unsigned long count; + u64 waitjiffies; + + /* + * If this is the first time round, we need to work out how fast + * the timer ticks + */ + if (twd_timer_rate == 0) { + printk(KERN_INFO "Calibrating local timer... "); + + /* Wait for a tick to start */ + waitjiffies = get_jiffies_64() + 1; + + while (get_jiffies_64() < waitjiffies) + udelay(10); + + /* OK, now the tick has started, let's get the timer going */ + waitjiffies += 5; + + /* enable, no interrupt or reload */ + __raw_writel(0x1, twd_base + TWD_TIMER_CONTROL); + + /* maximum value */ + __raw_writel(0xFFFFFFFFU, twd_base + TWD_TIMER_COUNTER); + + while (get_jiffies_64() < waitjiffies) + udelay(10); + + count = __raw_readl(twd_base + TWD_TIMER_COUNTER); + + twd_timer_rate = (0xFFFFFFFFU - count) * (HZ / 5); + + printk("%lu.%02luMHz.\n", twd_timer_rate / 1000000, + (twd_timer_rate / 10000) % 100); + } +} + +/* + * Setup the local clock events for a CPU. + */ +void __cpuinit twd_timer_setup(struct clock_event_device *clk) +{ + struct clock_event_device **this_cpu_clk; + + if (!twd_evt) { + + twd_evt = alloc_percpu(struct clock_event_device *); + if (!twd_evt) { + pr_err("twd: can't allocate memory\n"); + return; + } + } + + if (!twd_clk) + twd_clk = twd_get_clock(); + + if (!IS_ERR_OR_NULL(twd_clk)) + twd_timer_rate = clk_get_rate(twd_clk); + else + twd_calibrate_rate(); + + clk->name = "local_timer"; + clk->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | + CLOCK_EVT_FEAT_C3STOP; + clk->rating = 350; + clk->set_mode = twd_set_mode; + clk->set_next_event = twd_set_next_event; + + this_cpu_clk = __this_cpu_ptr(twd_evt); + *this_cpu_clk = clk; + + clockevents_config_and_register(clk, twd_timer_rate, + 0xf, 0xffffffff); + + /* Make sure our local interrupt controller has this enabled */ + gic_enable_ppi(clk->irq); +} diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c new file mode 100644 index 00000000..381d23a4 --- /dev/null +++ b/arch/arm/kernel/stacktrace.c @@ -0,0 +1,131 @@ +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/stacktrace.h> + +#include <asm/stacktrace.h> + +#if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND) +/* + * Unwind the current stack frame and store the new register values in the + * structure passed as argument. Unwinding is equivalent to a function return, + * hence the new PC value rather than LR should be used for backtrace. + * + * With framepointer enabled, a simple function prologue looks like this: + * mov ip, sp + * stmdb sp!, {fp, ip, lr, pc} + * sub fp, ip, #4 + * + * A simple function epilogue looks like this: + * ldm sp, {fp, sp, pc} + * + * Note that with framepointer enabled, even the leaf functions have the same + * prologue and epilogue, therefore we can ignore the LR value in this case. + */ +int notrace unwind_frame(struct stackframe *frame) +{ + unsigned long high, low; + unsigned long fp = frame->fp; + + /* only go to a higher address on the stack */ + low = frame->sp; + high = ALIGN(low, THREAD_SIZE); + + /* check current frame pointer is within bounds */ + if (fp < (low + 12) || fp + 4 >= high) + return -EINVAL; + + /* restore the registers from the stack frame */ + frame->fp = *(unsigned long *)(fp - 12); + frame->sp = *(unsigned long *)(fp - 8); + frame->pc = *(unsigned long *)(fp - 4); + + return 0; +} +#endif + +void notrace walk_stackframe(struct stackframe *frame, + int (*fn)(struct stackframe *, void *), void *data) +{ + while (1) { + int ret; + + if (fn(frame, data)) + break; + ret = unwind_frame(frame); + if (ret < 0) + break; + } +} +EXPORT_SYMBOL(walk_stackframe); + +#ifdef CONFIG_STACKTRACE +struct stack_trace_data { + struct stack_trace *trace; + unsigned int no_sched_functions; + unsigned int skip; +}; + +static int save_trace(struct stackframe *frame, void *d) +{ + struct stack_trace_data *data = d; + struct stack_trace *trace = data->trace; + unsigned long addr = frame->pc; + + if (data->no_sched_functions && in_sched_functions(addr)) + return 0; + if (data->skip) { + data->skip--; + return 0; + } + + trace->entries[trace->nr_entries++] = addr; + + return trace->nr_entries >= trace->max_entries; +} + +void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +{ + struct stack_trace_data data; + struct stackframe frame; + + data.trace = trace; + data.skip = trace->skip; + + if (tsk != current) { +#ifdef CONFIG_SMP + /* + * What guarantees do we have here that 'tsk' is not + * running on another CPU? For now, ignore it as we + * can't guarantee we won't explode. + */ + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; + return; +#else + data.no_sched_functions = 1; + frame.fp = thread_saved_fp(tsk); + frame.sp = thread_saved_sp(tsk); + frame.lr = 0; /* recovered from the stack */ + frame.pc = thread_saved_pc(tsk); +#endif + } else { + register unsigned long current_sp asm ("sp"); + + data.no_sched_functions = 0; + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_sp; + frame.lr = (unsigned long)__builtin_return_address(0); + frame.pc = (unsigned long)save_stack_trace_tsk; + } + + walk_stackframe(&frame, save_trace, &data); + if (trace->nr_entries < trace->max_entries) + trace->entries[trace->nr_entries++] = ULONG_MAX; +} + +void save_stack_trace(struct stack_trace *trace) +{ + save_stack_trace_tsk(current, trace); +} +EXPORT_SYMBOL_GPL(save_stack_trace); +#endif diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c new file mode 100644 index 00000000..40ee7e50 --- /dev/null +++ b/arch/arm/kernel/swp_emulate.c @@ -0,0 +1,267 @@ +/* + * linux/arch/arm/kernel/swp_emulate.c + * + * Copyright (C) 2009 ARM Limited + * __user_* functions adapted from include/asm/uaccess.h + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * Implements emulation of the SWP/SWPB instructions using load-exclusive and + * store-exclusive for processors that have them disabled (or future ones that + * might not implement them). + * + * Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>] + * Where: Rt = destination + * Rt2 = source + * Rn = address + */ + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/proc_fs.h> +#include <linux/sched.h> +#include <linux/syscalls.h> +#include <linux/perf_event.h> + +#include <asm/traps.h> +#include <asm/uaccess.h> + +/* + * Error-checking SWP macros implemented using ldrex{b}/strex{b} + */ +#define __user_swpX_asm(data, addr, res, temp, B) \ + __asm__ __volatile__( \ + " mov %2, %1\n" \ + "0: ldrex"B" %1, [%3]\n" \ + "1: strex"B" %0, %2, [%3]\n" \ + " cmp %0, #0\n" \ + " movne %0, %4\n" \ + "2:\n" \ + " .section .fixup,\"ax\"\n" \ + " .align 2\n" \ + "3: mov %0, %5\n" \ + " b 2b\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .align 3\n" \ + " .long 0b, 3b\n" \ + " .long 1b, 3b\n" \ + " .previous" \ + : "=&r" (res), "+r" (data), "=&r" (temp) \ + : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT) \ + : "cc", "memory") + +#define __user_swp_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "") +#define __user_swpb_asm(data, addr, res, temp) \ + __user_swpX_asm(data, addr, res, temp, "b") + +/* + * Macros/defines for extracting register numbers from instruction. + */ +#define EXTRACT_REG_NUM(instruction, offset) \ + (((instruction) & (0xf << (offset))) >> (offset)) +#define RN_OFFSET 16 +#define RT_OFFSET 12 +#define RT2_OFFSET 0 +/* + * Bit 22 of the instruction encoding distinguishes between + * the SWP and SWPB variants (bit set means SWPB). + */ +#define TYPE_SWPB (1 << 22) + +static unsigned long swpcounter; +static unsigned long swpbcounter; +static unsigned long abtcounter; +static pid_t previous_pid; + +#ifdef CONFIG_PROC_FS +static int proc_read_status(char *page, char **start, off_t off, int count, + int *eof, void *data) +{ + char *p = page; + int len; + + p += sprintf(p, "Emulated SWP:\t\t%lu\n", swpcounter); + p += sprintf(p, "Emulated SWPB:\t\t%lu\n", swpbcounter); + p += sprintf(p, "Aborted SWP{B}:\t\t%lu\n", abtcounter); + if (previous_pid != 0) + p += sprintf(p, "Last process:\t\t%d\n", previous_pid); + + len = (p - page) - off; + if (len < 0) + len = 0; + + *eof = (len <= count) ? 1 : 0; + *start = page + off; + + return len; +} +#endif + +/* + * Set up process info to signal segmentation fault - called on access error. + */ +static void set_segfault(struct pt_regs *regs, unsigned long addr) +{ + siginfo_t info; + + if (find_vma(current->mm, addr) == NULL) + info.si_code = SEGV_MAPERR; + else + info.si_code = SEGV_ACCERR; + + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_addr = (void *) instruction_pointer(regs); + + pr_debug("SWP{B} emulation: access caused memory abort!\n"); + arm_notify_die("Illegal memory access", regs, &info, 0, 0); + + abtcounter++; +} + +static int emulate_swpX(unsigned int address, unsigned int *data, + unsigned int type) +{ + unsigned int res = 0; + + if ((type != TYPE_SWPB) && (address & 0x3)) { + /* SWP to unaligned address not permitted */ + pr_debug("SWP instruction on unaligned pointer!\n"); + return -EFAULT; + } + + while (1) { + unsigned long temp; + + /* + * Barrier required between accessing protected resource and + * releasing a lock for it. Legacy code might not have done + * this, and we cannot determine that this is not the case + * being emulated, so insert always. + */ + smp_mb(); + + if (type == TYPE_SWPB) + __user_swpb_asm(*data, address, res, temp); + else + __user_swp_asm(*data, address, res, temp); + + if (likely(res != -EAGAIN) || signal_pending(current)) + break; + + cond_resched(); + } + + if (res == 0) { + /* + * Barrier also required between acquiring a lock for a + * protected resource and accessing the resource. Inserted for + * same reason as above. + */ + smp_mb(); + + if (type == TYPE_SWPB) + swpbcounter++; + else + swpcounter++; + } + + return res; +} + +/* + * swp_handler logs the id of calling process, dissects the instruction, sanity + * checks the memory location, calls emulate_swpX for the actual operation and + * deals with fixup/error handling before returning + */ +static int swp_handler(struct pt_regs *regs, unsigned int instr) +{ + unsigned int address, destreg, data, type; + unsigned int res = 0; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, regs->ARM_pc); + + if (current->pid != previous_pid) { + pr_debug("\"%s\" (%ld) uses deprecated SWP{B} instruction\n", + current->comm, (unsigned long)current->pid); + previous_pid = current->pid; + } + + address = regs->uregs[EXTRACT_REG_NUM(instr, RN_OFFSET)]; + data = regs->uregs[EXTRACT_REG_NUM(instr, RT2_OFFSET)]; + destreg = EXTRACT_REG_NUM(instr, RT_OFFSET); + + type = instr & TYPE_SWPB; + + pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n", + EXTRACT_REG_NUM(instr, RN_OFFSET), address, + destreg, EXTRACT_REG_NUM(instr, RT2_OFFSET), data); + + /* Check access in reasonable access range for both SWP and SWPB */ + if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) { + pr_debug("SWP{B} emulation: access to %p not allowed!\n", + (void *)address); + res = -EFAULT; + } else { + res = emulate_swpX(address, &data, type); + } + + if (res == 0) { + /* + * On successful emulation, revert the adjustment to the PC + * made in kernel/traps.c in order to resume execution at the + * instruction following the SWP{B}. + */ + regs->ARM_pc += 4; + regs->uregs[destreg] = data; + } else if (res == -EFAULT) { + /* + * Memory errors do not mean emulation failed. + * Set up signal info to return SEGV, then return OK + */ + set_segfault(regs, address); + } + + return 0; +} + +/* + * Only emulate SWP/SWPB executed in ARM state/User mode. + * The kernel must be SWP free and SWP{B} does not exist in Thumb/ThumbEE. + */ +static struct undef_hook swp_hook = { + .instr_mask = 0x0fb00ff0, + .instr_val = 0x01000090, + .cpsr_mask = MODE_MASK | PSR_T_BIT | PSR_J_BIT, + .cpsr_val = USR_MODE, + .fn = swp_handler +}; + +/* + * Register handler and create status file in /proc/cpu + * Invoked as late_initcall, since not needed before init spawned. + */ +static int __init swp_emulation_init(void) +{ +#ifdef CONFIG_PROC_FS + struct proc_dir_entry *res; + + res = create_proc_entry("cpu/swp_emulation", S_IRUGO, NULL); + + if (!res) + return -ENOMEM; + + res->read_proc = proc_read_status; +#endif /* CONFIG_PROC_FS */ + + printk(KERN_NOTICE "Registering SWP/SWPB emulation handler\n"); + register_undef_hook(&swp_hook); + + return 0; +} + +late_initcall(swp_emulation_init); diff --git a/arch/arm/kernel/sys_arm.c b/arch/arm/kernel/sys_arm.c new file mode 100644 index 00000000..0264ab43 --- /dev/null +++ b/arch/arm/kernel/sys_arm.c @@ -0,0 +1,133 @@ +/* + * linux/arch/arm/kernel/sys_arm.c + * + * Copyright (C) People who wrote linux/arch/i386/kernel/sys_i386.c + * Copyright (C) 1995, 1996 Russell King. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This file contains various random system calls that + * have a non-standard calling sequence on the Linux/arm + * platform. + */ +#include <linux/module.h> +#include <linux/errno.h> +#include <linux/sched.h> +#include <linux/mm.h> +#include <linux/sem.h> +#include <linux/msg.h> +#include <linux/shm.h> +#include <linux/stat.h> +#include <linux/syscalls.h> +#include <linux/mman.h> +#include <linux/fs.h> +#include <linux/file.h> +#include <linux/ipc.h> +#include <linux/uaccess.h> +#include <linux/slab.h> + +/* Fork a new task - this creates a new program thread. + * This is called indirectly via a small wrapper + */ +asmlinkage int sys_fork(struct pt_regs *regs) +{ +#ifdef CONFIG_MMU + return do_fork(SIGCHLD, regs->ARM_sp, regs, 0, NULL, NULL); +#else + /* can not support in nommu mode */ + return(-EINVAL); +#endif +} + +/* Clone a task - this clones the calling program thread. + * This is called indirectly via a small wrapper + */ +asmlinkage int sys_clone(unsigned long clone_flags, unsigned long newsp, + int __user *parent_tidptr, int tls_val, + int __user *child_tidptr, struct pt_regs *regs) +{ + if (!newsp) + newsp = regs->ARM_sp; + + return do_fork(clone_flags, newsp, regs, 0, parent_tidptr, child_tidptr); +} + +asmlinkage int sys_vfork(struct pt_regs *regs) +{ + return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->ARM_sp, regs, 0, NULL, NULL); +} + +/* sys_execve() executes a new program. + * This is called indirectly via a small wrapper + */ +asmlinkage int sys_execve(const char __user *filenamei, + const char __user *const __user *argv, + const char __user *const __user *envp, struct pt_regs *regs) +{ + int error; + char * filename; + + filename = getname(filenamei); + error = PTR_ERR(filename); + if (IS_ERR(filename)) + goto out; + error = do_execve(filename, argv, envp, regs); + putname(filename); +out: + return error; +} + +int kernel_execve(const char *filename, + const char *const argv[], + const char *const envp[]) +{ + struct pt_regs regs; + int ret; + + memset(®s, 0, sizeof(struct pt_regs)); + ret = do_execve(filename, + (const char __user *const __user *)argv, + (const char __user *const __user *)envp, ®s); + if (ret < 0) + goto out; + + /* + * Save argc to the register structure for userspace. + */ + regs.ARM_r0 = ret; + + /* + * We were successful. We won't be returning to our caller, but + * instead to user space by manipulating the kernel stack. + */ + asm( "add r0, %0, %1\n\t" + "mov r1, %2\n\t" + "mov r2, %3\n\t" + "bl memmove\n\t" /* copy regs to top of stack */ + "mov r8, #0\n\t" /* not a syscall */ + "mov r9, %0\n\t" /* thread structure */ + "mov sp, r0\n\t" /* reposition stack pointer */ + "b ret_to_user" + : + : "r" (current_thread_info()), + "Ir" (THREAD_START_SP - sizeof(regs)), + "r" (®s), + "Ir" (sizeof(regs)) + : "r0", "r1", "r2", "r3", "r8", "r9", "ip", "lr", "memory"); + + out: + return ret; +} +EXPORT_SYMBOL(kernel_execve); + +/* + * Since loff_t is a 64 bit type we avoid a lot of ABI hassle + * with a different argument ordering. + */ +asmlinkage long sys_arm_fadvise64_64(int fd, int advice, + loff_t offset, loff_t len) +{ + return sys_fadvise64_64(fd, offset, len, advice); +} diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c new file mode 100644 index 00000000..af0aaebf --- /dev/null +++ b/arch/arm/kernel/sys_oabi-compat.c @@ -0,0 +1,453 @@ +/* + * arch/arm/kernel/sys_oabi-compat.c + * + * Compatibility wrappers for syscalls that are used from + * old ABI user space binaries with an EABI kernel. + * + * Author: Nicolas Pitre + * Created: Oct 7, 2005 + * Copyright: MontaVista Software, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +/* + * The legacy ABI and the new ARM EABI have different rules making some + * syscalls incompatible especially with structure arguments. + * Most notably, Eabi says 64-bit members should be 64-bit aligned instead of + * simply word aligned. EABI also pads structures to the size of the largest + * member it contains instead of the invariant 32-bit. + * + * The following syscalls are affected: + * + * sys_stat64: + * sys_lstat64: + * sys_fstat64: + * sys_fstatat64: + * + * struct stat64 has different sizes and some members are shifted + * Compatibility wrappers are needed for them and provided below. + * + * sys_fcntl64: + * + * struct flock64 has different sizes and some members are shifted + * A compatibility wrapper is needed and provided below. + * + * sys_statfs64: + * sys_fstatfs64: + * + * struct statfs64 has extra padding with EABI growing its size from + * 84 to 88. This struct is now __attribute__((packed,aligned(4))) + * with a small assembly wrapper to force the sz argument to 84 if it is 88 + * to avoid copying the extra padding over user space unexpecting it. + * + * sys_newuname: + * + * struct new_utsname has no padding with EABI. No problem there. + * + * sys_epoll_ctl: + * sys_epoll_wait: + * + * struct epoll_event has its second member shifted also affecting the + * structure size. Compatibility wrappers are needed and provided below. + * + * sys_ipc: + * sys_semop: + * sys_semtimedop: + * + * struct sembuf loses its padding with EABI. Since arrays of them are + * used they have to be copyed to remove the padding. Compatibility wrappers + * provided below. + * + * sys_bind: + * sys_connect: + * sys_sendmsg: + * sys_sendto: + * sys_socketcall: + * + * struct sockaddr_un loses its padding with EABI. Since the size of the + * structure is used as a validation test in unix_mkname(), we need to + * change the length argument to 110 whenever it is 112. Compatibility + * wrappers provided below. + */ + +#include <linux/syscalls.h> +#include <linux/errno.h> +#include <linux/fs.h> +#include <linux/fcntl.h> +#include <linux/eventpoll.h> +#include <linux/sem.h> +#include <linux/socket.h> +#include <linux/net.h> +#include <linux/ipc.h> +#include <linux/uaccess.h> +#include <linux/slab.h> + +struct oldabi_stat64 { + unsigned long long st_dev; + unsigned int __pad1; + unsigned long __st_ino; + unsigned int st_mode; + unsigned int st_nlink; + + unsigned long st_uid; + unsigned long st_gid; + + unsigned long long st_rdev; + unsigned int __pad2; + + long long st_size; + unsigned long st_blksize; + unsigned long long st_blocks; + + unsigned long st_atime; + unsigned long st_atime_nsec; + + unsigned long st_mtime; + unsigned long st_mtime_nsec; + + unsigned long st_ctime; + unsigned long st_ctime_nsec; + + unsigned long long st_ino; +} __attribute__ ((packed,aligned(4))); + +static long cp_oldabi_stat64(struct kstat *stat, + struct oldabi_stat64 __user *statbuf) +{ + struct oldabi_stat64 tmp; + + tmp.st_dev = huge_encode_dev(stat->dev); + tmp.__pad1 = 0; + tmp.__st_ino = stat->ino; + tmp.st_mode = stat->mode; + tmp.st_nlink = stat->nlink; + tmp.st_uid = stat->uid; + tmp.st_gid = stat->gid; + tmp.st_rdev = huge_encode_dev(stat->rdev); + tmp.st_size = stat->size; + tmp.st_blocks = stat->blocks; + tmp.__pad2 = 0; + tmp.st_blksize = stat->blksize; + tmp.st_atime = stat->atime.tv_sec; + tmp.st_atime_nsec = stat->atime.tv_nsec; + tmp.st_mtime = stat->mtime.tv_sec; + tmp.st_mtime_nsec = stat->mtime.tv_nsec; + tmp.st_ctime = stat->ctime.tv_sec; + tmp.st_ctime_nsec = stat->ctime.tv_nsec; + tmp.st_ino = stat->ino; + return copy_to_user(statbuf,&tmp,sizeof(tmp)) ? -EFAULT : 0; +} + +asmlinkage long sys_oabi_stat64(const char __user * filename, + struct oldabi_stat64 __user * statbuf) +{ + struct kstat stat; + int error = vfs_stat(filename, &stat); + if (!error) + error = cp_oldabi_stat64(&stat, statbuf); + return error; +} + +asmlinkage long sys_oabi_lstat64(const char __user * filename, + struct oldabi_stat64 __user * statbuf) +{ + struct kstat stat; + int error = vfs_lstat(filename, &stat); + if (!error) + error = cp_oldabi_stat64(&stat, statbuf); + return error; +} + +asmlinkage long sys_oabi_fstat64(unsigned long fd, + struct oldabi_stat64 __user * statbuf) +{ + struct kstat stat; + int error = vfs_fstat(fd, &stat); + if (!error) + error = cp_oldabi_stat64(&stat, statbuf); + return error; +} + +asmlinkage long sys_oabi_fstatat64(int dfd, + const char __user *filename, + struct oldabi_stat64 __user *statbuf, + int flag) +{ + struct kstat stat; + int error; + + error = vfs_fstatat(dfd, filename, &stat, flag); + if (error) + return error; + return cp_oldabi_stat64(&stat, statbuf); +} + +struct oabi_flock64 { + short l_type; + short l_whence; + loff_t l_start; + loff_t l_len; + pid_t l_pid; +} __attribute__ ((packed,aligned(4))); + +asmlinkage long sys_oabi_fcntl64(unsigned int fd, unsigned int cmd, + unsigned long arg) +{ + struct oabi_flock64 user; + struct flock64 kernel; + mm_segment_t fs = USER_DS; /* initialized to kill a warning */ + unsigned long local_arg = arg; + int ret; + + switch (cmd) { + case F_GETLK64: + case F_SETLK64: + case F_SETLKW64: + if (copy_from_user(&user, (struct oabi_flock64 __user *)arg, + sizeof(user))) + return -EFAULT; + kernel.l_type = user.l_type; + kernel.l_whence = user.l_whence; + kernel.l_start = user.l_start; + kernel.l_len = user.l_len; + kernel.l_pid = user.l_pid; + local_arg = (unsigned long)&kernel; + fs = get_fs(); + set_fs(KERNEL_DS); + } + + ret = sys_fcntl64(fd, cmd, local_arg); + + switch (cmd) { + case F_GETLK64: + if (!ret) { + user.l_type = kernel.l_type; + user.l_whence = kernel.l_whence; + user.l_start = kernel.l_start; + user.l_len = kernel.l_len; + user.l_pid = kernel.l_pid; + if (copy_to_user((struct oabi_flock64 __user *)arg, + &user, sizeof(user))) + ret = -EFAULT; + } + case F_SETLK64: + case F_SETLKW64: + set_fs(fs); + } + + return ret; +} + +struct oabi_epoll_event { + __u32 events; + __u64 data; +} __attribute__ ((packed,aligned(4))); + +asmlinkage long sys_oabi_epoll_ctl(int epfd, int op, int fd, + struct oabi_epoll_event __user *event) +{ + struct oabi_epoll_event user; + struct epoll_event kernel; + mm_segment_t fs; + long ret; + + if (op == EPOLL_CTL_DEL) + return sys_epoll_ctl(epfd, op, fd, NULL); + if (copy_from_user(&user, event, sizeof(user))) + return -EFAULT; + kernel.events = user.events; + kernel.data = user.data; + fs = get_fs(); + set_fs(KERNEL_DS); + ret = sys_epoll_ctl(epfd, op, fd, &kernel); + set_fs(fs); + return ret; +} + +asmlinkage long sys_oabi_epoll_wait(int epfd, + struct oabi_epoll_event __user *events, + int maxevents, int timeout) +{ + struct epoll_event *kbuf; + mm_segment_t fs; + long ret, err, i; + + if (maxevents <= 0 || maxevents > (INT_MAX/sizeof(struct epoll_event))) + return -EINVAL; + kbuf = kmalloc(sizeof(*kbuf) * maxevents, GFP_KERNEL); + if (!kbuf) + return -ENOMEM; + fs = get_fs(); + set_fs(KERNEL_DS); + ret = sys_epoll_wait(epfd, kbuf, maxevents, timeout); + set_fs(fs); + err = 0; + for (i = 0; i < ret; i++) { + __put_user_error(kbuf[i].events, &events->events, err); + __put_user_error(kbuf[i].data, &events->data, err); + events++; + } + kfree(kbuf); + return err ? -EFAULT : ret; +} + +struct oabi_sembuf { + unsigned short sem_num; + short sem_op; + short sem_flg; + unsigned short __pad; +}; + +asmlinkage long sys_oabi_semtimedop(int semid, + struct oabi_sembuf __user *tsops, + unsigned nsops, + const struct timespec __user *timeout) +{ + struct sembuf *sops; + struct timespec local_timeout; + long err; + int i; + + if (nsops < 1 || nsops > SEMOPM) + return -EINVAL; + sops = kmalloc(sizeof(*sops) * nsops, GFP_KERNEL); + if (!sops) + return -ENOMEM; + err = 0; + for (i = 0; i < nsops; i++) { + __get_user_error(sops[i].sem_num, &tsops->sem_num, err); + __get_user_error(sops[i].sem_op, &tsops->sem_op, err); + __get_user_error(sops[i].sem_flg, &tsops->sem_flg, err); + tsops++; + } + if (timeout) { + /* copy this as well before changing domain protection */ + err |= copy_from_user(&local_timeout, timeout, sizeof(*timeout)); + timeout = &local_timeout; + } + if (err) { + err = -EFAULT; + } else { + mm_segment_t fs = get_fs(); + set_fs(KERNEL_DS); + err = sys_semtimedop(semid, sops, nsops, timeout); + set_fs(fs); + } + kfree(sops); + return err; +} + +asmlinkage long sys_oabi_semop(int semid, struct oabi_sembuf __user *tsops, + unsigned nsops) +{ + return sys_oabi_semtimedop(semid, tsops, nsops, NULL); +} + +asmlinkage int sys_oabi_ipc(uint call, int first, int second, int third, + void __user *ptr, long fifth) +{ + switch (call & 0xffff) { + case SEMOP: + return sys_oabi_semtimedop(first, + (struct oabi_sembuf __user *)ptr, + second, NULL); + case SEMTIMEDOP: + return sys_oabi_semtimedop(first, + (struct oabi_sembuf __user *)ptr, + second, + (const struct timespec __user *)fifth); + default: + return sys_ipc(call, first, second, third, ptr, fifth); + } +} + +asmlinkage long sys_oabi_bind(int fd, struct sockaddr __user *addr, int addrlen) +{ + sa_family_t sa_family; + if (addrlen == 112 && + get_user(sa_family, &addr->sa_family) == 0 && + sa_family == AF_UNIX) + addrlen = 110; + return sys_bind(fd, addr, addrlen); +} + +asmlinkage long sys_oabi_connect(int fd, struct sockaddr __user *addr, int addrlen) +{ + sa_family_t sa_family; + if (addrlen == 112 && + get_user(sa_family, &addr->sa_family) == 0 && + sa_family == AF_UNIX) + addrlen = 110; + return sys_connect(fd, addr, addrlen); +} + +asmlinkage long sys_oabi_sendto(int fd, void __user *buff, + size_t len, unsigned flags, + struct sockaddr __user *addr, + int addrlen) +{ + sa_family_t sa_family; + if (addrlen == 112 && + get_user(sa_family, &addr->sa_family) == 0 && + sa_family == AF_UNIX) + addrlen = 110; + return sys_sendto(fd, buff, len, flags, addr, addrlen); +} + +asmlinkage long sys_oabi_sendmsg(int fd, struct msghdr __user *msg, unsigned flags) +{ + struct sockaddr __user *addr; + int msg_namelen; + sa_family_t sa_family; + if (msg && + get_user(msg_namelen, &msg->msg_namelen) == 0 && + msg_namelen == 112 && + get_user(addr, &msg->msg_name) == 0 && + get_user(sa_family, &addr->sa_family) == 0 && + sa_family == AF_UNIX) + { + /* + * HACK ALERT: there is a limit to how much backward bending + * we should do for what is actually a transitional + * compatibility layer. This already has known flaws with + * a few ioctls that we don't intend to fix. Therefore + * consider this blatent hack as another one... and take care + * to run for cover. In most cases it will "just work fine". + * If it doesn't, well, tough. + */ + put_user(110, &msg->msg_namelen); + } + return sys_sendmsg(fd, msg, flags); +} + +asmlinkage long sys_oabi_socketcall(int call, unsigned long __user *args) +{ + unsigned long r = -EFAULT, a[6]; + + switch (call) { + case SYS_BIND: + if (copy_from_user(a, args, 3 * sizeof(long)) == 0) + r = sys_oabi_bind(a[0], (struct sockaddr __user *)a[1], a[2]); + break; + case SYS_CONNECT: + if (copy_from_user(a, args, 3 * sizeof(long)) == 0) + r = sys_oabi_connect(a[0], (struct sockaddr __user *)a[1], a[2]); + break; + case SYS_SENDTO: + if (copy_from_user(a, args, 6 * sizeof(long)) == 0) + r = sys_oabi_sendto(a[0], (void __user *)a[1], a[2], a[3], + (struct sockaddr __user *)a[4], a[5]); + break; + case SYS_SENDMSG: + if (copy_from_user(a, args, 3 * sizeof(long)) == 0) + r = sys_oabi_sendmsg(a[0], (struct msghdr __user *)a[1], a[2]); + break; + default: + r = sys_socketcall(call, args); + } + + return r; +} diff --git a/arch/arm/kernel/tcm.c b/arch/arm/kernel/tcm.c new file mode 100644 index 00000000..f5cf660e --- /dev/null +++ b/arch/arm/kernel/tcm.c @@ -0,0 +1,274 @@ +/* + * Copyright (C) 2008-2009 ST-Ericsson AB + * License terms: GNU General Public License (GPL) version 2 + * TCM memory handling for ARM systems + * + * Author: Linus Walleij <linus.walleij@stericsson.com> + * Author: Rickard Andersson <rickard.andersson@stericsson.com> + */ +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/stddef.h> +#include <linux/ioport.h> +#include <linux/genalloc.h> +#include <linux/string.h> /* memcpy */ +#include <asm/cputype.h> +#include <asm/mach/map.h> +#include <asm/memory.h> +#include "tcm.h" + +static struct gen_pool *tcm_pool; + +/* TCM section definitions from the linker */ +extern char __itcm_start, __sitcm_text, __eitcm_text; +extern char __dtcm_start, __sdtcm_data, __edtcm_data; + +/* These will be increased as we run */ +u32 dtcm_end = DTCM_OFFSET; +u32 itcm_end = ITCM_OFFSET; + +/* + * TCM memory resources + */ +static struct resource dtcm_res = { + .name = "DTCM RAM", + .start = DTCM_OFFSET, + .end = DTCM_OFFSET, + .flags = IORESOURCE_MEM +}; + +static struct resource itcm_res = { + .name = "ITCM RAM", + .start = ITCM_OFFSET, + .end = ITCM_OFFSET, + .flags = IORESOURCE_MEM +}; + +static struct map_desc dtcm_iomap[] __initdata = { + { + .virtual = DTCM_OFFSET, + .pfn = __phys_to_pfn(DTCM_OFFSET), + .length = 0, + .type = MT_MEMORY_DTCM + } +}; + +static struct map_desc itcm_iomap[] __initdata = { + { + .virtual = ITCM_OFFSET, + .pfn = __phys_to_pfn(ITCM_OFFSET), + .length = 0, + .type = MT_MEMORY_ITCM + } +}; + +/* + * Allocate a chunk of TCM memory + */ +void *tcm_alloc(size_t len) +{ + unsigned long vaddr; + + if (!tcm_pool) + return NULL; + + vaddr = gen_pool_alloc(tcm_pool, len); + if (!vaddr) + return NULL; + + return (void *) vaddr; +} +EXPORT_SYMBOL(tcm_alloc); + +/* + * Free a chunk of TCM memory + */ +void tcm_free(void *addr, size_t len) +{ + gen_pool_free(tcm_pool, (unsigned long) addr, len); +} +EXPORT_SYMBOL(tcm_free); + +static int __init setup_tcm_bank(u8 type, u8 bank, u8 banks, + u32 *offset) +{ + const int tcm_sizes[16] = { 0, -1, -1, 4, 8, 16, 32, 64, 128, + 256, 512, 1024, -1, -1, -1, -1 }; + u32 tcm_region; + int tcm_size; + + /* + * If there are more than one TCM bank of this type, + * select the TCM bank to operate on in the TCM selection + * register. + */ + if (banks > 1) + asm("mcr p15, 0, %0, c9, c2, 0" + : /* No output operands */ + : "r" (bank)); + + /* Read the special TCM region register c9, 0 */ + if (!type) + asm("mrc p15, 0, %0, c9, c1, 0" + : "=r" (tcm_region)); + else + asm("mrc p15, 0, %0, c9, c1, 1" + : "=r" (tcm_region)); + + tcm_size = tcm_sizes[(tcm_region >> 2) & 0x0f]; + if (tcm_size < 0) { + pr_err("CPU: %sTCM%d of unknown size\n", + type ? "I" : "D", bank); + return -EINVAL; + } else if (tcm_size > 32) { + pr_err("CPU: %sTCM%d larger than 32k found\n", + type ? "I" : "D", bank); + return -EINVAL; + } else { + pr_info("CPU: found %sTCM%d %dk @ %08x, %senabled\n", + type ? "I" : "D", + bank, + tcm_size, + (tcm_region & 0xfffff000U), + (tcm_region & 1) ? "" : "not "); + } + + /* Force move the TCM bank to where we want it, enable */ + tcm_region = *offset | (tcm_region & 0x00000ffeU) | 1; + + if (!type) + asm("mcr p15, 0, %0, c9, c1, 0" + : /* No output operands */ + : "r" (tcm_region)); + else + asm("mcr p15, 0, %0, c9, c1, 1" + : /* No output operands */ + : "r" (tcm_region)); + + /* Increase offset */ + *offset += (tcm_size << 10); + + pr_info("CPU: moved %sTCM%d %dk to %08x, enabled\n", + type ? "I" : "D", + bank, + tcm_size, + (tcm_region & 0xfffff000U)); + return 0; +} + +/* + * This initializes the TCM memory + */ +void __init tcm_init(void) +{ + u32 tcm_status = read_cpuid_tcmstatus(); + u8 dtcm_banks = (tcm_status >> 16) & 0x03; + u8 itcm_banks = (tcm_status & 0x03); + char *start; + char *end; + char *ram; + int ret; + int i; + + /* Setup DTCM if present */ + if (dtcm_banks > 0) { + for (i = 0; i < dtcm_banks; i++) { + ret = setup_tcm_bank(0, i, dtcm_banks, &dtcm_end); + if (ret) + return; + } + dtcm_res.end = dtcm_end - 1; + request_resource(&iomem_resource, &dtcm_res); + dtcm_iomap[0].length = dtcm_end - DTCM_OFFSET; + iotable_init(dtcm_iomap, 1); + /* Copy data from RAM to DTCM */ + start = &__sdtcm_data; + end = &__edtcm_data; + ram = &__dtcm_start; + /* This means you compiled more code than fits into DTCM */ + BUG_ON((end - start) > (dtcm_end - DTCM_OFFSET)); + memcpy(start, ram, (end-start)); + pr_debug("CPU DTCM: copied data from %p - %p\n", start, end); + } + + /* Setup ITCM if present */ + if (itcm_banks > 0) { + for (i = 0; i < itcm_banks; i++) { + ret = setup_tcm_bank(1, i, itcm_banks, &itcm_end); + if (ret) + return; + } + itcm_res.end = itcm_end - 1; + request_resource(&iomem_resource, &itcm_res); + itcm_iomap[0].length = itcm_end - ITCM_OFFSET; + iotable_init(itcm_iomap, 1); + /* Copy code from RAM to ITCM */ + start = &__sitcm_text; + end = &__eitcm_text; + ram = &__itcm_start; + /* This means you compiled more code than fits into ITCM */ + BUG_ON((end - start) > (itcm_end - ITCM_OFFSET)); + memcpy(start, ram, (end-start)); + pr_debug("CPU ITCM: copied code from %p - %p\n", start, end); + } +} + +/* + * This creates the TCM memory pool and has to be done later, + * during the core_initicalls, since the allocator is not yet + * up and running when the first initialization runs. + */ +static int __init setup_tcm_pool(void) +{ + u32 tcm_status = read_cpuid_tcmstatus(); + u32 dtcm_pool_start = (u32) &__edtcm_data; + u32 itcm_pool_start = (u32) &__eitcm_text; + int ret; + + /* + * Set up malloc pool, 2^2 = 4 bytes granularity since + * the TCM is sometimes just 4 KiB. NB: pages and cache + * line alignments does not matter in TCM! + */ + tcm_pool = gen_pool_create(2, -1); + + pr_debug("Setting up TCM memory pool\n"); + + /* Add the rest of DTCM to the TCM pool */ + if (tcm_status & (0x03 << 16)) { + if (dtcm_pool_start < dtcm_end) { + ret = gen_pool_add(tcm_pool, dtcm_pool_start, + dtcm_end - dtcm_pool_start, -1); + if (ret) { + pr_err("CPU DTCM: could not add DTCM " \ + "remainder to pool!\n"); + return ret; + } + pr_debug("CPU DTCM: Added %08x bytes @ %08x to " \ + "the TCM memory pool\n", + dtcm_end - dtcm_pool_start, + dtcm_pool_start); + } + } + + /* Add the rest of ITCM to the TCM pool */ + if (tcm_status & 0x03) { + if (itcm_pool_start < itcm_end) { + ret = gen_pool_add(tcm_pool, itcm_pool_start, + itcm_end - itcm_pool_start, -1); + if (ret) { + pr_err("CPU ITCM: could not add ITCM " \ + "remainder to pool!\n"); + return ret; + } + pr_debug("CPU ITCM: Added %08x bytes @ %08x to " \ + "the TCM memory pool\n", + itcm_end - itcm_pool_start, + itcm_pool_start); + } + } + return 0; +} + +core_initcall(setup_tcm_pool); diff --git a/arch/arm/kernel/tcm.h b/arch/arm/kernel/tcm.h new file mode 100644 index 00000000..8015ad43 --- /dev/null +++ b/arch/arm/kernel/tcm.h @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2008-2009 ST-Ericsson AB + * License terms: GNU General Public License (GPL) version 2 + * TCM memory handling for ARM systems + * + * Author: Linus Walleij <linus.walleij@stericsson.com> + * Author: Rickard Andersson <rickard.andersson@stericsson.com> + */ + +#ifdef CONFIG_HAVE_TCM +void __init tcm_init(void); +#else +/* No TCM support, just blank inlines to be optimized out */ +inline void tcm_init(void) +{ +} +#endif diff --git a/arch/arm/kernel/thumbee.c b/arch/arm/kernel/thumbee.c new file mode 100644 index 00000000..9cb7aaca --- /dev/null +++ b/arch/arm/kernel/thumbee.c @@ -0,0 +1,81 @@ +/* + * arch/arm/kernel/thumbee.c + * + * Copyright (C) 2008 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include <linux/kernel.h> +#include <linux/init.h> + +#include <asm/thread_notify.h> + +/* + * Access to the ThumbEE Handler Base register + */ +static inline unsigned long teehbr_read(void) +{ + unsigned long v; + asm("mrc p14, 6, %0, c1, c0, 0\n" : "=r" (v)); + return v; +} + +static inline void teehbr_write(unsigned long v) +{ + asm("mcr p14, 6, %0, c1, c0, 0\n" : : "r" (v)); +} + +static int thumbee_notifier(struct notifier_block *self, unsigned long cmd, void *t) +{ + struct thread_info *thread = t; + + switch (cmd) { + case THREAD_NOTIFY_FLUSH: + thread->thumbee_state = 0; + break; + case THREAD_NOTIFY_SWITCH: + current_thread_info()->thumbee_state = teehbr_read(); + teehbr_write(thread->thumbee_state); + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block thumbee_notifier_block = { + .notifier_call = thumbee_notifier, +}; + +static int __init thumbee_init(void) +{ + unsigned long pfr0; + unsigned int cpu_arch = cpu_architecture(); + + if (cpu_arch < CPU_ARCH_ARMv7) + return 0; + + /* processor feature register 0 */ + asm("mrc p15, 0, %0, c0, c1, 0\n" : "=r" (pfr0)); + if ((pfr0 & 0x0000f000) != 0x00001000) + return 0; + + printk(KERN_INFO "ThumbEE CPU extension supported.\n"); + elf_hwcap |= HWCAP_THUMBEE; + thread_register_notifier(&thumbee_notifier_block); + + return 0; +} + +late_initcall(thumbee_init); diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c new file mode 100644 index 00000000..cb634c3e --- /dev/null +++ b/arch/arm/kernel/time.c @@ -0,0 +1,158 @@ +/* + * linux/arch/arm/kernel/time.c + * + * Copyright (C) 1991, 1992, 1995 Linus Torvalds + * Modifications for ARM (C) 1994-2001 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This file contains the ARM-specific time handling details: + * reading the RTC at bootup, etc... + */ +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/time.h> +#include <linux/init.h> +#include <linux/sched.h> +#include <linux/smp.h> +#include <linux/timex.h> +#include <linux/errno.h> +#include <linux/profile.h> +#include <linux/syscore_ops.h> +#include <linux/timer.h> +#include <linux/irq.h> + +#include <linux/mc146818rtc.h> + +#include <asm/leds.h> +#include <asm/thread_info.h> +#include <asm/sched_clock.h> +#include <asm/stacktrace.h> +#include <asm/mach/arch.h> +#include <asm/mach/time.h> + +/* + * Our system timer. + */ +static struct sys_timer *system_timer; + +#if defined(CONFIG_RTC_DRV_CMOS) || defined(CONFIG_RTC_DRV_CMOS_MODULE) +/* this needs a better home */ +DEFINE_SPINLOCK(rtc_lock); + +#ifdef CONFIG_RTC_DRV_CMOS_MODULE +EXPORT_SYMBOL(rtc_lock); +#endif +#endif /* pc-style 'CMOS' RTC support */ + +/* change this if you have some constant time drift */ +#define USECS_PER_JIFFY (1000000/HZ) + +#ifdef CONFIG_SMP +unsigned long profile_pc(struct pt_regs *regs) +{ + struct stackframe frame; + + if (!in_lock_functions(regs->ARM_pc)) + return regs->ARM_pc; + + frame.fp = regs->ARM_fp; + frame.sp = regs->ARM_sp; + frame.lr = regs->ARM_lr; + frame.pc = regs->ARM_pc; + do { + int ret = unwind_frame(&frame); + if (ret < 0) + return 0; + } while (in_lock_functions(frame.pc)); + + return frame.pc; +} +EXPORT_SYMBOL(profile_pc); +#endif + +#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET +u32 arch_gettimeoffset(void) +{ + if (system_timer->offset != NULL) + return system_timer->offset() * 1000; + + return 0; +} +#endif /* CONFIG_ARCH_USES_GETTIMEOFFSET */ + +#ifdef CONFIG_LEDS_TIMER +static inline void do_leds(void) +{ + static unsigned int count = HZ/2; + + if (--count == 0) { + count = HZ/2; + leds_event(led_timer); + } +} +#else +#define do_leds() +#endif + + +#ifndef CONFIG_GENERIC_CLOCKEVENTS +/* + * Kernel system timer support. + */ +void timer_tick(void) +{ + profile_tick(CPU_PROFILING); + do_leds(); + xtime_update(1); +#ifndef CONFIG_SMP + update_process_times(user_mode(get_irq_regs())); +#endif +} +#endif + +#if defined(CONFIG_PM) && !defined(CONFIG_GENERIC_CLOCKEVENTS) +static int timer_suspend(void) +{ + if (system_timer->suspend) + system_timer->suspend(); + + return 0; +} + +static void timer_resume(void) +{ + if (system_timer->resume) + system_timer->resume(); +} +#else +#define timer_suspend NULL +#define timer_resume NULL +#endif + +static struct syscore_ops timer_syscore_ops = { + .suspend = timer_suspend, + .resume = timer_resume, +}; + +static int __init timer_init_syscore_ops(void) +{ + register_syscore_ops(&timer_syscore_ops); + + return 0; +} + +device_initcall(timer_init_syscore_ops); + +void __init time_init(void) +{ + system_timer = machine_desc->timer; + system_timer->init(); +#ifdef CONFIG_HAVE_SCHED_CLOCK + sched_clock_postinit(); +#endif +} + diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c new file mode 100644 index 00000000..56b27153 --- /dev/null +++ b/arch/arm/kernel/traps.c @@ -0,0 +1,796 @@ +/* + * linux/arch/arm/kernel/traps.c + * + * Copyright (C) 1995-2009 Russell King + * Fragments that appear the same as linux/arch/i386/kernel/traps.c (C) Linus Torvalds + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * 'traps.c' handles hardware exceptions after we have saved some state in + * 'linux/arch/arm/lib/traps.S'. Mostly a debugging aid, but will probably + * kill the offending process. + */ +#include <linux/signal.h> +#include <linux/personality.h> +#include <linux/kallsyms.h> +#include <linux/spinlock.h> +#include <linux/uaccess.h> +#include <linux/hardirq.h> +#include <linux/kdebug.h> +#include <linux/module.h> +#include <linux/kexec.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/sched.h> + +#include <asm/atomic.h> +#include <asm/cacheflush.h> +#include <asm/system.h> +#include <asm/unistd.h> +#include <asm/traps.h> +#include <asm/unwind.h> +#include <asm/tls.h> + +#include "signal.h" + +static const char *handler[]= { "prefetch abort", "data abort", "address exception", "interrupt" }; + +void *vectors_page; + +#ifdef CONFIG_DEBUG_USER +unsigned int user_debug; + +static int __init user_debug_setup(char *str) +{ + get_option(&str, &user_debug); + return 1; +} +__setup("user_debug=", user_debug_setup); +#endif + +static void dump_mem(const char *, const char *, unsigned long, unsigned long); + +void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame) +{ +#ifdef CONFIG_KALLSYMS + printk("[<%08lx>] (%pS) from [<%08lx>] (%pS)\n", where, (void *)where, from, (void *)from); +#else + printk("Function entered at [<%08lx>] from [<%08lx>]\n", where, from); +#endif + + if (in_exception_text(where)) + dump_mem("", "Exception stack", frame + 4, frame + 4 + sizeof(struct pt_regs)); +} + +#ifndef CONFIG_ARM_UNWIND +/* + * Stack pointers should always be within the kernels view of + * physical memory. If it is not there, then we can't dump + * out any information relating to the stack. + */ +static int verify_stack(unsigned long sp) +{ + if (sp < PAGE_OFFSET || + (sp > (unsigned long)high_memory && high_memory != NULL)) + return -EFAULT; + + return 0; +} +#endif + +/* + * Dump out the contents of some memory nicely... + */ +static void dump_mem(const char *lvl, const char *str, unsigned long bottom, + unsigned long top) +{ + unsigned long first; + mm_segment_t fs; + int i; + + /* + * We need to switch to kernel mode so that we can use __get_user + * to safely read from kernel space. Note that we now dump the + * code first, just in case the backtrace kills us. + */ + fs = get_fs(); + set_fs(KERNEL_DS); + + printk("%s%s(0x%08lx to 0x%08lx)\n", lvl, str, bottom, top); + + for (first = bottom & ~31; first < top; first += 32) { + unsigned long p; + char str[sizeof(" 12345678") * 8 + 1]; + + memset(str, ' ', sizeof(str)); + str[sizeof(str) - 1] = '\0'; + + for (p = first, i = 0; i < 8 && p < top; i++, p += 4) { + if (p >= bottom && p < top) { + unsigned long val; + if (__get_user(val, (unsigned long *)p) == 0) + sprintf(str + i * 9, " %08lx", val); + else + sprintf(str + i * 9, " ????????"); + } + } + printk("%s%04lx:%s\n", lvl, first & 0xffff, str); + } + + set_fs(fs); +} + +static void dump_instr(const char *lvl, struct pt_regs *regs) +{ + unsigned long addr = instruction_pointer(regs); + const int thumb = thumb_mode(regs); + const int width = thumb ? 4 : 8; + mm_segment_t fs; + char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str; + int i; + + /* + * We need to switch to kernel mode so that we can use __get_user + * to safely read from kernel space. Note that we now dump the + * code first, just in case the backtrace kills us. + */ + fs = get_fs(); + set_fs(KERNEL_DS); + + for (i = -4; i < 1 + !!thumb; i++) { + unsigned int val, bad; + + if (thumb) + bad = __get_user(val, &((u16 *)addr)[i]); + else + bad = __get_user(val, &((u32 *)addr)[i]); + + if (!bad) + p += sprintf(p, i == 0 ? "(%0*x) " : "%0*x ", + width, val); + else { + p += sprintf(p, "bad PC value"); + break; + } + } + printk("%sCode: %s\n", lvl, str); + + set_fs(fs); +} + +#ifdef CONFIG_ARM_UNWIND +static inline void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) +{ + unwind_backtrace(regs, tsk); +} +#else +static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk) +{ + unsigned int fp, mode; + int ok = 1; + + printk("Backtrace: "); + + if (!tsk) + tsk = current; + + if (regs) { + fp = regs->ARM_fp; + mode = processor_mode(regs); + } else if (tsk != current) { + fp = thread_saved_fp(tsk); + mode = 0x10; + } else { + asm("mov %0, fp" : "=r" (fp) : : "cc"); + mode = 0x10; + } + + if (!fp) { + printk("no frame pointer"); + ok = 0; + } else if (verify_stack(fp)) { + printk("invalid frame pointer 0x%08x", fp); + ok = 0; + } else if (fp < (unsigned long)end_of_stack(tsk)) + printk("frame pointer underflow"); + printk("\n"); + + if (ok) + c_backtrace(fp, mode); +} +#endif + +void dump_stack(void) +{ + dump_backtrace(NULL, NULL); +} + +EXPORT_SYMBOL(dump_stack); + +void show_stack(struct task_struct *tsk, unsigned long *sp) +{ + dump_backtrace(NULL, tsk); + barrier(); +} + +#ifdef CONFIG_PREEMPT +#define S_PREEMPT " PREEMPT" +#else +#define S_PREEMPT "" +#endif +#ifdef CONFIG_SMP +#define S_SMP " SMP" +#else +#define S_SMP "" +#endif + +static int __die(const char *str, int err, struct thread_info *thread, struct pt_regs *regs) +{ + struct task_struct *tsk = thread->task; + static int die_counter; + int ret; + + printk(KERN_EMERG "Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n", + str, err, ++die_counter); + + /* trap and error numbers are mostly meaningless on ARM */ + ret = notify_die(DIE_OOPS, str, regs, err, tsk->thread.trap_no, SIGSEGV); + if (ret == NOTIFY_STOP) + return ret; + + print_modules(); + __show_regs(regs); + printk(KERN_EMERG "Process %.*s (pid: %d, stack limit = 0x%p)\n", + TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk), thread + 1); + + if (!user_mode(regs) || in_interrupt()) { + dump_mem(KERN_EMERG, "Stack: ", regs->ARM_sp, + THREAD_SIZE + (unsigned long)task_stack_page(tsk)); + dump_backtrace(regs, tsk); + dump_instr(KERN_EMERG, regs); + } + + return ret; +} + +static DEFINE_SPINLOCK(die_lock); + +/* + * This function is protected against re-entrancy. + */ +void die(const char *str, struct pt_regs *regs, int err) +{ + struct thread_info *thread = current_thread_info(); + int ret; + + oops_enter(); + + spin_lock_irq(&die_lock); + console_verbose(); + bust_spinlocks(1); + ret = __die(str, err, thread, regs); + + if (regs && kexec_should_crash(thread->task)) + crash_kexec(regs); + + bust_spinlocks(0); + add_taint(TAINT_DIE); + spin_unlock_irq(&die_lock); + oops_exit(); + + if (in_interrupt()) + panic("Fatal exception in interrupt"); + if (panic_on_oops) + panic("Fatal exception"); + if (ret != NOTIFY_STOP) + do_exit(SIGSEGV); +} + +void arm_notify_die(const char *str, struct pt_regs *regs, + struct siginfo *info, unsigned long err, unsigned long trap) +{ + if (user_mode(regs)) { + current->thread.error_code = err; + current->thread.trap_no = trap; + + force_sig_info(info->si_signo, info, current); + } else { + die(str, regs, err); + } +} + +static LIST_HEAD(undef_hook); +static DEFINE_SPINLOCK(undef_lock); + +void register_undef_hook(struct undef_hook *hook) +{ + unsigned long flags; + + spin_lock_irqsave(&undef_lock, flags); + list_add(&hook->node, &undef_hook); + spin_unlock_irqrestore(&undef_lock, flags); +} + +void unregister_undef_hook(struct undef_hook *hook) +{ + unsigned long flags; + + spin_lock_irqsave(&undef_lock, flags); + list_del(&hook->node); + spin_unlock_irqrestore(&undef_lock, flags); +} + +static int call_undef_hook(struct pt_regs *regs, unsigned int instr) +{ + struct undef_hook *hook; + unsigned long flags; + int (*fn)(struct pt_regs *regs, unsigned int instr) = NULL; + + spin_lock_irqsave(&undef_lock, flags); + list_for_each_entry(hook, &undef_hook, node) + if ((instr & hook->instr_mask) == hook->instr_val && + (regs->ARM_cpsr & hook->cpsr_mask) == hook->cpsr_val) + fn = hook->fn; + spin_unlock_irqrestore(&undef_lock, flags); + + return fn ? fn(regs, instr) : 1; +} + +asmlinkage void __exception do_undefinstr(struct pt_regs *regs) +{ + unsigned int correction = thumb_mode(regs) ? 2 : 4; + unsigned int instr; + siginfo_t info; + void __user *pc; + + /* + * According to the ARM ARM, PC is 2 or 4 bytes ahead, + * depending whether we're in Thumb mode or not. + * Correct this offset. + */ + regs->ARM_pc -= correction; + + pc = (void __user *)instruction_pointer(regs); + + if (processor_mode(regs) == SVC_MODE) { + instr = *(u32 *) pc; + } else if (thumb_mode(regs)) { + get_user(instr, (u16 __user *)pc); + } else { + get_user(instr, (u32 __user *)pc); + } + + if (call_undef_hook(regs, instr) == 0) + return; + +#ifdef CONFIG_DEBUG_USER + if (user_debug & UDBG_UNDEFINED) { + printk(KERN_INFO "%s (%d): undefined instruction: pc=%p\n", + current->comm, task_pid_nr(current), pc); + dump_instr(KERN_INFO, regs); + } +#endif + + info.si_signo = SIGILL; + info.si_errno = 0; + info.si_code = ILL_ILLOPC; + info.si_addr = pc; + + arm_notify_die("Oops - undefined instruction", regs, &info, 0, 6); +} + +asmlinkage void do_unexp_fiq (struct pt_regs *regs) +{ + printk("Hmm. Unexpected FIQ received, but trying to continue\n"); + printk("You may have a hardware problem...\n"); +} + +/* + * bad_mode handles the impossible case in the vectors. If you see one of + * these, then it's extremely serious, and could mean you have buggy hardware. + * It never returns, and never tries to sync. We hope that we can at least + * dump out some state information... + */ +asmlinkage void bad_mode(struct pt_regs *regs, int reason) +{ + console_verbose(); + + printk(KERN_CRIT "Bad mode in %s handler detected\n", handler[reason]); + + die("Oops - bad mode", regs, 0); + local_irq_disable(); + panic("bad mode"); +} + +static int bad_syscall(int n, struct pt_regs *regs) +{ + struct thread_info *thread = current_thread_info(); + siginfo_t info; + + if ((current->personality & PER_MASK) != PER_LINUX && + thread->exec_domain->handler) { + thread->exec_domain->handler(n, regs); + return regs->ARM_r0; + } + +#ifdef CONFIG_DEBUG_USER + if (user_debug & UDBG_SYSCALL) { + printk(KERN_ERR "[%d] %s: obsolete system call %08x.\n", + task_pid_nr(current), current->comm, n); + dump_instr(KERN_ERR, regs); + } +#endif + + info.si_signo = SIGILL; + info.si_errno = 0; + info.si_code = ILL_ILLTRP; + info.si_addr = (void __user *)instruction_pointer(regs) - + (thumb_mode(regs) ? 2 : 4); + + arm_notify_die("Oops - bad syscall", regs, &info, n, 0); + + return regs->ARM_r0; +} + +static inline void +do_cache_op(unsigned long start, unsigned long end, int flags) +{ + struct mm_struct *mm = current->active_mm; + struct vm_area_struct *vma; + + if (end < start || flags) + return; + + down_read(&mm->mmap_sem); + vma = find_vma(mm, start); + if (vma && vma->vm_start < end) { + if (start < vma->vm_start) + start = vma->vm_start; + if (end > vma->vm_end) + end = vma->vm_end; + + up_read(&mm->mmap_sem); + flush_cache_user_range(start, end); + return; + } + up_read(&mm->mmap_sem); +} + +/* + * Handle all unrecognised system calls. + * 0x9f0000 - 0x9fffff are some more esoteric system calls + */ +#define NR(x) ((__ARM_NR_##x) - __ARM_NR_BASE) +asmlinkage int arm_syscall(int no, struct pt_regs *regs) +{ + struct thread_info *thread = current_thread_info(); + siginfo_t info; + + if ((no >> 16) != (__ARM_NR_BASE>> 16)) + return bad_syscall(no, regs); + + switch (no & 0xffff) { + case 0: /* branch through 0 */ + info.si_signo = SIGSEGV; + info.si_errno = 0; + info.si_code = SEGV_MAPERR; + info.si_addr = NULL; + + arm_notify_die("branch through zero", regs, &info, 0, 0); + return 0; + + case NR(breakpoint): /* SWI BREAK_POINT */ + regs->ARM_pc -= thumb_mode(regs) ? 2 : 4; + ptrace_break(current, regs); + return regs->ARM_r0; + + /* + * Flush a region from virtual address 'r0' to virtual address 'r1' + * _exclusive_. There is no alignment requirement on either address; + * user space does not need to know the hardware cache layout. + * + * r2 contains flags. It should ALWAYS be passed as ZERO until it + * is defined to be something else. For now we ignore it, but may + * the fires of hell burn in your belly if you break this rule. ;) + * + * (at a later date, we may want to allow this call to not flush + * various aspects of the cache. Passing '0' will guarantee that + * everything necessary gets flushed to maintain consistency in + * the specified region). + */ + case NR(cacheflush): + do_cache_op(regs->ARM_r0, regs->ARM_r1, regs->ARM_r2); + return 0; + + case NR(usr26): + if (!(elf_hwcap & HWCAP_26BIT)) + break; + regs->ARM_cpsr &= ~MODE32_BIT; + return regs->ARM_r0; + + case NR(usr32): + if (!(elf_hwcap & HWCAP_26BIT)) + break; + regs->ARM_cpsr |= MODE32_BIT; + return regs->ARM_r0; + + case NR(set_tls): + thread->tp_value = regs->ARM_r0; + if (tls_emu) + return 0; + if (has_tls_reg) { + asm ("mcr p15, 0, %0, c13, c0, 3" + : : "r" (regs->ARM_r0)); + } else { + /* + * User space must never try to access this directly. + * Expect your app to break eventually if you do so. + * The user helper at 0xffff0fe0 must be used instead. + * (see entry-armv.S for details) + */ + *((unsigned int *)0xffff0ff0) = regs->ARM_r0; + } + return 0; + +#ifdef CONFIG_NEEDS_SYSCALL_FOR_CMPXCHG + /* + * Atomically store r1 in *r2 if *r2 is equal to r0 for user space. + * Return zero in r0 if *MEM was changed or non-zero if no exchange + * happened. Also set the user C flag accordingly. + * If access permissions have to be fixed up then non-zero is + * returned and the operation has to be re-attempted. + * + * *NOTE*: This is a ghost syscall private to the kernel. Only the + * __kuser_cmpxchg code in entry-armv.S should be aware of its + * existence. Don't ever use this from user code. + */ + case NR(cmpxchg): + for (;;) { + extern void do_DataAbort(unsigned long addr, unsigned int fsr, + struct pt_regs *regs); + unsigned long val; + unsigned long addr = regs->ARM_r2; + struct mm_struct *mm = current->mm; + pgd_t *pgd; pmd_t *pmd; pte_t *pte; + spinlock_t *ptl; + + regs->ARM_cpsr &= ~PSR_C_BIT; + down_read(&mm->mmap_sem); + pgd = pgd_offset(mm, addr); + if (!pgd_present(*pgd)) + goto bad_access; + pmd = pmd_offset(pgd, addr); + if (!pmd_present(*pmd)) + goto bad_access; + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!pte_present(*pte) || !pte_write(*pte) || !pte_dirty(*pte)) { + pte_unmap_unlock(pte, ptl); + goto bad_access; + } + val = *(unsigned long *)addr; + val -= regs->ARM_r0; + if (val == 0) { + *(unsigned long *)addr = regs->ARM_r1; + regs->ARM_cpsr |= PSR_C_BIT; + } + pte_unmap_unlock(pte, ptl); + up_read(&mm->mmap_sem); + return val; + + bad_access: + up_read(&mm->mmap_sem); + /* simulate a write access fault */ + do_DataAbort(addr, 15 + (1 << 11), regs); + } +#endif + + default: + /* Calls 9f00xx..9f07ff are defined to return -ENOSYS + if not implemented, rather than raising SIGILL. This + way the calling program can gracefully determine whether + a feature is supported. */ + if ((no & 0xffff) <= 0x7ff) + return -ENOSYS; + break; + } +#ifdef CONFIG_DEBUG_USER + /* + * experience shows that these seem to indicate that + * something catastrophic has happened + */ + if (user_debug & UDBG_SYSCALL) { + printk("[%d] %s: arm syscall %d\n", + task_pid_nr(current), current->comm, no); + dump_instr("", regs); + if (user_mode(regs)) { + __show_regs(regs); + c_backtrace(regs->ARM_fp, processor_mode(regs)); + } + } +#endif + info.si_signo = SIGILL; + info.si_errno = 0; + info.si_code = ILL_ILLTRP; + info.si_addr = (void __user *)instruction_pointer(regs) - + (thumb_mode(regs) ? 2 : 4); + + arm_notify_die("Oops - bad syscall(2)", regs, &info, no, 0); + return 0; +} + +#ifdef CONFIG_TLS_REG_EMUL + +/* + * We might be running on an ARMv6+ processor which should have the TLS + * register but for some reason we can't use it, or maybe an SMP system + * using a pre-ARMv6 processor (there are apparently a few prototypes like + * that in existence) and therefore access to that register must be + * emulated. + */ + +static int get_tp_trap(struct pt_regs *regs, unsigned int instr) +{ + int reg = (instr >> 12) & 15; + if (reg == 15) + return 1; + regs->uregs[reg] = current_thread_info()->tp_value; + regs->ARM_pc += 4; + return 0; +} + +static struct undef_hook arm_mrc_hook = { + .instr_mask = 0x0fff0fff, + .instr_val = 0x0e1d0f70, + .cpsr_mask = PSR_T_BIT, + .cpsr_val = 0, + .fn = get_tp_trap, +}; + +static int __init arm_mrc_hook_init(void) +{ + register_undef_hook(&arm_mrc_hook); + return 0; +} + +late_initcall(arm_mrc_hook_init); + +#endif + +void __bad_xchg(volatile void *ptr, int size) +{ + printk("xchg: bad data size: pc 0x%p, ptr 0x%p, size %d\n", + __builtin_return_address(0), ptr, size); + BUG(); +} +EXPORT_SYMBOL(__bad_xchg); + +/* + * A data abort trap was taken, but we did not handle the instruction. + * Try to abort the user program, or panic if it was the kernel. + */ +asmlinkage void +baddataabort(int code, unsigned long instr, struct pt_regs *regs) +{ + unsigned long addr = instruction_pointer(regs); + siginfo_t info; + +#ifdef CONFIG_DEBUG_USER + if (user_debug & UDBG_BADABORT) { + printk(KERN_ERR "[%d] %s: bad data abort: code %d instr 0x%08lx\n", + task_pid_nr(current), current->comm, code, instr); + dump_instr(KERN_ERR, regs); + show_pte(current->mm, addr); + } +#endif + + info.si_signo = SIGILL; + info.si_errno = 0; + info.si_code = ILL_ILLOPC; + info.si_addr = (void __user *)addr; + + arm_notify_die("unknown data abort code", regs, &info, instr, 0); +} + +void __attribute__((noreturn)) __bug(const char *file, int line) +{ + printk(KERN_CRIT"kernel BUG at %s:%d!\n", file, line); + *(int *)0 = 0; + + /* Avoid "noreturn function does return" */ + for (;;); +} +EXPORT_SYMBOL(__bug); + +void __readwrite_bug(const char *fn) +{ + printk("%s called, but not implemented\n", fn); + BUG(); +} +EXPORT_SYMBOL(__readwrite_bug); + +void __pte_error(const char *file, int line, pte_t pte) +{ + printk("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte)); +} + +void __pmd_error(const char *file, int line, pmd_t pmd) +{ + printk("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd)); +} + +void __pgd_error(const char *file, int line, pgd_t pgd) +{ + printk("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd)); +} + +asmlinkage void __div0(void) +{ + printk("Division by zero in kernel.\n"); + dump_stack(); +} +EXPORT_SYMBOL(__div0); + +void abort(void) +{ + BUG(); + + /* if that doesn't kill us, halt */ + panic("Oops failed to kill thread"); +} +EXPORT_SYMBOL(abort); + +void __init trap_init(void) +{ + return; +} + +static void __init kuser_get_tls_init(unsigned long vectors) +{ + /* + * vectors + 0xfe0 = __kuser_get_tls + * vectors + 0xfe8 = hardware TLS instruction at 0xffff0fe8 + */ + if (tls_emu || has_tls_reg) + memcpy((void *)vectors + 0xfe0, (void *)vectors + 0xfe8, 4); +} + +void __init early_trap_init(void) +{ +#if defined(CONFIG_CPU_USE_DOMAINS) + unsigned long vectors = CONFIG_VECTORS_BASE; +#else + unsigned long vectors = (unsigned long)vectors_page; +#endif + extern char __stubs_start[], __stubs_end[]; + extern char __vectors_start[], __vectors_end[]; + extern char __kuser_helper_start[], __kuser_helper_end[]; + int kuser_sz = __kuser_helper_end - __kuser_helper_start; + + /* + * Copy the vectors, stubs and kuser helpers (in entry-armv.S) + * into the vector page, mapped at 0xffff0000, and ensure these + * are visible to the instruction stream. + */ + memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start); + memcpy((void *)vectors + 0x200, __stubs_start, __stubs_end - __stubs_start); + memcpy((void *)vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz); + + /* + * Do processor specific fixups for the kuser helpers + */ + kuser_get_tls_init(vectors); + + /* + * Copy signal return handlers into the vector page, and + * set sigreturn to be a pointer to these. + */ + memcpy((void *)(vectors + KERN_SIGRETURN_CODE - CONFIG_VECTORS_BASE), + sigreturn_codes, sizeof(sigreturn_codes)); + memcpy((void *)(vectors + KERN_RESTART_CODE - CONFIG_VECTORS_BASE), + syscall_restart_code, sizeof(syscall_restart_code)); + + flush_icache_range(vectors, vectors + PAGE_SIZE); + modify_domain(DOMAIN_USER, DOMAIN_CLIENT); +} diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c new file mode 100644 index 00000000..d2cb0b3c --- /dev/null +++ b/arch/arm/kernel/unwind.c @@ -0,0 +1,452 @@ +/* + * arch/arm/kernel/unwind.c + * + * Copyright (C) 2008 ARM Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * + * Stack unwinding support for ARM + * + * An ARM EABI version of gcc is required to generate the unwind + * tables. For information about the structure of the unwind tables, + * see "Exception Handling ABI for the ARM Architecture" at: + * + * http://infocenter.arm.com/help/topic/com.arm.doc.subset.swdev.abi/index.html + */ + +#ifndef __CHECKER__ +#if !defined (__ARM_EABI__) +#warning Your compiler does not have EABI support. +#warning ARM unwind is known to compile only with EABI compilers. +#warning Change compiler or disable ARM_UNWIND option. +#elif (__GNUC__ == 4 && __GNUC_MINOR__ <= 2) +#warning Your compiler is too buggy; it is known to not compile ARM unwind support. +#warning Change compiler or disable ARM_UNWIND option. +#endif +#endif /* __CHECKER__ */ + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/list.h> + +#include <asm/stacktrace.h> +#include <asm/traps.h> +#include <asm/unwind.h> + +/* Dummy functions to avoid linker complaints */ +void __aeabi_unwind_cpp_pr0(void) +{ +}; +EXPORT_SYMBOL(__aeabi_unwind_cpp_pr0); + +void __aeabi_unwind_cpp_pr1(void) +{ +}; +EXPORT_SYMBOL(__aeabi_unwind_cpp_pr1); + +void __aeabi_unwind_cpp_pr2(void) +{ +}; +EXPORT_SYMBOL(__aeabi_unwind_cpp_pr2); + +struct unwind_ctrl_block { + unsigned long vrs[16]; /* virtual register set */ + unsigned long *insn; /* pointer to the current instructions word */ + int entries; /* number of entries left to interpret */ + int byte; /* current byte number in the instructions word */ +}; + +enum regs { +#ifdef CONFIG_THUMB2_KERNEL + FP = 7, +#else + FP = 11, +#endif + SP = 13, + LR = 14, + PC = 15 +}; + +extern struct unwind_idx __start_unwind_idx[]; +extern struct unwind_idx __stop_unwind_idx[]; + +static DEFINE_SPINLOCK(unwind_lock); +static LIST_HEAD(unwind_tables); + +/* Convert a prel31 symbol to an absolute address */ +#define prel31_to_addr(ptr) \ +({ \ + /* sign-extend to 32 bits */ \ + long offset = (((long)*(ptr)) << 1) >> 1; \ + (unsigned long)(ptr) + offset; \ +}) + +/* + * Binary search in the unwind index. The entries entries are + * guaranteed to be sorted in ascending order by the linker. + */ +static struct unwind_idx *search_index(unsigned long addr, + struct unwind_idx *first, + struct unwind_idx *last) +{ + pr_debug("%s(%08lx, %p, %p)\n", __func__, addr, first, last); + + if (addr < first->addr) { + pr_warning("unwind: Unknown symbol address %08lx\n", addr); + return NULL; + } else if (addr >= last->addr) + return last; + + while (first < last - 1) { + struct unwind_idx *mid = first + ((last - first + 1) >> 1); + + if (addr < mid->addr) + last = mid; + else + first = mid; + } + + return first; +} + +static struct unwind_idx *unwind_find_idx(unsigned long addr) +{ + struct unwind_idx *idx = NULL; + unsigned long flags; + + pr_debug("%s(%08lx)\n", __func__, addr); + + if (core_kernel_text(addr)) + /* main unwind table */ + idx = search_index(addr, __start_unwind_idx, + __stop_unwind_idx - 1); + else { + /* module unwind tables */ + struct unwind_table *table; + + spin_lock_irqsave(&unwind_lock, flags); + list_for_each_entry(table, &unwind_tables, list) { + if (addr >= table->begin_addr && + addr < table->end_addr) { + idx = search_index(addr, table->start, + table->stop - 1); + /* Move-to-front to exploit common traces */ + list_move(&table->list, &unwind_tables); + break; + } + } + spin_unlock_irqrestore(&unwind_lock, flags); + } + + pr_debug("%s: idx = %p\n", __func__, idx); + return idx; +} + +static unsigned long unwind_get_byte(struct unwind_ctrl_block *ctrl) +{ + unsigned long ret; + + if (ctrl->entries <= 0) { + pr_warning("unwind: Corrupt unwind table\n"); + return 0; + } + + ret = (*ctrl->insn >> (ctrl->byte * 8)) & 0xff; + + if (ctrl->byte == 0) { + ctrl->insn++; + ctrl->entries--; + ctrl->byte = 3; + } else + ctrl->byte--; + + return ret; +} + +/* + * Execute the current unwind instruction. + */ +static int unwind_exec_insn(struct unwind_ctrl_block *ctrl) +{ + unsigned long insn = unwind_get_byte(ctrl); + + pr_debug("%s: insn = %08lx\n", __func__, insn); + + if ((insn & 0xc0) == 0x00) + ctrl->vrs[SP] += ((insn & 0x3f) << 2) + 4; + else if ((insn & 0xc0) == 0x40) + ctrl->vrs[SP] -= ((insn & 0x3f) << 2) + 4; + else if ((insn & 0xf0) == 0x80) { + unsigned long mask; + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int load_sp, reg = 4; + + insn = (insn << 8) | unwind_get_byte(ctrl); + mask = insn & 0x0fff; + if (mask == 0) { + pr_warning("unwind: 'Refuse to unwind' instruction %04lx\n", + insn); + return -URC_FAILURE; + } + + /* pop R4-R15 according to mask */ + load_sp = mask & (1 << (13 - 4)); + while (mask) { + if (mask & 1) + ctrl->vrs[reg] = *vsp++; + mask >>= 1; + reg++; + } + if (!load_sp) + ctrl->vrs[SP] = (unsigned long)vsp; + } else if ((insn & 0xf0) == 0x90 && + (insn & 0x0d) != 0x0d) + ctrl->vrs[SP] = ctrl->vrs[insn & 0x0f]; + else if ((insn & 0xf0) == 0xa0) { + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int reg; + + /* pop R4-R[4+bbb] */ + for (reg = 4; reg <= 4 + (insn & 7); reg++) + ctrl->vrs[reg] = *vsp++; + if (insn & 0x80) + ctrl->vrs[14] = *vsp++; + ctrl->vrs[SP] = (unsigned long)vsp; + } else if (insn == 0xb0) { + if (ctrl->vrs[PC] == 0) + ctrl->vrs[PC] = ctrl->vrs[LR]; + /* no further processing */ + ctrl->entries = 0; + } else if (insn == 0xb1) { + unsigned long mask = unwind_get_byte(ctrl); + unsigned long *vsp = (unsigned long *)ctrl->vrs[SP]; + int reg = 0; + + if (mask == 0 || mask & 0xf0) { + pr_warning("unwind: Spare encoding %04lx\n", + (insn << 8) | mask); + return -URC_FAILURE; + } + + /* pop R0-R3 according to mask */ + while (mask) { + if (mask & 1) + ctrl->vrs[reg] = *vsp++; + mask >>= 1; + reg++; + } + ctrl->vrs[SP] = (unsigned long)vsp; + } else if (insn == 0xb2) { + unsigned long uleb128 = unwind_get_byte(ctrl); + + ctrl->vrs[SP] += 0x204 + (uleb128 << 2); + } else { + pr_warning("unwind: Unhandled instruction %02lx\n", insn); + return -URC_FAILURE; + } + + pr_debug("%s: fp = %08lx sp = %08lx lr = %08lx pc = %08lx\n", __func__, + ctrl->vrs[FP], ctrl->vrs[SP], ctrl->vrs[LR], ctrl->vrs[PC]); + + return URC_OK; +} + +/* + * Unwind a single frame starting with *sp for the symbol at *pc. It + * updates the *pc and *sp with the new values. + */ +int unwind_frame(struct stackframe *frame) +{ + unsigned long high, low; + struct unwind_idx *idx; + struct unwind_ctrl_block ctrl; + + /* only go to a higher address on the stack */ + low = frame->sp; + high = ALIGN(low, THREAD_SIZE); + + pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__, + frame->pc, frame->lr, frame->sp); + + if (!kernel_text_address(frame->pc)) + return -URC_FAILURE; + + idx = unwind_find_idx(frame->pc); + if (!idx) { + pr_warning("unwind: Index not found %08lx\n", frame->pc); + return -URC_FAILURE; + } + + ctrl.vrs[FP] = frame->fp; + ctrl.vrs[SP] = frame->sp; + ctrl.vrs[LR] = frame->lr; + ctrl.vrs[PC] = 0; + + if (idx->insn == 1) + /* can't unwind */ + return -URC_FAILURE; + else if ((idx->insn & 0x80000000) == 0) + /* prel31 to the unwind table */ + ctrl.insn = (unsigned long *)prel31_to_addr(&idx->insn); + else if ((idx->insn & 0xff000000) == 0x80000000) + /* only personality routine 0 supported in the index */ + ctrl.insn = &idx->insn; + else { + pr_warning("unwind: Unsupported personality routine %08lx in the index at %p\n", + idx->insn, idx); + return -URC_FAILURE; + } + + /* check the personality routine */ + if ((*ctrl.insn & 0xff000000) == 0x80000000) { + ctrl.byte = 2; + ctrl.entries = 1; + } else if ((*ctrl.insn & 0xff000000) == 0x81000000) { + ctrl.byte = 1; + ctrl.entries = 1 + ((*ctrl.insn & 0x00ff0000) >> 16); + } else { + pr_warning("unwind: Unsupported personality routine %08lx at %p\n", + *ctrl.insn, ctrl.insn); + return -URC_FAILURE; + } + + while (ctrl.entries > 0) { + int urc = unwind_exec_insn(&ctrl); + if (urc < 0) + return urc; + if (ctrl.vrs[SP] < low || ctrl.vrs[SP] >= high) + return -URC_FAILURE; + } + + if (ctrl.vrs[PC] == 0) + ctrl.vrs[PC] = ctrl.vrs[LR]; + + /* check for infinite loop */ + if (frame->pc == ctrl.vrs[PC]) + return -URC_FAILURE; + + frame->fp = ctrl.vrs[FP]; + frame->sp = ctrl.vrs[SP]; + frame->lr = ctrl.vrs[LR]; + frame->pc = ctrl.vrs[PC]; + + return URC_OK; +} + +void unwind_backtrace(struct pt_regs *regs, struct task_struct *tsk) +{ + struct stackframe frame; + register unsigned long current_sp asm ("sp"); + + pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk); + + if (!tsk) + tsk = current; + + if (regs) { + frame.fp = regs->ARM_fp; + frame.sp = regs->ARM_sp; + frame.lr = regs->ARM_lr; + /* PC might be corrupted, use LR in that case. */ + frame.pc = kernel_text_address(regs->ARM_pc) + ? regs->ARM_pc : regs->ARM_lr; + } else if (tsk == current) { + frame.fp = (unsigned long)__builtin_frame_address(0); + frame.sp = current_sp; + frame.lr = (unsigned long)__builtin_return_address(0); + frame.pc = (unsigned long)unwind_backtrace; + } else { + /* task blocked in __switch_to */ + frame.fp = thread_saved_fp(tsk); + frame.sp = thread_saved_sp(tsk); + /* + * The function calling __switch_to cannot be a leaf function + * so LR is recovered from the stack. + */ + frame.lr = 0; + frame.pc = thread_saved_pc(tsk); + } + + while (1) { + int urc; + unsigned long where = frame.pc; + + urc = unwind_frame(&frame); + if (urc < 0) + break; + dump_backtrace_entry(where, frame.pc, frame.sp - 4); + } +} + +struct unwind_table *unwind_table_add(unsigned long start, unsigned long size, + unsigned long text_addr, + unsigned long text_size) +{ + unsigned long flags; + struct unwind_idx *idx; + struct unwind_table *tab = kmalloc(sizeof(*tab), GFP_KERNEL); + + pr_debug("%s(%08lx, %08lx, %08lx, %08lx)\n", __func__, start, size, + text_addr, text_size); + + if (!tab) + return tab; + + tab->start = (struct unwind_idx *)start; + tab->stop = (struct unwind_idx *)(start + size); + tab->begin_addr = text_addr; + tab->end_addr = text_addr + text_size; + + /* Convert the symbol addresses to absolute values */ + for (idx = tab->start; idx < tab->stop; idx++) + idx->addr = prel31_to_addr(&idx->addr); + + spin_lock_irqsave(&unwind_lock, flags); + list_add_tail(&tab->list, &unwind_tables); + spin_unlock_irqrestore(&unwind_lock, flags); + + return tab; +} + +void unwind_table_del(struct unwind_table *tab) +{ + unsigned long flags; + + if (!tab) + return; + + spin_lock_irqsave(&unwind_lock, flags); + list_del(&tab->list); + spin_unlock_irqrestore(&unwind_lock, flags); + + kfree(tab); +} + +int __init unwind_init(void) +{ + struct unwind_idx *idx; + + /* Convert the symbol addresses to absolute values */ + for (idx = __start_unwind_idx; idx < __stop_unwind_idx; idx++) + idx->addr = prel31_to_addr(&idx->addr); + + pr_debug("unwind: ARM stack unwinding initialised\n"); + + return 0; +} diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S new file mode 100644 index 00000000..e5287f21 --- /dev/null +++ b/arch/arm/kernel/vmlinux.lds.S @@ -0,0 +1,287 @@ +/* ld script to make ARM Linux kernel + * taken from the i386 version by Russell King + * Written by Martin Mares <mj@atrey.karlin.mff.cuni.cz> + */ + +#include <asm-generic/vmlinux.lds.h> +#include <asm/thread_info.h> +#include <asm/memory.h> +#include <asm/page.h> + +#define PROC_INFO \ + VMLINUX_SYMBOL(__proc_info_begin) = .; \ + *(.proc.info.init) \ + VMLINUX_SYMBOL(__proc_info_end) = .; + +#ifdef CONFIG_HOTPLUG_CPU +#define ARM_CPU_DISCARD(x) +#define ARM_CPU_KEEP(x) x +#else +#define ARM_CPU_DISCARD(x) x +#define ARM_CPU_KEEP(x) +#endif + +#if defined(CONFIG_SMP_ON_UP) && !defined(CONFIG_DEBUG_SPINLOCK) +#define ARM_EXIT_KEEP(x) x +#else +#define ARM_EXIT_KEEP(x) +#endif + +OUTPUT_ARCH(arm) +ENTRY(stext) + +#ifndef __ARMEB__ +jiffies = jiffies_64; +#else +jiffies = jiffies_64 + 4; +#endif + +SECTIONS +{ +#ifdef CONFIG_XIP_KERNEL + . = XIP_VIRT_ADDR(CONFIG_XIP_PHYS_ADDR); +#else + . = PAGE_OFFSET + TEXT_OFFSET; +#endif + + .init : { /* Init code and data */ + _stext = .; + _sinittext = .; + HEAD_TEXT + INIT_TEXT + ARM_EXIT_KEEP(EXIT_TEXT) + _einittext = .; + ARM_CPU_DISCARD(PROC_INFO) + __arch_info_begin = .; + *(.arch.info.init) + __arch_info_end = .; + __tagtable_begin = .; + *(.taglist.init) + __tagtable_end = .; +#ifdef CONFIG_SMP_ON_UP + __smpalt_begin = .; + *(.alt.smp.init) + __smpalt_end = .; +#endif + + __pv_table_begin = .; + *(.pv_table) + __pv_table_end = .; + + INIT_SETUP(16) + + INIT_CALLS + CON_INITCALL + SECURITY_INITCALL + INIT_RAM_FS + +#ifndef CONFIG_XIP_KERNEL + __init_begin = _stext; + INIT_DATA + ARM_EXIT_KEEP(EXIT_DATA) +#endif + } + + PERCPU_SECTION(32) + +#ifndef CONFIG_XIP_KERNEL + . = ALIGN(PAGE_SIZE); + __init_end = .; +#endif + + /* + * unwind exit sections must be discarded before the rest of the + * unwind sections get included. + */ + /DISCARD/ : { + *(.ARM.exidx.exit.text) + *(.ARM.extab.exit.text) + ARM_CPU_DISCARD(*(.ARM.exidx.cpuexit.text)) + ARM_CPU_DISCARD(*(.ARM.extab.cpuexit.text)) +#ifndef CONFIG_HOTPLUG + *(.ARM.exidx.devexit.text) + *(.ARM.extab.devexit.text) +#endif +#ifndef CONFIG_MMU + *(.fixup) + *(__ex_table) +#endif + } + + .text : { /* Real text segment */ + _text = .; /* Text and read-only data */ + __exception_text_start = .; + *(.exception.text) + __exception_text_end = .; + IRQENTRY_TEXT + TEXT_TEXT + SCHED_TEXT + LOCK_TEXT + KPROBES_TEXT +#ifdef CONFIG_MMU + *(.fixup) +#endif + *(.gnu.warning) + *(.rodata) + *(.rodata.*) + *(.glue_7) + *(.glue_7t) + . = ALIGN(4); + *(.got) /* Global offset table */ + ARM_CPU_KEEP(PROC_INFO) + } + + RO_DATA(PAGE_SIZE) + +#ifdef CONFIG_ARM_UNWIND + /* + * Stack unwinding tables + */ + . = ALIGN(8); + .ARM.unwind_idx : { + __start_unwind_idx = .; + *(.ARM.exidx*) + __stop_unwind_idx = .; + } + .ARM.unwind_tab : { + __start_unwind_tab = .; + *(.ARM.extab*) + __stop_unwind_tab = .; + } +#endif + + _etext = .; /* End of text and rodata section */ + +#ifdef CONFIG_XIP_KERNEL + __data_loc = ALIGN(4); /* location in binary */ + . = PAGE_OFFSET + TEXT_OFFSET; +#else + . = ALIGN(THREAD_SIZE); + __data_loc = .; +#endif + + .data : AT(__data_loc) { + _data = .; /* address in memory */ + _sdata = .; + + /* + * first, the init task union, aligned + * to an 8192 byte boundary. + */ + INIT_TASK_DATA(THREAD_SIZE) + +#ifdef CONFIG_XIP_KERNEL + . = ALIGN(PAGE_SIZE); + __init_begin = .; + INIT_DATA + ARM_EXIT_KEEP(EXIT_DATA) + . = ALIGN(PAGE_SIZE); + __init_end = .; +#endif + + NOSAVE_DATA + CACHELINE_ALIGNED_DATA(32) + READ_MOSTLY_DATA(32) + + /* + * The exception fixup table (might need resorting at runtime) + */ + . = ALIGN(32); + __start___ex_table = .; +#ifdef CONFIG_MMU + *(__ex_table) +#endif + __stop___ex_table = .; + + /* + * and the usual data section + */ + DATA_DATA + CONSTRUCTORS + + _edata = .; + } + _edata_loc = __data_loc + SIZEOF(.data); + +#ifdef CONFIG_HAVE_TCM + /* + * We align everything to a page boundary so we can + * free it after init has commenced and TCM contents have + * been copied to its destination. + */ + .tcm_start : { + . = ALIGN(PAGE_SIZE); + __tcm_start = .; + __itcm_start = .; + } + + /* + * Link these to the ITCM RAM + * Put VMA to the TCM address and LMA to the common RAM + * and we'll upload the contents from RAM to TCM and free + * the used RAM after that. + */ + .text_itcm ITCM_OFFSET : AT(__itcm_start) + { + __sitcm_text = .; + *(.tcm.text) + *(.tcm.rodata) + . = ALIGN(4); + __eitcm_text = .; + } + + /* + * Reset the dot pointer, this is needed to create the + * relative __dtcm_start below (to be used as extern in code). + */ + . = ADDR(.tcm_start) + SIZEOF(.tcm_start) + SIZEOF(.text_itcm); + + .dtcm_start : { + __dtcm_start = .; + } + + /* TODO: add remainder of ITCM as well, that can be used for data! */ + .data_dtcm DTCM_OFFSET : AT(__dtcm_start) + { + . = ALIGN(4); + __sdtcm_data = .; + *(.tcm.data) + . = ALIGN(4); + __edtcm_data = .; + } + + /* Reset the dot pointer or the linker gets confused */ + . = ADDR(.dtcm_start) + SIZEOF(.data_dtcm); + + /* End marker for freeing TCM copy in linked object */ + .tcm_end : AT(ADDR(.dtcm_start) + SIZEOF(.data_dtcm)){ + . = ALIGN(PAGE_SIZE); + __tcm_end = .; + } +#endif + + NOTES + + BSS_SECTION(0, 0, 0) + _end = .; + + STABS_DEBUG + .comment 0 : { *(.comment) } + + /* Default discards */ + DISCARDS + +#ifndef CONFIG_SMP_ON_UP + /DISCARD/ : { + *(.alt.smp.init) + } +#endif +} + +/* + * These must never be empty + * If you have to comment these two assert statements out, your + * binutils is too old (for other reasons as well) + */ +ASSERT((__proc_info_end - __proc_info_begin), "missing CPU support") +ASSERT((__arch_info_end - __arch_info_begin), "no machine record defined") diff --git a/arch/arm/kernel/xscale-cp0.c b/arch/arm/kernel/xscale-cp0.c new file mode 100644 index 00000000..1796157e --- /dev/null +++ b/arch/arm/kernel/xscale-cp0.c @@ -0,0 +1,179 @@ +/* + * linux/arch/arm/kernel/xscale-cp0.c + * + * XScale DSP and iWMMXt coprocessor context switching and handling + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/types.h> +#include <linux/kernel.h> +#include <linux/signal.h> +#include <linux/sched.h> +#include <linux/init.h> +#include <linux/io.h> +#include <asm/thread_notify.h> + +static inline void dsp_save_state(u32 *state) +{ + __asm__ __volatile__ ( + "mrrc p0, 0, %0, %1, c0\n" + : "=r" (state[0]), "=r" (state[1])); +} + +static inline void dsp_load_state(u32 *state) +{ + __asm__ __volatile__ ( + "mcrr p0, 0, %0, %1, c0\n" + : : "r" (state[0]), "r" (state[1])); +} + +static int dsp_do(struct notifier_block *self, unsigned long cmd, void *t) +{ + struct thread_info *thread = t; + + switch (cmd) { + case THREAD_NOTIFY_FLUSH: + thread->cpu_context.extra[0] = 0; + thread->cpu_context.extra[1] = 0; + break; + + case THREAD_NOTIFY_SWITCH: + dsp_save_state(current_thread_info()->cpu_context.extra); + dsp_load_state(thread->cpu_context.extra); + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block dsp_notifier_block = { + .notifier_call = dsp_do, +}; + + +#ifdef CONFIG_IWMMXT +static int iwmmxt_do(struct notifier_block *self, unsigned long cmd, void *t) +{ + struct thread_info *thread = t; + + switch (cmd) { + case THREAD_NOTIFY_FLUSH: + /* + * flush_thread() zeroes thread->fpstate, so no need + * to do anything here. + * + * FALLTHROUGH: Ensure we don't try to overwrite our newly + * initialised state information on the first fault. + */ + + case THREAD_NOTIFY_EXIT: + iwmmxt_task_release(thread); + break; + + case THREAD_NOTIFY_SWITCH: + iwmmxt_task_switch(thread); + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block iwmmxt_notifier_block = { + .notifier_call = iwmmxt_do, +}; +#endif + + +static u32 __init xscale_cp_access_read(void) +{ + u32 value; + + __asm__ __volatile__ ( + "mrc p15, 0, %0, c15, c1, 0\n\t" + : "=r" (value)); + + return value; +} + +static void __init xscale_cp_access_write(u32 value) +{ + u32 temp; + + __asm__ __volatile__ ( + "mcr p15, 0, %1, c15, c1, 0\n\t" + "mrc p15, 0, %0, c15, c1, 0\n\t" + "mov %0, %0\n\t" + "sub pc, pc, #4\n\t" + : "=r" (temp) : "r" (value)); +} + +/* + * Detect whether we have a MAC coprocessor (40 bit register) or an + * iWMMXt coprocessor (64 bit registers) by loading 00000100:00000000 + * into a coprocessor register and reading it back, and checking + * whether the upper word survived intact. + */ +static int __init cpu_has_iwmmxt(void) +{ + u32 lo; + u32 hi; + + /* + * This sequence is interpreted by the DSP coprocessor as: + * mar acc0, %2, %3 + * mra %0, %1, acc0 + * + * And by the iWMMXt coprocessor as: + * tmcrr wR0, %2, %3 + * tmrrc %0, %1, wR0 + */ + __asm__ __volatile__ ( + "mcrr p0, 0, %2, %3, c0\n" + "mrrc p0, 0, %0, %1, c0\n" + : "=r" (lo), "=r" (hi) + : "r" (0), "r" (0x100)); + + return !!hi; +} + + +/* + * If we detect that the CPU has iWMMXt (and CONFIG_IWMMXT=y), we + * disable CP0/CP1 on boot, and let call_fpe() and the iWMMXt lazy + * switch code handle iWMMXt context switching. If on the other + * hand the CPU has a DSP coprocessor, we keep access to CP0 enabled + * all the time, and save/restore acc0 on context switch in non-lazy + * fashion. + */ +static int __init xscale_cp0_init(void) +{ + u32 cp_access; + + cp_access = xscale_cp_access_read() & ~3; + xscale_cp_access_write(cp_access | 1); + + if (cpu_has_iwmmxt()) { +#ifndef CONFIG_IWMMXT + printk(KERN_WARNING "CAUTION: XScale iWMMXt coprocessor " + "detected, but kernel support is missing.\n"); +#else + printk(KERN_INFO "XScale iWMMXt coprocessor detected.\n"); + elf_hwcap |= HWCAP_IWMMXT; + thread_register_notifier(&iwmmxt_notifier_block); +#endif + } else { + printk(KERN_INFO "XScale DSP coprocessor detected.\n"); + thread_register_notifier(&dsp_notifier_block); + cp_access |= 1; + } + + xscale_cp_access_write(cp_access); + + return 0; +} + +late_initcall(xscale_cp0_init); |