x86: move struct bug_frame instances out of line

Just like Linux did many years ago, move them into a separate (data)
section, such that they no longer pollute instruction caches and TLBs.

Assertion frames, requiring two pointers to be stored, occupy two slots
in the array, with the second slot mimicking a frame the location
pointer of which doesn't match any address within .text or .init.text
(it effectively points back to the slot itself, which - being in a data
section - can't be reached by non-buggy execution).

Signed-off-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Keir Fraser <keir@xen.org>

5 files changed, 105 insertions, 84 deletions

diff --git a/xen/arch/x86/efi/stub.c b/xen/arch/x86/efi/stub.c
index 1520bae6e3..8916a2ec7b 100644
--- a/xen/arch/x86/efi/stub.c
+++ b/xen/arch/x86/efi/stub.c
@@ -1,7 +1,7 @@
 #include <xen/efi.h>
 #include <xen/errno.h>
 #include <xen/init.h>
-#include <asm/bug.h>
+#include <xen/lib.h>
 
 #ifndef efi_enabled
 const bool_t efi_enabled = 0;
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index b445b2fa3b..9db42c829d 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -952,11 +952,18 @@ static int emulate_forced_invalid_op(struct cpu_user_regs *regs)
 
 void do_invalid_op(struct cpu_user_regs *regs)
 {
-    struct bug_frame bug;
-    struct bug_frame_str bug_str;
-    const char *p, *filename, *predicate, *eip = (char *)regs->eip;
+    const struct bug_frame *bug;
+    u8 bug_insn[2];
+    const char *filename, *predicate, *eip = (char *)regs->eip;
     unsigned long fixup;
     int id, lineno;
+    static const struct bug_frame *const stop_frames[] = {
+        __stop_bug_frames_0,
+        __stop_bug_frames_1,
+        __stop_bug_frames_2,
+        __stop_bug_frames_3,
+        NULL
+    };
 
     DEBUGGER_trap_entry(TRAP_invalid_op, regs);
 
@@ -968,70 +975,65 @@ void do_invalid_op(struct cpu_user_regs *regs)
         return;
     }
 
-    if ( !is_kernel(eip) ||
-         __copy_from_user(&bug, eip, sizeof(bug)) ||
-         memcmp(bug.ud2, "\xf\xb", sizeof(bug.ud2)) ||
-         (bug.ret != 0xc2) )
+    if ( (!is_kernel_text(eip) &&
+          (system_state > SYS_STATE_boot || !is_kernel_inittext(eip))) ||
+         __copy_from_user(bug_insn, eip, sizeof(bug_insn)) ||
+         memcmp(bug_insn, "\xf\xb", sizeof(bug_insn)) )
         goto die;
-    eip += sizeof(bug);
 
-    /* Decode first pointer argument. */
-    if ( !is_kernel(eip) ||
-         __copy_from_user(&bug_str, eip, sizeof(bug_str)) ||
-         (bug_str.mov != 0xbc) )
-        goto die;
-    p = bug_str(bug_str, eip);
-    if ( !is_kernel(p) )
+    for ( bug = __start_bug_frames, id = 0; stop_frames[id]; ++bug )
+    {
+        while ( unlikely(bug == stop_frames[id]) )
+            ++id;
+        if ( bug_loc(bug) == eip )
+            break;
+    }
+    if ( !stop_frames[id] )
         goto die;
-    eip += sizeof(bug_str);
-
-    id = bug.id & 3;
 
+    eip += sizeof(bug_insn);
     if ( id == BUGFRAME_run_fn )
     {
-        void (*fn)(struct cpu_user_regs *) = (void *)p;
-        (*fn)(regs);
+        void (*fn)(struct cpu_user_regs *) = bug_ptr(bug);
+
+        fn(regs);
         regs->eip = (unsigned long)eip;
         return;
     }
 
     /* WARN, BUG or ASSERT: decode the filename pointer and line number. */
-    filename = p;
-    lineno = bug.id >> 2;
+    filename = bug_ptr(bug);
+    if ( !is_kernel(filename) )
+        goto die;
+    lineno = bug_line(bug);
 
-    if ( id == BUGFRAME_warn )
+    switch ( id )
     {
+    case BUGFRAME_warn:
         printk("Xen WARN at %.50s:%d\n", filename, lineno);
         show_execution_state(regs);
         regs->eip = (unsigned long)eip;
         return;
-    }
 
-    if ( id == BUGFRAME_bug )
-    {
+    case BUGFRAME_bug:
         printk("Xen BUG at %.50s:%d\n", filename, lineno);
         DEBUGGER_trap_fatal(TRAP_invalid_op, regs);
         show_execution_state(regs);
         panic("Xen BUG at %.50s:%d\n", filename, lineno);
-    }
 
-    /* ASSERT: decode the predicate string pointer. */
-    ASSERT(id == BUGFRAME_assert);
-    if ( !is_kernel(eip) ||
-         __copy_from_user(&bug_str, eip, sizeof(bug_str)) ||
-         (bug_str.mov != 0xbc) )
-        goto die;
-    predicate = bug_str(bug_str, eip);
-    eip += sizeof(bug_str);
+    case BUGFRAME_assert:
+        /* ASSERT: decode the predicate string pointer. */
+        predicate = bug_msg(bug);
+        if ( !is_kernel(predicate) )
+            predicate = "<unknown>";
 
-    if ( !is_kernel(predicate) )
-        predicate = "<unknown>";
-    printk("Assertion '%s' failed at %.50s:%d\n",
-           predicate, filename, lineno);
-    DEBUGGER_trap_fatal(TRAP_invalid_op, regs);
-    show_execution_state(regs);
-    panic("Assertion '%s' failed at %.50s:%d\n",
-          predicate, filename, lineno);
+        printk("Assertion '%s' failed at %.50s:%d\n",
+               predicate, filename, lineno);
+        DEBUGGER_trap_fatal(TRAP_invalid_op, regs);
+        show_execution_state(regs);
+        panic("Assertion '%s' failed at %.50s:%d\n",
+              predicate, filename, lineno);
+    }
 
  die:
     if ( (fixup = search_exception_table(regs->eip)) != 0 )
diff --git a/xen/arch/x86/xen.lds.S b/xen/arch/x86/xen.lds.S
index d959941d36..9600cdf19e 100644
--- a/xen/arch/x86/xen.lds.S
+++ b/xen/arch/x86/xen.lds.S
@@ -51,6 +51,18 @@ SECTIONS
   } :text = 0x9090
 
   .rodata : {
+       /* Bug frames table */
+       . = ALIGN(4);
+       __start_bug_frames = .;
+       *(.bug_frames.0)
+       __stop_bug_frames_0 = .;
+       *(.bug_frames.1)
+       __stop_bug_frames_1 = .;
+       *(.bug_frames.2)
+       __stop_bug_frames_2 = .;
+       *(.bug_frames.3)
+       __stop_bug_frames_3 = .;
+
        *(.rodata)
        *(.rodata.*)
   } :text
diff --git a/xen/include/asm-x86/bug.h b/xen/include/asm-x86/bug.h
index c1fc186be9..148975fde2 100644
--- a/xen/include/asm-x86/bug.h
+++ b/xen/include/asm-x86/bug.h
@@ -1,42 +1,60 @@
 #ifndef __X86_BUG_H__
 #define __X86_BUG_H__
 
-#include <asm/x86_64/bug.h>
+#define BUG_DISP_WIDTH    24
+#define BUG_LINE_LO_WIDTH (31 - BUG_DISP_WIDTH)
+#define BUG_LINE_HI_WIDTH (31 - BUG_DISP_WIDTH)
 
 struct bug_frame {
-    unsigned char ud2[2];
-    unsigned char ret;
-    unsigned short id; /* BUGFRAME_??? */
-} __attribute__((packed));
+    signed int loc_disp:BUG_DISP_WIDTH;
+    unsigned int line_hi:BUG_LINE_HI_WIDTH;
+    signed int ptr_disp:BUG_DISP_WIDTH;
+    unsigned int line_lo:BUG_LINE_LO_WIDTH;
+    signed int msg_disp[];
+};
+
+#define bug_loc(b) ((const void *)(b) + (b)->loc_disp)
+#define bug_ptr(b) ((const void *)(b) + (b)->ptr_disp)
+#define bug_line(b) ((((b)->line_hi + ((b)->loc_disp < 0)) <<                \
+                      BUG_LINE_LO_WIDTH) +                                   \
+                     (b)->line_lo + ((b)->ptr_disp < 0))
+#define bug_msg(b) ((const char *)(b) + (b)->msg_disp[1])
 
 #define BUGFRAME_run_fn 0
 #define BUGFRAME_warn   1
 #define BUGFRAME_bug    2
 #define BUGFRAME_assert 3
 
-#define run_in_exception_handler(fn)               \
-    asm volatile (                                 \
-        "ud2 ; ret %0" BUG_STR(1)                  \
-        : : "i" (BUGFRAME_run_fn),                 \
-            "i" (&(fn)) )
-
-#define WARN()                                     \
-    asm volatile (                                 \
-        "ud2 ; ret %0" BUG_STR(1)                  \
-        : : "i" (BUGFRAME_warn | (__LINE__<<2)),   \
-            "i" (__FILE__) )
-
-#define BUG()                                      \
-    asm volatile (                                 \
-        "ud2 ; ret %0" BUG_STR(1)                  \
-        : : "i" (BUGFRAME_bug | (__LINE__<<2)),    \
-            "i" (__FILE__) )
-
-#define assert_failed(p)                           \
-    asm volatile (                                 \
-        "ud2 ; ret %0" BUG_STR(1) BUG_STR(2)       \
-        : : "i" (BUGFRAME_assert | (__LINE__<<2)), \
-            "i" (__FILE__), "i" (p) )
-
+#define BUG_FRAME(type, line, ptr, msg) do {                                 \
+    BUILD_BUG_ON((line) >> (BUG_LINE_LO_WIDTH + BUG_LINE_HI_WIDTH));         \
+    asm volatile ( ".Lbug%=: ud2\n"                                          \
+                   ".pushsection .bug_frames.%c0, \"a\", @progbits\n"        \
+                   ".p2align 2\n"                                            \
+                   ".Lfrm%=:\n"                                              \
+                   ".long (.Lbug%= - .Lfrm%=) + %c4\n"                       \
+                   ".long (%c1 - .Lfrm%=) + %c3\n"                           \
+                   ".ifnes \"" msg "\", \"\"\n"                              \
+                   ".long 0, %c2 - .Lfrm%=\n"                                \
+                   ".endif\n"                                                \
+                   ".popsection"                                             \
+                   :                                                         \
+                   : "i" (type), "i" (ptr), "i" (msg),                       \
+                     "i" ((line & ((1 << BUG_LINE_LO_WIDTH) - 1))            \
+                          << BUG_DISP_WIDTH),                                \
+                     "i" (((line) >> BUG_LINE_LO_WIDTH) << BUG_DISP_WIDTH)); \
+} while (0)
+
+#define WARN() BUG_FRAME(BUGFRAME_warn, __LINE__, __FILE__, "")
+#define BUG()  BUG_FRAME(BUGFRAME_bug,  __LINE__, __FILE__, "")
+
+#define run_in_exception_handler(fn) BUG_FRAME(BUGFRAME_run_fn, 0, fn, "")
+
+#define assert_failed(msg) BUG_FRAME(BUGFRAME_assert, __LINE__, __FILE__, msg)
+
+extern const struct bug_frame __start_bug_frames[],
+                              __stop_bug_frames_0[],
+                              __stop_bug_frames_1[],
+                              __stop_bug_frames_2[],
+                              __stop_bug_frames_3[];
 
 #endif /* __X86_BUG_H__ */
diff --git a/xen/include/asm-x86/x86_64/bug.h b/xen/include/asm-x86/x86_64/bug.h
deleted file mode 100644
index ecae455b83..0000000000
--- a/xen/include/asm-x86/x86_64/bug.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef __X86_64_BUG_H__
-#define __X86_64_BUG_H__
-
-struct bug_frame_str {
-    unsigned char mov;
-    signed int str_disp;
-} __attribute__((packed));
-#define bug_str(b, rip) ((const char *)(rip) + (b).str_disp)
-#define BUG_STR(n) "; movl %" #n " - ., %%esp"
-
-#endif /* __X86_64_BUG_H__ */