Add PV-GRUB

This fetches GRUB1 sources, applies the {graphical, print function,
save default, and ext3_256byte} patches from debian, and applies a
patch to make it work on x86_64 and port it to Mini-OS.  By using
libxc, PV-GRUB can then "kexec" the loaded kernel from inside the
domain itself, hence permitting to avoid the security-concerned
pygrub.

Signed-off-by: Samuel Thibault <samuel.thibault@eu.citrix.com>

8 files changed, 1368 insertions, 0 deletions

diff --git a/stubdom/grub/Makefile b/stubdom/grub/Makefile
new file mode 100644
index 0000000000..4634c0927f
--- /dev/null
+++ b/stubdom/grub/Makefile
@@ -0,0 +1,76 @@
+XEN_ROOT = ../..
+
+include $(XEN_ROOT)/Config.mk
+vpath %.c ../grub-cvs
+
+BOOT=boot-$(XEN_TARGET_ARCH).o
+
+DEF_CPPFLAGS += -I$(XEN_ROOT)/tools/libxc -I.
+DEF_CPPFLAGS += -I../grub-cvs/stage1
+DEF_CPPFLAGS += -I../grub-cvs/stage2
+DEF_CPPFLAGS += -I../grub-cvs/netboot
+DEF_CPPFLAGS += -I$(XEN_ROOT)/tools/firmware/vgabios
+DEF_CPPFLAGS += -DWITHOUT_LIBC_STUBS
+DEF_CPPFLAGS += -DSUPPORT_NETBOOT
+DEF_CPPFLAGS += -DSUPPORT_GRAPHICS
+DEF_CPPFLAGS += -DSUPPORT_SERIAL
+DEF_CPPFLAGS += -DPRESET_MENU_STRING='""'
+DEF_CPPFLAGS += -DPACKAGE='"grubdom"' -DVERSION='"0.97"'
+
+all: main.a
+
+STAGE2_SOURCES=builtins.c char_io.c cmdline.c common.c console.c disk_io.c graphics.c gunzip.c md5.c serial.c stage2.c terminfo.c tparm.c
+
+NETBOOT_SOURCES=fsys_tftp.c main.c misc.c
+CPPFLAGS += -DFSYS_TFTP=1
+
+STAGE2_SOURCES+=fsys_ext2fs.c
+CPPFLAGS += -DFSYS_EXT2FS=1
+
+STAGE2_SOURCES+=fsys_fat.c
+CPPFLAGS += -DFSYS_FAT=1
+
+STAGE2_SOURCES+=fsys_ffs.c
+CPPFLAGS += -DFSYS_FFS=1
+
+STAGE2_SOURCES+=fsys_iso9660.c
+CPPFLAGS += -DFSYS_ISO9660=1
+
+STAGE2_SOURCES+=fsys_jfs.c
+CPPFLAGS += -DFSYS_JFS=1
+
+STAGE2_SOURCES+=fsys_minix.c
+CPPFLAGS += -DFSYS_MINIX=1
+
+STAGE2_SOURCES+=fsys_reiserfs.c
+CPPFLAGS += -DFSYS_REISERFS=1
+
+STAGE2_SOURCES+=fsys_ufs2.c
+CPPFLAGS += -DFSYS_UFS2=1
+
+STAGE2_SOURCES+=fsys_vstafs.c
+CPPFLAGS += -DFSYS_VSTAFS=1
+
+ifeq (0,1)
+STAGE2_SOURCES+=fsys_xfs.c
+CPPFLAGS += -DFSYS_XFS=1
+endif
+
+STAGE2_SOURCES:=$(addprefix stage2/,$(STAGE2_SOURCES))
+NETBOOT_SOURCES:=$(addprefix netboot/,$(NETBOOT_SOURCES))
+
+$(BOOT): DEF_CPPFLAGS+=-D__ASSEMBLY__
+
+OBJS = $(NETBOOT_SOURCES:.c=.o) $(STAGE2_SOURCES:.c=.o) kexec.o mini-os.o
+
+dirs:
+	mkdir -p netboot stage2
+	touch $@
+
+$(OBJS): dirs
+
+main.a: $(BOOT) $(OBJS)
+	$(AR) cr $@ $^
+
+clean:
+	rm -fr dirs *.a *.o stage2 netboot
diff --git a/stubdom/grub/boot-x86_32.S b/stubdom/grub/boot-x86_32.S
new file mode 100644
index 0000000000..c95fd30dd3
--- /dev/null
+++ b/stubdom/grub/boot-x86_32.S
@@ -0,0 +1,112 @@
+#include <os.h>
+#include <arch_limits.h>
+#include <xen/arch-x86_32.h>
+
+/* For simplicity, we keep all of this into just one data page */
+.data
+.globl _boot_page
+_boot_page:
+        .align __PAGE_SIZE
+
+/*
+ * The following data is initialized from C code
+ */
+
+/* Pte of this page */
+.globl _boot_page_entry
+_boot_page_entry:
+_boot_page_entry_lo:
+        .long 0
+_boot_page_entry_hi:
+        .long 0
+
+/* mmuext_op structure */
+/* Set new page directory */
+_boot_mmuext:
+        /* Op # */
+        .long MMUEXT_NEW_BASEPTR
+
+        /* MFN of target page table directory */
+.globl _boot_pdmfn
+_boot_pdmfn:
+        .long 0
+
+        /* Unused */
+        .long 0
+
+/* Unpin old page directory */
+        /* Op # */
+        .long MMUEXT_UNPIN_TABLE
+
+        /* MFN of old page table directory */
+.globl _boot_oldpdmfn
+_boot_oldpdmfn:
+        .long 0
+
+        /* Unused */
+        .long 0
+
+/* Target stack address, also target virtual address of this page */
+.globl _boot_stack
+_boot_stack:
+        .long 0
+        .long __KERNEL_SS
+.globl _boot_target
+_boot_target:
+        .long 0
+
+/* Target start info */
+.globl _boot_start_info
+_boot_start_info:
+        .long 0
+
+/* Target start address */
+.globl _boot_start
+_boot_start:
+        .long 0
+
+/*
+ * Boot target OS, does not return
+ */
+.globl _boot
+_boot:
+        /* Project ourselves at the target place. */
+        movl    _boot_target, %ebx
+        movl    %ebx, %ebp     /* also keep it in ebp for relative addressing */
+        movl    _boot_page_entry_lo, %ecx
+        movl    _boot_page_entry_hi, %edx
+        movl    $2, %esi /* UVMF_INVLPG */
+        movl    $__HYPERVISOR_update_va_mapping, %eax
+        int     $0x82
+        testl   %eax, %eax
+        jz      0f
+        ud2
+
+0:
+        /* Go there. */
+        movl    $(0f - _boot_page), %eax
+        movl    _boot_target, %ebx
+        addl    %ebx, %eax
+        jmpl    *%eax
+0:
+        
+        /* Load target page table and unpin old page table.  */
+        /* We shouldn't have any problem since in the new page table our page is
+           mapped at the same place.  */
+        leal    (_boot_mmuext-_boot_page)(%ebp), %ebx
+        movl    $2, %ecx
+        xorl    %edx, %edx
+        movl    $0x7FF0, %esi /* DOMID_SELF */
+        movl    $__HYPERVISOR_mmuext_op, %eax
+        int     $0x82
+        testl   %eax, %eax
+        jns     0f
+        ud2
+
+0:
+        /* Initialize registers.  */
+        lss     (_boot_stack-_boot_page)(%ebp), %esp
+        movl    (_boot_start_info-_boot_page)(%ebp), %esi
+
+        /* Jump!  */
+        jmpl    *(_boot_start-_boot_page)(%ebp)
diff --git a/stubdom/grub/boot-x86_64.S b/stubdom/grub/boot-x86_64.S
new file mode 100644
index 0000000000..2eae6c3437
--- /dev/null
+++ b/stubdom/grub/boot-x86_64.S
@@ -0,0 +1,108 @@
+#include <os.h>
+#include <arch_limits.h>
+#include <xen/arch-x86_64.h>
+
+/* For simplicity, we keep all of this into just one data page */
+.data
+.globl _boot_page
+_boot_page:
+        .align __PAGE_SIZE
+
+/*
+ * The following data is initialized from C code
+ */
+
+/* Pte of this page */
+.globl _boot_page_entry
+_boot_page_entry:
+        .quad 0
+
+/* mmuext_op structure */
+/* Set new page directory */
+_boot_mmuext:
+        /* Op # */
+        .long MMUEXT_NEW_BASEPTR
+        .long 0 /* pad */
+
+        /* MFN of target page table directory */
+.globl _boot_pdmfn
+_boot_pdmfn:
+        .quad 0
+
+        /* Unused */
+        .quad 0
+
+/* Unpin old page directory */
+        /* Op # */
+        .long MMUEXT_UNPIN_TABLE
+        .long 0 /* pad */
+
+        /* MFN of old page table directory */
+.globl _boot_oldpdmfn
+_boot_oldpdmfn:
+        .quad 0
+
+        /* Unused */
+        .quad 0
+
+/* Target stack address, also target virtual address of this page */
+.globl _boot_stack
+_boot_stack:
+        .quad 0
+.globl _boot_target
+_boot_target:
+        .quad 0
+
+/* Target start info */
+.globl _boot_start_info
+_boot_start_info:
+        .quad 0
+
+/* Target start address */
+.globl _boot_start
+_boot_start:
+        .quad 0
+
+/*
+ * Boot target OS, does not return
+ */
+.globl _boot
+_boot:
+        /* Project ourselves at the target place. */
+        movq    _boot_target, %rdi
+        movq    _boot_page_entry, %rsi
+        movq    $2, %rdx /* UVMF_INVLPG */
+        movq    $__HYPERVISOR_update_va_mapping, %rax
+        syscall
+        testq   %rax, %rax
+        jz      0f
+        ud2
+
+0:
+        /* Go there. */
+        movq    $(0f - _boot_page), %rax
+        movq    _boot_target, %rbx
+        addq    %rbx, %rax
+        jmpq    *%rax
+0:
+        
+        /* Load target page table and unpin old page table.  */
+        /* We shouldn't have any problem since in the new page table our page is
+           mapped at the same place.  */
+        leaq    _boot_mmuext(%rip), %rdi
+        movq    $2, %rsi
+        xorq    %rdx, %rdx
+        movq    $0x7FF0, %r10 /* DOMID_SELF */
+        movq    $__HYPERVISOR_mmuext_op, %rax
+        syscall
+        testq   %rax, %rax
+        jns     0f
+        ud2
+
+0:
+        /* Initialize registers.  */
+        movq    _boot_stack(%rip), %rsp
+        movq    _boot_start_info(%rip), %rsi
+
+        /* Jump!  */
+        jmpq    *_boot_start(%rip)
diff --git a/stubdom/grub/config.h b/stubdom/grub/config.h
new file mode 100644
index 0000000000..dcb2e7a15c
--- /dev/null
+++ b/stubdom/grub/config.h
@@ -0,0 +1,11 @@
+#include <stdio.h>
+#undef putchar
+#include <ctype.h>
+#include <string.h>
+#define debug _debug
+#define grub_halt(a) do_exit()
+#define printf grub_printf
+void kexec(void *kernel, long kernel_size, void *module, long module_size, char *cmdline);
+struct fbfront_dev *fb_open(void *fb, int width, int height, int depth);
+void fb_close(void);
+void pv_boot (void);
diff --git a/stubdom/grub/kexec.c b/stubdom/grub/kexec.c
new file mode 100644
index 0000000000..5400fe88cd
--- /dev/null
+++ b/stubdom/grub/kexec.c
@@ -0,0 +1,324 @@
+/*
+ * This supports booting another PV kernel from Mini-OS
+ *
+ * The idea is to setup it using libxc, answer to day0 memory allocation
+ * requests, and using a trampoline boot page to switch to the new page table.
+ *
+ * The procedure of the boot page is:
+ * - map itself at the target position (that may overwrite some C stuff, but we
+ *   do not care any more)
+ * - jump there
+ * - switch to the target page table
+ * - unpin the old page table
+ * - jump to the new kernel
+ *
+ * Samuel Thibault <Samuel.Thibault@eu.citrix.com>, May 2008
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+
+#include <xenctrl.h>
+#include <xc_dom.h>
+
+#include <kernel.h>
+#include <console.h>
+#include <os.h>
+#include <blkfront.h>
+#include <netfront.h>
+#include <fbfront.h>
+#include <shared.h>
+
+#include "mini-os.h"
+
+#if 0
+#define DEBUG(fmt, ...) printk(fmt, ## __VA_ARGS__)
+#else
+#define DEBUG(fmt, ...) (void)0
+#endif
+
+/* Assembly boot page from boot.S */
+extern void _boot_page;
+extern pgentry_t _boot_page_entry;
+extern unsigned long _boot_pdmfn;
+extern unsigned long _boot_stack, _boot_target, _boot_start_info, _boot_start;
+extern xen_pfn_t _boot_oldpdmfn;
+extern void _boot(void);
+
+static unsigned long *pages;
+static unsigned long *pages_mfns;
+static unsigned long allocated;
+
+int pin_table(int xc_handle, unsigned int type, unsigned long mfn,
+              domid_t dom);
+
+/* We need mfn to appear as target_pfn, so exchange with the MFN there */
+static void do_exchange(struct xc_dom_image *dom, xen_pfn_t target_pfn, xen_pfn_t source_mfn)
+{
+    xen_pfn_t source_pfn;
+    xen_pfn_t target_mfn;
+
+    for (source_pfn = 0; source_pfn < start_info.nr_pages; source_pfn++)
+        if (dom->p2m_host[source_pfn] == source_mfn)
+            break;
+    ASSERT(source_pfn < start_info.nr_pages);
+
+    target_mfn = dom->p2m_host[target_pfn];
+
+    /* Put target MFN at source PFN */
+    dom->p2m_host[source_pfn] = target_mfn;
+
+    /* Put source MFN at target PFN */
+    dom->p2m_host[target_pfn] = source_mfn;
+}
+
+int kexec_allocate(struct xc_dom_image *dom, xen_vaddr_t up_to)
+{
+    unsigned long new_allocated = (up_to - dom->parms.virt_base) / PAGE_SIZE;
+    unsigned long i;
+
+    pages = realloc(pages, new_allocated * sizeof(*pages));
+    pages_mfns = realloc(pages_mfns, new_allocated * sizeof(*pages_mfns));
+    for (i = allocated; i < new_allocated; i++) {
+        /* Exchange old page of PFN i with a newly allocated page.  */
+        xen_pfn_t old_mfn = dom->p2m_host[i];
+        xen_pfn_t new_pfn;
+        xen_pfn_t new_mfn;
+
+        pages[i] = alloc_page();
+        memset((void*) pages[i], 0, PAGE_SIZE);
+        new_pfn = PHYS_PFN(to_phys(pages[i]));
+        pages_mfns[i] = new_mfn = pfn_to_mfn(new_pfn);
+
+        /* Put old page at new PFN */
+        dom->p2m_host[new_pfn] = old_mfn;
+
+        /* Put new page at PFN i */
+        dom->p2m_host[i] = new_mfn;
+    }
+
+    allocated = new_allocated;
+
+    return 0;
+}
+
+void kexec(void *kernel, long kernel_size, void *module, long module_size, char *cmdline)
+{
+    struct xc_dom_image *dom;
+    int rc;
+    domid_t domid = DOMID_SELF;
+    xen_pfn_t pfn;
+    int xc_handle;
+    unsigned long i;
+    void *seg;
+    xen_pfn_t boot_page_mfn = virt_to_mfn(&_boot_page);
+    char features[] = "";
+    struct mmu_update *m2p_updates;
+    unsigned long nr_m2p_updates;
+
+    DEBUG("booting with cmdline %s\n", cmdline);
+    xc_handle = xc_interface_open();
+
+    dom = xc_dom_allocate(cmdline, features);
+    dom->allocate = kexec_allocate;
+
+    dom->kernel_blob = kernel;
+    dom->kernel_size = kernel_size;
+
+    dom->ramdisk_blob = module;
+    dom->ramdisk_size = module_size;
+
+    dom->flags = 0;
+    dom->console_evtchn = start_info.console.domU.evtchn;
+    dom->xenstore_evtchn = start_info.store_evtchn;
+
+    if ( (rc = xc_dom_boot_xen_init(dom, xc_handle, domid)) != 0 ) {
+        grub_printf("xc_dom_boot_xen_init returned %d\n", rc);
+        errnum = ERR_BOOT_FAILURE;
+        goto out;
+    }
+    if ( (rc = xc_dom_parse_image(dom)) != 0 ) {
+        grub_printf("xc_dom_parse_image returned %d\n", rc);
+        errnum = ERR_BOOT_FAILURE;
+        goto out;
+    }
+
+#ifdef __i386__
+    if (strcmp(dom->guest_type, "xen-3.0-x86_32p")) {
+        grub_printf("can only boot x86 32 PAE kernels, not %s\n", dom->guest_type);
+        errnum = ERR_EXEC_FORMAT;
+        goto out;
+    }
+#endif
+#ifdef __x86_64__
+    if (strcmp(dom->guest_type, "xen-3.0-x86_64")) {
+        grub_printf("can only boot x86 64 kernels, not %s\n", dom->guest_type);
+        errnum = ERR_EXEC_FORMAT;
+        goto out;
+    }
+#endif
+
+    /* equivalent of xc_dom_mem_init */
+    dom->arch_hooks = xc_dom_find_arch_hooks(dom->guest_type);
+    dom->total_pages = start_info.nr_pages;
+
+    /* equivalent of arch_setup_meminit */
+
+    /* setup initial p2m */
+    dom->p2m_host = malloc(sizeof(*dom->p2m_host) * dom->total_pages);
+
+    /* Start with our current P2M */
+    for (i = 0; i < dom->total_pages; i++)
+        dom->p2m_host[i] = pfn_to_mfn(i);
+
+    if ( (rc = xc_dom_build_image(dom)) != 0 ) {
+        grub_printf("xc_dom_build_image returned %d\n", rc);
+        errnum = ERR_BOOT_FAILURE;
+        goto out;
+    }
+
+    /* copy hypercall page */
+    /* TODO: domctl instead, but requires privileges */
+    if (dom->parms.virt_hypercall != -1) {
+        pfn = PHYS_PFN(dom->parms.virt_hypercall - dom->parms.virt_base);
+        memcpy((void *) pages[pfn], hypercall_page, PAGE_SIZE);
+    }
+
+    /* Equivalent of xc_dom_boot_image */
+    dom->shared_info_mfn = PHYS_PFN(start_info.shared_info);
+
+    if (!xc_dom_compat_check(dom)) {
+        grub_printf("xc_dom_compat_check failed\n");
+        errnum = ERR_EXEC_FORMAT;
+        goto out;
+    }
+
+    /* Move current console, xenstore and boot MFNs to the allocated place */
+    do_exchange(dom, dom->console_pfn, start_info.console.domU.mfn);
+    do_exchange(dom, dom->xenstore_pfn, start_info.store_mfn);
+    DEBUG("virt base at %llx\n", dom->parms.virt_base);
+    DEBUG("bootstack_pfn %lx\n", dom->bootstack_pfn);
+    _boot_target = dom->parms.virt_base + PFN_PHYS(dom->bootstack_pfn);
+    DEBUG("_boot_target %lx\n", _boot_target);
+    do_exchange(dom, PHYS_PFN(_boot_target - dom->parms.virt_base),
+            virt_to_mfn(&_boot_page));
+
+    /* Make sure the bootstrap page table does not RW-map any of our current
+     * page table frames */
+    kexec_allocate(dom, dom->virt_pgtab_end);
+
+    if ( (rc = xc_dom_update_guest_p2m(dom))) {
+        grub_printf("xc_dom_update_guest_p2m returned %d\n", rc);
+        errnum = ERR_BOOT_FAILURE;
+        goto out;
+    }
+
+    if ( dom->arch_hooks->setup_pgtables )
+        if ( (rc = dom->arch_hooks->setup_pgtables(dom))) {
+            grub_printf("setup_pgtables returned %d\n", rc);
+            errnum = ERR_BOOT_FAILURE;
+            goto out;
+        }
+
+    /* start info page */
+#undef start_info
+    if ( dom->arch_hooks->start_info )
+        dom->arch_hooks->start_info(dom);
+#define start_info (start_info_union.start_info)
+
+    xc_dom_log_memory_footprint(dom);
+
+    /* Unmap libxc's projection of the boot page table */
+    seg = xc_dom_seg_to_ptr(dom, &dom->pgtables_seg);
+    munmap(seg, dom->pgtables_seg.vend - dom->pgtables_seg.vstart);
+
+    /* Unmap day0 pages to avoid having a r/w mapping of the future page table */
+    for (pfn = 0; pfn < allocated; pfn++)
+        munmap((void*) pages[pfn], PAGE_SIZE);
+
+    /* Pin the boot page table base */
+    if ( (rc = pin_table(dom->guest_xc, 
+#ifdef __i386__
+                MMUEXT_PIN_L3_TABLE,
+#endif
+#ifdef __x86_64__
+                MMUEXT_PIN_L4_TABLE,
+#endif
+                xc_dom_p2m_host(dom, dom->pgtables_seg.pfn),
+                dom->guest_domid)) != 0 ) {
+        grub_printf("pin_table(%lx) returned %d\n", xc_dom_p2m_host(dom,
+                    dom->pgtables_seg.pfn), rc);
+        errnum = ERR_BOOT_FAILURE;
+        goto out_remap;
+    }
+
+    /* We populate the Mini-OS page table here so that boot.S can just call
+     * update_va_mapping to project itself there.  */
+    need_pgt(_boot_target);
+    DEBUG("day0 pages %lx\n", allocated);
+    DEBUG("boot target page %lx\n", _boot_target);
+    DEBUG("boot page %p\n", &_boot_page);
+    DEBUG("boot page mfn %lx\n", boot_page_mfn);
+    _boot_page_entry = PFN_PHYS(boot_page_mfn) | L1_PROT;
+    DEBUG("boot page entry %llx\n", _boot_page_entry);
+    _boot_oldpdmfn = virt_to_mfn(start_info.pt_base);
+    DEBUG("boot old pd mfn %lx\n", _boot_oldpdmfn);
+    DEBUG("boot pd virt %lx\n", dom->pgtables_seg.vstart);
+    _boot_pdmfn = dom->p2m_host[PHYS_PFN(dom->pgtables_seg.vstart - dom->parms.virt_base)];
+    DEBUG("boot pd mfn %lx\n", _boot_pdmfn);
+    _boot_stack = _boot_target + PAGE_SIZE;
+    DEBUG("boot stack %lx\n", _boot_stack);
+    _boot_start_info = dom->parms.virt_base + PFN_PHYS(dom->start_info_pfn);
+    DEBUG("boot start info %lx\n", _boot_start_info);
+    _boot_start = dom->parms.virt_entry;
+    DEBUG("boot start %lx\n", _boot_start);
+
+    /* Keep only useful entries */
+    for (nr_m2p_updates = pfn = 0; pfn < start_info.nr_pages; pfn++)
+        if (dom->p2m_host[pfn] != pfn_to_mfn(pfn))
+            nr_m2p_updates++;
+
+    m2p_updates = malloc(sizeof(*m2p_updates) * nr_m2p_updates);
+    for (i = pfn = 0; pfn < start_info.nr_pages; pfn++)
+        if (dom->p2m_host[pfn] != pfn_to_mfn(pfn)) {
+            m2p_updates[i].ptr = PFN_PHYS(dom->p2m_host[pfn]) | MMU_MACHPHYS_UPDATE;
+            m2p_updates[i].val = pfn;
+            i++;
+        }
+
+    for (i = 0; i < blk_nb; i++)
+        shutdown_blkfront(blk_dev[i]);
+    if (net_dev)
+        shutdown_netfront(net_dev);
+    if (kbd_dev)
+        shutdown_kbdfront(kbd_dev);
+    stop_kernel();
+
+    /* Update M2P */
+    if ((rc = HYPERVISOR_mmu_update(m2p_updates, nr_m2p_updates, NULL, DOMID_SELF)) < 0) {
+        xprintk("Could not update M2P\n");
+        ASSERT(0);
+    }
+
+    xprintk("go!\n");
+
+    /* Jump to trampoline boot page */
+    _boot();
+
+    ASSERT(0);
+
+out_remap:
+    for (pfn = 0; pfn < allocated; pfn++)
+        do_map_frames(pages[pfn], &pages_mfns[pfn], 1, 0, 0, DOMID_SELF, 0, L1_PROT);
+out:
+    xc_dom_release(dom);
+    for (pfn = 0; pfn < allocated; pfn++)
+        free_page((void*)pages[pfn]);
+    free(pages);
+    free(pages_mfns);
+    pages = NULL;
+    pages_mfns = NULL;
+    allocated = 0;
+    xc_interface_close(xc_handle );
+}
diff --git a/stubdom/grub/mini-os.c b/stubdom/grub/mini-os.c
new file mode 100644
index 0000000000..49e1f21f86
--- /dev/null
+++ b/stubdom/grub/mini-os.c
@@ -0,0 +1,702 @@
+/*
+ * Mini-OS support for GRUB.
+ *
+ * Samuel Thibault <Samuel.Thibault@eu.citrix.com>, May 2008
+ */
+#include <sys/types.h>
+#include <sys/time.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <malloc.h>
+#include <unistd.h>
+
+#include <hypervisor.h>
+#include <blkfront.h>
+#include <netfront.h>
+#include <fbfront.h>
+#include <semaphore.h>
+
+#include <osdep.h>
+#include <shared.h>
+#include <nic.h>
+#include <etherboot.h>
+#include <terminfo.h>
+#include <term.h>
+
+#include "mini-os.h"
+
+extern const char *preset_menu;
+char config_file[DEFAULT_FILE_BUFLEN] = "(hd0,0)/boot/grub/menu.lst";
+unsigned long boot_drive = NETWORK_DRIVE;
+unsigned long install_partition = 0xFFFFFF;
+
+char version_string[] = VERSION;
+
+/* Variables from asm.S */
+int saved_entryno;
+
+/*
+ * Disk
+ */
+
+struct blkfront_dev **blk_dev;
+int blk_nb;
+static struct blkfront_info *blk_info;
+
+static int vbdcmp(const void *_vbd1, const void *_vbd2) {
+    char *vbd1 = *(char **)_vbd1;
+    char *vbd2 = *(char **)_vbd2;
+    int vbdn1 = atoi(vbd1);
+    int vbdn2 = atoi(vbd2);
+    return vbdn1 - vbdn2;
+}
+
+void init_disk (void)
+{
+    char **list;
+    char *msg;
+    int i;
+    char *path;
+
+    msg = xenbus_ls(XBT_NIL, "device/vbd", &list);
+    if (msg) {
+        printk("Error %s while reading list of disks\n", msg);
+        free(msg);
+        return;
+    }
+    blk_nb = 0;
+    while (list[blk_nb])
+        blk_nb++;
+    blk_dev = malloc(blk_nb * sizeof(*blk_dev));
+    blk_info = malloc(blk_nb * sizeof(*blk_info));
+
+    qsort(list, blk_nb, sizeof(*list), vbdcmp);
+
+    for (i = 0; i < blk_nb; i++) {
+        printk("vbd %s is hd%d\n", list[i], i);
+        asprintf(&path, "device/vbd/%s", list[i]);
+        blk_dev[i] = init_blkfront(path, &blk_info[i]);
+        free(path);
+        free(list[i]);
+    }
+}
+
+/* Return the geometry of DRIVE in GEOMETRY. If an error occurs, return
+   non-zero, otherwise zero.  */
+int get_diskinfo (int drive, struct geometry *geometry)
+{
+    int i;
+    if (!(drive & 0x80))
+        return -1;
+
+    i = drive - 0x80;
+    if (i >= blk_nb)
+        return -1;
+
+    /* Bogus geometry */
+    geometry->cylinders = 65535;
+    geometry->heads = 255;
+    geometry->sectors = 63;
+
+    geometry->total_sectors = blk_info[i].sectors;
+    geometry->sector_size = blk_info[i].sector_size;
+    geometry->flags = BIOSDISK_FLAG_LBA_EXTENSION;
+    if (blk_info[i].info & VDISK_CDROM)
+        geometry->flags |= BIOSDISK_FLAG_CDROM;
+    return 0;
+}
+
+/* Read/write NSEC sectors starting from SECTOR in DRIVE disk with GEOMETRY
+   from/into SEGMENT segment. If READ is BIOSDISK_READ, then read it,
+   else if READ is BIOSDISK_WRITE, then write it. If an geometry error
+   occurs, return BIOSDISK_ERROR_GEOMETRY, and if other error occurs, then
+   return the error number. Otherwise, return 0.  */
+int
+biosdisk (int read, int drive, struct geometry *geometry,
+          unsigned int sector, int nsec, int segment)
+{
+    void *addr = (void *) ((unsigned long)segment << 4);
+    struct blkfront_aiocb aiocb;
+    int i;
+
+    if (!(drive & 0x80))
+        return -1;
+
+    i = drive - 0x80;
+    if (i >= blk_nb)
+        return -1;
+
+    aiocb.aio_dev = blk_dev[i];
+    aiocb.aio_buf = addr;
+    aiocb.aio_nbytes = (size_t)nsec * blk_info[i].sector_size;
+    aiocb.aio_offset = (off_t)sector * blk_info[i].sector_size;
+    aiocb.aio_cb = NULL;
+
+    blkfront_io(&aiocb, read == BIOSDISK_WRITE);
+
+    return 0;
+}
+
+static int
+load_file(char *name, void **ptr, long *size)
+{
+    char *buf = NULL;
+    int allocated = 1 * 1024 * 1024;
+    int len, filled = 0;
+
+    if (!grub_open (name))
+        return -1;
+
+    buf = malloc(allocated);
+
+    errnum = 0;
+    while (1) {
+        len = grub_read (buf + filled, allocated - filled);
+        if (! len) {
+            if (!errnum)
+                break;
+            grub_close ();
+            return -1;
+        }
+        filled += len;
+        if (filled < allocated)
+            break;
+        allocated *= 2;
+        buf = realloc(buf, allocated);
+    }
+    grub_close ();
+    *ptr = buf;
+    *size = filled;
+    return 0;
+}
+
+void *kernel_image, *module_image;
+long  kernel_size, module_size;
+char *kernel_arg, *module_arg;
+
+kernel_t
+load_image (char *kernel, char *arg, kernel_t suggested_type,
+            unsigned long load_flags)
+{
+    arg = skip_to(0, arg);
+    if (kernel_image)
+        free(kernel_image);
+    kernel_image = NULL;
+    if (load_file (kernel, &kernel_image, &kernel_size))
+        return KERNEL_TYPE_NONE;
+    if (kernel_arg)
+        free(kernel_arg);
+    kernel_arg = strdup(arg);
+    return KERNEL_TYPE_PV;
+}
+
+int
+load_initrd (char *initrd)
+{
+    if (module_image)
+        free(module_image);
+    module_image = NULL;
+    load_file (initrd, &module_image, &module_size);
+    return ! errnum;
+}
+
+int
+load_module (char *module, char *arg)
+{
+    if (module_image)
+        free(module_image);
+    module_image = NULL;
+    load_file (module, &module_image, &module_size);
+    if (module_arg)
+        free(module_arg);
+    module_arg = strdup(arg);
+    return ! errnum;
+}
+
+void
+pv_boot (void)
+{
+    kexec(kernel_image, kernel_size, module_image, module_size, kernel_arg);
+}
+
+/*
+ * Network
+ */
+
+struct netfront_dev *net_dev;
+
+int
+minios_probe (struct nic *nic)
+{
+    char *ip;
+
+    if (net_dev)
+        return 1;
+
+    /* Clear the ARP table.  */
+    grub_memset ((char *) arptable, 0,
+                 MAX_ARP * sizeof (struct arptable_t));
+
+    net_dev = init_netfront(NULL, (void*) -1, nic->node_addr, &ip);
+    if (!net_dev)
+        return 0;
+
+    return 1;
+}
+
+/* reset adapter */
+static void minios_reset(struct nic *nic)
+{
+    /* TODO? */
+}
+
+static void minios_disable(struct nic *nic)
+{
+}
+
+/* Wait for a frame */
+static int minios_poll(struct nic *nic)
+{
+    return !! (nic->packetlen = netfront_receive(net_dev, (void*) nic->packet, ETH_FRAME_LEN));
+}
+
+/* Transmit a frame */
+struct frame {
+        uint8_t dest[ETH_ALEN];
+        uint8_t src[ETH_ALEN];
+        uint16_t type;
+        unsigned char data[];
+};
+static void minios_transmit (struct nic *nic, const char *d, unsigned int t,
+                             unsigned int s, const char *p)
+{
+    struct frame *frame = alloca(sizeof(frame) + s);
+
+    memcpy(frame->dest, d, ETH_ALEN);
+    memcpy(frame->src, nic->node_addr, ETH_ALEN);
+    frame->type = htons(t);
+    memcpy(frame->data, p, s);
+
+    netfront_xmit(net_dev, (void*) frame, sizeof(*frame) + s);
+}
+
+static char packet[ETH_FRAME_LEN];
+
+struct nic nic = {
+    .reset = minios_reset,
+    .poll = minios_poll,
+    .transmit = minios_transmit,
+    .disable = minios_disable,
+    .flags = 0,
+    .rom_info = NULL,
+    .node_addr = arptable[ARP_CLIENT].node,
+    .packet = packet,
+    .packetlen = 0,
+    .priv_data = NULL,
+};
+
+int
+eth_probe (void)
+{
+    return minios_probe(&nic);
+}
+
+int
+eth_poll (void)
+{
+    return minios_poll (&nic);
+}
+
+void
+eth_disable (void)
+{
+    minios_disable (&nic);
+}
+
+void
+eth_transmit (const char *d, unsigned int t,
+              unsigned int s, const void *p)
+{
+    minios_transmit (&nic, d, t, s, p);
+    if (t == IP)
+        twiddle();
+}
+
+/*
+ * Console
+ */
+void
+serial_hw_put (int _c)
+{
+  char c = _c;
+  console_print(&c, 1);
+}
+
+int
+serial_hw_fetch (void)
+{
+    char key;
+
+    if (!xencons_ring_avail())
+        return -1;
+
+    read(STDIN_FILENO, &key, 1);
+    switch (key) {
+    case 0x7f: key = '\b'; break;
+    }
+    return key;
+}
+
+/*
+ * PVFB
+ */
+struct kbdfront_dev *kbd_dev;
+struct fbfront_dev *fb_dev;
+static union xenkbd_in_event ev;
+static int has_ev;
+int console_checkkey (void)
+{
+    if (has_ev)
+        return 1;
+    has_ev = kbdfront_receive(kbd_dev, &ev, 1);
+    return has_ev;
+}
+
+/* static QWERTY layout, that's what most PC BIOSes do anyway */
+static char linux2ascii[] = {
+    [ 1 ] = 27,
+    [ 2 ] = '1',
+    [ 3 ] = '2',
+    [ 4 ] = '3',
+    [ 5 ] = '4',
+    [ 6 ] = '5',
+    [ 7 ] = '6',
+    [ 8 ] = '7',
+    [ 9 ] = '8',
+    [ 10 ] = '9',
+    [ 11 ] = '0',
+    [ 12 ] = '-',
+    [ 13 ] = '=',
+    [ 14 ] = '\b',
+    [ 15 ] = '\t',
+    [ 16 ] = 'q',
+    [ 17 ] = 'w',
+    [ 18 ] = 'e',
+    [ 19 ] = 'r',
+    [ 20 ] = 't',
+    [ 21 ] = 'y',
+    [ 22 ] = 'u',
+    [ 23 ] = 'i',
+    [ 24 ] = 'o',
+    [ 25 ] = 'p',
+    [ 26 ] = '[',
+    [ 27 ] = ']',
+    [ 28 ] = '\n',
+
+    [ 30 ] = 'a',
+    [ 31 ] = 's',
+    [ 32 ] = 'd',
+    [ 33 ] = 'f',
+    [ 34 ] = 'g',
+    [ 35 ] = 'h',
+    [ 36 ] = 'j',
+    [ 37 ] = 'k',
+    [ 38 ] = 'l',
+    [ 39 ] = ';',
+    [ 40 ] = '\'',
+    [ 41 ] = '`',
+
+    [ 43 ] = '\\',
+    [ 44 ] = 'z',
+    [ 45 ] = 'x',
+    [ 46 ] = 'c',
+    [ 47 ] = 'v',
+    [ 48 ] = 'b',
+    [ 49 ] = 'n',
+    [ 50 ] = 'm',
+    [ 51 ] = ',',
+    [ 52 ] = '.',
+    [ 53 ] = '/',
+
+    [ 55 ] = '*',
+    [ 57 ] = ' ',
+
+    [ 71 ] = '7',
+    [ 72 ] = '8',
+    [ 73 ] = '9',
+    [ 74 ] = '-',
+    [ 75 ] = '4',
+    [ 76 ] = '5',
+    [ 77 ] = '6',
+    [ 78 ] = '+',
+    [ 79 ] = '1',
+    [ 80 ] = '2',
+    [ 81 ] = '3',
+    [ 82 ] = '0',
+    [ 83 ] = '.',
+
+    [ 86 ] = '<',
+
+    [ 96 ] = '\n',
+
+    [ 98 ] = '/',
+
+    [ 102 ] = 1,  /* home */
+    [ 103 ] = 16, /* up */
+    [ 104 ] = 7,  /* page up */
+    [ 105 ] = 2,  /* left */
+    [ 106 ] = 6,  /* right */
+    [ 107 ] = 5,  /* end */
+    [ 108 ] = 14, /* down */
+    [ 109 ] = 3,  /* page down */
+
+    [ 111 ] = 4,  /* delete */
+};
+
+static char linux2ascii_shifted[] = {
+    [ 1 ] = 27,
+    [ 2 ] = '!',
+    [ 3 ] = '@',
+    [ 4 ] = '#',
+    [ 5 ] = '$',
+    [ 6 ] = '%',
+    [ 7 ] = '^',
+    [ 8 ] = '&',
+    [ 9 ] = '*',
+    [ 10 ] = '(',
+    [ 11 ] = ')',
+    [ 12 ] = '_',
+    [ 13 ] = '+',
+    [ 14 ] = '\b',
+    [ 15 ] = '\t',
+    [ 16 ] = 'Q',
+    [ 17 ] = 'W',
+    [ 18 ] = 'E',
+    [ 19 ] = 'R',
+    [ 20 ] = 'T',
+    [ 21 ] = 'Y',
+    [ 22 ] = 'U',
+    [ 23 ] = 'I',
+    [ 24 ] = 'O',
+    [ 25 ] = 'P',
+    [ 26 ] = '{',
+    [ 27 ] = '}',
+    [ 28 ] = '\n',
+
+    [ 30 ] = 'A',
+    [ 31 ] = 'S',
+    [ 32 ] = 'D',
+    [ 33 ] = 'F',
+    [ 34 ] = 'G',
+    [ 35 ] = 'H',
+    [ 36 ] = 'J',
+    [ 37 ] = 'K',
+    [ 38 ] = 'L',
+    [ 39 ] = ':',
+    [ 40 ] = '"',
+    [ 41 ] = '~',
+
+    [ 43 ] = '|',
+    [ 44 ] = 'Z',
+    [ 45 ] = 'X',
+    [ 46 ] = 'C',
+    [ 47 ] = 'V',
+    [ 48 ] = 'B',
+    [ 49 ] = 'N',
+    [ 50 ] = 'M',
+    [ 51 ] = '<',
+    [ 52 ] = '>',
+    [ 53 ] = '?',
+
+    [ 55 ] = '*',
+    [ 57 ] = ' ',
+
+    [ 71 ] = '7',
+    [ 72 ] = '8',
+    [ 73 ] = '9',
+    [ 74 ] = '-',
+    [ 75 ] = '4',
+    [ 76 ] = '5',
+    [ 77 ] = '6',
+    [ 78 ] = '+',
+    [ 79 ] = '1',
+    [ 80 ] = '2',
+    [ 81 ] = '3',
+    [ 82 ] = '0',
+    [ 83 ] = '.',
+
+    [ 86 ] = '>',
+
+    [ 96 ] = '\n',
+
+    [ 98 ] = '/',
+
+    [ 102 ] = 1,  /* home */
+    [ 103 ] = 16, /* up */
+    [ 104 ] = 7,  /* page up */
+    [ 105 ] = 2,  /* left */
+    [ 106 ] = 6,  /* right */
+    [ 107 ] = 5,  /* end */
+    [ 108 ] = 14, /* down */
+    [ 109 ] = 3,  /* page down */
+
+    [ 111 ] = 4,  /* delete */
+};
+
+int console_getkey (void)
+{
+    static int shift, control, alt, caps_lock;
+
+    if (!has_ev)
+        has_ev = kbdfront_receive(kbd_dev, &ev, 1);
+    if (!has_ev)
+        return 0;
+
+    has_ev = 0;
+    if (ev.type != XENKBD_TYPE_KEY)
+        return 0;
+
+    if (ev.key.keycode == 42 || ev.key.keycode == 54) {
+        caps_lock = 0;
+        shift = ev.key.pressed;
+        return 0;
+    }
+    if (ev.key.keycode == 58) {
+        caps_lock ^= 1;
+        return 0;
+    }
+    if (ev.key.keycode == 29 || ev.key.keycode == 97) {
+        control = ev.key.pressed;
+        return 0;
+    }
+    if (ev.key.keycode == 56) {
+        alt = ev.key.pressed;
+        return 0;
+    }
+
+    if (!ev.key.pressed)
+        return 0;
+
+    if (ev.key.keycode < sizeof(linux2ascii) / sizeof(*linux2ascii)) {
+        char val;
+        if (shift || caps_lock)
+            val = linux2ascii_shifted[ev.key.keycode];
+        else
+            val = linux2ascii[ev.key.keycode];
+        if (control)
+            val &= ~0x60;
+        return val;
+    }
+
+    return 0;
+}
+
+static void kbd_thread(void *p)
+{
+    struct semaphore *sem = p;
+
+    kbd_dev = init_kbdfront(NULL, 1);
+    up(sem);
+}
+
+struct fbfront_dev *fb_open(void *fb, int width, int height, int depth)
+{
+    unsigned long *mfns;
+    int linesize = width * (depth / 8);
+    int memsize = linesize * height;
+    int numpages = (memsize + PAGE_SIZE - 1) / PAGE_SIZE;
+    DECLARE_MUTEX_LOCKED(sem);
+    int i;
+
+    create_thread("kbdfront", kbd_thread, &sem);
+
+    mfns = malloc(numpages * sizeof(*mfns));
+    for (i = 0; i < numpages; i++) {
+        memset(fb + i * PAGE_SIZE, 0, PAGE_SIZE);
+        mfns[i] = virtual_to_mfn(fb + i * PAGE_SIZE);
+    }
+    fb_dev = init_fbfront(NULL, mfns, width, height, depth, linesize, numpages);
+    free(mfns);
+
+    if (!fb_dev)
+        return NULL;
+
+    down(&sem);
+    if (!kbd_dev)
+        return NULL;
+
+    return fb_dev;
+}
+
+void kbd_close(void *foo)
+{
+    shutdown_kbdfront(kbd_dev);
+    kbd_dev = NULL;
+}
+
+void fb_close(void)
+{
+    create_thread("kbdfront close", kbd_close, NULL);
+    shutdown_fbfront(fb_dev);
+    fb_dev = NULL;
+}
+
+/*
+ * Misc
+ */
+
+int getrtsecs (void)
+{
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    return tv.tv_sec;
+}
+
+int currticks (void)
+{
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    return ((tv.tv_sec * 1000000ULL + tv.tv_usec) * TICKS_PER_SEC) / 1000000;
+}
+
+void __attribute__ ((noreturn)) grub_reboot (void)
+{
+    for ( ;; )
+    {
+        struct sched_shutdown sched_shutdown = { .reason = SHUTDOWN_reboot };
+        HYPERVISOR_sched_op(SCHEDOP_shutdown, &sched_shutdown);
+    }
+}
+
+#define SCRATCH_MEMSIZE (4 * 1024 * 1024)
+
+/* Note: not allocating it dynamically permits to make sure it lays below 4G
+ * for grub's 32bit pointers to work */
+char grub_scratch_mem[SCRATCH_MEMSIZE] __attribute__((aligned(PAGE_SIZE)));
+
+int main(int argc, char *argv[])
+{
+    if (argc > 1) {
+        strncpy(config_file, argv[1], sizeof(config_file) - 1);
+        config_file[sizeof(config_file) - 1] = 0;
+        if (!strncmp(config_file, "(nd)", 4))
+            preset_menu = "dhcp";
+    } else
+        preset_menu = "dhcp --with-configfile";
+
+    mbi.drives_addr = BOOTSEC_LOCATION + (60 * 1024);
+    mbi.drives_length = 0;
+
+    mbi.boot_loader_name = (unsigned long) "GNU GRUB " VERSION;
+    mbi.mem_lower = (start_info.nr_pages * PAGE_SIZE) / 1024;
+    mbi.mem_upper = 0;
+    saved_drive = boot_drive;
+    saved_partition = install_partition;
+
+    init_disk();
+
+    /* Try to make sure the client part got launched */
+    sleep(1);
+    cmain();
+    printk("cmain returned!\n");
+}
diff --git a/stubdom/grub/mini-os.h b/stubdom/grub/mini-os.h
new file mode 100644
index 0000000000..6c6844122e
--- /dev/null
+++ b/stubdom/grub/mini-os.h
@@ -0,0 +1,5 @@
+extern int blk_nb;
+extern struct blkfront_dev **blk_dev;
+extern struct netfront_dev *net_dev;
+extern struct kbdfront_dev *kbd_dev;
+extern struct fbfront_dev *fb_dev;
diff --git a/stubdom/grub/osdep.h b/stubdom/grub/osdep.h
new file mode 100644
index 0000000000..1d3e50adbf
--- /dev/null
+++ b/stubdom/grub/osdep.h
@@ -0,0 +1,30 @@
+#ifndef __OSDEP_H__
+#define __OSDEP_H__
+
+#include <byteswap.h>
+#define swap32(x) bswap_32(x)
+#define swap16(x) bswap_16(x)
+
+#include <machine/endian.h>
+#if BYTE_ORDER == BIG_ENDIAN
+#define htons(x) (x)
+#define ntohs(x) (x)
+#define htonl(x) (x)
+#define ntohl(x) (x)
+#else
+#define htons(x) swap16(x)
+#define ntohs(x) swap16(x)
+#define htonl(x) swap32(x)
+#define ntohl(x) swap32(x)
+#endif
+
+typedef unsigned long Address;
+
+/* ANSI prototyping macro */
+#ifdef  __STDC__
+#define P(x)	x
+#else
+#define P(x)	()
+#endif
+
+#endif