From d2781af07b4d42e5e02620c067c1da627baee567 Mon Sep 17 00:00:00 2001 From: "cwc22@centipede.cl.cam.ac.uk" Date: Mon, 4 Apr 2005 20:22:17 +0000 Subject: bitkeeper revision 1.1236.56.1 (4251a1f9OIyZY2I2LqBlxl0mi64FkA) Grant tables: substantially more robust. Block front and back drivers: support for using grant tables for interdomain communication. --- docs/misc/grant-tables.txt | 325 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 docs/misc/grant-tables.txt (limited to 'docs/misc/grant-tables.txt') diff --git a/docs/misc/grant-tables.txt b/docs/misc/grant-tables.txt new file mode 100644 index 0000000000..604be3bb08 --- /dev/null +++ b/docs/misc/grant-tables.txt @@ -0,0 +1,325 @@ +******************************************************************************** + A Rough Introduction to Using Grant Tables + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Christopher Clark, March, 2005. + +Grant tables are a mechanism for sharing and transferring frames between +domains, without requiring the participating domains to be privileged. + +The first mode of use allows domA to grant domB access to a specific frame, +whilst retaining ownership. The block front driver uses this to grant memory +access to the block back driver, so that it may read or write as requested. + + 1. domA creates a grant access reference, and transmits the ref id to domB. + 2. domB uses the reference to map the granted frame. + 3. domB performs the memory access. + 4. domB unmaps the granted frame. + 5. domA removes its grant. + + +The second mode allows domA to accept a transfer of ownership of a frame from +domB. The net front and back driver will use this for packet tx/rx. This +mechanism is still being implemented, though the xen<->guest interface design +is complete. + + 1. domA creates an accept transfer grant reference, and transmits it to domB. + 2. domB uses the ref to hand over a frame it owns. + 3. domA accepts the transfer + 4. domA clears the used reference. + + +******************************************************************************** + Data structures + ~~~~~~~~~~~~~~~ + + The following data structures are used by Xen and the guests to implement + grant tables: + + 1. Shared grant entries + 2. Active grant entries + 3. Map tracking + + These are not the users primary interface to grant tables, but are discussed + because an understanding of how they work may be useful. Each of these is a + finite resource. + + Shared grant entries + ~~~~~~~~~~~~~~~~~~~~ + + A set of pages are shared between Xen and a guest, holding the shared grant + entries. The guest writes into these entries to create grant references. The + index of the entry is transmitted to the remote domain: this is the + reference used to activate an entry. Xen will write into a shared entry to + indicate to a guest that its grant is in use. + sha->domid : remote domain being granted rights + sha->frame : machine frame being granted + sha->flags : allow access, allow transfer, remote is reading/writing, etc. + + Active grant entries + ~~~~~~~~~~~~~~~~~~~~ + + Xen maintains a set of private frames per domain, holding the active grant + entries for safety, and to reference count mappings. + act->domid : remote domain being granted rights + act->frame : machine frame being granted + act->pin : used to hold reference counts + + Map tracking + ~~~~~~~~~~~~ + + Every time a frame is mapped, a map track entry is stored in the metadata of + the mapping domain. The index of this entry is returned from the map call, + and is used to unmap the frame. Map track entries are also searched whenever a + page table entry containing a foreign frame number is overwritten: the first + matching map track entry is then removed, as if unmap had been invoked. + These are not used by the transfer mechanism. + map->domid : owner of the mapped frame + map->ref_and_flags : grant reference, ro/rw, mapped for host or device access + +******************************************************************************** + + Granting a foreign domain access to frames + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + domA [frame]--> domB + + + domA: #include + grant_ref_t gref[BATCH_SIZE]; + + for ( i = 0; i < BATCH_SIZE; i++ ) + gref[i] = gnttab_grant_foreign_access( domBid, mfn, (readonly ? 1 : 0) ); + + + .. gref is then somehow transmitted to domB for use. + + + Mapping foreign frames + ~~~~~~~~~~~~~~~~~~~~~~ + + domB: #include + unsigned long mmap_vstart; + gnttab_op_t aop[BATCH_SIZE]; + grant_ref_t mapped_handle[BATCH_SIZE]; + + if ( (mmap_vstart = allocate_empty_lowmem_region(BATCH_SIZE)) == 0 ) + BUG(); + + for ( i = 0; i < BATCH_SIZE; i++ ) + { + aop[i].u.map_grant_ref.host_virt_addr = + mmap_vstart + (i * PAGE_SIZE); + aop[i].u.map_grant_ref.dom = domAid; + aop[i].u.map_grant_ref.ref = gref[i]; + aop[i].u.map_grant_ref.flags = ( GNTMAP_host_map | GNTMAP_readonly ); + } + + if ( unlikely(HYPERVISOR_grant_table_op( + GNTTABOP_map_grant_ref, aop, BATCH_SIZE))) + BUG(); + + for ( i = 0; i < BATCH_SIZE; i++ ) + { + if ( unlikely(aop[i].u.map_grant_ref.dev_bus_addr == 0) ) + { + tidyup_all(aop, i); + goto panic; + } + + phys_to_machine_mapping[__pa(mmap_vstart + (i * PAGE_SIZE))>>PAGE_SHIFT] = + FOREIGN_FRAME(aop[i].u.map_grant_ref.dev_bus_addr); + + mapped_handle[i] = aop[i].u.map_grant_ref.handle; + } + + + + Unmapping foreign frames + ~~~~~~~~~~~~~~~~~~~~~~~~ + + domB: + for ( i = 0; i < BATCH_SIZE; i++ ) + { + aop[i].u.unmap_grant_ref.host_virt_addr = mmap_vstart + (i * PAGE_SIZE); + aop[i].u.unmap_grant_ref.dev_bus_addr = 0; + aop[i].u.unmap_grant_ref.handle = mapped_handle[i]; + } + if ( unlikely(HYPERVISOR_grant_table_op( + GNTTABOP_unmap_grant_ref, aop, BATCH_SIZE))) + BUG(); + + + Ending foreign access + ~~~~~~~~~~~~~~~~~~~~~ + + Note that this only prevents further mappings; it does _not_ revoke access. + Should _only_ be used when the remote domain has unmapped the frame. + gnttab_query_foreign_access( gref ) will indicate the state of any mapping. + + domA: + if ( gnttab_query_foreign_access( gref[i] ) == 0 ) + gnttab_end_foreign_access( gref[i], readonly ); + + TODO: readonly yet to be implemented. + + +******************************************************************************** + + Transferring ownership of a frame to another domain + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + [ XXX: Transfer mechanism is alpha-calibre code, untested, use at own risk XXX ] + [ XXX: show use of batch operations below, rather than single frame XXX ] + [ XXX: linux internal interface could/should be wrapped to be tidier XXX ] + + + Prepare to accept a frame from a foreign domain + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + domA: + if ( (p = alloc_page(GFP_HIGHUSER)) == NULL ) + { + printk("Cannot alloc a frame to surrender\n"); + break; + } + pfn = p - mem_map; + mfn = phys_to_machine_mapping[pfn]; + + if ( !PageHighMem(p) ) + { + v = phys_to_virt(pfn << PAGE_SHIFT); + scrub_pages(v, 1); + queue_l1_entry_update(get_ptep((unsigned long)v), 0); + } + + /* Ensure that ballooned highmem pages don't have cached mappings. */ + kmap_flush_unused(); + + /* Flush updates through and flush the TLB. */ + xen_tlb_flush(); + + phys_to_machine_mapping[pfn] = INVALID_P2M_ENTRY; + + if ( HYPERVISOR_dom_mem_op( + MEMOP_decrease_reservation, &mfn, 1, 0) != 1 ) + { + printk("MEMOP_decrease_reservation failed\n"); + /* er... ok. free the page then */ + __free_page(p); + break; + } + + accepting_pfn = pfn; + ref = gnttab_grant_foreign_transfer( (domid_t) args.arg[0], pfn ); + printk("Accepting dom %lu frame at ref (%d)\n", args.arg[0], ref); + + + Transfer a frame to a foreign domain + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + domB: + mmu_update_t update; + domid_t domid; + grant_ref_t gref; + unsigned long pfn, mfn, *v; + struct page *transfer_page = 0; + + /* alloc a page and grant access. + * alloc page returns a page struct. */ + if ( (transfer_page = alloc_page(GFP_HIGHUSER)) == NULL ) + return -ENOMEM; + + pfn = transfer_page - mem_map; + mfn = phys_to_machine_mapping[pfn]; + + /* need to remove all references to this page */ + if ( !PageHighMem(transfer_page) ) + { + v = phys_to_virt(pfn << PAGE_SHIFT); + scrub_pages(v, 1); + sprintf((char *)v, "This page (%lx) was transferred.\n", mfn); + queue_l1_entry_update(get_ptep((unsigned long)v), 0); + } +#ifdef CONFIG_XEN_SCRUB_PAGES + else + { + v = kmap(transfer_page); + scrub_pages(v, 1); + sprintf((char *)v, "This page (%lx) was transferred.\n", mfn); + kunmap(transfer_page); + } +#endif + /* Delete any cached kmappings */ + kmap_flush_unused(); + + /* Flush updates through and flush the TLB */ + xen_tlb_flush(); + + /* invalidate in P2M */ + phys_to_machine_mapping[pfn] = INVALID_P2M_ENTRY; + + domid = (domid_t)args.arg[0]; + gref = (grant_ref_t)args.arg[1]; + + update.ptr = MMU_EXTENDED_COMMAND; + update.ptr |= ((gref & 0x00FF) << 2); + update.ptr |= mfn << PAGE_SHIFT; + + update.val = MMUEXT_TRANSFER_PAGE; + update.val |= (domid << 16); + update.val |= (gref & 0xFF00); + + ret = HYPERVISOR_mmu_update(&update, 1, NULL); + + + Map a transferred frame + ~~~~~~~~~~~~~~~~~~~~~~~ + + TODO: + + + Clear the used transfer reference + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + TODO: + + +******************************************************************************** + + Using a private reserve of grant references + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Where it is known in advance how many grant references are required, and +failure to allocate them on demand would cause difficulty, a batch can be +allocated and held in a private reserve. + +To reserve a private batch: + + /* housekeeping data - treat as opaque: */ + grant_ref_t gref_head, gref_terminal; + + if ( 0 > gnttab_alloc_grant_references( number_to_reserve, + &gref_head, &gref_terminal )) + return -ENOSPC; + + +To release a batch back to the shared pool: + + gnttab_free_grant_references( number_reserved, gref_head ); + + +To claim a reserved reference: + + ref = gnttab_claim_grant_reference( &gref_head, gref_terminal ); + + +To release a claimed reference back to the reserve pool: + + gnttab_release_grant_reference( &gref_head, gref ); + + +To use a claimed reference to grant access, use these alternative functions +that take an additional parameter of the grant reference to use: + + gnttab_grant_foreign_access_ref + gnttab_grant_foreign_transfer_ref -- cgit v1.2.3