From c3da952a0261cb2fa381154a6f22b758a4bd843b Mon Sep 17 00:00:00 2001 From: Keir Fraser Date: Wed, 27 May 2009 11:15:08 +0100 Subject: Pass cpumasks by reference always. Rather than passing cpumasks by value in all cases (which is problematic for large NR_CPUS configurations), pass them 'by reference' (i.e. through a pointer to a const cpumask). On x86 this changes send_IPI_mask() to always only send IPIs to remote CPUs (meaning any caller needing to handle the current CPU as well has to do so on its own). Signed-off-by: Jan Beulich Signed-off-by: Keir Fraser --- xen/arch/ia64/linux-xen/smp.c | 15 +++-- xen/arch/ia64/linux-xen/sn/kernel/sn2_smp.c | 2 +- xen/arch/ia64/xen/mm.c | 2 +- xen/arch/ia64/xen/vhpt.c | 15 +++-- xen/arch/x86/acpi/cpufreq/cpufreq.c | 8 +-- xen/arch/x86/acpi/cpufreq/powernow.c | 2 +- xen/arch/x86/cpu/mcheck/mce.c | 8 +-- xen/arch/x86/cpu/mtrr/main.c | 2 +- xen/arch/x86/crash.c | 14 ----- xen/arch/x86/domain.c | 4 +- xen/arch/x86/genapic/x2apic.c | 7 ++- xen/arch/x86/hpet.c | 2 +- xen/arch/x86/hvm/hvm.c | 2 +- xen/arch/x86/hvm/vmx/vmcs.c | 4 +- xen/arch/x86/hvm/vmx/vmx.c | 2 +- xen/arch/x86/irq.c | 6 +- xen/arch/x86/machine_kexec.c | 6 +- xen/arch/x86/mm.c | 22 +++---- xen/arch/x86/mm/hap/hap.c | 6 +- xen/arch/x86/mm/shadow/common.c | 18 +++--- xen/arch/x86/mm/shadow/multi.c | 8 +-- xen/arch/x86/shutdown.c | 2 +- xen/arch/x86/smp.c | 90 ++++++++++++++++------------- xen/arch/x86/time.c | 2 +- xen/common/Makefile | 1 + xen/common/cpu.c | 26 +++++++++ xen/common/grant_table.c | 10 ++-- xen/common/keyhandler.c | 2 +- xen/common/page_alloc.c | 2 +- xen/include/asm-ia64/tlbflush.h | 2 +- xen/include/asm-x86/flushtlb.h | 10 ++-- xen/include/asm-x86/genapic.h | 8 +-- xen/include/asm-x86/ipi.h | 8 --- xen/include/asm-x86/smp.h | 2 + xen/include/xen/cpumask.h | 29 ++++++---- xen/include/xen/smp.h | 8 +-- xen/include/xen/softirq.h | 2 +- 37 files changed, 187 insertions(+), 172 deletions(-) create mode 100644 xen/common/cpu.c delete mode 100644 xen/include/asm-x86/ipi.h diff --git a/xen/arch/ia64/linux-xen/smp.c b/xen/arch/ia64/linux-xen/smp.c index 43786124f1..03d673f1c2 100644 --- a/xen/arch/ia64/linux-xen/smp.c +++ b/xen/arch/ia64/linux-xen/smp.c @@ -57,19 +57,18 @@ //#if CONFIG_SMP || IA64 #if CONFIG_SMP //Huh? This seems to be used on ia64 even if !CONFIG_SMP -void smp_send_event_check_mask(cpumask_t mask) +void smp_send_event_check_mask(const cpumask_t *mask) { int cpu; /* Not for me. */ - cpu_clear(smp_processor_id(), mask); - if (cpus_empty(mask)) + if (cpus_subset(*mask, *cpumask_of(smp_processor_id()))) return; //printf("smp_send_event_check_mask called\n"); for (cpu = 0; cpu < NR_CPUS; ++cpu) - if (cpu_isset(cpu, mask)) + if (cpu_isset(cpu, *mask) && cpu != smp_processor_id()) platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0); } #endif @@ -438,11 +437,11 @@ EXPORT_SYMBOL(smp_call_function); #ifdef XEN int -on_selected_cpus(cpumask_t selected, void (*func) (void *info), void *info, - int retry, int wait) +on_selected_cpus(const cpumask_t *selected, void (*func) (void *info), + void *info, int retry, int wait) { struct call_data_struct data; - unsigned int cpu, nr_cpus = cpus_weight(selected); + unsigned int cpu, nr_cpus = cpus_weight(*selected); ASSERT(local_irq_is_enabled()); @@ -460,7 +459,7 @@ on_selected_cpus(cpumask_t selected, void (*func) (void *info), void *info, call_data = &data; wmb(); - for_each_cpu_mask(cpu, selected) + for_each_cpu_mask(cpu, *selected) send_IPI_single(cpu, IPI_CALL_FUNC); while (atomic_read(wait ? &data.finished : &data.started) != nr_cpus) diff --git a/xen/arch/ia64/linux-xen/sn/kernel/sn2_smp.c b/xen/arch/ia64/linux-xen/sn/kernel/sn2_smp.c index 707c55ce17..b525c8b411 100644 --- a/xen/arch/ia64/linux-xen/sn/kernel/sn2_smp.c +++ b/xen/arch/ia64/linux-xen/sn/kernel/sn2_smp.c @@ -239,7 +239,7 @@ sn2_global_tlb_purge(unsigned long start, flush_data.start = start; flush_data.end = end; flush_data.nbits = nbits; - on_selected_cpus(selected_cpus, sn_flush_ptcga_cpu, + on_selected_cpus(&selected_cpus, sn_flush_ptcga_cpu, &flush_data, 1, 1); } spin_unlock(&sn2_ptcg_lock2); diff --git a/xen/arch/ia64/xen/mm.c b/xen/arch/ia64/xen/mm.c index 20071061f2..de18232388 100644 --- a/xen/arch/ia64/xen/mm.c +++ b/xen/arch/ia64/xen/mm.c @@ -3189,7 +3189,7 @@ int get_page_type(struct page_info *page, unsigned long type) if ( unlikely(!cpus_empty(mask)) ) { perfc_incr(need_flush_tlb_flush); - flush_tlb_mask(mask); + flush_tlb_mask(&mask); } /* We lose existing type, back pointer, and validity. */ diff --git a/xen/arch/ia64/xen/vhpt.c b/xen/arch/ia64/xen/vhpt.c index 525b76fc37..212f550960 100644 --- a/xen/arch/ia64/xen/vhpt.c +++ b/xen/arch/ia64/xen/vhpt.c @@ -548,22 +548,21 @@ void flush_tlb_for_log_dirty(struct domain *d) cpus_clear (d->domain_dirty_cpumask); } -void flush_tlb_mask(cpumask_t mask) +void flush_tlb_mask(const cpumask_t *mask) { int cpu; cpu = smp_processor_id(); - if (cpu_isset (cpu, mask)) { - cpu_clear(cpu, mask); + if (cpu_isset(cpu, *mask)) flush_tlb_vhpt_all (NULL); - } - if (cpus_empty(mask)) + if (cpus_subset(*mask, *cpumask_of(cpu))) return; - for_each_cpu_mask (cpu, mask) - smp_call_function_single - (cpu, (void (*)(void *))flush_tlb_vhpt_all, NULL, 1, 1); + for_each_cpu_mask (cpu, *mask) + if (cpu != smp_processor_id()) + smp_call_function_single + (cpu, (void (*)(void *))flush_tlb_vhpt_all, NULL, 1, 1); } #ifdef PERF_COUNTERS diff --git a/xen/arch/x86/acpi/cpufreq/cpufreq.c b/xen/arch/x86/acpi/cpufreq/cpufreq.c index 1631a30935..ccf7bffa75 100644 --- a/xen/arch/x86/acpi/cpufreq/cpufreq.c +++ b/xen/arch/x86/acpi/cpufreq/cpufreq.c @@ -186,7 +186,7 @@ static void drv_read(struct drv_cmd *cmd) if (likely(cpu_isset(smp_processor_id(), cmd->mask))) do_drv_read((void *)cmd); else - on_selected_cpus( cmd->mask, do_drv_read, (void *)cmd, 0, 1); + on_selected_cpus(&cmd->mask, do_drv_read, (void *)cmd, 0, 1); } static void drv_write(struct drv_cmd *cmd) @@ -195,7 +195,7 @@ static void drv_write(struct drv_cmd *cmd) cpu_isset(smp_processor_id(), cmd->mask)) do_drv_write((void *)cmd); else - on_selected_cpus( cmd->mask, do_drv_write, (void *)cmd, 0, 0); + on_selected_cpus(&cmd->mask, do_drv_write, (void *)cmd, 0, 0); } static u32 get_cur_val(cpumask_t mask) @@ -274,7 +274,6 @@ static unsigned int get_measured_perf(unsigned int cpu, unsigned int flag) struct cpufreq_policy *policy; struct perf_pair readin, cur, *saved; unsigned int perf_percent; - cpumask_t cpumask; unsigned int retval; if (!cpu_online(cpu)) @@ -303,8 +302,7 @@ static unsigned int get_measured_perf(unsigned int cpu, unsigned int flag) if (cpu == smp_processor_id()) { read_measured_perf_ctrs((void *)&readin); } else { - cpumask = cpumask_of_cpu(cpu); - on_selected_cpus(cpumask, read_measured_perf_ctrs, + on_selected_cpus(cpumask_of(cpu), read_measured_perf_ctrs, (void *)&readin, 0, 1); } diff --git a/xen/arch/x86/acpi/cpufreq/powernow.c b/xen/arch/x86/acpi/cpufreq/powernow.c index f69b425a0c..98b8ed2df6 100644 --- a/xen/arch/x86/acpi/cpufreq/powernow.c +++ b/xen/arch/x86/acpi/cpufreq/powernow.c @@ -121,7 +121,7 @@ static int powernow_cpufreq_target(struct cpufreq_policy *policy, cmd.val = next_perf_state; - on_selected_cpus( cmd.mask, transition_pstate, (void *) &cmd, 0, 0); + on_selected_cpus(&cmd.mask, transition_pstate, (void *) &cmd, 0, 0); perf->state = next_perf_state; policy->cur = freqs.new; diff --git a/xen/arch/x86/cpu/mcheck/mce.c b/xen/arch/x86/cpu/mcheck/mce.c index 49f3f84899..655c168d42 100644 --- a/xen/arch/x86/cpu/mcheck/mce.c +++ b/xen/arch/x86/cpu/mcheck/mce.c @@ -1205,8 +1205,8 @@ long do_mca(XEN_GUEST_HANDLE(xen_mc_t) u_xen_mc) add_taint(TAINT_ERROR_INJECT); - on_selected_cpus(cpumask_of_cpu(target), - x86_mc_msrinject, mc_msrinject, 1, 1); + on_selected_cpus(cpumask_of(target), x86_mc_msrinject, + mc_msrinject, 1, 1); break; @@ -1225,8 +1225,8 @@ long do_mca(XEN_GUEST_HANDLE(xen_mc_t) u_xen_mc) add_taint(TAINT_ERROR_INJECT); - on_selected_cpus(cpumask_of_cpu(target), x86_mc_mceinject, - mc_mceinject, 1, 1); + on_selected_cpus(cpumask_of(target), x86_mc_mceinject, + mc_mceinject, 1, 1); break; default: diff --git a/xen/arch/x86/cpu/mtrr/main.c b/xen/arch/x86/cpu/mtrr/main.c index 16133218a9..0f15eb8688 100644 --- a/xen/arch/x86/cpu/mtrr/main.c +++ b/xen/arch/x86/cpu/mtrr/main.c @@ -688,7 +688,7 @@ void mtrr_save_state(void) if (cpu == 0) mtrr_save_fixed_ranges(NULL); else - on_selected_cpus(cpumask_of_cpu(0), mtrr_save_fixed_ranges, NULL, 1, 1); + on_selected_cpus(cpumask_of(0), mtrr_save_fixed_ranges, NULL, 1, 1); put_cpu(); } diff --git a/xen/arch/x86/crash.c b/xen/arch/x86/crash.c index d38f969d02..009ae9ab7e 100644 --- a/xen/arch/x86/crash.c +++ b/xen/arch/x86/crash.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -51,19 +50,6 @@ static int crash_nmi_callback(struct cpu_user_regs *regs, int cpu) return 1; } -/* - * By using the NMI code instead of a vector we just sneak thru the - * word generator coming out with just what we want. AND it does - * not matter if clustered_apic_mode is set or not. - */ -static void smp_send_nmi_allbutself(void) -{ - cpumask_t allbutself = cpu_online_map; - cpu_clear(smp_processor_id(), allbutself); - if ( !cpus_empty(allbutself) ) - send_IPI_mask(allbutself, APIC_DM_NMI); -} - static void nmi_shootdown_cpus(void) { unsigned long msecs; diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c index 170e61d4b8..0004259eb6 100644 --- a/xen/arch/x86/domain.c +++ b/xen/arch/x86/domain.c @@ -1316,7 +1316,7 @@ void context_switch(struct vcpu *prev, struct vcpu *next) if ( unlikely(!cpu_isset(cpu, dirty_mask) && !cpus_empty(dirty_mask)) ) { /* Other cpus call __sync_lazy_execstate from flush ipi handler. */ - flush_tlb_mask(dirty_mask); + flush_tlb_mask(&dirty_mask); } if ( is_hvm_vcpu(prev) && !list_empty(&prev->arch.hvm_vcpu.tm_list) ) @@ -1410,7 +1410,7 @@ void sync_vcpu_execstate(struct vcpu *v) (void)__sync_lazy_execstate(); /* Other cpus call __sync_lazy_execstate from flush ipi handler. */ - flush_tlb_mask(v->vcpu_dirty_cpumask); + flush_tlb_mask(&v->vcpu_dirty_cpumask); } struct migrate_info { diff --git a/xen/arch/x86/genapic/x2apic.c b/xen/arch/x86/genapic/x2apic.c index 3dd97250c7..08ef983699 100644 --- a/xen/arch/x86/genapic/x2apic.c +++ b/xen/arch/x86/genapic/x2apic.c @@ -56,7 +56,7 @@ unsigned int cpu_mask_to_apicid_x2apic(cpumask_t cpumask) return cpu_physical_id(first_cpu(cpumask)); } -void send_IPI_mask_x2apic(cpumask_t cpumask, int vector) +void send_IPI_mask_x2apic(const cpumask_t *cpumask, int vector) { unsigned int cpu, cfg; unsigned long flags; @@ -76,8 +76,9 @@ void send_IPI_mask_x2apic(cpumask_t cpumask, int vector) local_irq_save(flags); cfg = APIC_DM_FIXED | 0 /* no shorthand */ | APIC_DEST_PHYSICAL | vector; - for_each_cpu_mask ( cpu, cpumask ) - apic_wrmsr(APIC_ICR, cfg, cpu_physical_id(cpu)); + for_each_cpu_mask ( cpu, *cpumask ) + if ( cpu != smp_processor_id() ) + apic_wrmsr(APIC_ICR, cfg, cpu_physical_id(cpu)); local_irq_restore(flags); } diff --git a/xen/arch/x86/hpet.c b/xen/arch/x86/hpet.c index e66eedee73..6877a80aea 100644 --- a/xen/arch/x86/hpet.c +++ b/xen/arch/x86/hpet.c @@ -617,7 +617,7 @@ void hpet_disable_legacy_broadcast(void) spin_unlock_irq(&legacy_hpet_event.lock); - smp_send_event_check_mask(cpu_online_map); + smp_send_event_check_mask(&cpu_online_map); } void hpet_broadcast_enter(void) diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c index 5156fbf11e..6c9fadca0d 100644 --- a/xen/arch/x86/hvm/hvm.c +++ b/xen/arch/x86/hvm/hvm.c @@ -2426,7 +2426,7 @@ static int hvmop_flush_tlb_all(void) paging_update_cr3(v); /* Flush all dirty TLBs. */ - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); /* Done. */ for_each_vcpu ( d, v ) diff --git a/xen/arch/x86/hvm/vmx/vmcs.c b/xen/arch/x86/hvm/vmx/vmcs.c index 5401e5638d..786a22796e 100644 --- a/xen/arch/x86/hvm/vmx/vmcs.c +++ b/xen/arch/x86/hvm/vmx/vmcs.c @@ -264,7 +264,7 @@ static void vmx_clear_vmcs(struct vcpu *v) int cpu = v->arch.hvm_vmx.active_cpu; if ( cpu != -1 ) - on_selected_cpus(cpumask_of_cpu(cpu), __vmx_clear_vmcs, v, 1, 1); + on_selected_cpus(cpumask_of(cpu), __vmx_clear_vmcs, v, 1, 1); } static void vmx_load_vmcs(struct vcpu *v) @@ -900,7 +900,7 @@ void vmx_do_resume(struct vcpu *v) { int cpu = v->arch.hvm_vmx.active_cpu; if ( cpu != -1 ) - on_selected_cpus(cpumask_of_cpu(cpu), wbinvd_ipi, NULL, 1, 1); + on_selected_cpus(cpumask_of(cpu), wbinvd_ipi, NULL, 1, 1); } vmx_clear_vmcs(v); diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c index 5fb1421ac6..c4eee9b44e 100644 --- a/xen/arch/x86/hvm/vmx/vmx.c +++ b/xen/arch/x86/hvm/vmx/vmx.c @@ -2164,7 +2164,7 @@ static void ept_handle_violation(unsigned long qualification, paddr_t gpa) { paging_mark_dirty(d, mfn_x(mfn)); p2m_change_type(d, gfn, p2m_ram_logdirty, p2m_ram_rw); - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); } return; } diff --git a/xen/arch/x86/irq.c b/xen/arch/x86/irq.c index 971117c010..65e69001cd 100644 --- a/xen/arch/x86/irq.c +++ b/xen/arch/x86/irq.c @@ -522,7 +522,7 @@ static void __pirq_guest_eoi(struct domain *d, int irq) } if ( !cpus_empty(cpu_eoi_map) ) - on_selected_cpus(cpu_eoi_map, set_eoi_ready, desc, 1, 0); + on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 1, 0); } int pirq_guest_eoi(struct domain *d, int irq) @@ -761,7 +761,7 @@ static irq_guest_action_t *__pirq_guest_unbind( { cpu_eoi_map = action->cpu_eoi_map; spin_unlock_irq(&desc->lock); - on_selected_cpus(cpu_eoi_map, set_eoi_ready, desc, 1, 0); + on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 1, 0); spin_lock_irq(&desc->lock); } break; @@ -799,7 +799,7 @@ static irq_guest_action_t *__pirq_guest_unbind( { BUG_ON(action->ack_type != ACKTYPE_EOI); spin_unlock_irq(&desc->lock); - on_selected_cpus(cpu_eoi_map, set_eoi_ready, desc, 1, 1); + on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 1, 1); spin_lock_irq(&desc->lock); } diff --git a/xen/arch/x86/machine_kexec.c b/xen/arch/x86/machine_kexec.c index 4d15e4b063..af7663e3fc 100644 --- a/xen/arch/x86/machine_kexec.c +++ b/xen/arch/x86/machine_kexec.c @@ -91,7 +91,6 @@ static void __machine_reboot_kexec(void *data) void machine_reboot_kexec(xen_kexec_image_t *image) { int reboot_cpu_id; - cpumask_t reboot_cpu; reboot_cpu_id = 0; @@ -100,9 +99,8 @@ void machine_reboot_kexec(xen_kexec_image_t *image) if ( reboot_cpu_id != smp_processor_id() ) { - cpus_clear(reboot_cpu); - cpu_set(reboot_cpu_id, reboot_cpu); - on_selected_cpus(reboot_cpu, __machine_reboot_kexec, image, 1, 0); + on_selected_cpus(cpumask_of(reboot_cpu_id), __machine_reboot_kexec, + image, 1, 0); for (;;) ; /* nothing */ } diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 1f4199d55f..10adea8808 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -517,7 +517,7 @@ static void invalidate_shadow_ldt(struct vcpu *v, int flush) /* Rid TLBs of stale mappings (guest mappings and shadow mappings). */ if ( flush ) - flush_tlb_mask(v->vcpu_dirty_cpumask); + flush_tlb_mask(&v->vcpu_dirty_cpumask); out: spin_unlock(&v->arch.shadow_ldt_lock); @@ -1250,7 +1250,7 @@ static void pae_flush_pgd( paging_update_cr3(v); cpus_or(m, m, v->vcpu_dirty_cpumask); } - flush_tlb_mask(m); + flush_tlb_mask(&m); } /* If below 4GB then the pgdir is not shadowed in low memory. */ @@ -1275,7 +1275,7 @@ static void pae_flush_pgd( spin_unlock(&cache->lock); } - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); } #else # define pae_flush_pgd(mfn, idx, nl3e) ((void)0) @@ -2290,7 +2290,7 @@ static int __get_page_type(struct page_info *page, unsigned long type, ((nx & PGT_type_mask) == PGT_writable_page)) ) { perfc_incr(need_flush_tlb_flush); - flush_tlb_mask(mask); + flush_tlb_mask(&mask); } /* We lose existing type and validity. */ @@ -2489,7 +2489,7 @@ static void process_deferred_ops(void) if ( deferred_ops & (DOP_FLUSH_ALL_TLBS|DOP_FLUSH_TLB) ) { if ( deferred_ops & DOP_FLUSH_ALL_TLBS ) - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); else flush_tlb_local(); } @@ -2824,9 +2824,9 @@ int do_mmuext_op( } pmask = vcpumask_to_pcpumask(d, vmask); if ( op.cmd == MMUEXT_TLB_FLUSH_MULTI ) - flush_tlb_mask(pmask); + flush_tlb_mask(&pmask); else - flush_tlb_one_mask(pmask, op.arg1.linear_addr); + flush_tlb_one_mask(&pmask, op.arg1.linear_addr); break; } @@ -2835,7 +2835,7 @@ int do_mmuext_op( break; case MMUEXT_INVLPG_ALL: - flush_tlb_one_mask(d->domain_dirty_cpumask, op.arg1.linear_addr); + flush_tlb_one_mask(&d->domain_dirty_cpumask, op.arg1.linear_addr); break; case MMUEXT_FLUSH_CACHE: @@ -3688,7 +3688,7 @@ int do_update_va_mapping(unsigned long va, u64 val64, pmask = vcpumask_to_pcpumask(d, vmask); if ( cpu_isset(smp_processor_id(), pmask) ) this_cpu(percpu_mm_info).deferred_ops &= ~DOP_FLUSH_TLB; - flush_tlb_mask(pmask); + flush_tlb_mask(&pmask); break; } break; @@ -3706,7 +3706,7 @@ int do_update_va_mapping(unsigned long va, u64 val64, flush_tlb_one_local(va); break; case UVMF_ALL: - flush_tlb_one_mask(d->domain_dirty_cpumask, va); + flush_tlb_one_mask(&d->domain_dirty_cpumask, va); break; default: if ( unlikely(!is_pv_32on64_domain(d) ? @@ -3716,7 +3716,7 @@ int do_update_va_mapping(unsigned long va, u64 val64, pmask = vcpumask_to_pcpumask(d, vmask); if ( this_cpu(percpu_mm_info).deferred_ops & DOP_FLUSH_TLB ) cpu_clear(smp_processor_id(), pmask); - flush_tlb_one_mask(pmask, va); + flush_tlb_one_mask(&pmask, va); break; } break; diff --git a/xen/arch/x86/mm/hap/hap.c b/xen/arch/x86/mm/hap/hap.c index b7615f0d69..e9d38103fc 100644 --- a/xen/arch/x86/mm/hap/hap.c +++ b/xen/arch/x86/mm/hap/hap.c @@ -64,7 +64,7 @@ int hap_enable_log_dirty(struct domain *d) /* set l1e entries of P2M table to be read-only. */ p2m_change_entry_type_global(d, p2m_ram_rw, p2m_ram_logdirty); - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); return 0; } @@ -83,7 +83,7 @@ void hap_clean_dirty_bitmap(struct domain *d) { /* set l1e entries of P2M table to be read-only. */ p2m_change_entry_type_global(d, p2m_ram_rw, p2m_ram_logdirty); - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); } /************************************************/ @@ -643,7 +643,7 @@ hap_write_p2m_entry(struct vcpu *v, unsigned long gfn, l1_pgentry_t *p, safe_write_pte(p, new); if ( (old_flags & _PAGE_PRESENT) && (level == 1 || (level == 2 && (old_flags & _PAGE_PSE))) ) - flush_tlb_mask(v->domain->domain_dirty_cpumask); + flush_tlb_mask(&v->domain->domain_dirty_cpumask); #if CONFIG_PAGING_LEVELS == 3 /* install P2M in monitor table for PAE Xen */ diff --git a/xen/arch/x86/mm/shadow/common.c b/xen/arch/x86/mm/shadow/common.c index ca4cb13ac4..875566d7c9 100644 --- a/xen/arch/x86/mm/shadow/common.c +++ b/xen/arch/x86/mm/shadow/common.c @@ -695,7 +695,7 @@ static int oos_remove_write_access(struct vcpu *v, mfn_t gmfn, } if ( ftlb ) - flush_tlb_mask(v->domain->domain_dirty_cpumask); + flush_tlb_mask(&v->domain->domain_dirty_cpumask); return 0; } @@ -1145,7 +1145,7 @@ sh_validate_guest_pt_write(struct vcpu *v, mfn_t gmfn, rc = sh_validate_guest_entry(v, gmfn, entry, size); if ( rc & SHADOW_SET_FLUSH ) /* Need to flush TLBs to pick up shadow PT changes */ - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); if ( rc & SHADOW_SET_ERROR ) { /* This page is probably not a pagetable any more: tear it out of the @@ -1393,7 +1393,7 @@ static void _shadow_prealloc( /* See if that freed up enough space */ if ( space_is_available(d, order, count) ) { - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); return; } } @@ -1447,7 +1447,7 @@ static void shadow_blow_tables(struct domain *d) pagetable_get_mfn(v->arch.shadow_table[i])); /* Make sure everyone sees the unshadowings */ - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); } void shadow_blow_tables_per_domain(struct domain *d) @@ -1554,7 +1554,7 @@ mfn_t shadow_alloc(struct domain *d, if ( unlikely(!cpus_empty(mask)) ) { perfc_incr(shadow_alloc_tlbflush); - flush_tlb_mask(mask); + flush_tlb_mask(&mask); } /* Now safe to clear the page for reuse */ p = sh_map_domain_page(page_to_mfn(sp+i)); @@ -2803,7 +2803,7 @@ void sh_remove_shadows(struct vcpu *v, mfn_t gmfn, int fast, int all) /* Need to flush TLBs now, so that linear maps are safe next time we * take a fault. */ - flush_tlb_mask(v->domain->domain_dirty_cpumask); + flush_tlb_mask(&v->domain->domain_dirty_cpumask); if ( do_locking ) shadow_unlock(v->domain); } @@ -3435,7 +3435,7 @@ shadow_write_p2m_entry(struct vcpu *v, unsigned long gfn, { sh_remove_all_shadows_and_parents(v, mfn); if ( sh_remove_all_mappings(v, mfn) ) - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); } } @@ -3474,7 +3474,7 @@ shadow_write_p2m_entry(struct vcpu *v, unsigned long gfn, } omfn = _mfn(mfn_x(omfn) + 1); } - flush_tlb_mask(flushmask); + flush_tlb_mask(&flushmask); if ( npte ) unmap_domain_page(npte); @@ -3752,7 +3752,7 @@ int shadow_track_dirty_vram(struct domain *d, } } if ( flush_tlb ) - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); goto out; out_sl1ma: diff --git a/xen/arch/x86/mm/shadow/multi.c b/xen/arch/x86/mm/shadow/multi.c index 072dda0b6b..69752f879d 100644 --- a/xen/arch/x86/mm/shadow/multi.c +++ b/xen/arch/x86/mm/shadow/multi.c @@ -3146,7 +3146,7 @@ static int sh_page_fault(struct vcpu *v, */ perfc_incr(shadow_rm_write_flush_tlb); atomic_inc(&d->arch.paging.shadow.gtable_dirty_version); - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); } #if (SHADOW_OPTIMIZATIONS & SHOPT_OUT_OF_SYNC) @@ -4135,7 +4135,7 @@ sh_update_cr3(struct vcpu *v, int do_locking) * (old) shadow linear maps in the writeable mapping heuristics. */ #if GUEST_PAGING_LEVELS == 2 if ( sh_remove_write_access(v, gmfn, 2, 0) != 0 ) - flush_tlb_mask(v->domain->domain_dirty_cpumask); + flush_tlb_mask(&v->domain->domain_dirty_cpumask); sh_set_toplevel_shadow(v, 0, gmfn, SH_type_l2_shadow); #elif GUEST_PAGING_LEVELS == 3 /* PAE guests have four shadow_table entries, based on the @@ -4158,7 +4158,7 @@ sh_update_cr3(struct vcpu *v, int do_locking) } } if ( flush ) - flush_tlb_mask(v->domain->domain_dirty_cpumask); + flush_tlb_mask(&v->domain->domain_dirty_cpumask); /* Now install the new shadows. */ for ( i = 0; i < 4; i++ ) { @@ -4179,7 +4179,7 @@ sh_update_cr3(struct vcpu *v, int do_locking) } #elif GUEST_PAGING_LEVELS == 4 if ( sh_remove_write_access(v, gmfn, 4, 0) != 0 ) - flush_tlb_mask(v->domain->domain_dirty_cpumask); + flush_tlb_mask(&v->domain->domain_dirty_cpumask); sh_set_toplevel_shadow(v, 0, gmfn, SH_type_l4_shadow); #else #error This should never happen diff --git a/xen/arch/x86/shutdown.c b/xen/arch/x86/shutdown.c index 18340e5b25..1d8218fa27 100644 --- a/xen/arch/x86/shutdown.c +++ b/xen/arch/x86/shutdown.c @@ -310,7 +310,7 @@ void machine_restart(unsigned int delay_millisecs) if ( get_apic_id() != boot_cpu_physical_apicid ) { /* Send IPI to the boot CPU (logical cpu 0). */ - on_selected_cpus(cpumask_of_cpu(0), __machine_restart, + on_selected_cpus(cpumask_of(0), __machine_restart, &delay_millisecs, 1, 0); for ( ; ; ) halt(); diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c index dba5cb58b6..0bf82da494 100644 --- a/xen/arch/x86/smp.c +++ b/xen/arch/x86/smp.c @@ -19,10 +19,15 @@ #include #include #include -#include #include #include +/* + * send_IPI_mask(cpumask, vector): sends @vector IPI to CPUs in @cpumask, + * excluding the local CPU. @cpumask may be empty. + */ +#define send_IPI_mask (genapic->send_IPI_mask) + /* * Some notes on x86 processor bugs affecting SMP operation: * @@ -84,14 +89,15 @@ void apic_wait_icr_idle(void) cpu_relax(); } -void send_IPI_mask_flat(cpumask_t cpumask, int vector) +void send_IPI_mask_flat(const cpumask_t *cpumask, int vector) { - unsigned long mask = cpus_addr(cpumask)[0]; + unsigned long mask = cpus_addr(*cpumask)[0]; unsigned long cfg; unsigned long flags; - /* An IPI with no target generates a send accept error from P5/P6 APICs. */ - WARN_ON(mask == 0); + mask &= ~(1UL << smp_processor_id()); + if ( mask == 0 ) + return; local_irq_save(flags); @@ -119,15 +125,18 @@ void send_IPI_mask_flat(cpumask_t cpumask, int vector) local_irq_restore(flags); } -void send_IPI_mask_phys(cpumask_t mask, int vector) +void send_IPI_mask_phys(const cpumask_t *mask, int vector) { unsigned long cfg, flags; unsigned int query_cpu; local_irq_save(flags); - for_each_cpu_mask ( query_cpu, mask ) + for_each_cpu_mask ( query_cpu, *mask ) { + if ( query_cpu == smp_processor_id() ) + continue; + /* * Wait for idle. */ @@ -170,20 +179,17 @@ fastcall void smp_invalidate_interrupt(void) irq_exit(); } -void flush_area_mask(cpumask_t mask, const void *va, unsigned int flags) +void flush_area_mask(const cpumask_t *mask, const void *va, unsigned int flags) { ASSERT(local_irq_is_enabled()); - if ( cpu_isset(smp_processor_id(), mask) ) - { + if ( cpu_isset(smp_processor_id(), *mask) ) flush_area_local(va, flags); - cpu_clear(smp_processor_id(), mask); - } - if ( !cpus_empty(mask) ) + if ( !cpus_subset(*mask, *cpumask_of(smp_processor_id())) ) { spin_lock(&flush_lock); - flush_cpumask = mask; + cpus_andnot(flush_cpumask, *mask, *cpumask_of(smp_processor_id())); flush_va = va; flush_flags = flags; send_IPI_mask(mask, INVALIDATE_TLB_VECTOR); @@ -201,18 +207,16 @@ void new_tlbflush_clock_period(void) /* Flush everyone else. We definitely flushed just before entry. */ allbutself = cpu_online_map; cpu_clear(smp_processor_id(), allbutself); - flush_mask(allbutself, FLUSH_TLB); + flush_mask(&allbutself, FLUSH_TLB); /* No need for atomicity: we are the only possible updater. */ ASSERT(tlbflush_clock == 0); tlbflush_clock++; } -void smp_send_event_check_mask(cpumask_t mask) +void smp_send_event_check_mask(const cpumask_t *mask) { - cpu_clear(smp_processor_id(), mask); - if ( !cpus_empty(mask) ) - send_IPI_mask(mask, EVENT_CHECK_VECTOR); + send_IPI_mask(mask, EVENT_CHECK_VECTOR); } /* @@ -225,11 +229,12 @@ struct call_data_struct { int wait; atomic_t started; atomic_t finished; - cpumask_t selected; + const cpumask_t *selected; }; static DEFINE_SPINLOCK(call_lock); static struct call_data_struct *call_data; +static void __smp_call_function_interrupt(void); int smp_call_function( void (*func) (void *info), @@ -239,34 +244,21 @@ int smp_call_function( { cpumask_t allbutself = cpu_online_map; cpu_clear(smp_processor_id(), allbutself); - return on_selected_cpus(allbutself, func, info, retry, wait); + return on_selected_cpus(&allbutself, func, info, retry, wait); } int on_selected_cpus( - cpumask_t selected, + const cpumask_t *selected, void (*func) (void *info), void *info, int retry, int wait) { struct call_data_struct data; - unsigned int nr_cpus = cpus_weight(selected); + unsigned int nr_cpus = cpus_weight(*selected); ASSERT(local_irq_is_enabled()); - /* Legacy UP system with no APIC to deliver IPIs? */ - if ( unlikely(!cpu_has_apic) ) - { - ASSERT(num_online_cpus() == 1); - if ( cpu_isset(0, selected) ) - { - local_irq_disable(); - func(info); - local_irq_enable(); - } - return 0; - } - if ( nr_cpus == 0 ) return 0; @@ -283,6 +275,13 @@ int on_selected_cpus( send_IPI_mask(selected, CALL_FUNCTION_VECTOR); + if ( cpu_isset(smp_processor_id(), *call_data->selected) ) + { + local_irq_disable(); + __smp_call_function_interrupt(); + local_irq_enable(); + } + while ( atomic_read(wait ? &data.finished : &data.started) != nr_cpus ) cpu_relax(); @@ -335,21 +334,23 @@ void smp_send_stop(void) local_irq_enable(); } +void smp_send_nmi_allbutself(void) +{ + send_IPI_mask(&cpu_online_map, APIC_DM_NMI); +} + fastcall void smp_event_check_interrupt(struct cpu_user_regs *regs) { ack_APIC_irq(); perfc_incr(ipis); } -fastcall void smp_call_function_interrupt(struct cpu_user_regs *regs) +static void __smp_call_function_interrupt(void) { void (*func)(void *info) = call_data->func; void *info = call_data->info; - ack_APIC_irq(); - perfc_incr(ipis); - - if ( !cpu_isset(smp_processor_id(), call_data->selected) ) + if ( !cpu_isset(smp_processor_id(), *call_data->selected) ) return; irq_enter(); @@ -369,3 +370,10 @@ fastcall void smp_call_function_interrupt(struct cpu_user_regs *regs) irq_exit(); } + +fastcall void smp_call_function_interrupt(struct cpu_user_regs *regs) +{ + ack_APIC_irq(); + perfc_incr(ipis); + __smp_call_function_interrupt(); +} diff --git a/xen/arch/x86/time.c b/xen/arch/x86/time.c index 9e9ff9b456..495ee844dc 100644 --- a/xen/arch/x86/time.c +++ b/xen/arch/x86/time.c @@ -1189,7 +1189,7 @@ static void time_calibration(void *unused) }; /* @wait=1 because we must wait for all cpus before freeing @r. */ - on_selected_cpus(r.cpu_calibration_map, + on_selected_cpus(&r.cpu_calibration_map, opt_consistent_tscs ? time_calibration_tsc_rendezvous : time_calibration_std_rendezvous, diff --git a/xen/common/Makefile b/xen/common/Makefile index 08b9e2b00e..27b13f3e34 100644 --- a/xen/common/Makefile +++ b/xen/common/Makefile @@ -1,4 +1,5 @@ obj-y += bitmap.o +obj-y += cpu.o obj-y += domctl.o obj-y += domain.o obj-y += event_channel.o diff --git a/xen/common/cpu.c b/xen/common/cpu.c new file mode 100644 index 0000000000..96cba72e9e --- /dev/null +++ b/xen/common/cpu.c @@ -0,0 +1,26 @@ +#include +#include + +/* + * cpu_bit_bitmap[] is a special, "compressed" data structure that + * represents all NR_CPUS bits binary values of 1< 32 + MASK_DECLARE_8(32), MASK_DECLARE_8(40), + MASK_DECLARE_8(48), MASK_DECLARE_8(56), +#endif +}; diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c index 530635469d..7bbc05d896 100644 --- a/xen/common/grant_table.c +++ b/xen/common/grant_table.c @@ -715,7 +715,7 @@ gnttab_unmap_grant_ref( goto fault; } - flush_tlb_mask(current->domain->domain_dirty_cpumask); + flush_tlb_mask(¤t->domain->domain_dirty_cpumask); for ( i = 0; i < partial_done; i++ ) __gnttab_unmap_common_complete(&(common[i])); @@ -727,7 +727,7 @@ gnttab_unmap_grant_ref( return 0; fault: - flush_tlb_mask(current->domain->domain_dirty_cpumask); + flush_tlb_mask(¤t->domain->domain_dirty_cpumask); for ( i = 0; i < partial_done; i++ ) __gnttab_unmap_common_complete(&(common[i])); @@ -774,7 +774,7 @@ gnttab_unmap_and_replace( goto fault; } - flush_tlb_mask(current->domain->domain_dirty_cpumask); + flush_tlb_mask(¤t->domain->domain_dirty_cpumask); for ( i = 0; i < partial_done; i++ ) __gnttab_unmap_common_complete(&(common[i])); @@ -786,7 +786,7 @@ gnttab_unmap_and_replace( return 0; fault: - flush_tlb_mask(current->domain->domain_dirty_cpumask); + flush_tlb_mask(¤t->domain->domain_dirty_cpumask); for ( i = 0; i < partial_done; i++ ) __gnttab_unmap_common_complete(&(common[i])); @@ -1123,7 +1123,7 @@ gnttab_transfer( #ifndef __ia64__ /* IA64 implicitly replaces the old page in steal_page(). */ guest_physmap_remove_page(d, gop.mfn, mfn, 0); #endif - flush_tlb_mask(d->domain_dirty_cpumask); + flush_tlb_mask(&d->domain_dirty_cpumask); /* Find the target domain. */ if ( unlikely((e = rcu_lock_domain_by_id(gop.domid)) == NULL) ) diff --git a/xen/common/keyhandler.c b/xen/common/keyhandler.c index c481df0f3f..5407456290 100644 --- a/xen/common/keyhandler.c +++ b/xen/common/keyhandler.c @@ -119,7 +119,7 @@ static void dump_registers(unsigned char key, struct cpu_user_regs *regs) if ( cpu == smp_processor_id() ) continue; printk("\n*** Dumping CPU%d host state: ***\n", cpu); - on_selected_cpus(cpumask_of_cpu(cpu), __dump_execstate, NULL, 1, 1); + on_selected_cpus(cpumask_of(cpu), __dump_execstate, NULL, 1, 1); } printk("\n"); diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c index bb143aedd6..bd514cfeed 100644 --- a/xen/common/page_alloc.c +++ b/xen/common/page_alloc.c @@ -431,7 +431,7 @@ static struct page_info *alloc_heap_pages( if ( unlikely(!cpus_empty(mask)) ) { perfc_incr(need_flush_tlb_flush); - flush_tlb_mask(mask); + flush_tlb_mask(&mask); } return pg; diff --git a/xen/include/asm-ia64/tlbflush.h b/xen/include/asm-ia64/tlbflush.h index bb9f2e3772..705b8717e1 100644 --- a/xen/include/asm-ia64/tlbflush.h +++ b/xen/include/asm-ia64/tlbflush.h @@ -39,7 +39,7 @@ void domain_flush_tlb_vhpt(struct domain *d); void flush_tlb_for_log_dirty(struct domain *d); /* Flush v-tlb on cpus set in mask for current domain. */ -void flush_tlb_mask(cpumask_t mask); +void flush_tlb_mask(const cpumask_t *mask); /* Flush local machine TLB. */ void local_flush_tlb_all (void); diff --git a/xen/include/asm-x86/flushtlb.h b/xen/include/asm-x86/flushtlb.h index 09a8f8bd9a..9442009ee8 100644 --- a/xen/include/asm-x86/flushtlb.h +++ b/xen/include/asm-x86/flushtlb.h @@ -90,12 +90,12 @@ void flush_area_local(const void *va, unsigned int flags); #define flush_local(flags) flush_area_local(NULL, flags) /* Flush specified CPUs' TLBs/caches */ -void flush_area_mask(cpumask_t, const void *va, unsigned int flags); +void flush_area_mask(const cpumask_t *, const void *va, unsigned int flags); #define flush_mask(mask, flags) flush_area_mask(mask, NULL, flags) /* Flush all CPUs' TLBs/caches */ -#define flush_area_all(va, flags) flush_area_mask(cpu_online_map, va, flags) -#define flush_all(flags) flush_mask(cpu_online_map, flags) +#define flush_area_all(va, flags) flush_area_mask(&cpu_online_map, va, flags) +#define flush_all(flags) flush_mask(&cpu_online_map, flags) /* Flush local TLBs */ #define flush_tlb_local() \ @@ -111,8 +111,8 @@ void flush_area_mask(cpumask_t, const void *va, unsigned int flags); /* Flush all CPUs' TLBs */ #define flush_tlb_all() \ - flush_tlb_mask(cpu_online_map) + flush_tlb_mask(&cpu_online_map) #define flush_tlb_one_all(v) \ - flush_tlb_one_mask(cpu_online_map, v) + flush_tlb_one_mask(&cpu_online_map, v) #endif /* __FLUSHTLB_H__ */ diff --git a/xen/include/asm-x86/genapic.h b/xen/include/asm-x86/genapic.h index 0653335be9..f99af3aa07 100644 --- a/xen/include/asm-x86/genapic.h +++ b/xen/include/asm-x86/genapic.h @@ -35,7 +35,7 @@ struct genapic { void (*clustered_apic_check)(void); cpumask_t (*target_cpus)(void); unsigned int (*cpu_mask_to_apicid)(cpumask_t cpumask); - void (*send_IPI_mask)(cpumask_t mask, int vector); + void (*send_IPI_mask)(const cpumask_t *mask, int vector); }; #define APICFUNC(x) .x = x @@ -52,7 +52,7 @@ void init_apic_ldr_flat(void); void clustered_apic_check_flat(void); cpumask_t target_cpus_flat(void); unsigned int cpu_mask_to_apicid_flat(cpumask_t cpumask); -void send_IPI_mask_flat(cpumask_t mask, int vector); +void send_IPI_mask_flat(const cpumask_t *mask, int vector); #define GENAPIC_FLAT \ .int_delivery_mode = dest_LowestPrio, \ .int_dest_mode = 1 /* logical delivery */, \ @@ -66,7 +66,7 @@ void init_apic_ldr_x2apic(void); void clustered_apic_check_x2apic(void); cpumask_t target_cpus_x2apic(void); unsigned int cpu_mask_to_apicid_x2apic(cpumask_t cpumask); -void send_IPI_mask_x2apic(cpumask_t mask, int vector); +void send_IPI_mask_x2apic(const cpumask_t *mask, int vector); #define GENAPIC_X2APIC \ .int_delivery_mode = dest_Fixed, \ .int_dest_mode = 0 /* physical delivery */, \ @@ -80,7 +80,7 @@ void init_apic_ldr_phys(void); void clustered_apic_check_phys(void); cpumask_t target_cpus_phys(void); unsigned int cpu_mask_to_apicid_phys(cpumask_t cpumask); -void send_IPI_mask_phys(cpumask_t mask, int vector); +void send_IPI_mask_phys(const cpumask_t *mask, int vector); #define GENAPIC_PHYS \ .int_delivery_mode = dest_Fixed, \ .int_dest_mode = 0 /* physical delivery */, \ diff --git a/xen/include/asm-x86/ipi.h b/xen/include/asm-x86/ipi.h deleted file mode 100644 index da8c94031f..0000000000 --- a/xen/include/asm-x86/ipi.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef __ASM_IPI_H -#define __ASM_IPI_H - -#include - -#define send_IPI_mask (genapic->send_IPI_mask) - -#endif /* __ASM_IPI_H */ diff --git a/xen/include/asm-x86/smp.h b/xen/include/asm-x86/smp.h index c62c53fce6..6d566d1a05 100644 --- a/xen/include/asm-x86/smp.h +++ b/xen/include/asm-x86/smp.h @@ -35,6 +35,8 @@ extern int pic_mode; extern cpumask_t cpu_sibling_map[]; extern cpumask_t cpu_core_map[]; +void smp_send_nmi_allbutself(void); + extern void (*mtrr_hook) (void); #ifdef CONFIG_X86_64 diff --git a/xen/include/xen/cpumask.h b/xen/include/xen/cpumask.h index 23dfd09a14..43f538eba4 100644 --- a/xen/include/xen/cpumask.h +++ b/xen/include/xen/cpumask.h @@ -80,7 +80,6 @@ #include typedef struct { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t; -extern cpumask_t _unused_cpumask_arg_; #define cpu_set(cpu, dst) __cpu_set((cpu), &(dst)) static inline void __cpu_set(int cpu, volatile cpumask_t *dstp) @@ -244,17 +243,23 @@ static inline int __cycle_cpu(int n, const cpumask_t *srcp, int nbits) return nxt; } -#define cpumask_of_cpu(cpu) \ -({ \ - typeof(_unused_cpumask_arg_) m; \ - if (sizeof(m) == sizeof(unsigned long)) { \ - m.bits[0] = 1UL<<(cpu); \ - } else { \ - cpus_clear(m); \ - cpu_set((cpu), m); \ - } \ - m; \ -}) +/* + * Special-case data structure for "single bit set only" constant CPU masks. + * + * We pre-generate all the 64 (or 32) possible bit positions, with enough + * padding to the left and the right, and return the constant pointer + * appropriately offset. + */ +extern const unsigned long + cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)]; + +static inline const cpumask_t *cpumask_of(unsigned int cpu) +{ + const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG]; + return (const cpumask_t *)(p - cpu / BITS_PER_LONG); +} + +#define cpumask_of_cpu(cpu) (*cpumask_of(cpu)) #define CPU_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(NR_CPUS) diff --git a/xen/include/xen/smp.h b/xen/include/xen/smp.h index 6ab913a430..93b0713205 100644 --- a/xen/include/xen/smp.h +++ b/xen/include/xen/smp.h @@ -9,9 +9,9 @@ */ extern void smp_send_stop(void); -extern void smp_send_event_check_mask(cpumask_t mask); +extern void smp_send_event_check_mask(const cpumask_t *mask); #define smp_send_event_check_cpu(cpu) \ - smp_send_event_check_mask(cpumask_of_cpu(cpu)) + smp_send_event_check_mask(cpumask_of(cpu)) /* * Prepare machine for booting other CPUs. @@ -41,7 +41,7 @@ extern int smp_call_function( * Call a function on a selection of processors */ extern int on_selected_cpus( - cpumask_t selected, + const cpumask_t *selected, void (*func) (void *info), void *info, int retry, @@ -62,7 +62,7 @@ static inline int on_each_cpu( int retry, int wait) { - return on_selected_cpus(cpu_online_map, func, info, retry, wait); + return on_selected_cpus(&cpu_online_map, func, info, retry, wait); } #define smp_processor_id() raw_smp_processor_id() diff --git a/xen/include/xen/softirq.h b/xen/include/xen/softirq.h index 0b59f63b22..efd0a54443 100644 --- a/xen/include/xen/softirq.h +++ b/xen/include/xen/softirq.h @@ -39,7 +39,7 @@ static inline void cpumask_raise_softirq(cpumask_t mask, unsigned int nr) cpu_clear(cpu, mask); } - smp_send_event_check_mask(mask); + smp_send_event_check_mask(&mask); } static inline void cpu_raise_softirq(unsigned int cpu, unsigned int nr) -- cgit v1.2.3