/*
 * tg3.c: Broadcom Tigon3 ethernet driver.
 *
 * Copyright (C) 2001, 2002 David S. Miller (davem@redhat.com)
 * Copyright (C) 2001, 2002 Jeff Garzik (jgarzik@pobox.com)
 */

#include <linux/config.h>

#include <linux/module.h>

#include <linux/lib.h>
#include <linux/types.h>
#include <linux/tqueue.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/init.h>
#include <linux/ioport.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/skbuff.h>
#include <linux/ethtool.h>
#include <linux/mii.h>
#include <linux/if_vlan.h>

#include <asm/system.h>
#include <asm/io.h>
#include <asm/byteorder.h>
#include <asm/uaccess.h>

#ifndef PCI_DMA_BUS_IS_PHYS
#define PCI_DMA_BUS_IS_PHYS 1
#endif

#if defined(CONFIG_VLAN_8021Q) || defined(CONFIG_VLAN_8021Q_MODULE)
#define TG3_VLAN_TAG_USED 1
#else
#define TG3_VLAN_TAG_USED 0
#endif

#ifdef NETIF_F_TSO
/* XXX some bug in tso firmware hangs tx cpu, disabled until fixed */
#define TG3_DO_TSO	0
#else
#define TG3_DO_TSO	0
#endif

#include "tg3.h"

#define DRV_MODULE_NAME		"tg3"
#define PFX DRV_MODULE_NAME	": "
#define DRV_MODULE_VERSION	"1.4c"
#define DRV_MODULE_RELDATE	"Feb 18, 2003"

#define TG3_DEF_MAC_MODE	0
#define TG3_DEF_RX_MODE		0
#define TG3_DEF_TX_MODE		0
#define TG3_DEF_MSG_ENABLE	  \
	(NETIF_MSG_DRV		| \
	 NETIF_MSG_PROBE	| \
	 NETIF_MSG_LINK		| \
	 NETIF_MSG_TIMER	| \
	 NETIF_MSG_IFDOWN	| \
	 NETIF_MSG_IFUP		| \
	 NETIF_MSG_RX_ERR	| \
	 NETIF_MSG_TX_ERR)

/* length of time before we decide the hardware is borked,
 * and dev->tx_timeout() should be called to fix the problem
 */
#define TG3_TX_TIMEOUT			(5 * HZ)

/* hardware minimum and maximum for a single frame's data payload */
#define TG3_MIN_MTU			60
#define TG3_MAX_MTU			9000

/* These numbers seem to be hard coded in the NIC firmware somehow.
 * You can't change the ring sizes, but you can change where you place
 * them in the NIC onboard memory.
 */
#define TG3_RX_RING_SIZE		512
#define TG3_DEF_RX_RING_PENDING		200
#define TG3_RX_JUMBO_RING_SIZE		256
#define TG3_DEF_RX_JUMBO_RING_PENDING	100
#define TG3_RX_RCB_RING_SIZE		1024
#define TG3_TX_RING_SIZE		512
#define TG3_DEF_TX_RING_PENDING		(TG3_TX_RING_SIZE - 1)

#define TG3_RX_RING_BYTES	(sizeof(struct tg3_rx_buffer_desc) * \
				 TG3_RX_RING_SIZE)
#define TG3_RX_JUMBO_RING_BYTES	(sizeof(struct tg3_rx_buffer_desc) * \
			         TG3_RX_JUMBO_RING_SIZE)
#define TG3_RX_RCB_RING_BYTES	(sizeof(struct tg3_rx_buffer_desc) * \
			         TG3_RX_RCB_RING_SIZE)
#define TG3_TX_RING_BYTES	(sizeof(struct tg3_tx_buffer_desc) * \
				 TG3_TX_RING_SIZE)
#define TX_RING_GAP(TP)	\
	(TG3_TX_RING_SIZE - (TP)->tx_pending)
#define TX_BUFFS_AVAIL(TP)						\
	(((TP)->tx_cons <= (TP)->tx_prod) ?				\
	  (TP)->tx_cons + (TP)->tx_pending - (TP)->tx_prod :		\
	  (TP)->tx_cons - (TP)->tx_prod - TX_RING_GAP(TP))
#define NEXT_TX(N)		(((N) + 1) & (TG3_TX_RING_SIZE - 1))

#define RX_PKT_BUF_SZ		(1536 + tp->rx_offset + 64)
#define RX_JUMBO_PKT_BUF_SZ	(9046 + tp->rx_offset + 64)

/* minimum number of free TX descriptors required to wake up TX process */
#define TG3_TX_WAKEUP_THRESH		(TG3_TX_RING_SIZE / 4)

static char version[] __devinitdata =
	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";

MODULE_AUTHOR("David S. Miller (davem@redhat.com) and Jeff Garzik (jgarzik@pobox.com)");
MODULE_DESCRIPTION("Broadcom Tigon3 ethernet driver");
MODULE_LICENSE("GPL");
MODULE_PARM(tg3_debug, "i");
MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");

static int tg3_debug = -1;	/* -1 == use TG3_DEF_MSG_ENABLE as value */

static struct pci_device_id tg3_pci_tbl[] __devinitdata = {
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_SYSKONNECT, 0x4400,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100,
	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
	{ 0, }
};

MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);

static void tg3_write_indirect_reg32(struct tg3 *tp, u32 off, u32 val)
{
	if ((tp->tg3_flags & TG3_FLAG_PCIX_TARGET_HWBUG) != 0) {
		unsigned long flags;

		spin_lock_irqsave(&tp->indirect_lock, flags);
		pci_write_config_dword(tp->pdev, TG3PCI_REG_BASE_ADDR, off);
		pci_write_config_dword(tp->pdev, TG3PCI_REG_DATA, val);
		spin_unlock_irqrestore(&tp->indirect_lock, flags);
	} else {
		writel(val, tp->regs + off);
		if ((tp->tg3_flags & TG3_FLAG_5701_REG_WRITE_BUG) != 0)
			readl(tp->regs + off);
	}
}

#define tw32(reg,val)		tg3_write_indirect_reg32(tp,(reg),(val))
#define tw32_mailbox(reg, val)	writel(((val) & 0xffffffff), tp->regs + (reg))
#define tw16(reg,val)		writew(((val) & 0xffff), tp->regs + (reg))
#define tw8(reg,val)		writeb(((val) & 0xff), tp->regs + (reg))
#define tr32(reg)		readl(tp->regs + (reg))
#define tr16(reg)		readw(tp->regs + (reg))
#define tr8(reg)		readb(tp->regs + (reg))

static void tg3_write_mem(struct tg3 *tp, u32 off, u32 val)
{
	unsigned long flags;

	spin_lock_irqsave(&tp->indirect_lock, flags);
	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);

	/* Always leave this as zero. */
	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
	spin_unlock_irqrestore(&tp->indirect_lock, flags);
}

static void tg3_read_mem(struct tg3 *tp, u32 off, u32 *val)
{
	unsigned long flags;

	spin_lock_irqsave(&tp->indirect_lock, flags);
	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, off);
	pci_read_config_dword(tp->pdev, TG3PCI_MEM_WIN_DATA, val);

	/* Always leave this as zero. */
	pci_write_config_dword(tp->pdev, TG3PCI_MEM_WIN_BASE_ADDR, 0);
	spin_unlock_irqrestore(&tp->indirect_lock, flags);
}

static void tg3_disable_ints(struct tg3 *tp)
{
	tw32(TG3PCI_MISC_HOST_CTRL,
	     (tp->misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT));
	tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000001);
	tr32(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW);
}

static inline void tg3_cond_int(struct tg3 *tp)
{
	if (tp->hw_status->status & SD_STATUS_UPDATED)
		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl | GRC_LCLCTRL_SETINT);
}

static void tg3_enable_ints(struct tg3 *tp)
{
	tw32(TG3PCI_MISC_HOST_CTRL,
	     (tp->misc_host_ctrl & ~MISC_HOST_CTRL_MASK_PCI_INT));
	tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW, 0x00000000);
	tr32(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW);

	tg3_cond_int(tp);
}

#ifdef NAPI
/* these netif_xxx funcs should be moved into generic net layer */
static void netif_poll_disable(struct net_device *dev)
{
	while (test_and_set_bit(__LINK_STATE_RX_SCHED, &dev->state)) {
		current->state = TASK_INTERRUPTIBLE;
		schedule_timeout(1);
	}
}

static inline void netif_poll_enable(struct net_device *dev)
{
	clear_bit(__LINK_STATE_RX_SCHED, &dev->state);
}

/* same as netif_rx_complete, except that local_irq_save(flags)
 * has already been issued
 */
static inline void __netif_rx_complete(struct net_device *dev)
{
	if (!test_bit(__LINK_STATE_RX_SCHED, &dev->state)) BUG();
	list_del(&dev->poll_list);
	clear_bit(__LINK_STATE_RX_SCHED, &dev->state);
}
#endif

static inline void netif_tx_disable(struct net_device *dev)
{
	spin_lock_bh(&dev->xmit_lock);
	netif_stop_queue(dev);
	spin_unlock_bh(&dev->xmit_lock);
}

static inline void tg3_netif_stop(struct tg3 *tp)
{
#ifdef NAPI
	netif_poll_disable(tp->dev);
#endif
	netif_tx_disable(tp->dev);
}

static inline void tg3_netif_start(struct tg3 *tp)
{
	netif_wake_queue(tp->dev);
	/* NOTE: unconditional netif_wake_queue is only appropriate
	 * so long as all callers are assured to have free tx slots
	 * (such as after tg3_init_hw)
	 */
#ifdef NAPI
	netif_poll_enable(tp->dev);
#endif
	tg3_cond_int(tp);
}

static void tg3_switch_clocks(struct tg3 *tp)
{
	if (tr32(TG3PCI_CLOCK_CTRL) & CLOCK_CTRL_44MHZ_CORE) {
		tw32(TG3PCI_CLOCK_CTRL,
		     (CLOCK_CTRL_44MHZ_CORE | CLOCK_CTRL_ALTCLK));
		tr32(TG3PCI_CLOCK_CTRL);
		udelay(40);
		tw32(TG3PCI_CLOCK_CTRL,
		     (CLOCK_CTRL_ALTCLK));
		tr32(TG3PCI_CLOCK_CTRL);
		udelay(40);
	}
	tw32(TG3PCI_CLOCK_CTRL, 0);
	tr32(TG3PCI_CLOCK_CTRL);
	udelay(40);
}

#define PHY_BUSY_LOOPS	5000

static int tg3_readphy(struct tg3 *tp, int reg, u32 *val)
{
	u32 frame_val;
	int loops, ret;

	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
		tw32(MAC_MI_MODE,
		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
		tr32(MAC_MI_MODE);
		udelay(40);
	}

	*val = 0xffffffff;

	frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
		      MI_COM_PHY_ADDR_MASK);
	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
		      MI_COM_REG_ADDR_MASK);
	frame_val |= (MI_COM_CMD_READ | MI_COM_START);
	
	tw32(MAC_MI_COM, frame_val);
	tr32(MAC_MI_COM);

	loops = PHY_BUSY_LOOPS;
	while (loops-- > 0) {
		udelay(10);
		frame_val = tr32(MAC_MI_COM);

		if ((frame_val & MI_COM_BUSY) == 0) {
			udelay(5);
			frame_val = tr32(MAC_MI_COM);
			break;
		}
	}

	ret = -EBUSY;
	if (loops > 0) {
		*val = frame_val & MI_COM_DATA_MASK;
		ret = 0;
	}

	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
		tw32(MAC_MI_MODE, tp->mi_mode);
		tr32(MAC_MI_MODE);
		udelay(40);
	}

	return ret;
}

static int tg3_writephy(struct tg3 *tp, int reg, u32 val)
{
	u32 frame_val;
	int loops, ret;

	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
		tw32(MAC_MI_MODE,
		     (tp->mi_mode & ~MAC_MI_MODE_AUTO_POLL));
		tr32(MAC_MI_MODE);
		udelay(40);
	}

	frame_val  = ((PHY_ADDR << MI_COM_PHY_ADDR_SHIFT) &
		      MI_COM_PHY_ADDR_MASK);
	frame_val |= ((reg << MI_COM_REG_ADDR_SHIFT) &
		      MI_COM_REG_ADDR_MASK);
	frame_val |= (val & MI_COM_DATA_MASK);
	frame_val |= (MI_COM_CMD_WRITE | MI_COM_START);
	
	tw32(MAC_MI_COM, frame_val);
	tr32(MAC_MI_COM);

	loops = PHY_BUSY_LOOPS;
	while (loops-- > 0) {
		udelay(10);
		frame_val = tr32(MAC_MI_COM);
		if ((frame_val & MI_COM_BUSY) == 0) {
			udelay(5);
			frame_val = tr32(MAC_MI_COM);
			break;
		}
	}

	ret = -EBUSY;
	if (loops > 0)
		ret = 0;

	if ((tp->mi_mode & MAC_MI_MODE_AUTO_POLL) != 0) {
		tw32(MAC_MI_MODE, tp->mi_mode);
		tr32(MAC_MI_MODE);
		udelay(40);
	}

	return ret;
}

/* This will reset the tigon3 PHY if there is no valid
 * link unless the FORCE argument is non-zero.
 */
static int tg3_phy_reset(struct tg3 *tp, int force)
{
	u32 phy_status, phy_control;
	int err, limit;

	err  = tg3_readphy(tp, MII_BMSR, &phy_status);
	err |= tg3_readphy(tp, MII_BMSR, &phy_status);
	if (err != 0)
		return -EBUSY;

	/* If we have link, and not forcing a reset, then nothing
	 * to do.
	 */
	if ((phy_status & BMSR_LSTATUS) != 0 && (force == 0))
		return 0;

	/* OK, reset it, and poll the BMCR_RESET bit until it
	 * clears or we time out.
	 */
	phy_control = BMCR_RESET;
	err = tg3_writephy(tp, MII_BMCR, phy_control);
	if (err != 0)
		return -EBUSY;

	limit = 5000;
	while (limit--) {
		err = tg3_readphy(tp, MII_BMCR, &phy_control);
		if (err != 0)
			return -EBUSY;

		if ((phy_control & BMCR_RESET) == 0) {
			udelay(40);
			return 0;
		}
		udelay(10);
	}

	return -EBUSY;
}

static int tg3_setup_phy(struct tg3 *);

static int tg3_set_power_state(struct tg3 *tp, int state)
{
	u32 misc_host_ctrl;
	u16 power_control, power_caps;
	int pm = tp->pm_cap;

	/* Make sure register accesses (indirect or otherwise)
	 * will function correctly.
	 */
	pci_write_config_dword(tp->pdev,
			       TG3PCI_MISC_HOST_CTRL,
			       tp->misc_host_ctrl);

	pci_read_config_word(tp->pdev,
			     pm + PCI_PM_CTRL,
			     &power_control);
	power_control |= PCI_PM_CTRL_PME_STATUS;
	power_control &= ~(PCI_PM_CTRL_STATE_MASK);
	switch (state) {
	case 0:
		power_control |= 0;
		pci_write_config_word(tp->pdev,
				      pm + PCI_PM_CTRL,
				      power_control);
		tw32(GRC_LOCAL_CTRL, tp->grc_local_ctrl);
		tr32(GRC_LOCAL_CTRL);
		udelay(100);

		return 0;

	case 1:
		power_control |= 1;
		break;

	case 2:
		power_control |= 2;
		break;

	case 3:
		power_control |= 3;
		break;

	default:
		printk(KERN_WARNING PFX "%s: Invalid power state (%d) "
		       "requested.\n",
		       tp->dev->name, state);
		return -EINVAL;
	};

	power_control |= PCI_PM_CTRL_PME_ENABLE;

	misc_host_ctrl = tr32(TG3PCI_MISC_HOST_CTRL);
	tw32(TG3PCI_MISC_HOST_CTRL,
	     misc_host_ctrl | MISC_HOST_CTRL_MASK_PCI_INT);

	if (tp->link_config.phy_is_low_power == 0) {
		tp->link_config.phy_is_low_power = 1;
		tp->link_config.orig_speed = tp->link_config.speed;
		tp->link_config.orig_duplex = tp->link_config.duplex;
		tp->link_config.orig_autoneg = tp->link_config.autoneg;
	}

	if (tp->phy_id != PHY_ID_SERDES) {
		tp->link_config.speed = SPEED_10;
		tp->link_config.duplex = DUPLEX_HALF;
		tp->link_config.autoneg = AUTONEG_ENABLE;
		tg3_setup_phy(tp);
	}

	pci_read_config_word(tp->pdev, pm + PCI_PM_PMC, &power_caps);

	if (tp->tg3_flags & TG3_FLAG_WOL_ENABLE) {
		u32 mac_mode;

		if (tp->phy_id != PHY_ID_SERDES) {
			tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x5a);
			udelay(40);

			mac_mode = MAC_MODE_PORT_MODE_MII;

			if (GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5700 ||
			    !(tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB))
				mac_mode |= MAC_MODE_LINK_POLARITY;
		} else {
			mac_mode = MAC_MODE_PORT_MODE_TBI;
		}


		if (((power_caps & PCI_PM_CAP_PME_D3cold) &&
		     (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)))
			mac_mode |= MAC_MODE_MAGIC_PKT_ENABLE;

		tw32(MAC_MODE, mac_mode);
		tr32(MAC_MODE);
		udelay(100);

		tw32(MAC_RX_MODE, RX_MODE_ENABLE);
		tr32(MAC_RX_MODE);
		udelay(10);
	}

	if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB) {
		u32 base_val;

		base_val = 0;
		if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
		    GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)
			base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
				     CLOCK_CTRL_TXCLK_DISABLE);

		tw32(TG3PCI_CLOCK_CTRL, base_val |
		     CLOCK_CTRL_ALTCLK);
		tr32(TG3PCI_CLOCK_CTRL);
		udelay(40);

		tw32(TG3PCI_CLOCK_CTRL, base_val |
		     CLOCK_CTRL_ALTCLK |
		     CLOCK_CTRL_44MHZ_CORE);
		tr32(TG3PCI_CLOCK_CTRL);
		udelay(40);

		tw32(TG3PCI_CLOCK_CTRL, base_val |
		     CLOCK_CTRL_44MHZ_CORE);
		tr32(TG3PCI_CLOCK_CTRL);
		udelay(40);
	} else {
		u32 base_val;

		base_val = 0;
		if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
		    GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701)
			base_val |= (CLOCK_CTRL_RXCLK_DISABLE |
				     CLOCK_CTRL_TXCLK_DISABLE);

		tw32(TG3PCI_CLOCK_CTRL, base_val |
		     CLOCK_CTRL_ALTCLK |
		     CLOCK_CTRL_PWRDOWN_PLL133);
		tr32(TG3PCI_CLOCK_CTRL);
		udelay(40);
	}

	if (!(tp->tg3_flags & TG3_FLAG_EEPROM_WRITE_PROT) &&
	    (tp->tg3_flags & TG3_FLAG_WOL_ENABLE)) {
		if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 ||
		    GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5701) {
			tw32(GRC_LOCAL_CTRL,
			     (GRC_LCLCTRL_GPIO_OE0 |
			      GRC_LCLCTRL_GPIO_OE1 |
			      GRC_LCLCTRL_GPIO_OE2 |
			      GRC_LCLCTRL_GPIO_OUTPUT0 |
			      GRC_LCLCTRL_GPIO_OUTPUT1));
			tr32(GRC_LOCAL_CTRL);
			udelay(100);
		} else {
			tw32(GRC_LOCAL_CTRL,
			     (GRC_LCLCTRL_GPIO_OE0 |
			      GRC_LCLCTRL_GPIO_OE1 |
			      GRC_LCLCTRL_GPIO_OE2 |
			      GRC_LCLCTRL_GPIO_OUTPUT1 |
			      GRC_LCLCTRL_GPIO_OUTPUT2));
			tr32(GRC_LOCAL_CTRL);
			udelay(100);

			tw32(GRC_LOCAL_CTRL,
			     (GRC_LCLCTRL_GPIO_OE0 |
			      GRC_LCLCTRL_GPIO_OE1 |
			      GRC_LCLCTRL_GPIO_OE2 |
			      GRC_LCLCTRL_GPIO_OUTPUT0 |
			      GRC_LCLCTRL_GPIO_OUTPUT1 |
			      GRC_LCLCTRL_GPIO_OUTPUT2));
			tr32(GRC_LOCAL_CTRL);
			udelay(100);

			tw32(GRC_LOCAL_CTRL,
			     (GRC_LCLCTRL_GPIO_OE0 |
			      GRC_LCLCTRL_GPIO_OE1 |
			      GRC_LCLCTRL_GPIO_OE2 |
			      GRC_LCLCTRL_GPIO_OUTPUT0 |
			      GRC_LCLCTRL_GPIO_OUTPUT1));
			tr32(GRC_LOCAL_CTRL);
			udelay(100);
		}
	}

	/* Finally, set the new power state. */
	pci_write_config_word(tp->pdev, pm + PCI_PM_CTRL, power_control);

	return 0;
}

static void tg3_link_report(struct tg3 *tp)
{
	if (!netif_carrier_ok(tp->dev)) {
		printk(KERN_INFO PFX "%s: Link is down.\n", tp->dev->name);
	} else {
		printk(KERN_INFO PFX "%s: Link is up at %d Mbps, %s duplex.\n",
		       tp->dev->name,
		       (tp->link_config.active_speed == SPEED_1000 ?
			1000 :
			(tp->link_config.active_speed == SPEED_100 ?
			 100 : 10)),
		       (tp->link_config.active_duplex == DUPLEX_FULL ?
			"full" : "half"));

		printk(KERN_INFO PFX "%s: Flow control is %s for TX and "
		       "%s for RX.\n",
		       tp->dev->name,
		       (tp->tg3_flags & TG3_FLAG_TX_PAUSE) ? "on" : "off",
		       (tp->tg3_flags & TG3_FLAG_RX_PAUSE) ? "on" : "off");
	}
}

static void tg3_setup_flow_control(struct tg3 *tp, u32 local_adv, u32 remote_adv)
{
	u32 new_tg3_flags = 0;

	if (local_adv & ADVERTISE_PAUSE_CAP) {
		if (local_adv & ADVERTISE_PAUSE_ASYM) {
			if (remote_adv & LPA_PAUSE_CAP)
				new_tg3_flags |=
					(TG3_FLAG_RX_PAUSE |
					 TG3_FLAG_TX_PAUSE);
			else if (remote_adv & LPA_PAUSE_ASYM)
				new_tg3_flags |=
					(TG3_FLAG_RX_PAUSE);
		} else {
			if (remote_adv & LPA_PAUSE_CAP)
				new_tg3_flags |=
					(TG3_FLAG_RX_PAUSE |
					 TG3_FLAG_TX_PAUSE);
		}
	} else if (local_adv & ADVERTISE_PAUSE_ASYM) {
		if ((remote_adv & LPA_PAUSE_CAP) &&
		    (remote_adv & LPA_PAUSE_ASYM))
			new_tg3_flags |= TG3_FLAG_TX_PAUSE;
	}

	tp->tg3_flags &= ~(TG3_FLAG_RX_PAUSE | TG3_FLAG_TX_PAUSE);
	tp->tg3_flags |= new_tg3_flags;

	if (new_tg3_flags & TG3_FLAG_RX_PAUSE)
		tp->rx_mode |= RX_MODE_FLOW_CTRL_ENABLE;
	else
		tp->rx_mode &= ~RX_MODE_FLOW_CTRL_ENABLE;

	if (new_tg3_flags & TG3_FLAG_TX_PAUSE)
		tp->tx_mode |= TX_MODE_FLOW_CTRL_ENABLE;
	else
		tp->tx_mode &= ~TX_MODE_FLOW_CTRL_ENABLE;
}

static void tg3_aux_stat_to_speed_duplex(struct tg3 *tp, u32 val, u16 *speed, u8 *duplex)
{
	switch (val & MII_TG3_AUX_STAT_SPDMASK) {
	case MII_TG3_AUX_STAT_10HALF:
		*speed = SPEED_10;
		*duplex = DUPLEX_HALF;
		break;

	case MII_TG3_AUX_STAT_10FULL:
		*speed = SPEED_10;
		*duplex = DUPLEX_FULL;
		break;

	case MII_TG3_AUX_STAT_100HALF:
		*speed = SPEED_100;
		*duplex = DUPLEX_HALF;
		break;

	case MII_TG3_AUX_STAT_100FULL:
		*speed = SPEED_100;
		*duplex = DUPLEX_FULL;
		break;

	case MII_TG3_AUX_STAT_1000HALF:
		*speed = SPEED_1000;
		*duplex = DUPLEX_HALF;
		break;

	case MII_TG3_AUX_STAT_1000FULL:
		*speed = SPEED_1000;
		*duplex = DUPLEX_FULL;
		break;

	default:
		*speed = SPEED_INVALID;
		*duplex = DUPLEX_INVALID;
		break;
	};
}

static int tg3_phy_copper_begin(struct tg3 *tp, int wait_for_link)
{
	u32 new_adv;
	int i;

	if (tp->link_config.phy_is_low_power) {
		/* Entering low power mode.  Disable gigabit and
		 * 100baseT advertisements.
		 */
		tg3_writephy(tp, MII_TG3_CTRL, 0);

		new_adv = (ADVERTISE_10HALF | ADVERTISE_10FULL |
			   ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
		if (tp->tg3_flags & TG3_FLAG_WOL_SPEED_100MB)
			new_adv |= (ADVERTISE_100HALF | ADVERTISE_100FULL);

		tg3_writephy(tp, MII_ADVERTISE, new_adv);
	} else if (tp->link_config.speed == SPEED_INVALID) {
		tp->link_config.advertising =
			(ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full |
			 ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full |
			 ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full |
			 ADVERTISED_Autoneg | ADVERTISED_MII);

		if (tp->tg3_flags & TG3_FLAG_10_100_ONLY)
			tp->link_config.advertising &=
				~(ADVERTISED_1000baseT_Half |
				  ADVERTISED_1000baseT_Full);

		new_adv = (ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP);
		if (tp->link_config.advertising & ADVERTISED_10baseT_Half)
			new_adv |= ADVERTISE_10HALF;
		if (tp->link_config.advertising & ADVERTISED_10baseT_Full)
			new_adv |= ADVERTISE_10FULL;
		if (tp->link_config.advertising & ADVERTISED_100baseT_Half)
			new_adv |= ADVERTISE_100HALF;
		if (tp->link_config.advertising & ADVERTISED_100baseT_Full)
			new_adv |= ADVERTISE_100FULL;
		tg3_writephy(tp, MII_ADVERTISE, new_adv);

		if (tp->link_config.advertising &
		    (ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full)) {
			new_adv = 0;
			if (tp->link_config.advertising & ADVERTISED_1000baseT_Half)
				new_adv |= MII_TG3_CTRL_ADV_1000_HALF;
			if (tp->link_config.advertising & ADVERTISED_1000baseT_Full)
				new_adv |= MII_TG3_CTRL_ADV_1000_FULL;
			if (!(tp->tg3_flags & TG3_FLAG_10_100_ONLY) &&
			    (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
			     tp->pci_chip_rev_id == CHIPREV_ID_5701_B0))
				new_adv |= (MII_TG3_CTRL_AS_MASTER |
					    MII_TG3_CTRL_ENABLE_AS_MASTER);
			tg3_writephy(tp, MII_TG3_CTRL, new_adv);
		} else {
			tg3_writephy(tp, MII_TG3_CTRL, 0);
		}
	} else {
		/* Asking for a specific link mode. */
		if (tp->link_config.speed == SPEED_1000) {
			new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
			tg3_writephy(tp, MII_ADVERTISE, new_adv);

			if (tp->link_config.duplex == DUPLEX_FULL)
				new_adv = MII_TG3_CTRL_ADV_1000_FULL;
			else
				new_adv = MII_TG3_CTRL_ADV_1000_HALF;
			if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
			    tp->pci_chip_rev_id == CHIPREV_ID_5701_B0)
				new_adv |= (MII_TG3_CTRL_AS_MASTER |
					    MII_TG3_CTRL_ENABLE_AS_MASTER);
			tg3_writephy(tp, MII_TG3_CTRL, new_adv);
		} else {
			tg3_writephy(tp, MII_TG3_CTRL, 0);

			new_adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
			if (tp->link_config.speed == SPEED_100) {
				if (tp->link_config.duplex == DUPLEX_FULL)
					new_adv |= ADVERTISE_100FULL;
				else
					new_adv |= ADVERTISE_100HALF;
			} else {
				if (tp->link_config.duplex == DUPLEX_FULL)
					new_adv |= ADVERTISE_10FULL;
				else
					new_adv |= ADVERTISE_10HALF;
			}
			tg3_writephy(tp, MII_ADVERTISE, new_adv);
		}
	}

	if (tp->link_config.autoneg == AUTONEG_DISABLE &&
	    tp->link_config.speed != SPEED_INVALID) {
		u32 bmcr, orig_bmcr;

		tp->link_config.active_speed = tp->link_config.speed;
		tp->link_config.active_duplex = tp->link_config.duplex;

		bmcr = 0;
		switch (tp->link_config.speed) {
		default:
		case SPEED_10:
			break;

		case SPEED_100:
			bmcr |= BMCR_SPEED100;
			break;

		case SPEED_1000:
			bmcr |= TG3_BMCR_SPEED1000;
			break;
		};

		if (tp->link_config.duplex == DUPLEX_FULL)
			bmcr |= BMCR_FULLDPLX;

		tg3_readphy(tp, MII_BMCR, &orig_bmcr);
		if (bmcr != orig_bmcr) {
			tg3_writephy(tp, MII_BMCR, BMCR_LOOPBACK);
			for (i = 0; i < 15000; i++) {
				u32 tmp;

				udelay(10);
				tg3_readphy(tp, MII_BMSR, &tmp);
				tg3_readphy(tp, MII_BMSR, &tmp);
				if (!(tmp & BMSR_LSTATUS)) {
					udelay(40);
					break;
				}
			}
			tg3_writephy(tp, MII_BMCR, bmcr);
			udelay(40);
		}
	} else {
		tg3_writephy(tp, MII_BMCR,
			     BMCR_ANENABLE | BMCR_ANRESTART);
	}

	if (wait_for_link) {
		tp->link_config.active_speed = SPEED_INVALID;
		tp->link_config.active_duplex = DUPLEX_INVALID;
		for (i = 0; i < 300000; i++) {
			u32 tmp;

			udelay(10);
			tg3_readphy(tp, MII_BMSR, &tmp);
			tg3_readphy(tp, MII_BMSR, &tmp);
			if (!(tmp & BMSR_LSTATUS))
				continue;

			tg3_readphy(tp, MII_TG3_AUX_STAT, &tmp);
			tg3_aux_stat_to_speed_duplex(tp, tmp,
						     &tp->link_config.active_speed,
						     &tp->link_config.active_duplex);
		}
		if (tp->link_config.active_speed == SPEED_INVALID)
			return -EINVAL;
	}

	return 0;
}

static int tg3_init_5401phy_dsp(struct tg3 *tp)
{
	int err;

	/* Turn off tap power management. */
	err  = tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x0c20);

	err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0012);
	err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1804);

	err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x0013);
	err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x1204);

	err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
	err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0132);

	err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x8006);
	err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0232);

	err |= tg3_writephy(tp, MII_TG3_DSP_ADDRESS, 0x201f);
	err |= tg3_writephy(tp, MII_TG3_DSP_RW_PORT, 0x0a20);

	udelay(40);

	return err;
}

static int tg3_setup_copper_phy(struct tg3 *tp)
{
	int current_link_up;
	u32 bmsr, dummy;
	u16 current_speed;
	u8 current_duplex;
	int i, err;

	tw32(MAC_STATUS,
	     (MAC_STATUS_SYNC_CHANGED |
	      MAC_STATUS_CFG_CHANGED));
	tr32(MAC_STATUS);
	udelay(40);

	tp->mi_mode = MAC_MI_MODE_BASE;
	tw32(MAC_MI_MODE, tp->mi_mode);
	tr32(MAC_MI_MODE);
	udelay(40);

	tg3_writephy(tp, MII_TG3_AUX_CTRL, 0x02);

	/* Some third-party PHYs need to be reset on link going
	 * down.
	 *
	 * XXX 5705 note: This workaround also applies to 5705_a0
	 */
	if ((GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5703 ||
	     GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5704) &&
	    netif_carrier_ok(tp->dev)) {
		tg3_readphy(tp, MII_BMSR, &bmsr);
		tg3_readphy(tp, MII_BMSR, &bmsr);
		if (!(bmsr & BMSR_LSTATUS))
			tg3_phy_reset(tp, 1);
	}

	if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5401) {
		tg3_readphy(tp, MII_BMSR, &bmsr);
		tg3_readphy(tp, MII_BMSR, &bmsr);

		if (!(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE))
			bmsr = 0;

		if (!(bmsr & BMSR_LSTATUS)) {
			err = tg3_init_5401phy_dsp(tp);
			if (err)
				return err;

			tg3_readphy(tp, MII_BMSR, &bmsr);
			for (i = 0; i < 1000; i++) {
				udelay(10);
				tg3_readphy(tp, MII_BMSR, &bmsr);
				if (bmsr & BMSR_LSTATUS) {
					udelay(40);
					break;
				}
			}

			if ((tp->phy_id & PHY_ID_REV_MASK) == PHY_REV_BCM5401_B0 &&
			    !(bmsr & BMSR_LSTATUS) &&
			    tp->link_config.active_speed == SPEED_1000) {
				err = tg3_phy_reset(tp, 1);
				if (!err)
					err = tg3_init_5401phy_dsp(tp);
				if (err)
					return err;
			}
		}
	} else if (tp->pci_chip_rev_id == CHIPREV_ID_5701_A0 ||
		   tp->pci_chip_rev_id == CHIPREV_ID_5701_B0) {
		/* 5701 {A0,B0} CRC bug workaround */
		tg3_writephy(tp, 0x15, 0x0a75);
		tg3_writephy(tp, 0x1c, 0x8c68);
		tg3_writephy(tp, 0x1c, 0x8d68);
		tg3_writephy(tp, 0x1c, 0x8c68);
	}

	/* Clear pending interrupts... */
	tg3_readphy(tp, MII_TG3_ISTAT, &dummy);
	tg3_readphy(tp, MII_TG3_ISTAT, &dummy);

	if (tp->tg3_flags & TG3_FLAG_USE_MI_INTERRUPT)
		tg3_writephy(tp, MII_TG3_IMASK, ~MII_TG3_INT_LINKCHG);
	else
		tg3_writephy(tp, MII_TG3_IMASK, ~0);

	if (tp->led_mode == led_mode_three_link)
		tg3_writephy(tp, MII_TG3_EXT_CTRL,
			     MII_TG3_EXT_CTRL_LNK3_LED_MODE);
	else
		tg3_writephy(tp, MII_TG3_EXT_CTRL, 0);

	current_link_up = 0;
	current_speed = SPEED_INVALID;
	current_duplex = DUPLEX_INVALID;

	tg3_readphy(tp, MII_BMSR, &bmsr);
	tg3_readphy(tp, MII_BMSR, &bmsr);

	if (bmsr & BMSR_LSTATUS) {
		u32 aux_stat, bmcr;

		tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
		for (i = 0; i < 2000; i++) {
			udelay(10);
			tg3_readphy(tp, MII_TG3_AUX_STAT, &aux_stat);
			if (aux_stat)
				break;
		}

		tg3_aux_stat_to_speed_duplex(tp, aux_stat,
					     &current_speed,
					     &current_duplex);
		tg3_readphy(tp, MII_BMCR, &bmcr);
		tg3_readphy(tp, MII_BMCR, &bmcr);
		if (tp->link_config.autoneg == AUTONEG_ENABLE) {
			if (bmcr & BMCR_ANENABLE) {
				u32 gig_ctrl;

				current_link_up = 1;

				/* Force autoneg restart if we are exiting
				 * low power mode.
				 */
				tg3_readphy(tp, MII_TG3_CTRL, &gig_ctrl);
				if (!(gig_ctrl & (MII_TG3_CTRL_ADV_1000_HALF |
						  MII_TG3_CTRL_ADV_1000_FULL))) {
					current_link_up = 0;
				}
			} else {
				current_link_up = 0;
			}
		} else {
			if (!(bmcr & BMCR_ANENABLE) &&
			    tp->link_config.speed == current_speed &&
			    tp->link_config.duplex == current_duplex) {
				current_link_up = 1;
			} else {
				current_link_up = 0;
			}
		}

		tp->link_config.active_speed = current_speed;
		tp->link_config.active_duplex = current_duplex;
	}

	if (current_link_up == 1 &&
	    (tp->link_config.active_duplex == DUPLEX_FULL) &&
	    (tp->link_config.autoneg == AUTONEG_ENABLE)) {
		u32 local_adv, remote_adv;

		tg3_readphy(tp, MII_ADVERTISE, &local_adv);
		local_adv &= (ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM);

		tg3_readphy(tp, MII_LPA, &remote_adv);
		remote_adv &= (LPA_PAUSE_CAP | LPA_PAUSE_ASYM);

		/* If we are not advertising full pause capability,
		 * something is wrong.  Bring the link down and reconfigure.
		 */
		if (local_adv != ADVERTISE_PAUSE_CAP) {
			current_link_up = 0;
		} else {
			tg3_setup_flow_control(tp, local_adv, remote_adv);
		}
	}

	if (current_link_up == 0) {
		u32 tmp;

		tg3_phy_copper_begin(tp, 0);

		tg3_readphy(tp, MII_BMSR, &tmp);
		tg3_readphy(tp, MII_BMSR, &tmp);
		if (tmp & BMSR_LSTATUS)
			current_link_up = 1;
	}

	tp->mac_mode &= ~MAC_MODE_PORT_MODE_MASK;
	if (current_link_up == 1) {
		if (tp->link_config.active_speed == SPEED_100 ||
		    tp->link_config.active_speed == SPEED_10)
			tp->mac_mode |= MAC_MODE_PORT_MODE_MII;
		else
			tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;
	} else
		tp->mac_mode |= MAC_MODE_PORT_MODE_GMII;

	tp->mac_mode &= ~MAC_MODE_HALF_DUPLEX;
	if (tp->link_config.active_duplex == DUPLEX_HALF)
		tp->mac_mode |= MAC_MODE_HALF_DUPLEX;

	tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
	if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700) {
		if ((tp->led_mode == led_mode_link10) ||
		    (current_link_up == 1 &&
		     tp->link_config.active_speed == SPEED_10))
			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
	} else {
		if (current_link_up == 1)
			tp->mac_mode |= MAC_MODE_LINK_POLARITY;
		tw32(MAC_LED_CTRL, LED_CTRL_PHY_MODE_1);
	}

	/* ??? Without this setting Netgear GA302T PHY does not
	 * ??? send/receive packets...
	 */
	if ((tp->phy_id & PHY_ID_MASK) == PHY_ID_BCM5411 &&
	    tp->pci_chip_rev_id == CHIPREV_ID_5700_ALTIMA) {
		tp->mi_mode |= MAC_MI_MODE_AUTO_POLL;
		tw32(MAC_MI_MODE, tp->mi_mode);
		tr32(MAC_MI_MODE);
		udelay(40);
	}

	tw32(MAC_MODE, tp->mac_mode);
	tr32(MAC_MODE);
	udelay(40);

	if (tp->tg3_flags &
	    (TG3_FLAG_USE_LINKCHG_REG |
	     TG3_FLAG_POLL_SERDES)) {
		/* Polled via timer. */
		tw32(MAC_EVENT, 0);
	} else {
		tw32(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
	}
	tr32(MAC_EVENT);
	udelay(40);

	if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5700 &&
	    current_link_up == 1 &&
	    tp->link_config.active_speed == SPEED_1000 &&
	    ((tp->tg3_flags & TG3_FLAG_PCIX_MODE) ||
	     (tp->tg3_flags & TG3_FLAG_PCI_HIGH_SPEED))) {
		udelay(120);
		tw32(MAC_STATUS,
		     (MAC_STATUS_SYNC_CHANGED |
		      MAC_STATUS_CFG_CHANGED));
		tr32(MAC_STATUS);
		udelay(40);
		tg3_write_mem(tp,
			      NIC_SRAM_FIRMWARE_MBOX,
			      NIC_SRAM_FIRMWARE_MBOX_MAGIC2);
	}

	if (current_link_up != netif_carrier_ok(tp->dev)) {
		if (current_link_up)
			netif_carrier_on(tp->dev);
		else
			netif_carrier_off(tp->dev);
		tg3_link_report(tp);
	}

	return 0;
}

struct tg3_fiber_aneginfo {
	int state;
#define ANEG_STATE_UNKNOWN		0
#define ANEG_STATE_AN_ENABLE		1
#define ANEG_STATE_RESTART_INIT		2
#define ANEG_STATE_RESTART		3
#define ANEG_STATE_DISABLE_LINK_OK	4
#define ANEG_STATE_ABILITY_DETECT_INIT	5
#define ANEG_STATE_ABILITY_DETECT	6
#define ANEG_STATE_ACK_DETECT_INIT	7
#define ANEG_STATE_ACK_DETECT		8
#define ANEG_STATE_COMPLETE_ACK_INIT	9
#define ANEG_STATE_COMPLETE_ACK		10
#define ANEG_STATE_IDLE_DETECT_INIT	11
#define ANEG_STATE_IDLE_DETECT		12
#define ANEG_STATE_LINK_OK		13
#define ANEG_STATE_NEXT_PAGE_WAIT_INIT	14
#define ANEG_STATE_NEXT_PAGE_WAIT	15

	u32 flags;
#define MR_AN_ENABLE		0x00000001
#define MR_RESTART_AN		0x00000002
#define MR_AN_COMPLETE		0x00000004
#define MR_PAGE_RX		0x00000008
#define MR_NP_LOADED		0x00000010
#define MR_TOGGLE_TX		0x00000020
#define MR_LP_ADV_FULL_DUPLEX	0x00000040
#define MR_LP_ADV_HALF_DUPLEX	0x00000080
#define MR_LP_ADV_SYM_PAUSE	0x00000100
#define MR_LP_ADV_ASYM_PAUSE	0x00000200
#define MR_LP_ADV_REMOTE_FAULT1	0x00000400
#define MR_LP_ADV_REMOTE_FAULT2	0x00000800
#define MR_LP_ADV_NEXT_PAGE	0x00001000
#define MR_TOGGLE_RX		0x00002000
#define MR_NP_RX		0x00004000

#define MR_LINK_OK		0x80000000

	unsigned long link_time, cur_time;

	u32 ability_match_cfg;
	int ability_match_count;

	char ability_match, idle_match, ack_match;

	u32 txconfig, rxconfig;
#define ANEG_CFG_NP		0x00000080
#define ANEG_CFG_ACK		0x00000040
#define ANEG_CFG_RF2		0x00000020
#define ANEG_CFG_RF1		0x00000010
#define ANEG_CFG_PS2		0x00000001
#define ANEG_CFG_PS1		0x00008000
#define ANEG_CFG_HD		0x00004000
#define ANEG_CFG_FD		0x00002000
#define ANEG_CFG_INVAL		0x00001f06

};
#define ANEG_OK		0
#define ANEG_DONE	1
#define ANEG_TIMER_ENAB	2
#define ANEG_FAILED	-1

#define ANEG_STATE_SETTLE_TIME	10000

static int tg3_fiber_aneg_smachine(struct tg3 *tp,
				   struct tg3_fiber_aneginfo *ap)
{
	unsigned long delta;
	u32 rx_cfg_reg;
	int ret;

	if (ap->state == ANEG_STATE_UNKNOWN) {
		ap->rxconfig = 0;
		ap->link_time = 0;
		ap->cur_time = 0;
		ap->ability_match_cfg = 0;
		ap->ability_match_count = 0;
		ap->ability_match = 0;
		ap->idle_match = 0;
		ap->ack_match = 0;
	}
	ap->cur_time++;

	if (tr32(MAC_STATUS) & MAC_STATUS_RCVD_CFG) {
		rx_cfg_reg = tr32(MAC_RX_AUTO_NEG);

		if (rx_cfg_reg != ap->ability_match_cfg) {
			ap->ability_match_cfg = rx_cfg_reg;
			ap->ability_match = 0;
			ap->ability_match_count = 0;
		} else {
			if (++ap->ability_match_count > 1) {
				ap->ability_match = 1;
				ap->ability_match_cfg = rx_cfg_reg;
			}
		}
		if (rx_cfg_reg & ANEG_CFG_ACK)
			ap->ack_match = 1;
		else
			ap->ack_match = 0;

		ap->idle_match = 0;
	} else {
		ap->idle_match = 1;
		ap->ability_match_cfg = 0;
		ap->ability_match_count = 0;
		ap->ability_match = 0;
		ap->ack_match = 0;

		rx_cfg_reg = 0;
	}

	ap->rxconfig = rx_cfg_reg;
	ret = ANEG_OK;

	switch(ap->state) {
	case ANEG_STATE_UNKNOWN:
		if (ap->flags & (MR_AN_ENABLE | MR_RESTART_AN))
			ap->state = ANEG_STATE_AN_ENABLE;

		/* fallthru */
	case ANEG_STATE_AN_ENABLE:
		ap->flags &= ~(MR_AN_COMPLETE | MR_PAGE_RX);
		if (ap->flags & MR_AN_ENABLE) {
			ap->link_time = 0;
			ap->cur_time = 0;
			ap->ability_match_cfg = 0;
			ap->ability_match_count = 0;
			ap->ability_match = 0;
			ap->idle_match = 0;
			ap->ack_match = 0;

			ap->state = ANEG_STATE_RESTART_INIT;
		} else {
			ap->state = ANEG_STATE_DISABLE_LINK_OK;
		}
		break;

	case ANEG_STATE_RESTART_INIT:
		ap->link_time = ap->cur_time;
		ap->flags &= ~(MR_NP_LOADED);
		ap->txconfig = 0;
		tw32(MAC_TX_AUTO_NEG, 0);
		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
		tw32(MAC_MODE, tp->mac_mode);
		tr32(MAC_MODE);
		udelay(40);

		ret = ANEG_TIMER_ENAB;
		ap->state = ANEG_STATE_RESTART;

		/* fallthru */
	case ANEG_STATE_RESTART:
		delta = ap->cur_time - ap->link_time;
		if (delta > ANEG_STATE_SETTLE_TIME) {
			ap->state = ANEG_STATE_ABILITY_DETECT_INIT;
		} else {
			ret = ANEG_TIMER_ENAB;
		}
		break;

	case ANEG_STATE_DISABLE_LINK_OK:
		ret = ANEG_DONE;
		break;

	case ANEG_STATE_ABILITY_DETECT_INIT:
		ap->flags &= ~(MR_TOGGLE_TX);
		ap->txconfig = (ANEG_CFG_FD | ANEG_CFG_PS1);
		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
		tw32(MAC_MODE, tp->mac_mode);
		tr32(MAC_MODE);
		udelay(40);

		ap->state = ANEG_STATE_ABILITY_DETECT;
		break;

	case ANEG_STATE_ABILITY_DETECT:
		if (ap->ability_match != 0 && ap->rxconfig != 0) {
			ap->state = ANEG_STATE_ACK_DETECT_INIT;
		}
		break;

	case ANEG_STATE_ACK_DETECT_INIT:
		ap->txconfig |= ANEG_CFG_ACK;
		tw32(MAC_TX_AUTO_NEG, ap->txconfig);
		tp->mac_mode |= MAC_MODE_SEND_CONFIGS;
		tw32(MAC_MODE, tp->mac_mode);
		tr32(MAC_MODE);
		udelay(40);

		ap->state = ANEG_STATE_ACK_DETECT;

		/* fallthru */
	case ANEG_STATE_ACK_DETECT:
		if (ap->ack_match != 0) {
			if ((ap->rxconfig & ~ANEG_CFG_ACK) ==
			    (ap->ability_match_cfg & ~ANEG_CFG_ACK)) {
				ap->state = ANEG_STATE_COMPLETE_ACK_INIT;
			} else {
				ap->state = ANEG_STATE_AN_ENABLE;
			}
		} else if (ap->ability_match != 0 &&
			   ap->rxconfig == 0) {
			ap->state = ANEG_STATE_AN_ENABLE;
		}
		break;

	case ANEG_STATE_COMPLETE_ACK_INIT:
		if (ap->rxconfig & ANEG_CFG_INVAL) {
			ret = ANEG_FAILED;
			break;
		}
		ap->flags &= ~(MR_LP_ADV_FULL_DUPLEX |
			       MR_LP_ADV_HALF_DUPLEX |
			       MR_LP_ADV_SYM_PAUSE |
			       MR_LP_ADV_ASYM_PAUSE |
			       MR_LP_ADV_REMOTE_FAULT1 |
			       MR_LP_ADV_REMOTE_FAULT2 |
			       MR_LP_ADV_NEXT_PAGE |
			       MR_TOGGLE_RX |
			       MR_NP_RX);
		if (ap->rxconfig & ANEG_CFG_FD)
			ap->flags |= MR_LP_ADV_FULL_DUPLEX;
		if (ap->rxconfig & ANEG_CFG_HD)
			ap->flags |= MR_LP_ADV_HALF_DUPLEX;
		if (ap->rxconfig & ANEG_CFG_PS1)
			ap->flags |= MR_LP_ADV_SYM_PAUSE;
		if (ap->rxconfig & ANEG_CFG_PS2)
			ap->flags |= MR_LP_ADV_ASYM_PAUSE;
		if (ap->rxconfig & ANEG_CFG_RF1)
			ap->flags |= MR_LP_ADV_REMOTE_FAULT1;
		if (ap->rxconfig & ANEG_CFG_RF2)
			ap->flags |= MR_LP_ADV_REMOTE_FAULT2;
		if (ap->rxconfig & ANEG_CFG_NP)
			ap->flags |= MR_LP_ADV_NEXT_PAGE;

		ap->link_time = ap->cur_time;

		ap->flags ^= (MR_TOGGLE_TX);
		if (ap->rxconfig & 0x0008)
			ap->flags |= MR_TOGGLE_RX;
		if (ap->rxconfig & ANEG_CFG_NP)
			ap->flags |= MR_NP_RX;
		ap->flags |= MR_PAGE_RX;

		ap->state = ANEG_STATE_COMPLETE_ACK;
		ret = ANEG_TIMER_ENAB;
		break;

	case ANEG_STATE_COMPLETE_ACK:
		if (ap->ability_match != 0 &&
		    ap->rxconfig == 0) {
			ap->state = ANEG_STATE_AN_ENABLE;
			break;
		}
		delta = ap->cur_time - ap->link_time;
		if (delta > ANEG_STATE_SETTLE_TIME) {
			if (!(ap->flags & (MR_LP_ADV_NEXT_PAGE))) {
				ap->state = ANEG_STATE_IDLE_DETECT_INIT;
			} else {
				if ((ap->txconfig & ANEG_CFG_NP) == 0 &&
				    !(ap->flags & MR_NP_RX)) {
					ap->state = ANEG_STATE_IDLE_DETECT_INIT;
				} else {
					ret = ANEG_FAILED;
				}
			}
		}
		break;

	case ANEG_STATE_IDLE_DETECT_INIT:
		ap->link_time = ap->cur_time;
		tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
		tw32(MAC_MODE, tp->mac_mode);
		tr32(MAC_MODE);
		udelay(40);

		ap->state = ANEG_STATE_IDLE_DETECT;
		ret = ANEG_TIMER_ENAB;
		break;

	case ANEG_STATE_IDLE_DETECT:
		if (ap->ability_match != 0 &&
		    ap->rxconfig == 0) {
			ap->state = ANEG_STATE_AN_ENABLE;
			break;
		}
		delta = ap->cur_time - ap->link_time;
		if (delta > ANEG_STATE_SETTLE_TIME) {
			/* XXX another gem from the Broadcom driver :( */
			ap->state = ANEG_STATE_LINK_OK;
		}
		break;

	case ANEG_STATE_LINK_OK:
		ap->flags |= (MR_AN_COMPLETE | MR_LINK_OK);
		ret = ANEG_DONE;
		break;

	case ANEG_STATE_NEXT_PAGE_WAIT_INIT:
		/* ??? unimplemented */
		break;

	case ANEG_STATE_NEXT_PAGE_WAIT:
		/* ??? unimplemented */
		break;

	default:
		ret = ANEG_FAILED;
		break;
	};

	return ret;
}

static int tg3_setup_fiber_phy(struct tg3 *tp)
{
	u32 orig_pause_cfg;
	u16 orig_active_speed;
	u8 orig_active_duplex;
	int current_link_up;
	int i;

	orig_pause_cfg =
		(tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
				  TG3_FLAG_TX_PAUSE));
	orig_active_speed = tp->link_config.active_speed;
	orig_active_duplex = tp->link_config.active_duplex;

	tp->mac_mode &= ~(MAC_MODE_PORT_MODE_MASK | MAC_MODE_HALF_DUPLEX);
	tp->mac_mode |= MAC_MODE_PORT_MODE_TBI;
	tw32(MAC_MODE, tp->mac_mode);
	tr32(MAC_MODE);
	udelay(40);

	/* Reset when initting first time or we have a link. */
	if (!(tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) ||
	    (tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED)) {
		/* Set PLL lock range. */
		tg3_writephy(tp, 0x16, 0x8007);

		/* SW reset */
		tg3_writephy(tp, MII_BMCR, BMCR_RESET);

		/* Wait for reset to complete. */
		/* XXX schedule_timeout() ... */
		for (i = 0; i < 500; i++)
			udelay(10);

		/* Config mode; select PMA/Ch 1 regs. */
		tg3_writephy(tp, 0x10, 0x8411);

		/* Enable auto-lock and comdet, select txclk for tx. */
		tg3_writephy(tp, 0x11, 0x0a10);

		tg3_writephy(tp, 0x18, 0x00a0);
		tg3_writephy(tp, 0x16, 0x41ff);

		/* Assert and deassert POR. */
		tg3_writephy(tp, 0x13, 0x0400);
		udelay(40);
		tg3_writephy(tp, 0x13, 0x0000);

		tg3_writephy(tp, 0x11, 0x0a50);
		udelay(40);
		tg3_writephy(tp, 0x11, 0x0a10);

		/* Wait for signal to stabilize */
		/* XXX schedule_timeout() ... */
		for (i = 0; i < 15000; i++)
			udelay(10);

		/* Deselect the channel register so we can read the PHYID
		 * later.
		 */
		tg3_writephy(tp, 0x10, 0x8011);
	}

	/* Enable link change interrupt unless serdes polling.  */
	if (!(tp->tg3_flags & TG3_FLAG_POLL_SERDES))
		tw32(MAC_EVENT, MAC_EVENT_LNKSTATE_CHANGED);
	else
		tw32(MAC_EVENT, 0);
	tr32(MAC_EVENT);
	udelay(40);

	current_link_up = 0;
	if (tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED) {
		if (tp->link_config.autoneg == AUTONEG_ENABLE &&
		    !(tp->tg3_flags & TG3_FLAG_GOT_SERDES_FLOWCTL)) {
			struct tg3_fiber_aneginfo aninfo;
			int status = ANEG_FAILED;
			unsigned int tick;
			u32 tmp;

			memset(&aninfo, 0, sizeof(aninfo));
			aninfo.flags |= (MR_AN_ENABLE);

			tw32(MAC_TX_AUTO_NEG, 0);

			tmp = tp->mac_mode & ~MAC_MODE_PORT_MODE_MASK;
			tw32(MAC_MODE, tmp | MAC_MODE_PORT_MODE_GMII);
			tr32(MAC_MODE);
			udelay(40);

			tw32(MAC_MODE, tp->mac_mode | MAC_MODE_SEND_CONFIGS);
			tr32(MAC_MODE);
			udelay(40);

			aninfo.state = ANEG_STATE_UNKNOWN;
			aninfo.cur_time = 0;
			tick = 0;
			while (++tick < 195000) {
				status = tg3_fiber_aneg_smachine(tp, &aninfo);
				if (status == ANEG_DONE ||
				    status == ANEG_FAILED)
					break;

				udelay(1);
			}

			tp->mac_mode &= ~MAC_MODE_SEND_CONFIGS;
			tw32(MAC_MODE, tp->mac_mode);
			tr32(MAC_MODE);
			udelay(40);

			if (status == ANEG_DONE &&
			    (aninfo.flags &
			     (MR_AN_COMPLETE | MR_LINK_OK |
			      MR_LP_ADV_FULL_DUPLEX))) {
				u32 local_adv, remote_adv;

				local_adv = ADVERTISE_PAUSE_CAP;
				remote_adv = 0;
				if (aninfo.flags & MR_LP_ADV_SYM_PAUSE)
					remote_adv |= LPA_PAUSE_CAP;
				if (aninfo.flags & MR_LP_ADV_ASYM_PAUSE)
					remote_adv |= LPA_PAUSE_ASYM;

				tg3_setup_flow_control(tp, local_adv, remote_adv);

				tp->tg3_flags |=
					TG3_FLAG_GOT_SERDES_FLOWCTL;
				current_link_up = 1;
			}
			for (i = 0; i < 60; i++) {
				udelay(20);
				tw32(MAC_STATUS,
				     (MAC_STATUS_SYNC_CHANGED |
				      MAC_STATUS_CFG_CHANGED));
				tr32(MAC_STATUS);
				udelay(40);
				if ((tr32(MAC_STATUS) &
				     (MAC_STATUS_SYNC_CHANGED |
				      MAC_STATUS_CFG_CHANGED)) == 0)
					break;
			}
			if (current_link_up == 0 &&
			    (tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED)) {
				current_link_up = 1;
			}
		} else {
			/* Forcing 1000FD link up. */
			current_link_up = 1;
		}
	}

	tp->mac_mode &= ~MAC_MODE_LINK_POLARITY;
	tw32(MAC_MODE, tp->mac_mode);
	tr32(MAC_MODE);
	udelay(40);

	tp->hw_status->status =
		(SD_STATUS_UPDATED |
		 (tp->hw_status->status & ~SD_STATUS_LINK_CHG));

	for (i = 0; i < 100; i++) {
		udelay(20);
		tw32(MAC_STATUS,
		     (MAC_STATUS_SYNC_CHANGED |
		      MAC_STATUS_CFG_CHANGED));
		tr32(MAC_STATUS);
		udelay(40);
		if ((tr32(MAC_STATUS) &
		     (MAC_STATUS_SYNC_CHANGED |
		      MAC_STATUS_CFG_CHANGED)) == 0)
			break;
	}

	if ((tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED) == 0)
		current_link_up = 0;

	if (current_link_up == 1) {
		tp->link_config.active_speed = SPEED_1000;
		tp->link_config.active_duplex = DUPLEX_FULL;
	} else {
		tp->link_config.active_speed = SPEED_INVALID;
		tp->link_config.active_duplex = DUPLEX_INVALID;
	}

	if (current_link_up != netif_carrier_ok(tp->dev)) {
		if (current_link_up)
			netif_carrier_on(tp->dev);
		else
			netif_carrier_off(tp->dev);
		tg3_link_report(tp);
	} else {
		u32 now_pause_cfg =
			tp->tg3_flags & (TG3_FLAG_RX_PAUSE |
					 TG3_FLAG_TX_PAUSE);
		if (orig_pause_cfg != now_pause_cfg ||
		    orig_active_speed != tp->link_config.active_speed ||
		    orig_active_duplex != tp->link_config.active_duplex)
			tg3_link_report(tp);
	}

	if ((tr32(MAC_STATUS) & MAC_STATUS_PCS_SYNCED) == 0) {
		tw32(MAC_MODE, tp->mac_mode | MAC_MODE_LINK_POLARITY);
		tr32(MAC_MODE);
		udelay(40);
		if (tp->tg3_flags & TG3_FLAG_INIT_COMPLETE) {
			tw32(MAC_MODE, tp->mac_mode);
			tr32(MAC_MODE);
			udelay(40);
		}
	}

	return 0;
}

static int tg3_setup_phy(struct tg3 *tp)
{
	int err;

	if (tp->phy_id == PHY_ID_SERDES) {
		err = tg3_setup_fiber_phy(tp);
	} else {
		err = tg3_setup_copper_phy(tp);
	}

	if (tp->link_config.active_speed == SPEED_1000 &&
	    tp->link_config.active_duplex == DUPLEX_HALF)
		tw32(MAC_TX_LENGTHS,
		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
		      (6 << TX_LENGTHS_IPG_SHIFT) |
		      (0xff << TX_LENGTHS_SLOT_TIME_SHIFT)));
	else
		tw32(MAC_TX_LENGTHS,
		     ((2 << TX_LENGTHS_IPG_CRS_SHIFT) |
		      (6 << TX_LENGTHS_IPG_SHIFT) |
		      (32 << TX_LENGTHS_SLOT_TIME_SHIFT)));

	return err;
}

/* Tigon3 never reports partial packet sends.  So we do not
 * need special logic to handle SKBs that have not had all
 * of their frags sent yet, like SunGEM does.
 */
static void tg3_tx(struct tg3 *tp)
{
	u32 hw_idx = tp->hw_status->idx[0].tx_consumer;
	u32 sw_idx = tp->tx_cons;

	while (sw_idx != hw_idx) {
		struct tx_ring_info *ri = &tp->tx_buffers[sw_idx];
		struct sk_buff *skb = ri->skb;
		int i;

		if (unlikely(skb == NULL))
			BUG();

		pci_unmap_single(tp->pdev,
				 pci_unmap_addr(ri, mapping),
				 (skb->len - skb->data_len),
				 PCI_DMA_TODEVICE);

		ri->skb = NULL;

		sw_idx = NEXT_TX(sw_idx);

		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
			if (unlikely(sw_idx == hw_idx))
				BUG();

			ri = &tp->tx_buffers[sw_idx];
			if (unlikely(ri->skb != NULL))
				BUG();

			pci_unmap_page(tp->pdev,
				       pci_unmap_addr(ri, mapping),
				       skb_shinfo(skb)->frags[i].size,
				       PCI_DMA_TODEVICE);

			sw_idx = NEXT_TX(sw_idx);
		}

		dev_kfree_skb_irq(skb);
	}

	tp->tx_cons = sw_idx;

	if (netif_queue_stopped(tp->dev) &&
	    (TX_BUFFS_AVAIL(tp) > TG3_TX_WAKEUP_THRESH))
		netif_wake_queue(tp->dev);
}

/* Returns size of skb allocated or < 0 on error.
 *
 * We only need to fill in the address because the other members
 * of the RX descriptor are invariant, see tg3_init_rings.
 *
 * Note the purposeful assymetry of cpu vs. chip accesses.  For
 * posting buffers we only dirty the first cache line of the RX
 * descriptor (containing the address).  Whereas for the RX status
 * buffers the cpu only reads the last cacheline of the RX descriptor
 * (to fetch the error flags, vlan tag, checksum, and opaque cookie).
 */
static int tg3_alloc_rx_skb(struct tg3 *tp, u32 opaque_key,
			    int src_idx, u32 dest_idx_unmasked)
{
	struct tg3_rx_buffer_desc *desc;
	struct ring_info *map, *src_map;
	struct sk_buff *skb;
	dma_addr_t mapping;
	int skb_size, dest_idx;

	src_map = NULL;
	switch (opaque_key) {
	case RXD_OPAQUE_RING_STD:
		dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
		desc = &tp->rx_std[dest_idx];
		map = &tp->rx_std_buffers[dest_idx];
		if (src_idx >= 0)
			src_map = &tp->rx_std_buffers[src_idx];
		skb_size = RX_PKT_BUF_SZ;
		break;

	case RXD_OPAQUE_RING_JUMBO:
		dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
		desc = &tp->rx_jumbo[dest_idx];
		map = &tp->rx_jumbo_buffers[dest_idx];
		if (src_idx >= 0)
			src_map = &tp->rx_jumbo_buffers[src_idx];
		skb_size = RX_JUMBO_PKT_BUF_SZ;
		break;

	default:
		return -EINVAL;
	};

	/* Do not overwrite any of the map or rp information
	 * until we are sure we can commit to a new buffer.
	 *
	 * Callers depend upon this behavior and assume that
	 * we leave everything unchanged if we fail.
	 */
	skb = dev_alloc_skb(skb_size);
	if (skb == NULL)
		return -ENOMEM;

	skb->dev = tp->dev;
	skb_reserve(skb, tp->rx_offset);

	mapping = pci_map_single(tp->pdev, skb->data,
				 skb_size - tp->rx_offset,
				 PCI_DMA_FROMDEVICE);

	map->skb = skb;
	pci_unmap_addr_set(map, mapping, mapping);

	if (src_map != NULL)
		src_map->skb = NULL;

	desc->addr_hi = ((u64)mapping >> 32);
	desc->addr_lo = ((u64)mapping & 0xffffffff);

	return skb_size;
}

/* We only need to move over in the address because the other
 * members of the RX descriptor are invariant.  See notes above
 * tg3_alloc_rx_skb for full details.
 */
static void tg3_recycle_rx(struct tg3 *tp, u32 opaque_key,
			   int src_idx, u32 dest_idx_unmasked)
{
	struct tg3_rx_buffer_desc *src_desc, *dest_desc;
	struct ring_info *src_map, *dest_map;
	int dest_idx;

	switch (opaque_key) {
	case RXD_OPAQUE_RING_STD:
		dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE;
		dest_desc = &tp->rx_std[dest_idx];
		dest_map = &tp->rx_std_buffers[dest_idx];
		src_desc = &tp->rx_std[src_idx];
		src_map = &tp->rx_std_buffers[src_idx];
		break;

	case RXD_OPAQUE_RING_JUMBO:
		dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE;
		dest_desc = &tp->rx_jumbo[dest_idx];
		dest_map = &tp->rx_jumbo_buffers[dest_idx];
		src_desc = &tp->rx_jumbo[src_idx];
		src_map = &tp->rx_jumbo_buffers[src_idx];
		break;

	default:
		return;
	};

	dest_map->skb = src_map->skb;
	pci_unmap_addr_set(dest_map, mapping,
			   pci_unmap_addr(src_map, mapping));
	dest_desc->addr_hi = src_desc->addr_hi;
	dest_desc->addr_lo = src_desc->addr_lo;

	src_map->skb = NULL;
}

#if TG3_VLAN_TAG_USED
static int tg3_vlan_rx(struct tg3 *tp, struct sk_buff *skb, u16 vlan_tag)
{
	return vlan_hwaccel_receive_skb(skb, tp->vlgrp, vlan_tag);
}
#endif

/* The RX ring scheme is composed of multiple rings which post fresh
 * buffers to the chip, and one special ring the chip uses to report
 * status back to the host.
 *
 * The special ring reports the status of received packets to the
 * host.  The chip does not write into the original descriptor the
 * RX buffer was obtained from.  The chip simply takes the original
 * descriptor as provided by the host, updates the status and length
 * field, then writes this into the next status ring entry.
 *
 * Each ring the host uses to post buffers to the chip is described
 * by a TG3_BDINFO entry in the chips SRAM area.  When a packet arrives,
 * it is first placed into the on-chip ram.  When the packet's length
 * is known, it walks down the TG3_BDINFO entries to select the ring.
 * Each TG3_BDINFO specifies a MAXLEN field and the first TG3_BDINFO
 * which is within the range of the new packet's length is chosen.
 *
 * The "seperate ring for rx status" scheme may sound queer, but it makes
 * sense from a cache coherency perspective.  If only the host writes
 * to the buffer post rings, and only the chip writes to the rx status
 * rings, then cache lines never move beyond shared-modified state.
 * If both the host and chip were to write into the same ring, cache line
 * eviction could occur since both entities want it in an exclusive state.
 */
static int tg3_rx(struct tg3 *tp, int budget)
{
	u32 work_mask;
	u32 rx_rcb_ptr = tp->rx_rcb_ptr;
	u16 hw_idx, sw_idx;
	int received;

	hw_idx = tp->hw_status->idx[0].rx_producer;
	sw_idx = rx_rcb_ptr % TG3_RX_RCB_RING_SIZE;
	work_mask = 0;
	received = 0;
	while (sw_idx != hw_idx && budget > 0) {
		struct tg3_rx_buffer_desc *desc = &tp->rx_rcb[sw_idx];
		unsigned int len;
		struct sk_buff *skb;
		dma_addr_t dma_addr;
		u32 opaque_key, desc_idx, *post_ptr;

		desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK;
		opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK;
		if (opaque_key == RXD_OPAQUE_RING_STD) {
			dma_addr = pci_unmap_addr(&tp->rx_std_buffers[desc_idx],
						  mapping);
			skb = tp->rx_std_buffers[desc_idx].skb;
			post_ptr = &tp->rx_std_ptr;
		} else if (opaque_key == RXD_OPAQUE_RING_JUMBO) {
			dma_addr = pci_unmap_addr(&tp->rx_jumbo_buffers[desc_idx],
						  mapping);
			skb = tp->rx_jumbo_buffers[desc_idx].skb;
			post_ptr = &tp->rx_jumbo_ptr;
		}
		else {
			goto next_pkt_nopost;
		}

		work_mask |= opaque_key;

		if ((desc->err_vlan & RXD_ERR_MASK) != 0 &&
		    (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) {
		drop_it:
			tg3_recycle_rx(tp, opaque_key,
				       desc_idx, *post_ptr);
		drop_it_no_recycle:
			/* Other statistics kept track of by card. */
			tp->net_stats.rx_dropped++;
			goto next_pkt;
		}

		len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - 4; /* omit crc */

		if (len > RX_COPY_THRESHOLD) {
			int skb_size;

			skb_size = tg3_alloc_rx_skb(tp, opaque_key,
						    desc_idx, *post_ptr);
			if (skb_size < 0)
				goto drop_it;

			pci_unmap_single(tp->pdev, dma_addr,
					 skb_size - tp->rx_offset,
					 PCI_DMA_FROMDEVICE);

			skb_put(skb, len);
		} else {
			struct sk_buff *copy_skb;

			tg3_recycle_rx(tp, opaque_key,
				       desc_idx, *post_ptr);

			copy_skb = dev_alloc_skb(len + 2);
			if (copy_skb == NULL)
				goto drop_it_no_recycle;

			copy_skb->dev = tp->dev;
			skb_reserve(copy_skb, 2);
			skb_put(copy_skb, len);
			pci_dma_sync_single(tp->pdev, dma_addr, len, PCI_DMA_FROMDEVICE);
			memcpy(copy_skb->data, skb->data, len);

			/* We'll reuse the original ring buffer. */
			skb = copy_skb;
		}

		if ((tp->tg3_flags & TG3_FLAG_RX_CHECKSUMS) &&
		    (desc->type_flags & RXD_FLAG_TCPUDP_CSUM) &&
		    (((desc->ip_tcp_csum & RXD_TCPCSUM_MASK)
		      >> RXD_TCPCSUM_SHIFT) == 0xffff))
			skb->ip_summed = CHECKSUM_UNNECESSARY;
		else
			skb->ip_summed = CHECKSUM_NONE;

		skb->protocol = eth_type_trans(skb, tp->dev);
#if TG3_VLAN_TAG_USED
		if (tp->vlgrp != NULL &&
		    desc->type_flags & RXD_FLAG_VLAN) {
			tg3_vlan_rx(tp, skb,
				    desc->err_vlan & RXD_VLAN_MASK);
		} else
#endif
#ifdef NAPI
			netif_receive_skb(skb);
#else
			netif_rx(skb);
#endif
		tp->dev->last_rx = jiffies;
		received++;
		budget--;

next_pkt:
		(*post_ptr)++;
next_pkt_nopost:
		rx_rcb_ptr++;
		sw_idx = rx_rcb_ptr % TG3_RX_RCB_RING_SIZE;
	}

	/* ACK the status ring. */
	tp->rx_rcb_ptr = rx_rcb_ptr;
	tw32_mailbox(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW,
		     (rx_rcb_ptr % TG3_RX_RCB_RING_SIZE));
	if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
		tr32(MAILBOX_RCVRET_CON_IDX_0 + TG3_64BIT_REG_LOW);

	/* Refill RX ring(s). */
	if (work_mask & RXD_OPAQUE_RING_STD) {
		sw_idx = tp->rx_std_ptr % TG3_RX_RING_SIZE;
		tw32_mailbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW,
			     sw_idx);
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW);
	}
	if (work_mask & RXD_OPAQUE_RING_JUMBO) {
		sw_idx = tp->rx_jumbo_ptr % TG3_RX_JUMBO_RING_SIZE;
		tw32_mailbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW,
			     sw_idx);
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW);
	}

	return received;
}

static int tg3_poll(struct net_device *netdev, int *budget)
{
	struct tg3 *tp = netdev->priv;
	struct tg3_hw_status *sblk = tp->hw_status;
	int done;
#ifdef NAPI
	unsigned long flags;
	spin_lock_irqsave(&tp->lock, flags);
#endif

	/* handle link change and other phy events */
	if (!(tp->tg3_flags &
	      (TG3_FLAG_USE_LINKCHG_REG |
	       TG3_FLAG_POLL_SERDES))) {
		if (sblk->status & SD_STATUS_LINK_CHG) {
			sblk->status = SD_STATUS_UPDATED |
				(sblk->status & ~SD_STATUS_LINK_CHG);
			tg3_setup_phy(tp);
		}
	}

	/* run TX completion thread */
	if (sblk->idx[0].tx_consumer != tp->tx_cons) {
		spin_lock(&tp->tx_lock);
		tg3_tx(tp);
		spin_unlock(&tp->tx_lock);
	}

#ifdef NAPI
	spin_unlock_irqrestore(&tp->lock, flags);
#endif

	/* run RX thread, within the bounds set by NAPI.
	 * All RX "locking" is done by ensuring outside
	 * code synchronizes with dev->poll()
	 */
	done = 1;
	if (sblk->idx[0].rx_producer != tp->rx_rcb_ptr) {
		int work_done;
#ifdef NAPI
		int orig_budget = *budget;

		if (orig_budget > netdev->quota)
			orig_budget = netdev->quota;

		work_done = tg3_rx(tp, orig_budget);

		*budget -= work_done;
		netdev->quota -= work_done;

		if (work_done >= orig_budget)
			done = 0;
#else
		work_done = tg3_rx(tp, 1000);
#endif
	}
#ifdef NAPI
	/* if no more work, tell net stack and NIC we're done */
	if (done) {
		spin_lock_irqsave(&tp->lock, flags);
		__netif_rx_complete(netdev);
		tg3_enable_ints(tp);
		spin_unlock_irqrestore(&tp->lock, flags);
	}
#endif
	return (done ? 0 : 1);
}

static inline unsigned int tg3_has_work(struct net_device *dev, struct tg3 *tp)
{
	struct tg3_hw_status *sblk = tp->hw_status;
	unsigned int work_exists = 0;

	/* check for phy events */
	if (!(tp->tg3_flags &
	      (TG3_FLAG_USE_LINKCHG_REG |
	       TG3_FLAG_POLL_SERDES))) {
		if (sblk->status & SD_STATUS_LINK_CHG)
			work_exists = 1;
	}
	/* check for RX/TX work to do */
	if (sblk->idx[0].tx_consumer != tp->tx_cons ||
	    sblk->idx[0].rx_producer != tp->rx_rcb_ptr)
		work_exists = 1;

	return work_exists;
}

static void tg3_interrupt(int irq, void *dev_id, struct pt_regs *regs)
{
	struct net_device *dev = dev_id;
	struct tg3 *tp = dev->priv;
#ifdef NAPI
	struct tg3_hw_status *sblk = tp->hw_status;
#endif
	unsigned long flags;

	spin_lock_irqsave(&tp->lock, flags);
#ifdef NAPI
	if (sblk->status & SD_STATUS_UPDATED) {
		/*
		 * writing any value to intr-mbox-0 clears PCI INTA# and
		 * chip-internal interrupt pending events.
		 * writing non-zero to intr-mbox-0 additional tells the
		 * NIC to stop sending us irqs, engaging "in-intr-handler"
		 * event coalescing.
		 */
		tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
			     0x00000001);
		/*
		 * Flush PCI write.  This also guarantees that our
		 * status block has been flushed to host memory.
		 */
		tr32(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW);
		sblk->status &= ~SD_STATUS_UPDATED;

		if (likely(tg3_has_work(dev, tp)))
			netif_rx_schedule(dev);		/* schedule NAPI poll */
		else {
			/* no work, shared interrupt perhaps?  re-enable
			 * interrupts, and flush that PCI write
			 */
			tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
			     	0x00000000);
			tr32(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW);
		}
	}
#else
	{
		int budget = 1000;
		tg3_poll( dev, &budget );

		tw32_mailbox(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW,
		       0x00000000);
		tr32(MAILBOX_INTERRUPT_0 + TG3_64BIT_REG_LOW);
	}
#endif
	spin_unlock_irqrestore(&tp->lock, flags);
}

static void tg3_init_rings(struct tg3 *);
static int tg3_init_hw(struct tg3 *);
static int tg3_halt(struct tg3 *);

static void tg3_reset_task(void *_data)
{
	struct tg3 *tp = _data;
	unsigned int restart_timer;

	tg3_netif_stop(tp);

	spin_lock_irq(&tp->lock);
	spin_lock(&tp->tx_lock);

	restart_timer = tp->tg3_flags2 & TG3_FLG2_RESTART_TIMER;
	tp->tg3_flags2 &= ~TG3_FLG2_RESTART_TIMER;

	tg3_halt(tp);
	tg3_init_rings(tp);
	tg3_init_hw(tp);

	spin_unlock(&tp->tx_lock);
	spin_unlock_irq(&tp->lock);

	tg3_netif_start(tp);

	if (restart_timer)
		mod_timer(&tp->timer, jiffies + 1);
}

static void tg3_tx_timeout(struct net_device *dev)
{
        struct tg3 *tp = dev->priv;

        printk(KERN_ERR PFX "%s: transmit timed out, resetting\n",
               dev->name);

        spin_lock_irq(&tp->lock);
        spin_lock(&tp->tx_lock);

        tg3_halt(tp);
        tg3_init_rings(tp);
        tg3_init_hw(tp);

        spin_unlock(&tp->tx_lock);
        spin_unlock_irq(&tp->lock);

        netif_wake_queue(dev);
}

#if !PCI_DMA_BUS_IS_PHYS
static void tg3_set_txd_addr(struct tg3 *tp, int entry, dma_addr_t mapping)
{
	if (tp->tg3_flags & TG3_FLAG_HOST_TXDS) {
		struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];

		txd->addr_hi = ((u64) mapping >> 32);
		txd->addr_lo = ((u64) mapping & 0xffffffff);
	} else {
		unsigned long txd;

		txd = (tp->regs +
		       NIC_SRAM_WIN_BASE +
		       NIC_SRAM_TX_BUFFER_DESC);
		txd += (entry * TXD_SIZE);

		if (sizeof(dma_addr_t) != sizeof(u32))
			writel(((u64) mapping >> 32),
			       txd + TXD_ADDR + TG3_64BIT_REG_HIGH);

		writel(((u64) mapping & 0xffffffff),
		       txd + TXD_ADDR + TG3_64BIT_REG_LOW);
	}
}
#endif

static void tg3_set_txd(struct tg3 *, int, dma_addr_t, int, u32, u32);

static int tigon3_4gb_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb,
				       u32 guilty_entry, int guilty_len,
				       u32 last_plus_one, u32 *start, u32 mss)
{
	dma_addr_t new_addr;
	u32 entry = *start;
	int i;

#if !PCI_DMA_BUS_IS_PHYS
	/* IOMMU, just map the guilty area again which is guarenteed to
	 * use different addresses.
	 */

	i = 0;
	while (entry != guilty_entry) {
		entry = NEXT_TX(entry);
		i++;
	}
	if (i == 0) {
		new_addr = pci_map_single(tp->pdev, skb->data, guilty_len,
					  PCI_DMA_TODEVICE);
	} else {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];

		new_addr = pci_map_page(tp->pdev,
					frag->page, frag->page_offset,
					guilty_len, PCI_DMA_TODEVICE);
	}
	pci_unmap_single(tp->pdev, pci_unmap_addr(&tp->tx_buffers[guilty_entry],
						  mapping),
			 guilty_len, PCI_DMA_TODEVICE);
	tg3_set_txd_addr(tp, guilty_entry, new_addr);
	pci_unmap_addr_set(&tp->tx_buffers[guilty_entry], mapping,
			   new_addr);
	*start = last_plus_one;
#else
	/* Oh well, no IOMMU, have to allocate a whole new SKB. */
	struct sk_buff *new_skb = skb_copy(skb, GFP_ATOMIC);

	if (!new_skb) {
		dev_kfree_skb(skb);
		return -1;
	}

	/* New SKB is guarenteed to be linear. */
	entry = *start;
	new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len,
				  PCI_DMA_TODEVICE);
	tg3_set_txd(tp, entry, new_addr, new_skb->len,
		    (skb->ip_summed == CHECKSUM_HW) ?
		    TXD_FLAG_TCPUDP_CSUM : 0, 1 | (mss << 1));
	*start = NEXT_TX(entry);

	/* Now clean up the sw ring entries. */
	i = 0;
	while (entry != last_plus_one) {
		int len;

		if (i == 0)
			len = skb->len - skb->data_len;
		else
			len = skb_shinfo(skb)->frags[i-1].size;
		pci_unmap_single(tp->pdev,
				 pci_unmap_addr(&tp->tx_buffers[entry], mapping),
				 len, PCI_DMA_TODEVICE);
		if (i == 0) {
			tp->tx_buffers[entry].skb = new_skb;
			pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, new_addr);
		} else {
			tp->tx_buffers[entry].skb = NULL;
		}
		entry = NEXT_TX(entry);
	}

	dev_kfree_skb(skb);
#endif

	return 0;
}

static void tg3_set_txd(struct tg3 *tp, int entry,
			dma_addr_t mapping, int len, u32 flags,
			u32 mss_and_is_end)
{
	int is_end = (mss_and_is_end & 0x1);
	u32 mss = (mss_and_is_end >> 1);
	u32 vlan_tag = 0;

	if (is_end)
		flags |= TXD_FLAG_END;
	if (flags & TXD_FLAG_VLAN) {
		vlan_tag = flags >> 16;
		flags &= 0xffff;
	}
	vlan_tag |= (mss << TXD_MSS_SHIFT);
	if (tp->tg3_flags & TG3_FLAG_HOST_TXDS) {
		struct tg3_tx_buffer_desc *txd = &tp->tx_ring[entry];

		txd->addr_hi = ((u64) mapping >> 32);
		txd->addr_lo = ((u64) mapping & 0xffffffff);
		txd->len_flags = (len << TXD_LEN_SHIFT) | flags;
		txd->vlan_tag = vlan_tag << TXD_VLAN_TAG_SHIFT;
	} else {
		struct tx_ring_info *txr = &tp->tx_buffers[entry];
		unsigned long txd;

		txd = (tp->regs +
		       NIC_SRAM_WIN_BASE +
		       NIC_SRAM_TX_BUFFER_DESC);
		txd += (entry * TXD_SIZE);

		/* Save some PIOs */
		if (sizeof(dma_addr_t) != sizeof(u32))
			writel(((u64) mapping >> 32),
			       txd + TXD_ADDR + TG3_64BIT_REG_HIGH);

		writel(((u64) mapping & 0xffffffff),
		       txd + TXD_ADDR + TG3_64BIT_REG_LOW);
		writel(len << TXD_LEN_SHIFT | flags, txd + TXD_LEN_FLAGS);
		if (txr->prev_vlan_tag != vlan_tag) {
			writel(vlan_tag << TXD_VLAN_TAG_SHIFT, txd + TXD_VLAN_TAG);
			txr->prev_vlan_tag = vlan_tag;
		}
	}
}

static inline int tg3_4g_overflow_test(dma_addr_t mapping, int len)
{
	u32 base = (u32) mapping & 0xffffffff;

	return ((base > 0xffffdcc0) &&
		((u64) mapping >> 32) == 0 &&
		(base + len + 8 < base));
}

static int tg3_start_xmit_4gbug(struct sk_buff *skb, struct net_device *dev)
{
	struct tg3 *tp = dev->priv;
	dma_addr_t mapping;
	unsigned int i;
	u32 len, entry, base_flags, mss;
	int would_hit_hwbug;
	unsigned long flags;

	len = (skb->len - skb->data_len);

	/* No BH disabling for tx_lock here.  We are running in BH disabled
	 * context and TX reclaim runs via tp->poll inside of a software
	 * interrupt.  Rejoice!
	 *
	 * Actually, things are not so simple.  If we are to take a hw
	 * IRQ here, we can deadlock, consider:
	 *
	 *       CPU1		CPU2
	 *   tg3_start_xmit
	 *   take tp->tx_lock
	 *			tg3_timer
	 *			take tp->lock
	 *   tg3_interrupt
	 *   spin on tp->lock
	 *			spin on tp->tx_lock
	 *
	 * So we really do need to disable interrupts when taking
	 * tx_lock here.
	 */
	spin_lock_irqsave(&tp->tx_lock, flags);

	/* This is a hard error, log it. */
	if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
		netif_stop_queue(dev);
		spin_unlock_irqrestore(&tp->tx_lock, flags);
		printk(KERN_ERR PFX "%s: BUG! Tx Ring full when queue awake!\n",
		       dev->name);
		return 1;
	}

	entry = tp->tx_prod;
	base_flags = 0;
	if (skb->ip_summed == CHECKSUM_HW)
		base_flags |= TXD_FLAG_TCPUDP_CSUM;
#if TG3_DO_TSO != 0
	if ((mss = skb_shinfo(skb)->tso_size) != 0)
		base_flags |= (TXD_FLAG_CPU_PRE_DMA |
			       TXD_FLAG_CPU_POST_DMA);
#else
	mss = 0;
#endif
#if TG3_VLAN_TAG_USED
	if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
		base_flags |= (TXD_FLAG_VLAN |
			       (vlan_tx_tag_get(skb) << 16));
#endif

	/* Queue skb data, a.k.a. the main skb fragment. */
	mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);

	tp->tx_buffers[entry].skb = skb;
	pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);

	would_hit_hwbug = 0;

	if (tg3_4g_overflow_test(mapping, len))
		would_hit_hwbug = entry + 1;

	tg3_set_txd(tp, entry, mapping, len, base_flags,
		    (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));

	entry = NEXT_TX(entry);

	/* Now loop through additional data fragments, and queue them. */
	if (skb_shinfo(skb)->nr_frags > 0) {
		unsigned int i, last;

		last = skb_shinfo(skb)->nr_frags - 1;
		for (i = 0; i <= last; i++) {
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

			len = frag->size;
			mapping = pci_map_page(tp->pdev,
					       frag->page,
					       frag->page_offset,
					       len, PCI_DMA_TODEVICE);

			tp->tx_buffers[entry].skb = NULL;
			pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);

			if (tg3_4g_overflow_test(mapping, len)) {
				/* Only one should match. */
				if (would_hit_hwbug)
					BUG();
				would_hit_hwbug = entry + 1;
			}

			tg3_set_txd(tp, entry, mapping, len,
				    base_flags, (i == last) | (mss << 1));

			entry = NEXT_TX(entry);
		}
	}

	if (would_hit_hwbug) {
		u32 last_plus_one = entry;
		u32 start;
		unsigned int len = 0;

		would_hit_hwbug -= 1;
		entry = entry - 1 - skb_shinfo(skb)->nr_frags;
		entry &= (TG3_TX_RING_SIZE - 1);
		start = entry;
		i = 0;
		while (entry != last_plus_one) {
			if (i == 0)
				len = skb->len - skb->data_len;
			else
				len = skb_shinfo(skb)->frags[i-1].size;

			if (entry == would_hit_hwbug)
				break;

			i++;
			entry = NEXT_TX(entry);

		}

		/* If the workaround fails due to memory/mapping
		 * failure, silently drop this packet.
		 */
		if (tigon3_4gb_hwbug_workaround(tp, skb,
						entry, len,
						last_plus_one,
						&start, mss))
			goto out_unlock;

		entry = start;
	}

	/* Packets are ready, update Tx producer idx local and on card. */
	if (tp->tg3_flags & TG3_FLAG_HOST_TXDS) {
		tw32_mailbox((MAILBOX_SNDHOST_PROD_IDX_0 +
			      TG3_64BIT_REG_LOW), entry);
		if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
			tw32_mailbox((MAILBOX_SNDHOST_PROD_IDX_0 +
				      TG3_64BIT_REG_LOW), entry);
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_SNDHOST_PROD_IDX_0 +
			     TG3_64BIT_REG_LOW);
	} else {
		/* First, make sure tg3 sees last descriptor fully
		 * in SRAM.
		 */
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_SNDNIC_PROD_IDX_0 +
			     TG3_64BIT_REG_LOW);

		tw32_mailbox((MAILBOX_SNDNIC_PROD_IDX_0 +
			      TG3_64BIT_REG_LOW), entry);
		if (tp->tg3_flags & TG3_FLAG_TXD_MBOX_HWBUG)
			tw32_mailbox((MAILBOX_SNDNIC_PROD_IDX_0 +
				      TG3_64BIT_REG_LOW), entry);

		/* Now post the mailbox write itself.  */
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_SNDNIC_PROD_IDX_0 +
			     TG3_64BIT_REG_LOW);
	}

	tp->tx_prod = entry;
	if (TX_BUFFS_AVAIL(tp) <= (MAX_SKB_FRAGS + 1))
		netif_stop_queue(dev);

out_unlock:
	spin_unlock_irqrestore(&tp->tx_lock, flags);

	dev->trans_start = jiffies;

	return 0;
}

static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct tg3 *tp = dev->priv;
	dma_addr_t mapping;
	u32 len, entry, base_flags, mss;
	unsigned long flags;

	len = (skb->len - skb->data_len);

	/* No BH disabling for tx_lock here.  We are running in BH disabled
	 * context and TX reclaim runs via tp->poll inside of a software
	 * interrupt.  Rejoice!
	 *
	 * Actually, things are not so simple.  If we are to take a hw
	 * IRQ here, we can deadlock, consider:
	 *
	 *       CPU1		CPU2
	 *   tg3_start_xmit
	 *   take tp->tx_lock
	 *			tg3_timer
	 *			take tp->lock
	 *   tg3_interrupt
	 *   spin on tp->lock
	 *			spin on tp->tx_lock
	 *
	 * So we really do need to disable interrupts when taking
	 * tx_lock here.
	 */
	spin_lock_irqsave(&tp->tx_lock, flags);

	/* This is a hard error, log it. */
	if (unlikely(TX_BUFFS_AVAIL(tp) <= (skb_shinfo(skb)->nr_frags + 1))) {
		netif_stop_queue(dev);
		spin_unlock_irqrestore(&tp->tx_lock, flags);
		printk(KERN_ERR PFX "%s: BUG! Tx Ring full when queue awake!\n",
		       dev->name);
		return 1;
	}

	entry = tp->tx_prod;
	base_flags = 0;
	if (skb->ip_summed == CHECKSUM_HW)
		base_flags |= TXD_FLAG_TCPUDP_CSUM;
#if TG3_DO_TSO != 0
	if ((mss = skb_shinfo(skb)->tso_size) != 0)
		base_flags |= (TXD_FLAG_CPU_PRE_DMA |
			       TXD_FLAG_CPU_POST_DMA);
#else
	mss = 0;
#endif
#if TG3_VLAN_TAG_USED
	if (tp->vlgrp != NULL && vlan_tx_tag_present(skb))
		base_flags |= (TXD_FLAG_VLAN |
			       (vlan_tx_tag_get(skb) << 16));
#endif

	/* Queue skb data, a.k.a. the main skb fragment. */
	mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE);

	tp->tx_buffers[entry].skb = skb;
	pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);

	tg3_set_txd(tp, entry, mapping, len, base_flags,
		    (skb_shinfo(skb)->nr_frags == 0) | (mss << 1));

	entry = NEXT_TX(entry);

	/* Now loop through additional data fragments, and queue them. */
	if (skb_shinfo(skb)->nr_frags > 0) {
		unsigned int i, last;

		last = skb_shinfo(skb)->nr_frags - 1;
		for (i = 0; i <= last; i++) {
			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];


			len = frag->size;
			mapping = pci_map_page(tp->pdev,
					       frag->page,
					       frag->page_offset,
					       len, PCI_DMA_TODEVICE);

			tp->tx_buffers[entry].skb = NULL;
			pci_unmap_addr_set(&tp->tx_buffers[entry], mapping, mapping);

			tg3_set_txd(tp, entry, mapping, len,
				    base_flags, (i == last) | (mss << 1));

			entry = NEXT_TX(entry);
		}
	}

	/* Packets are ready, update Tx producer idx local and on card.
	 * We know this is not a 5700 (by virtue of not being a chip
	 * requiring the 4GB overflow workaround) so we can safely omit
	 * the double-write bug tests.
	 */
	if (tp->tg3_flags & TG3_FLAG_HOST_TXDS) {
		tw32_mailbox((MAILBOX_SNDHOST_PROD_IDX_0 +
			      TG3_64BIT_REG_LOW), entry);
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_SNDHOST_PROD_IDX_0 +
			     TG3_64BIT_REG_LOW);
	} else {
		/* First, make sure tg3 sees last descriptor fully
		 * in SRAM.
		 */
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_SNDNIC_PROD_IDX_0 +
			     TG3_64BIT_REG_LOW);

		tw32_mailbox((MAILBOX_SNDNIC_PROD_IDX_0 +
			      TG3_64BIT_REG_LOW), entry);

		/* Now post the mailbox write itself.  */
		if (tp->tg3_flags & TG3_FLAG_MBOX_WRITE_REORDER)
			tr32(MAILBOX_SNDNIC_PROD_IDX_0 +
			     TG3_64BIT_REG_LOW);
	}