ixgbe: Implement PCI SR-IOV sysfs callback operation

Implement callbacks in the driver for the new PCI bus driver
interface that allows the user to enable/disable SR-IOV VFs
in a device via the sysfs interface.

Signed-off-by: Greg Rose <gregory.v.rose@intel.com>
CC: Don Dutile <ddutile@redhat.com>
Tested-by: Phil Schmitt <phillip.j.schmitt@intel.com>
Tested-by: Sibai Li <sibai.li@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
index cd4c83b1a977d84ace2b4068f40fe6e5d4fd39d6..8371ae4265fe6d3ccc1101bf3f07df9c5205b8a9 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h
@@ -778,5 +778,8 @@ extern int ixgbe_ptp_hwtstamp_ioctl(struct ixgbe_adapter *adapter,
 extern void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter);
 extern void ixgbe_ptp_reset(struct ixgbe_adapter *adapter);
 extern void ixgbe_ptp_check_pps_event(struct ixgbe_adapter *adapter, u32 eicr);
 extern void ixgbe_ptp_start_cyclecounter(struct ixgbe_adapter *adapter);
 extern void ixgbe_ptp_reset(struct ixgbe_adapter *adapter);
 extern void ixgbe_ptp_check_pps_event(struct ixgbe_adapter *adapter, u32 eicr);

+#ifdef CONFIG_PCI_IOV
+void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter);
+#endif

 
 #endif /* _IXGBE_H_ */
 
 #endif /* _IXGBE_H_ */

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 2472dc85db0d3b070b3ad6ec58a119ff7f09a33a..5e3ed33215f796f7e3516902c247cd356b65d690 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -6836,6 +6836,26 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc)
 }
 
 #endif /* CONFIG_IXGBE_DCB */
 }
 
 #endif /* CONFIG_IXGBE_DCB */

+#ifdef CONFIG_PCI_IOV
+void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)
+{
+       struct net_device *netdev = adapter->netdev;
+
+       rtnl_lock();
+#ifdef CONFIG_IXGBE_DCB
+       ixgbe_setup_tc(netdev, netdev_get_num_tc(netdev));
+#else
+       if (netif_running(netdev))
+               ixgbe_close(netdev);
+       ixgbe_clear_interrupt_scheme(adapter);
+       ixgbe_init_interrupt_scheme(adapter);
+       if (netif_running(netdev))
+               ixgbe_open(netdev);
+#endif
+       rtnl_unlock();
+}
+
+#endif

 void ixgbe_do_reset(struct net_device *netdev)
 {
        struct ixgbe_adapter *adapter = netdev_priv(netdev);
 void ixgbe_do_reset(struct net_device *netdev)
 {
        struct ixgbe_adapter *adapter = netdev_priv(netdev);


@@ -7625,8 +7645,14 @@ static void ixgbe_remove(struct pci_dev *pdev)
        if (netdev->reg_state == NETREG_REGISTERED)
                unregister_netdev(netdev);
 
        if (netdev->reg_state == NETREG_REGISTERED)
                unregister_netdev(netdev);
 

-       ixgbe_disable_sriov(adapter);
-
+#ifdef CONFIG_PCI_IOV
+       /*
+        * Only disable SR-IOV on unload if the user specified the now
+        * deprecated max_vfs module parameter.
+        */
+       if (max_vfs)
+               ixgbe_disable_sriov(adapter);
+#endif

        ixgbe_clear_interrupt_scheme(adapter);
 
        ixgbe_release_hw_control(adapter);
        ixgbe_clear_interrupt_scheme(adapter);
 
        ixgbe_release_hw_control(adapter);


@@ -7840,6 +7866,7 @@ static struct pci_driver ixgbe_driver = {
        .resume   = ixgbe_resume,
 #endif
        .shutdown = ixgbe_shutdown,
        .resume   = ixgbe_resume,
 #endif
        .shutdown = ixgbe_shutdown,

+       .sriov_configure = ixgbe_pci_sriov_configure,

        .err_handler = &ixgbe_err_handler
 };
 
        .err_handler = &ixgbe_err_handler
 };
 

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
index 8a284ac4b4009b590e2dd39580b0370dd144ad56..ee3507f0ea530c309475c9799f2b2067e998501f 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c
@@ -207,11 +207,12 @@ static bool ixgbe_vfs_are_assigned(struct ixgbe_adapter *adapter)
 }
 
 #endif /* #ifdef CONFIG_PCI_IOV */
 }
 
 #endif /* #ifdef CONFIG_PCI_IOV */

-void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
+int ixgbe_disable_sriov(struct ixgbe_adapter *adapter)

 {
        struct ixgbe_hw *hw = &adapter->hw;
        u32 gpie;
        u32 vmdctl;
 {
        struct ixgbe_hw *hw = &adapter->hw;
        u32 gpie;
        u32 vmdctl;

+       int rss;

 
        /* set num VFs to 0 to prevent access to vfinfo */
        adapter->num_vfs = 0;
 
        /* set num VFs to 0 to prevent access to vfinfo */
        adapter->num_vfs = 0;


@@ -226,7 +227,7 @@ void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
 
        /* if SR-IOV is already disabled then there is nothing to do */
        if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
 
        /* if SR-IOV is already disabled then there is nothing to do */
        if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))

-               return;
+               return 0;

 
 #ifdef CONFIG_PCI_IOV
        /*
 
 #ifdef CONFIG_PCI_IOV
        /*


@@ -236,7 +237,7 @@ void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
         */
        if (ixgbe_vfs_are_assigned(adapter)) {
                e_dev_warn("Unloading driver while VFs are assigned - VFs will not be deallocated\n");
         */
        if (ixgbe_vfs_are_assigned(adapter)) {
                e_dev_warn("Unloading driver while VFs are assigned - VFs will not be deallocated\n");

-               return;
+               return -EPERM;

        }
        /* disable iov and allow time for transactions to clear */
        pci_disable_sriov(adapter->pdev);
        }
        /* disable iov and allow time for transactions to clear */
        pci_disable_sriov(adapter->pdev);


@@ -259,10 +260,94 @@ void ixgbe_disable_sriov(struct ixgbe_adapter *adapter)
                adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
        adapter->ring_feature[RING_F_VMDQ].offset = 0;
 
                adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
        adapter->ring_feature[RING_F_VMDQ].offset = 0;
 

+       rss = min_t(int, IXGBE_MAX_RSS_INDICES, num_online_cpus());
+       adapter->ring_feature[RING_F_RSS].limit = rss;
+

        /* take a breather then clean up driver data */
        msleep(100);
 
        adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;
        /* take a breather then clean up driver data */
        msleep(100);
 
        adapter->flags &= ~IXGBE_FLAG_SRIOV_ENABLED;

+       return 0;
+}
+
+static int ixgbe_pci_sriov_enable(struct pci_dev *dev, int num_vfs)
+{
+#ifdef CONFIG_PCI_IOV
+       struct ixgbe_adapter *adapter = pci_get_drvdata(dev);
+       int err = 0;
+       int i;
+       int pre_existing_vfs = pci_num_vf(dev);
+
+       if (pre_existing_vfs && pre_existing_vfs != num_vfs)
+               err = ixgbe_disable_sriov(adapter);
+       else if (pre_existing_vfs && pre_existing_vfs == num_vfs)
+               goto out;
+
+       if (err)
+               goto err_out;
+
+       /* While the SR-IOV capability structure reports total VFs to be
+        * 64 we limit the actual number that can be allocated to 63 so
+        * that some transmit/receive resources can be reserved to the
+        * PF.  The PCI bus driver already checks for other values out of
+        * range.
+        */
+       if (num_vfs > 63) {
+               err = -EPERM;
+               goto err_out;
+       }
+
+       adapter->num_vfs = num_vfs;
+
+       err = __ixgbe_enable_sriov(adapter);
+       if (err)
+               goto err_out;
+
+       for (i = 0; i < adapter->num_vfs; i++)
+               ixgbe_vf_configuration(dev, (i | 0x10000000));
+
+       err = pci_enable_sriov(dev, num_vfs);
+       if (err) {
+               e_dev_warn("Failed to enable PCI sriov: %d\n", err);
+               goto err_out;
+       }
+       ixgbe_sriov_reinit(adapter);
+
+out:
+       return num_vfs;
+
+err_out:
+       return err;
+#endif
+       return 0;
+}
+
+static int ixgbe_pci_sriov_disable(struct pci_dev *dev)
+{
+       struct ixgbe_adapter *adapter = pci_get_drvdata(dev);
+       int err;
+       u32 current_flags = adapter->flags;
+
+       err = ixgbe_disable_sriov(adapter);
+
+       /* Only reinit if no error and state changed */
+       if (!err && current_flags != adapter->flags) {
+               /* ixgbe_disable_sriov() doesn't clear VMDQ flag */
+               adapter->flags &= ~IXGBE_FLAG_VMDQ_ENABLED;
+#ifdef CONFIG_PCI_IOV
+               ixgbe_sriov_reinit(adapter);
+#endif
+       }
+
+       return err;
+}
+
+int ixgbe_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
+{
+       if (num_vfs == 0)
+               return ixgbe_pci_sriov_disable(dev);
+       else
+               return ixgbe_pci_sriov_enable(dev, num_vfs);

 }
 
 static int ixgbe_set_vf_multicasts(struct ixgbe_adapter *adapter,
 }
 
 static int ixgbe_set_vf_multicasts(struct ixgbe_adapter *adapter,

diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
index f693469b81ef061760f1f916de83a67a57e76129..008f9cea68d118074a05418e025cef4a177b0a81 100644 (file)
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.h
@@ -41,10 +41,11 @@ int ixgbe_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, bool setting);
 int ixgbe_ndo_get_vf_config(struct net_device *netdev,
                            int vf, struct ifla_vf_info *ivi);
 void ixgbe_check_vf_rate_limit(struct ixgbe_adapter *adapter);
 int ixgbe_ndo_get_vf_config(struct net_device *netdev,
                            int vf, struct ifla_vf_info *ivi);
 void ixgbe_check_vf_rate_limit(struct ixgbe_adapter *adapter);

-void ixgbe_disable_sriov(struct ixgbe_adapter *adapter);
+int ixgbe_disable_sriov(struct ixgbe_adapter *adapter);

 #ifdef CONFIG_PCI_IOV
 void ixgbe_enable_sriov(struct ixgbe_adapter *adapter);
 #endif
 #ifdef CONFIG_PCI_IOV
 void ixgbe_enable_sriov(struct ixgbe_adapter *adapter);
 #endif

+int ixgbe_pci_sriov_configure(struct pci_dev *dev, int num_vfs);

 
 static inline void ixgbe_set_vmvir(struct ixgbe_adapter *adapter,
                                   u16 vid, u16 qos, u32 vf)
 
 static inline void ixgbe_set_vmvir(struct ixgbe_adapter *adapter,
                                   u16 vid, u16 qos, u32 vf)