[dpdk-dev] [PATCH 1/5] ethdev: introduce device removal event

[Gaetan Rivet]

This new API allows reacting to a device removal.
A device removal is the sudden disappearance of a device from its
bus.

PMDs implementing support for this notification guarantee that the removal
of the underlying device does not incur a risk to the application.

In particular, Rx/Tx bursts and all other functions can still be called
(albeit likely returning errors) without triggering a crash, irrespective
of an application handling this event.

Signed-off-by: Gaetan Rivet <gaetan.rivet at 6wind.com>
Signed-off-by: Elad Persiko <eladpe at mellanox.com>
---
 doc/guides/nics/features/default.ini            |  1 +
 doc/guides/prog_guide/env_abstraction_layer.rst | 21 +++++++++++++++++++--
 lib/librte_eal/common/include/rte_pci.h         |  2 ++
 lib/librte_ether/rte_ethdev.c                   | 13 +++++++++----
 lib/librte_ether/rte_ethdev.h                   |  9 +++++++--
 5 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/doc/guides/nics/features/default.ini b/doc/guides/nics/features/default.ini
index 9e363ff..7bb56ec 100644
--- a/doc/guides/nics/features/default.ini
+++ b/doc/guides/nics/features/default.ini
@@ -9,6 +9,7 @@
 Speed capabilities   =
 Link status          =
 Link status event    =
+Removal event        =
 Queue status event   =
 Rx interrupt         =
 Queue start/stop     =
diff --git a/doc/guides/prog_guide/env_abstraction_layer.rst b/doc/guides/prog_guide/env_abstraction_layer.rst
index 10a10a8..f72674f 100644
--- a/doc/guides/prog_guide/env_abstraction_layer.rst
+++ b/doc/guides/prog_guide/env_abstraction_layer.rst
@@ -178,8 +178,8 @@ The EAL also allows timed callbacks to be used in the same way as for NIC interr
 
 .. note::
 
-    In DPDK PMD, the only interrupts handled by the dedicated host thread are those for link status change,
-    i.e. link up and link down notification.
+    In DPDK PMD, the only interrupts handled by the dedicated host thread are those for link status change
+    (link up and link down notification) and for sudden device removal.
 
 
 + RX Interrupt Event
@@ -207,6 +207,23 @@ The eth_dev driver takes responsibility to program the latter mapping.
 The RX interrupt are controlled/enabled/disabled by ethdev APIs - 'rte_eth_dev_rx_intr_*'. They return failure if the PMD
 hasn't support them yet. The intr_conf.rxq flag is used to turn on the capability of RX interrupt per device.
 
++ Device Removal Event
+
+This event is triggered by a device being removed at a bus level. Its
+underlying resources may have been made unavailable (i.e. PCI mappings
+unmapped). The PMD must make sure that on such occurrence, the application can
+still safely use its callbacks.
+
+This event can be subscribed to in the same way one would subscribe to a link
+status change event. The execution context is thus the same, i.e. it is the
+dedicated interrupt host thread.
+
+Considering this, it is likely that an application would want to close a
+device having emitted a Device Removal Event. In such case, calling
+``rte_eth_dev_close()`` can trigger it to unregister its own Device Removal Event
+callback. Care must be taken not to close the device from the interrupt handler
+context. It is necessary to reschedule such closing operation.
+
 Blacklisting
 ~~~~~~~~~~~~
 
diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h
index 8557e47..4a67883 100644
--- a/lib/librte_eal/common/include/rte_pci.h
+++ b/lib/librte_eal/common/include/rte_pci.h
@@ -216,6 +216,8 @@ struct rte_pci_driver {
 #define RTE_PCI_DRV_NEED_MAPPING 0x0001
 /** Device driver supports link state interrupt */
 #define RTE_PCI_DRV_INTR_LSC	0x0008
+/** Device driver supports device removal interrupt */
+#define RTE_PCI_DRV_INTR_RMV 0x0010
 
 /**
  * A structure describing a PCI mapping.
diff --git a/lib/librte_ether/rte_ethdev.c b/lib/librte_ether/rte_ethdev.c
index 3a52d0a..6c4b796 100644
--- a/lib/librte_ether/rte_ethdev.c
+++ b/lib/librte_ether/rte_ethdev.c
@@ -848,16 +848,19 @@ rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 		return -EINVAL;
 	}
 
-	/*
-	 * If link state interrupt is enabled, check that the
-	 * device supports it.
-	 */
+	/* Check that the device supports requested interrupts */
 	if ((dev_conf->intr_conf.lsc == 1) &&
 		(!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
 			RTE_PMD_DEBUG_TRACE("driver %s does not support lsc\n",
 					dev->data->drv_name);
 			return -EINVAL;
 	}
+	if ((dev_conf->intr_conf.rmv == 1) &&
+	    (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
+		RTE_PMD_DEBUG_TRACE("driver %s does not support rmv\n",
+				    dev->data->drv_name);
+		return -EINVAL;
+	}
 
 	/*
 	 * If jumbo frames are enabled, check that the maximum RX packet
@@ -3222,6 +3225,8 @@ rte_eth_copy_pci_info(struct rte_eth_dev *eth_dev, struct rte_pci_device *pci_de
 	eth_dev->data->dev_flags = 0;
 	if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC)
 		eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
+	if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_RMV)
+		eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_RMV;
 
 	eth_dev->data->kdrv = pci_dev->kdrv;
 	eth_dev->data->numa_node = pci_dev->device.numa_node;
diff --git a/lib/librte_ether/rte_ethdev.h b/lib/librte_ether/rte_ethdev.h
index bdad81b..ed99660 100644
--- a/lib/librte_ether/rte_ethdev.h
+++ b/lib/librte_ether/rte_ethdev.h
@@ -816,9 +816,11 @@ struct rte_eth_udp_tunnel {
  */
 struct rte_intr_conf {
 	/** enable/disable lsc interrupt. 0 (default) - disable, 1 enable */
-	uint16_t lsc;
+	uint32_t lsc:1;
 	/** enable/disable rxq interrupt. 0 (default) - disable, 1 enable */
-	uint16_t rxq;
+	uint32_t rxq:1;
+	/** enable/disable rmv interrupt. 0 (default) - disable, 1 enable */
+	uint32_t rmv:1;
 };
 
 /**
@@ -1722,6 +1724,8 @@ struct rte_eth_dev_data {
 #define RTE_ETH_DEV_INTR_LSC     0x0002
 /** Device is a bonded slave */
 #define RTE_ETH_DEV_BONDED_SLAVE 0x0004
+/** Device supports device removal interrupt */
+#define RTE_ETH_DEV_INTR_RMV     0x0008
 
 /**
  * @internal
@@ -3212,6 +3216,7 @@ enum rte_eth_event_type {
 			/**< reset interrupt event, sent to VF on PF reset */
 	RTE_ETH_EVENT_VF_MBOX,  /**< message from the VF received by PF */
 	RTE_ETH_EVENT_MACSEC,   /**< MACsec offload related event */
+	RTE_ETH_EVENT_INTR_RMV, /**< device removal event */
 	RTE_ETH_EVENT_MAX       /**< max value of this enum */
 };
 
-- 
2.1.4