Merge pull request #1453 from microsoft/xdp-nic-split

[catpowder] Support NetVSC/VF dual NIC in hyper-V VMs

scripts/config/default.yaml

@@ -7,6 +7,8 @@ demikernel:
 raw_socket:
   linux_interface_name: "abcde"
   xdp_interface_index: 0
+  # Enable the following line if you have a VF interface
+  # xdp_vf_interface_index: 0
 dpdk:
   eal_init: ["", "-c", "0xff", "-n", "4", "-a", "WW:WW.W","--proc-type=auto"]
 tcp_socket_options:

src/rust/catpowder/win/runtime.rs

@@ -44,6 +44,7 @@ struct CatpowderRuntimeInner {
     api: XdpApi,
     tx: TxRing,
     rx_rings: Vec<RxRing>,
+    vf_rx_rings: Vec<RxRing>,
 }
 //======================================================================================================================
 // Implementations
@@ -60,6 +61,7 @@ impl SharedCatpowderRuntime {
     /// Instantiates a new XDP runtime.
     pub fn new(config: &Config) -> Result<Self, Fail> {
         let ifindex: u32 = config.local_interface_index()?;
+        let vf_if_index = config.local_vf_interface_index(); // N.B. this one is optional
 
         trace!("Creating XDP runtime.");
         let mut api: XdpApi = XdpApi::new()?;
@@ -72,9 +74,30 @@ impl SharedCatpowderRuntime {
         for queueid in 0..queue_count {
             rx_rings.push(RxRing::new(&mut api, Self::RING_LENGTH, ifindex, queueid as u32)?);
         }
-        trace!("Created {} RX rings.", rx_rings.len());
+        trace!("Created {} RX rings on interface {}", rx_rings.len(), ifindex);
 
-        Ok(Self(SharedObject::new(CatpowderRuntimeInner { api, tx, rx_rings })))
+        if let Ok(vf_if_index) = vf_if_index { // Optionally create VF RX rings
+            let vf_queue_count = deduce_rss_settings(&mut api, vf_if_index)?;
+            let mut vf_rx_rings = Vec::with_capacity(vf_queue_count as usize);
+            for queueid in 0..vf_queue_count {
+                vf_rx_rings.push(RxRing::new(&mut api, Self::RING_LENGTH, vf_if_index, queueid as u32)?);
+            }
+            trace!("Created {} RX rings on VF interface {}.", vf_rx_rings.len(), vf_if_index);
+
+            Ok(Self(SharedObject::new(CatpowderRuntimeInner {
+                api,
+                tx,
+                rx_rings,
+                vf_rx_rings,
+            })))
+        } else {
+            Ok(Self(SharedObject::new(CatpowderRuntimeInner {
+                api,
+                tx,
+                rx_rings,
+                vf_rx_rings: Vec::new(),
+            })))
+        }
     }
 }
 
@@ -153,6 +176,25 @@ impl PhysicalLayer for SharedCatpowderRuntime {
             }
         }
 
+        for rx in self.0.borrow_mut().vf_rx_rings.iter_mut() {
+            if rx.reserve_rx(Self::RING_LENGTH, &mut idx) == Self::RING_LENGTH {
+                let xdp_buffer: XdpBuffer = rx.get_buffer(idx);
+                let dbuf: DemiBuffer = DemiBuffer::from_slice(&*xdp_buffer)?;
+                rx.release_rx(Self::RING_LENGTH);
+
+                ret.push(dbuf);
+
+                rx.reserve_rx_fill(Self::RING_LENGTH, &mut idx);
+                // NB for now there is only ever one element in the fill ring, so we don't have to
+                // change the ring contents.
+                rx.submit_rx_fill(Self::RING_LENGTH);
+
+                if ret.is_full() {
+                    break;
+                }
+            }
+        }
+
         Ok(ret)
     }
 }

src/rust/demikernel/config.rs

@@ -60,8 +60,15 @@ mod raw_socket_config {
     pub const SECTION_NAME: &str = "raw_socket";
     #[cfg(target_os = "linux")]
     pub const LOCAL_INTERFACE_NAME: &str = "linux_interface_name";
+
+    // The primary interface index. This should be the virtualized interface for VMs.
     #[cfg(target_os = "windows")]
     pub const LOCAL_INTERFACE_INDEX: &str = "xdp_interface_index";
+
+    // N.B. hyper-V VMs can have both NetVSC and VF interfaces working in tandem, in which case
+    // we need to listen to the corresponding VF interface as well.
+    #[cfg(target_os = "windows")]
+    pub const LOCAL_VF_INTERFACE_INDEX: &str = "xdp_vf_interface_index";
 }
 
 //======================================================================================================================
@@ -292,6 +299,18 @@ impl Config {
         }
     }
 
+    #[cfg(all(feature = "catpowder-libos", target_os = "windows"))]
+    pub fn local_vf_interface_index(&self) -> Result<u32, Fail> {
+        if let Some(addr) = Self::get_typed_env_option(raw_socket_config::LOCAL_VF_INTERFACE_INDEX)? {
+            Ok(addr)
+        } else {
+            Self::get_int_option(
+                self.get_raw_socket_config()?,
+                raw_socket_config::LOCAL_VF_INTERFACE_INDEX,
+            )
+        }
+    }
+
     #[cfg(feature = "catnip-libos")]
     /// DPDK Config: Reads the "DPDK EAL" parameter the underlying configuration file.
     pub fn eal_init_args(&self) -> Result<Vec<CString>, Fail> {