improve direct connector (#685)

* support ipv6 stun
* show interface and public ip in cli node info
* direct conn should keep trying unless already direct connected
* peer should use conn with smallest latency
* deprecate ipv6_listener, use -l instead

easytier/src/peers/foreign_network_manager.rs

@@ -695,7 +695,8 @@ mod tests {
 
         let (a_ring, b_ring) = crate::tunnel::ring::create_ring_tunnel_pair();
         let b_mgr_copy = pm_center.clone();
-        let s_ret = tokio::spawn(async move { b_mgr_copy.add_tunnel_as_server(b_ring).await });
+        let s_ret =
+            tokio::spawn(async move { b_mgr_copy.add_tunnel_as_server(b_ring, true).await });
 
         pma_net1.add_client_tunnel(a_ring).await.unwrap();
 

easytier/src/peers/peer.rs

@@ -11,7 +11,7 @@ use super::{
     peer_conn::{PeerConn, PeerConnId},
     PacketRecvChan,
 };
-use crate::proto::cli::PeerConnInfo;
+use crate::{common::scoped_task::ScopedTask, proto::cli::PeerConnInfo};
 use crate::{
     common::{
         error::Error,
@@ -36,7 +36,8 @@ pub struct Peer {
 
     shutdown_notifier: Arc<tokio::sync::Notify>,
 
-    default_conn_id: AtomicCell<PeerConnId>,
+    default_conn_id: Arc<AtomicCell<PeerConnId>>,
+    default_conn_id_clear_task: ScopedTask<()>,
 }
 
 impl Peer {
@@ -88,6 +89,19 @@ impl Peer {
             )),
         );
 
+        let default_conn_id = Arc::new(AtomicCell::new(PeerConnId::default()));
+
+        let conns_copy = conns.clone();
+        let default_conn_id_copy = default_conn_id.clone();
+        let default_conn_id_clear_task = ScopedTask::from(tokio::spawn(async move {
+            loop {
+                tokio::time::sleep(std::time::Duration::from_secs(5)).await;
+                if conns_copy.len() > 1 {
+                    default_conn_id_copy.store(PeerConnId::default());
+                }
+            }
+        }));
+
         Peer {
             peer_node_id,
             conns: conns.clone(),
@@ -98,7 +112,8 @@ impl Peer {
             close_event_listener,
 
             shutdown_notifier,
-            default_conn_id: AtomicCell::new(PeerConnId::default()),
+            default_conn_id,
+            default_conn_id_clear_task,
         }
     }
 
@@ -117,14 +132,19 @@ impl Peer {
             return Some(conn.clone());
         }
 
-        let conn = self.conns.iter().next();
-        if conn.is_none() {
-            return None;
+        // find a conn with the smallest latency
+        let mut min_latency = std::u64::MAX;
+        for conn in self.conns.iter() {
+            let latency = conn.value().get_stats().latency_us;
+            if latency < min_latency {
+                min_latency = latency;
+                self.default_conn_id.store(conn.get_conn_id());
+            }
         }
 
-        let conn = conn.unwrap().clone();
-        self.default_conn_id.store(conn.get_conn_id());
-        Some(conn)
+        self.conns
+            .get(&self.default_conn_id.load())
+            .map(|conn| conn.clone())
     }
 
     pub async fn send_msg(&self, msg: ZCPacket) -> Result<(), Error> {
@@ -158,6 +178,10 @@ impl Peer {
         }
         ret
     }
+
+    pub fn get_default_conn_id(&self) -> PeerConnId {
+        self.default_conn_id.load()
+    }
 }
 
 // pritn on drop

easytier/src/peers/peer_manager.rs

@@ -8,7 +8,7 @@ use std::{
 use anyhow::Context;
 use async_trait::async_trait;
 
-use dashmap::DashMap;
+use dashmap::{DashMap, DashSet};
 
 use tokio::{
     sync::{
@@ -23,7 +23,7 @@ use crate::{
         compressor::{Compressor as _, DefaultCompressor},
         constants::EASYTIER_VERSION,
         error::Error,
-        global_ctx::{ArcGlobalCtx, NetworkIdentity},
+        global_ctx::{ArcGlobalCtx, GlobalCtxEvent, NetworkIdentity},
         stun::StunInfoCollectorTrait,
         PeerId,
     },
@@ -141,6 +141,9 @@ pub struct PeerManager {
     data_compress_algo: CompressorAlgo,
 
     exit_nodes: Vec<Ipv4Addr>,
+
+    // conns that are directly connected (which are not hole punched)
+    directly_connected_conn_map: Arc<DashMap<PeerId, DashSet<uuid::Uuid>>>,
 }
 
 impl Debug for PeerManager {
@@ -267,6 +270,8 @@ impl PeerManager {
             data_compress_algo,
 
             exit_nodes,
+
+            directly_connected_conn_map: Arc::new(DashMap::new()),
         }
     }
 
@@ -325,8 +330,48 @@ impl PeerManager {
         Ok((peer_id, conn_id))
     }
 
+    fn add_directly_connected_conn(&self, peer_id: PeerId, conn_id: uuid::Uuid) {
+        let _ = self
+            .directly_connected_conn_map
+            .entry(peer_id)
+            .or_insert_with(DashSet::new)
+            .insert(conn_id);
+    }
+
+    pub fn has_directly_connected_conn(&self, peer_id: PeerId) -> bool {
+        self.directly_connected_conn_map
+            .get(&peer_id)
+            .map_or(false, |x| !x.is_empty())
+    }
+
+    async fn start_peer_conn_close_event_handler(&self) {
+        let dmap = self.directly_connected_conn_map.clone();
+        let mut event_recv = self.global_ctx.subscribe();
+        self.tasks.lock().await.spawn(async move {
+            while let Ok(event) = event_recv.recv().await {
+                match event {
+                    GlobalCtxEvent::PeerConnRemoved(info) => {
+                        if let Some(set) = dmap.get_mut(&info.peer_id) {
+                            let conn_id = info.conn_id.parse().unwrap();
+                            let old = set.remove(&conn_id);
+                            tracing::info!(
+                                ?old,
+                                ?info,
+                                "try remove conn id from directly connected map"
+                            );
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        });
+    }
+
     #[tracing::instrument]
-    pub async fn try_connect<C>(&self, mut connector: C) -> Result<(PeerId, PeerConnId), Error>
+    pub async fn try_direct_connect<C>(
+        &self,
+        mut connector: C,
+    ) -> Result<(PeerId, PeerConnId), Error>
     where
         C: TunnelConnector + Debug,
     {
@@ -334,18 +379,28 @@ impl PeerManager {
         let t = ns
             .run_async(|| async move { connector.connect().await })
             .await?;
-        self.add_client_tunnel(t).await
+        let (peer_id, conn_id) = self.add_client_tunnel(t).await?;
+        self.add_directly_connected_conn(peer_id, conn_id);
+        Ok((peer_id, conn_id))
     }
 
     #[tracing::instrument]
-    pub async fn add_tunnel_as_server(&self, tunnel: Box<dyn Tunnel>) -> Result<(), Error> {
+    pub async fn add_tunnel_as_server(
+        &self,
+        tunnel: Box<dyn Tunnel>,
+        is_directly_connected: bool,
+    ) -> Result<(), Error> {
         tracing::info!("add tunnel as server start");
         let mut peer = PeerConn::new(self.my_peer_id, self.global_ctx.clone(), tunnel);
         peer.do_handshake_as_server().await?;
         if peer.get_network_identity().network_name
             == self.global_ctx.get_network_identity().network_name
         {
+            let (peer_id, conn_id) = (peer.get_peer_id(), peer.get_conn_id());
             self.add_new_peer_conn(peer).await?;
+            if is_directly_connected {
+                self.add_directly_connected_conn(peer_id, conn_id);
+            }
         } else {
             self.foreign_network_manager.add_peer_conn(peer).await?;
         }
@@ -857,9 +912,11 @@ impl PeerManager {
 
     async fn run_clean_peer_without_conn_routine(&self) {
         let peer_map = self.peers.clone();
+        let dmap = self.directly_connected_conn_map.clone();
         self.tasks.lock().await.spawn(async move {
             loop {
                 peer_map.clean_peer_without_conn().await;
+                dmap.retain(|p, v| peer_map.has_peer(*p) && !v.is_empty());
                 tokio::time::sleep(std::time::Duration::from_secs(3)).await;
             }
         });
@@ -876,6 +933,8 @@ impl PeerManager {
     }
 
     pub async fn run(&self) -> Result<(), Error> {
+        self.start_peer_conn_close_event_handler().await;
+
         match &self.route_algo_inst {
             RouteAlgoInst::Ospf(route) => self.add_route(route.clone()).await,
             RouteAlgoInst::None => {}
@@ -924,7 +983,7 @@ impl PeerManager {
         self.foreign_network_client.clone()
     }
 
-    pub fn get_my_info(&self) -> cli::NodeInfo {
+    pub async fn get_my_info(&self) -> cli::NodeInfo {
         cli::NodeInfo {
             peer_id: self.my_peer_id,
             ipv4_addr: self
@@ -950,6 +1009,7 @@ impl PeerManager {
             config: self.global_ctx.config.dump(),
             version: EASYTIER_VERSION.to_string(),
             feature_flag: Some(self.global_ctx.get_feature_flags()),
+            ip_list: Some(self.global_ctx.get_ip_collector().collect_ip_addrs().await),
         }
     }
 
@@ -958,6 +1018,13 @@ impl PeerManager {
             tokio::time::sleep(std::time::Duration::from_secs(1)).await;
         }
     }
+
+    pub fn get_directly_connections(&self, peer_id: PeerId) -> DashSet<uuid::Uuid> {
+        self.directly_connected_conn_map
+            .get(&peer_id)
+            .map(|x| x.clone())
+            .unwrap_or_default()
+    }
 }
 
 #[cfg(test)]
@@ -1026,7 +1093,7 @@ mod tests {
 
         tokio::spawn(async move {
             client.set_bind_addrs(vec![]);
-            client_mgr.try_connect(client).await.unwrap();
+            client_mgr.try_direct_connect(client).await.unwrap();
         });
 
         server_mgr

easytier/src/peers/peer_map.rs

@@ -212,6 +212,11 @@ impl PeerMap {
         }
     }
 
+    pub async fn get_peer_default_conn_id(&self, peer_id: PeerId) -> Option<PeerConnId> {
+        self.get_peer_by_id(peer_id)
+            .map(|p| p.get_default_conn_id())
+    }
+
     pub async fn close_peer_conn(
         &self,
         peer_id: PeerId,

easytier/src/peers/rpc_service.rs

@@ -32,12 +32,23 @@ impl PeerManagerRpcService {
                 .await
                 .iter(),
         );
+        let peer_map = self.peer_manager.get_peer_map();
         let mut peer_infos = Vec::new();
         for peer in peers {
             let mut peer_info = PeerInfo::default();
             peer_info.peer_id = peer;
+            peer_info.default_conn_id = peer_map
+                .get_peer_default_conn_id(peer)
+                .await
+                .map(Into::into);
+            peer_info.directly_connected_conns = self
+                .peer_manager
+                .get_directly_connections(peer)
+                .into_iter()
+                .map(Into::into)
+                .collect();
 
-            if let Some(conns) = self.peer_manager.get_peer_map().list_peer_conns(peer).await {
+            if let Some(conns) = peer_map.list_peer_conns(peer).await {
                 peer_info.conns = conns;
             } else if let Some(conns) = self
                 .peer_manager
@@ -121,7 +132,7 @@ impl PeerManageRpc for PeerManagerRpcService {
         _request: ShowNodeInfoRequest, // Accept request of type HelloRequest
     ) -> Result<ShowNodeInfoResponse, rpc_types::error::Error> {
         Ok(ShowNodeInfoResponse {
-            node_info: Some(self.peer_manager.get_my_info()),
+            node_info: Some(self.peer_manager.get_my_info().await),
         })
     }
 }

easytier/src/peers/tests.rs

@@ -45,7 +45,7 @@ pub async fn connect_peer_manager(client: Arc<PeerManager>, server: Arc<PeerMana
     });
     let b_mgr_copy = server.clone();
     tokio::spawn(async move {
-        b_mgr_copy.add_tunnel_as_server(b_ring).await.unwrap();
+        b_mgr_copy.add_tunnel_as_server(b_ring, true).await.unwrap();
     });
 }