add more debug info for route (#155)

1. use info log for sync_route_info
2. allow dump route info with cli tool.
3. dump route info every 60s

easytier/src/peers/peer_manager.rs

@@ -535,6 +535,10 @@ impl PeerManager {
         self.get_route().list_routes().await
     }
 
+    pub async fn dump_route(&self) -> String {
+        self.get_route().dump().await
+    }
+
     async fn run_nic_packet_process_pipeline(&self, data: &mut ZCPacket) {
         for pipeline in self.nic_packet_process_pipeline.read().await.iter().rev() {
             pipeline.try_process_packet_from_nic(data).await;

easytier/src/peers/peer_ospf_route.rs

@@ -700,9 +700,13 @@ impl Debug for PeerRouteServiceImpl {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("PeerRouteServiceImpl")
             .field("my_peer_id", &self.my_peer_id)
+            .field("sessions", &self.sessions)
+            .field("route_table", &self.route_table)
+            .field("route_table_with_cost", &self.route_table_with_cost)
+            .field("synced_route_info", &self.synced_route_info)
             .field(
-                "sessions",
-                &self.sessions.iter().map(|x| *x.key()).collect::<Vec<_>>(),
+                "cached_local_conn_map",
+                &self.cached_local_conn_map.lock().unwrap(),
             )
             .finish()
     }
@@ -842,9 +846,7 @@ impl PeerRouteServiceImpl {
 
         let all_peer_ids = &conn_bitmap.peer_ids;
         for (peer_idx, (peer_id, _)) in all_peer_ids.iter().enumerate() {
-            let Some(connected) = self.synced_route_info.conn_map.get(peer_id) else {
-                continue;
-            };
+            let connected = self.synced_route_info.conn_map.get(peer_id).unwrap();
 
             for (idx, (other_peer_id, _)) in all_peer_ids.iter().enumerate() {
                 if connected.0.contains(other_peer_id) {
@@ -942,7 +944,7 @@ impl PeerRouteServiceImpl {
         let my_peer_id = self.my_peer_id;
 
         let (peer_infos, conn_bitmap) = self.build_sync_request(&session);
-        tracing::trace!("my_id {:?}, pper_id: {:?}, peer_infos: {:?}, conn_bitmap: {:?}, synced_route_info: {:?} session: {:?}",
+        tracing::info!("my_id {:?}, pper_id: {:?}, peer_infos: {:?}, conn_bitmap: {:?}, synced_route_info: {:?} session: {:?}",
                        my_peer_id, dst_peer_id, peer_infos, conn_bitmap, self.synced_route_info, session);
 
         if peer_infos.is_none()
@@ -952,6 +954,10 @@ impl PeerRouteServiceImpl {
             return true;
         }
 
+        session
+            .need_sync_initiator_info
+            .store(false, Ordering::Relaxed);
+
         let ret = peer_rpc
             .do_client_rpc_scoped(SERVICE_ID, dst_peer_id, |c| async {
                 let client = RouteServiceClient::new(tarpc::client::Config::default(), c).spawn();
@@ -978,10 +984,6 @@ impl PeerRouteServiceImpl {
                     .dst_is_initiator
                     .store(ret.is_initiator, Ordering::Relaxed);
 
-                session
-                    .need_sync_initiator_info
-                    .store(false, Ordering::Relaxed);
-
                 session.update_dst_session_id(ret.session_id);
 
                 if let Some(peer_infos) = &peer_infos {
@@ -1014,6 +1016,22 @@ struct RouteSessionManager {
     sync_now_broadcast: tokio::sync::broadcast::Sender<()>,
 }
 
+impl Debug for RouteSessionManager {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RouteSessionManager")
+            .field(
+                "session_tasks",
+                &self
+                    .session_tasks
+                    .iter()
+                    .map(|x| *x.key())
+                    .collect::<Vec<_>>(),
+            )
+            .field("dump_sessions", &self.dump_sessions())
+            .finish()
+    }
+}
+
 #[tarpc::server]
 impl RouteService for RouteSessionManager {
     async fn sync_route_info(
@@ -1052,7 +1070,7 @@ impl RouteService for RouteSessionManager {
 
         service_impl.update_route_table_and_cached_local_conn_bitmap();
 
-        tracing::debug!(
+        tracing::info!(
             "sync_route_info: from_peer_id: {:?}, is_initiator: {:?}, peer_infos: {:?}, conn_bitmap: {:?}, synced_route_info: {:?} session: {:?}, new_route_table: {:?}",
             from_peer_id, is_initiator, peer_infos, conn_bitmap, service_impl.synced_route_info, session, service_impl.route_table);
 
@@ -1297,6 +1315,16 @@ pub struct PeerRoute {
     tasks: std::sync::Mutex<JoinSet<()>>,
 }
 
+impl Debug for PeerRoute {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("PeerRoute")
+            .field("my_peer_id", &self.my_peer_id)
+            .field("service_impl", &self.service_impl)
+            .field("session_mgr", &self.session_mgr)
+            .finish()
+    }
+}
+
 impl PeerRoute {
     pub fn new(
         my_peer_id: PeerId,
@@ -1322,6 +1350,8 @@ impl PeerRoute {
         loop {
             tokio::time::sleep(Duration::from_secs(60)).await;
             service_impl.clear_expired_peer();
+            // TODO: use debug log level for this.
+            tracing::info!(?service_impl, "clear_expired_peer");
         }
     }
 
@@ -1449,6 +1479,10 @@ impl Route for PeerRoute {
         *self.service_impl.cost_calculator.lock().unwrap() = Some(_cost_fn);
         self.service_impl.update_route_table();
     }
+
+    async fn dump(&self) -> String {
+        format!("{:#?}", self)
+    }
 }
 
 impl PeerPacketFilter for Arc<PeerRoute> {}

easytier/src/peers/route_trait.rs

@@ -78,6 +78,10 @@ pub trait Route {
     }
 
     async fn set_route_cost_fn(&self, _cost_fn: RouteCostCalculator) {}
+
+    async fn dump(&self) -> String {
+        "this route implementation does not support dump".to_string()
+    }
 }
 
 pub type ArcRoute = Arc<Box<dyn Route + Send + Sync>>;

easytier/src/peers/rpc_service.rs

@@ -1,9 +1,8 @@
 use std::sync::Arc;
 
 use crate::rpc::{
-    cli::PeerInfo,
-    peer_manage_rpc_server::PeerManageRpc,
-    {ListPeerRequest, ListPeerResponse, ListRouteRequest, ListRouteResponse},
+    cli::PeerInfo, peer_manage_rpc_server::PeerManageRpc, DumpRouteRequest, DumpRouteResponse,
+    ListPeerRequest, ListPeerResponse, ListRouteRequest, ListRouteResponse,
 };
 use tonic::{Request, Response, Status};
 
@@ -60,4 +59,13 @@ impl PeerManageRpc for PeerManagerRpcService {
         reply.routes = self.peer_manager.list_routes().await;
         Ok(Response::new(reply))
     }
+
+    async fn dump_route(
+        &self,
+        _request: Request<DumpRouteRequest>, // Accept request of type HelloRequest
+    ) -> Result<Response<DumpRouteResponse>, Status> {
+        let mut reply = DumpRouteResponse::default();
+        reply.result = self.peer_manager.dump_route().await;
+        Ok(Response::new(reply))
+    }
 }