From 756d498b900562ea90eab7526a19d73c51c406f3 Mon Sep 17 00:00:00 2001
From: "Sijie.Sun" <sijie.sun@smartx.com>
Date: Thu, 8 Feb 2024 23:44:51 +0800
Subject: [PATCH] Stun fix (#18)

* make easytier-core a lib
* add stun command to easytier cli
* fix stun test for musl
---
 easytier-cli/Cargo.toml                       |   8 ++
 easytier-cli/src/main.rs                      |  51 +++++++--
 easytier-core/Cargo.toml                      |  13 ++-
 easytier-core/src/common/global_ctx.rs        |  26 +----
 easytier-core/src/common/netns.rs             |   4 -
 easytier-core/src/common/network.rs           |   4 +-
 easytier-core/src/common/stun.rs              | 106 +++++++++++++++---
 easytier-core/src/connector/direct.rs         |   2 +-
 easytier-core/src/connector/manual.rs         |  10 +-
 easytier-core/src/connector/udp_hole_punch.rs |   4 +-
 easytier-core/src/instance/instance.rs        |   4 +-
 easytier-core/src/lib.rs                      |   6 +
 easytier-core/src/main.rs                     |   1 +
 easytier-core/src/peer_center/service.rs      |   2 +-
 easytier-core/src/peers/peer.rs               |   5 +-
 easytier-core/src/peers/peer_conn.rs          |   2 +-
 easytier-core/src/peers/peer_manager.rs       |   2 +-
 easytier-core/src/peers/peer_map.rs           |   2 +-
 easytier-core/src/peers/peer_rip_route.rs     |   6 +-
 easytier-core/src/peers/route_trait.rs        |   2 +-
 easytier-core/src/peers/rpc_service.rs        |  10 +-
 easytier-core/src/rpc/{lib.rs => mod.rs}      |   0
 easytier-core/src/tests/mod.rs                |   2 +-
 easytier-core/src/tunnels/mod.rs              |   2 +-
 easytier-core/src/tunnels/tunnel_filter.rs    |   2 +-
 easytier-core/src/tunnels/udp_tunnel.rs       |   2 +-
 26 files changed, 192 insertions(+), 86 deletions(-)
 create mode 100644 easytier-core/src/lib.rs
 rename easytier-core/src/rpc/{lib.rs => mod.rs} (100%)

diff --git a/easytier-cli/Cargo.toml b/easytier-cli/Cargo.toml
index 296f7a8..4dcff3e 100644
--- a/easytier-cli/Cargo.toml
+++ b/easytier-cli/Cargo.toml
@@ -6,6 +6,14 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+tracing = { version = "0.1", features = ["log"] }
+tracing-subscriber = { version = "0.3", features = [
+    "env-filter",
+    "local-time",
+    "time",
+] }
+tracing-appender = "0.2.3"
+
 futures = "0.3"
 tokio = { version = "1", features = ["full"] }
 anyhow = "1.0"
diff --git a/easytier-cli/src/main.rs b/easytier-cli/src/main.rs
index 6aeeefa..674952f 100644
--- a/easytier-cli/src/main.rs
+++ b/easytier-cli/src/main.rs
@@ -1,12 +1,17 @@
 use std::vec;
 
 use clap::{command, Args, Parser, Subcommand};
-use easytier_rpc::{
-    connector_manage_rpc_client::ConnectorManageRpcClient,
-    peer_manage_rpc_client::PeerManageRpcClient, *,
+use easytier_core::{
+    common::stun::{StunInfoCollector, UdpNatTypeDetector},
+    rpc::{
+        connector_manage_rpc_client::ConnectorManageRpcClient,
+        peer_manage_rpc_client::PeerManageRpcClient, *,
+    },
 };
 use humansize::format_size;
 use tabled::settings::Style;
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer};
 
 #[derive(Parser, Debug)]
 #[command(author, version, about, long_about = None)]
@@ -16,13 +21,14 @@ struct Cli {
     instance_name: String,
 
     #[command(subcommand)]
-    sub_command: Option<SubCommand>,
+    sub_command: SubCommand,
 }
 
 #[derive(Subcommand, Debug)]
 enum SubCommand {
     Peer(PeerArgs),
     Connector(ConnectorArgs),
+    Stun,
     Route,
 }
 
@@ -344,15 +350,41 @@ impl CommandHandler {
     }
 }
 
+fn init_logger() {
+    // logger to rolling file
+    let file_filter = EnvFilter::builder()
+        .with_default_directive(LevelFilter::INFO.into())
+        .from_env()
+        .unwrap();
+    let file_appender = tracing_appender::rolling::Builder::new()
+        .rotation(tracing_appender::rolling::Rotation::DAILY)
+        .max_log_files(1)
+        .filename_prefix("cli.log")
+        .build("/tmp")
+        .expect("failed to initialize rolling file appender");
+    let mut file_layer = tracing_subscriber::fmt::layer();
+    file_layer.set_ansi(false);
+    let file_layer = file_layer
+        .with_writer(file_appender)
+        .with_filter(file_filter);
+
+    tracing_subscriber::Registry::default()
+        .with(file_layer)
+        .init();
+}
+
 #[tokio::main]
+#[tracing::instrument]
 async fn main() -> Result<(), Error> {
+    init_logger();
+
     let cli = Cli::parse();
     let handler = CommandHandler {
         addr: "http://127.0.0.1:15888".to_string(),
     };
 
     match cli.sub_command {
-        Some(SubCommand::Peer(peer_args)) => match &peer_args.sub_command {
+        SubCommand::Peer(peer_args) => match &peer_args.sub_command {
             Some(PeerSubCommand::Add) => {
                 println!("add peer");
             }
@@ -370,7 +402,7 @@ async fn main() -> Result<(), Error> {
                 handler.handle_peer_list(&peer_args).await?;
             }
         },
-        Some(SubCommand::Connector(conn_args)) => match conn_args.sub_command {
+        SubCommand::Connector(conn_args) => match conn_args.sub_command {
             Some(ConnectorSubCommand::Add) => {
                 println!("add connector");
             }
@@ -384,11 +416,12 @@ async fn main() -> Result<(), Error> {
                 handler.handle_connector_list().await?;
             }
         },
-        Some(SubCommand::Route) => {
+        SubCommand::Route => {
             handler.handle_route_list().await?;
         }
-        None => {
-            println!("list peer");
+        SubCommand::Stun => {
+            let stun = UdpNatTypeDetector::new(StunInfoCollector::get_default_servers());
+            println!("udp type: {:?}", stun.get_udp_nat_type(0).await);
         }
     }
 
diff --git a/easytier-core/Cargo.toml b/easytier-core/Cargo.toml
index 11da408..22d5be7 100644
--- a/easytier-core/Cargo.toml
+++ b/easytier-core/Cargo.toml
@@ -8,8 +8,8 @@ rust-version = "1.75"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [lib]
-name = "easytier_rpc"
-path = "src/rpc/lib.rs"
+name = "easytier_core"
+test = false
 
 [dependencies]
 tracing = { version = "0.1", features = ["log"] }
@@ -87,8 +87,15 @@ public-ip = { version = "0.2", features = ["default"] }
 
 clap = { version = "4.4", features = ["derive"] }
 
+async-recursion = "1.0.5"
+
 [target.'cfg(windows)'.dependencies]
-windows-sys = { version = "0.52", features = ["Win32_Networking_WinSock", "Win32_NetworkManagement_IpHelper", "Win32_Foundation", "Win32_System_IO"] }
+windows-sys = { version = "0.52", features = [
+    "Win32_Networking_WinSock",
+    "Win32_NetworkManagement_IpHelper",
+    "Win32_Foundation",
+    "Win32_System_IO",
+] }
 
 [build-dependencies]
 tonic-build = "0.10"
diff --git a/easytier-core/src/common/global_ctx.rs b/easytier-core/src/common/global_ctx.rs
index eadfa67..4887eca 100644
--- a/easytier-core/src/common/global_ctx.rs
+++ b/easytier-core/src/common/global_ctx.rs
@@ -1,7 +1,7 @@
 use std::{io::Write, sync::Arc};
 
+use crate::rpc::PeerConnInfo;
 use crossbeam::atomic::AtomicCell;
-use easytier_rpc::PeerConnInfo;
 
 use super::{
     config_fs::ConfigFs,
@@ -62,27 +62,6 @@ impl GlobalCtx {
 
         let (event_bus, _) = tokio::sync::broadcast::channel(100);
 
-        // NOTICE: we may need to choose stun stun server based on geo location
-        // stun server cross nation may return a external ip address with high latency and loss rate
-        let default_stun_servers = vec![
-            "stun.miwifi.com:3478".to_string(),
-            "stun.qq.com:3478".to_string(),
-            "stun.chat.bilibili.com:3478".to_string(),
-            "fwa.lifesizecloud.com:3478".to_string(),
-            "stun.isp.net.au:3478".to_string(),
-            "stun.nextcloud.com:3478".to_string(),
-            "stun.freeswitch.org:3478".to_string(),
-            "stun.voip.blackberry.com:3478".to_string(),
-            "stunserver.stunprotocol.org:3478".to_string(),
-            "stun.sipnet.com:3478".to_string(),
-            "stun.radiojar.com:3478".to_string(),
-            "stun.sonetel.com:3478".to_string(),
-            "stun.voipgate.com:3478".to_string(),
-            "stun.counterpath.com:3478".to_string(),
-            "180.235.108.91:3478".to_string(),
-            "193.22.2.248:3478".to_string(),
-        ];
-
         GlobalCtx {
             inst_name: inst_name.to_string(),
             id,
@@ -96,7 +75,7 @@ impl GlobalCtx {
 
             hotname: AtomicCell::new(None),
 
-            stun_info_collection: Box::new(StunInfoCollector::new(default_stun_servers)),
+            stun_info_collection: Box::new(StunInfoCollector::new_with_default_servers()),
         }
     }
 
@@ -206,6 +185,7 @@ impl GlobalCtx {
         let ptr = ptr as *mut Box<dyn StunInfoCollectorTrait>;
         unsafe {
             std::ptr::drop_in_place(ptr);
+            #[allow(invalid_reference_casting)]
             std::ptr::write(ptr, collector);
         }
     }
diff --git a/easytier-core/src/common/netns.rs b/easytier-core/src/common/netns.rs
index 71c4a4b..7ce384a 100644
--- a/easytier-core/src/common/netns.rs
+++ b/easytier-core/src/common/netns.rs
@@ -1,6 +1,4 @@
 use futures::Future;
-use once_cell::sync::Lazy;
-use tokio::sync::Mutex;
 
 #[cfg(target_os = "linux")]
 use nix::sched::{setns, CloneFlags};
@@ -12,8 +10,6 @@ pub struct NetNSGuard {
     old_ns: Option<std::fs::File>,
 }
 
-type NetNSLock = Mutex<()>;
-static LOCK: Lazy<NetNSLock> = Lazy::new(|| Mutex::new(()));
 pub static ROOT_NETNS_NAME: &str = "_root_ns";
 
 #[cfg(target_os = "linux")]
diff --git a/easytier-core/src/common/network.rs b/easytier-core/src/common/network.rs
index ded53d4..4e336db 100644
--- a/easytier-core/src/common/network.rs
+++ b/easytier-core/src/common/network.rs
@@ -1,6 +1,6 @@
 use std::{ops::Deref, sync::Arc};
 
-use easytier_rpc::peer::GetIpListResponse;
+use crate::rpc::peer::GetIpListResponse;
 use pnet::datalink::NetworkInterface;
 use tokio::{
     sync::{Mutex, RwLock},
@@ -156,7 +156,7 @@ impl IPCollector {
 
     #[tracing::instrument(skip(net_ns))]
     async fn do_collect_ip_addrs(with_public: bool, net_ns: NetNS) -> GetIpListResponse {
-        let mut ret = easytier_rpc::peer::GetIpListResponse {
+        let mut ret = crate::rpc::peer::GetIpListResponse {
             public_ipv4: "".to_string(),
             interface_ipv4s: vec![],
             public_ipv6: "".to_string(),
diff --git a/easytier-core/src/common/stun.rs b/easytier-core/src/common/stun.rs
index bde9425..17a01ea 100644
--- a/easytier-core/src/common/stun.rs
+++ b/easytier-core/src/common/stun.rs
@@ -2,19 +2,50 @@ use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
 use std::sync::Arc;
 use std::time::Duration;
 
+use crate::rpc::{NatType, StunInfo};
 use crossbeam::atomic::AtomicCell;
-use easytier_rpc::{NatType, StunInfo};
 use stun_format::Attr;
 use tokio::net::{lookup_host, UdpSocket};
 use tokio::sync::RwLock;
 use tokio::task::JoinSet;
+use tracing::Level;
 
 use crate::common::error::Error;
 
-struct Stun {
-    stun_server: String,
-    req_repeat: u8,
-    resp_timeout: Duration,
+struct HostResolverIter {
+    hostnames: Vec<String>,
+    ips: Vec<SocketAddr>,
+}
+
+impl HostResolverIter {
+    fn new(hostnames: Vec<String>) -> Self {
+        Self {
+            hostnames,
+            ips: vec![],
+        }
+    }
+
+    #[async_recursion::async_recursion]
+    async fn next(&mut self) -> Option<SocketAddr> {
+        if self.ips.is_empty() {
+            if self.hostnames.is_empty() {
+                return None;
+            }
+
+            let host = self.hostnames.remove(0);
+            match lookup_host(&host).await {
+                Ok(ips) => {
+                    self.ips = ips.collect();
+                }
+                Err(e) => {
+                    tracing::warn!(?host, ?e, "lookup host for stun failed");
+                    return self.next().await;
+                }
+            };
+        }
+
+        Some(self.ips.remove(0))
+    }
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -36,8 +67,15 @@ impl BindRequestResponse {
     }
 }
 
+#[derive(Debug, Clone)]
+struct Stun {
+    stun_server: SocketAddr,
+    req_repeat: u8,
+    resp_timeout: Duration,
+}
+
 impl Stun {
-    pub fn new(stun_server: String) -> Self {
+    pub fn new(stun_server: SocketAddr) -> Self {
         Self {
             stun_server,
             req_repeat: 3,
@@ -73,7 +111,7 @@ impl Stun {
             unsafe { std::ptr::copy(udp_buf.as_ptr(), buf.as_ptr() as *mut u8, len) };
 
             let msg = stun_format::Msg::<'a>::from(&buf[..]);
-            tracing::trace!(b = ?&udp_buf[..len], ?msg, ?tids, ?remote_addr, "recv stun response");
+            tracing::info!(b = ?&udp_buf[..len], ?msg, ?tids, ?remote_addr, ?stun_host, "recv stun response");
 
             if msg.typ().is_none() || msg.tid().is_none() {
                 continue;
@@ -151,16 +189,14 @@ impl Stun {
         changed_addr
     }
 
+    #[tracing::instrument(ret, err, level = Level::INFO)]
     pub async fn bind_request(
         &self,
         source_port: u16,
         change_ip: bool,
         change_port: bool,
     ) -> Result<BindRequestResponse, Error> {
-        let stun_host = lookup_host(&self.stun_server)
-            .await?
-            .next()
-            .ok_or(Error::NotFound)?;
+        let stun_host = self.stun_server;
         let udp = UdpSocket::bind(format!("0.0.0.0:{}", source_port)).await?;
 
         // repeat req in case of packet loss
@@ -218,7 +254,7 @@ impl Stun {
     }
 }
 
-struct UdpNatTypeDetector {
+pub struct UdpNatTypeDetector {
     stun_servers: Vec<String>,
 }
 
@@ -227,7 +263,7 @@ impl UdpNatTypeDetector {
         Self { stun_servers }
     }
 
-    async fn get_udp_nat_type(&self, mut source_port: u16) -> NatType {
+    pub async fn get_udp_nat_type(&self, mut source_port: u16) -> NatType {
         // Like classic STUN (rfc3489). Detect NAT behavior for UDP.
         // Modified from rfc3489. Requires at least two STUN servers.
         let mut ret_test1_1 = None;
@@ -241,7 +277,8 @@ impl UdpNatTypeDetector {
         }
 
         let mut succ = false;
-        for server_ip in &self.stun_servers {
+        let mut ips = HostResolverIter::new(self.stun_servers.clone());
+        while let Some(server_ip) = ips.next().await {
             let stun = Stun::new(server_ip.clone());
             let ret = stun.bind_request(source_port, false, false).await;
             if ret.is_err() {
@@ -255,13 +292,19 @@ impl UdpNatTypeDetector {
             ret_test1_2 = ret.ok();
             let ret = stun.bind_request(source_port, true, true).await;
             if let Ok(resp) = ret {
-                if !resp.ip_changed || !resp.port_changed {
+                if !resp.real_ip_changed || !resp.real_port_changed {
+                    tracing::info!(
+                        ?server_ip,
+                        ?ret,
+                        "stun bind request return with unchanged ip and port"
+                    );
                     // Try another STUN server
                     continue;
                 }
             }
             ret_test2 = ret.ok();
             ret_test3 = stun.bind_request(source_port, false, true).await.ok();
+            tracing::info!(?ret_test3, "stun bind request with changed port");
             succ = true;
             break;
         }
@@ -340,7 +383,8 @@ impl StunInfoCollectorTrait for StunInfoCollector {
 
     async fn get_udp_port_mapping(&self, local_port: u16) -> Result<SocketAddr, Error> {
         let stun_servers = self.stun_servers.read().await.clone();
-        for server in stun_servers.iter() {
+        let mut ips = HostResolverIter::new(stun_servers.clone());
+        while let Some(server) = ips.next().await {
             let stun = Stun::new(server.clone());
             let Ok(ret) = stun.bind_request(local_port, false, false).await else {
                 tracing::warn!(?server, "stun bind request failed");
@@ -371,6 +415,33 @@ impl StunInfoCollector {
         ret
     }
 
+    pub fn new_with_default_servers() -> Self {
+        Self::new(Self::get_default_servers())
+    }
+
+    pub fn get_default_servers() -> Vec<String> {
+        // NOTICE: we may need to choose stun stun server based on geo location
+        // stun server cross nation may return a external ip address with high latency and loss rate
+        vec![
+            "stun.miwifi.com:3478".to_string(),
+            "stun.qq.com:3478".to_string(),
+            // "stun.chat.bilibili.com:3478".to_string(), // bilibili's stun server doesn't repond to change_ip and change_port
+            "fwa.lifesizecloud.com:3478".to_string(),
+            "stun.isp.net.au:3478".to_string(),
+            "stun.nextcloud.com:3478".to_string(),
+            "stun.freeswitch.org:3478".to_string(),
+            "stun.voip.blackberry.com:3478".to_string(),
+            "stunserver.stunprotocol.org:3478".to_string(),
+            "stun.sipnet.com:3478".to_string(),
+            "stun.radiojar.com:3478".to_string(),
+            "stun.sonetel.com:3478".to_string(),
+            "stun.voipgate.com:3478".to_string(),
+            "stun.counterpath.com:3478".to_string(),
+            "180.235.108.91:3478".to_string(),
+            "193.22.2.248:3478".to_string(),
+        ]
+    }
+
     fn start_stun_routine(&mut self) {
         let stun_servers = self.stun_servers.clone();
         let udp_nat_type = self.udp_nat_type.clone();
@@ -412,7 +483,8 @@ mod tests {
     #[tokio::test]
     async fn test_stun_bind_request() {
         // miwifi / qq seems not correctly responde to change_ip and change_port, they always try to change the src ip and port.
-        let stun = Stun::new("stun1.l.google.com:19302".to_string());
+        let mut ips = HostResolverIter::new(vec!["stun1.l.google.com:19302".to_string()]);
+        let stun = Stun::new(ips.next().await.unwrap());
         // let stun = Stun::new("180.235.108.91:3478".to_string());
         // let stun = Stun::new("193.22.2.248:3478".to_string());
         // let stun = Stun::new("stun.chat.bilibili.com:3478".to_string());
diff --git a/easytier-core/src/connector/direct.rs b/easytier-core/src/connector/direct.rs
index 1c58b77..b24dcb8 100644
--- a/easytier-core/src/connector/direct.rs
+++ b/easytier-core/src/connector/direct.rs
@@ -12,7 +12,7 @@ use crate::{
     peers::{peer_manager::PeerManager, peer_rpc::PeerRpcManager, PeerId},
 };
 
-use easytier_rpc::{peer::GetIpListResponse, PeerConnInfo};
+use crate::rpc::{peer::GetIpListResponse, PeerConnInfo};
 use tokio::{task::JoinSet, time::timeout};
 use tracing::Instrument;
 
diff --git a/easytier-core/src/connector/manual.rs b/easytier-core/src/connector/manual.rs
index 0be083f..8ea9808 100644
--- a/easytier-core/src/connector/manual.rs
+++ b/easytier-core/src/connector/manual.rs
@@ -1,16 +1,14 @@
 use std::{collections::BTreeSet, sync::Arc};
 
 use dashmap::{DashMap, DashSet};
-use easytier_rpc::{
-    connector_manage_rpc_server::ConnectorManageRpc, Connector, ConnectorStatus,
-    ListConnectorRequest, ManageConnectorRequest,
-};
 use tokio::{
     sync::{broadcast::Receiver, mpsc, Mutex},
     task::JoinSet,
     time::timeout,
 };
 
+use crate::rpc as easytier_rpc;
+
 use crate::{
     common::{
         error::Error,
@@ -19,6 +17,10 @@ use crate::{
     },
     connector::set_bind_addr_for_peer_connector,
     peers::peer_manager::PeerManager,
+    rpc::{
+        connector_manage_rpc_server::ConnectorManageRpc, Connector, ConnectorStatus,
+        ListConnectorRequest, ManageConnectorRequest,
+    },
     tunnels::{Tunnel, TunnelConnector},
     use_global_var,
 };
diff --git a/easytier-core/src/connector/udp_hole_punch.rs b/easytier-core/src/connector/udp_hole_punch.rs
index a7b2a51..b7db5c8 100644
--- a/easytier-core/src/connector/udp_hole_punch.rs
+++ b/easytier-core/src/connector/udp_hole_punch.rs
@@ -2,7 +2,6 @@ use std::{net::SocketAddr, sync::Arc};
 
 use anyhow::Context;
 use crossbeam::atomic::AtomicCell;
-use easytier_rpc::NatType;
 use rand::{seq::SliceRandom, Rng, SeedableRng};
 use tokio::{net::UdpSocket, sync::Mutex, task::JoinSet};
 use tracing::Instrument;
@@ -13,6 +12,7 @@ use crate::{
         stun::StunInfoCollectorTrait,
     },
     peers::{peer_manager::PeerManager, PeerId},
+    rpc::NatType,
     tunnels::{
         common::setup_sokcet2,
         udp_tunnel::{UdpPacket, UdpTunnelConnector, UdpTunnelListener},
@@ -459,7 +459,7 @@ impl UdpHolePunchConnector {
 mod tests {
     use std::sync::Arc;
 
-    use easytier_rpc::{NatType, StunInfo};
+    use crate::rpc::{NatType, StunInfo};
 
     use crate::{
         common::{error::Error, stun::StunInfoCollectorTrait},
diff --git a/easytier-core/src/instance/instance.rs b/easytier-core/src/instance/instance.rs
index 5cc418b..e18319c 100644
--- a/easytier-core/src/instance/instance.rs
+++ b/easytier-core/src/instance/instance.rs
@@ -318,12 +318,12 @@ impl Instance {
             log::info!("[INIT RPC] start rpc server. addr: {}", addr);
             Server::builder()
                 .add_service(
-                    easytier_rpc::peer_manage_rpc_server::PeerManageRpcServer::new(
+                    crate::rpc::peer_manage_rpc_server::PeerManageRpcServer::new(
                         PeerManagerRpcService::new(peer_mgr),
                     ),
                 )
                 .add_service(
-                    easytier_rpc::connector_manage_rpc_server::ConnectorManageRpcServer::new(
+                    crate::rpc::connector_manage_rpc_server::ConnectorManageRpcServer::new(
                         ConnectorManagerRpcService(conn_manager.clone()),
                     ),
                 )
diff --git a/easytier-core/src/lib.rs b/easytier-core/src/lib.rs
new file mode 100644
index 0000000..8e4346d
--- /dev/null
+++ b/easytier-core/src/lib.rs
@@ -0,0 +1,6 @@
+#![allow(dead_code)]
+
+pub mod arch;
+pub mod common;
+pub mod rpc;
+pub mod tunnels;
diff --git a/easytier-core/src/main.rs b/easytier-core/src/main.rs
index d673370..1bb7941 100644
--- a/easytier-core/src/main.rs
+++ b/easytier-core/src/main.rs
@@ -12,6 +12,7 @@ mod gateway;
 mod instance;
 mod peer_center;
 mod peers;
+mod rpc;
 mod tunnels;
 
 use instance::instance::{Instance, InstanceConfigWriter};
diff --git a/easytier-core/src/peer_center/service.rs b/easytier-core/src/peer_center/service.rs
index 3c0ddf6..58b07a8 100644
--- a/easytier-core/src/peer_center/service.rs
+++ b/easytier-core/src/peer_center/service.rs
@@ -3,7 +3,7 @@ use std::collections::BTreeMap;
 use crate::peers::PeerId;
 
 use super::{Digest, Error};
-use easytier_rpc::PeerInfo;
+use crate::rpc::PeerInfo;
 
 #[derive(Debug, Clone, Hash, serde::Deserialize, serde::Serialize)]
 pub enum LatencyLevel {
diff --git a/easytier-core/src/peers/peer.rs b/easytier-core/src/peers/peer.rs
index 1edbd2b..f14d96c 100644
--- a/easytier-core/src/peers/peer.rs
+++ b/easytier-core/src/peers/peer.rs
@@ -1,7 +1,6 @@
 use std::sync::Arc;
 
 use dashmap::DashMap;
-use easytier_rpc::PeerConnInfo;
 
 use tokio::{
     select,
@@ -12,12 +11,12 @@ use tokio_util::bytes::Bytes;
 use tracing::Instrument;
 use uuid::Uuid;
 
+use super::peer_conn::PeerConn;
 use crate::common::{
     error::Error,
     global_ctx::{ArcGlobalCtx, GlobalCtxEvent},
 };
-
-use super::peer_conn::PeerConn;
+use crate::rpc::PeerConnInfo;
 
 type ArcPeerConn = Arc<Mutex<PeerConn>>;
 type ConnMap = Arc<DashMap<Uuid, ArcPeerConn>>;
diff --git a/easytier-core/src/peers/peer_conn.rs b/easytier-core/src/peers/peer_conn.rs
index 06453cb..687c750 100644
--- a/easytier-core/src/peers/peer_conn.rs
+++ b/easytier-core/src/peers/peer_conn.rs
@@ -7,7 +7,6 @@ use std::{
     },
 };
 
-use easytier_rpc::{PeerConnInfo, PeerConnStats};
 use futures::{SinkExt, StreamExt};
 use pnet::datalink::NetworkInterface;
 
@@ -26,6 +25,7 @@ use tracing::Instrument;
 use crate::{
     common::global_ctx::ArcGlobalCtx,
     define_tunnel_filter_chain,
+    rpc::{PeerConnInfo, PeerConnStats},
     tunnels::{
         stats::{Throughput, WindowLatency},
         tunnel_filter::StatsRecorderTunnelFilter,
diff --git a/easytier-core/src/peers/peer_manager.rs b/easytier-core/src/peers/peer_manager.rs
index fd1fc04..a1ed2cd 100644
--- a/easytier-core/src/peers/peer_manager.rs
+++ b/easytier-core/src/peers/peer_manager.rs
@@ -353,7 +353,7 @@ impl PeerManager {
         self.peers.add_route(arc_route).await;
     }
 
-    pub async fn list_routes(&self) -> Vec<easytier_rpc::Route> {
+    pub async fn list_routes(&self) -> Vec<crate::rpc::Route> {
         self.basic_route.list_routes().await
     }
 
diff --git a/easytier-core/src/peers/peer_map.rs b/easytier-core/src/peers/peer_map.rs
index 3a7922a..119a40c 100644
--- a/easytier-core/src/peers/peer_map.rs
+++ b/easytier-core/src/peers/peer_map.rs
@@ -2,12 +2,12 @@ use std::{net::Ipv4Addr, sync::Arc};
 
 use anyhow::Context;
 use dashmap::DashMap;
-use easytier_rpc::PeerConnInfo;
 use tokio::sync::{mpsc, RwLock};
 use tokio_util::bytes::Bytes;
 
 use crate::{
     common::{error::Error, global_ctx::ArcGlobalCtx},
+    rpc::PeerConnInfo,
     tunnels::TunnelError,
 };
 
diff --git a/easytier-core/src/peers/peer_rip_route.rs b/easytier-core/src/peers/peer_rip_route.rs
index c22eaa4..4e1f878 100644
--- a/easytier-core/src/peers/peer_rip_route.rs
+++ b/easytier-core/src/peers/peer_rip_route.rs
@@ -2,7 +2,6 @@ use std::{net::Ipv4Addr, sync::Arc, time::Duration};
 
 use async_trait::async_trait;
 use dashmap::DashMap;
-use easytier_rpc::{NatType, StunInfo};
 use rkyv::{Archive, Deserialize, Serialize};
 use tokio::{sync::Mutex, task::JoinSet};
 use tokio_util::bytes::Bytes;
@@ -21,6 +20,7 @@ use crate::{
         route_trait::{Route, RouteInterfaceBox},
         PeerId,
     },
+    rpc::{NatType, StunInfo},
 };
 
 use super::{packet::ArchivedPacketBody, peer_manager::PeerPacketFilter};
@@ -431,11 +431,11 @@ impl Route for BasicRoute {
         }
     }
 
-    async fn list_routes(&self) -> Vec<easytier_rpc::Route> {
+    async fn list_routes(&self) -> Vec<crate::rpc::Route> {
         let mut routes = Vec::new();
 
         let parse_route_info = |real_peer_id: &Uuid, route_info: &SyncPeerInfo| {
-            let mut route = easytier_rpc::Route::default();
+            let mut route = crate::rpc::Route::default();
             route.ipv4_addr = if let Some(ipv4_addr) = route_info.ipv4_addr {
                 ipv4_addr.to_string()
             } else {
diff --git a/easytier-core/src/peers/route_trait.rs b/easytier-core/src/peers/route_trait.rs
index ef1ac28..3bb59b6 100644
--- a/easytier-core/src/peers/route_trait.rs
+++ b/easytier-core/src/peers/route_trait.rs
@@ -27,7 +27,7 @@ pub trait Route {
     async fn close(&self);
 
     async fn get_next_hop(&self, peer_id: &PeerId) -> Option<PeerId>;
-    async fn list_routes(&self) -> Vec<easytier_rpc::Route>;
+    async fn list_routes(&self) -> Vec<crate::rpc::Route>;
 
     async fn get_peer_id_by_ipv4(&self, _ipv4: &Ipv4Addr) -> Option<PeerId> {
         None
diff --git a/easytier-core/src/peers/rpc_service.rs b/easytier-core/src/peers/rpc_service.rs
index 2bf4d17..4363e97 100644
--- a/easytier-core/src/peers/rpc_service.rs
+++ b/easytier-core/src/peers/rpc_service.rs
@@ -1,8 +1,10 @@
 use std::sync::Arc;
 
-use easytier_rpc::cli::PeerInfo;
-use easytier_rpc::peer_manage_rpc_server::PeerManageRpc;
-use easytier_rpc::{ListPeerRequest, ListPeerResponse, ListRouteRequest, ListRouteResponse};
+use crate::rpc::{
+    cli::PeerInfo,
+    peer_manage_rpc_server::PeerManageRpc,
+    {ListPeerRequest, ListPeerResponse, ListRouteRequest, ListRouteResponse},
+};
 use tonic::{Request, Response, Status};
 
 use super::peer_manager::PeerManager;
@@ -20,7 +22,7 @@ impl PeerManagerRpcService {
         let peers = self.peer_manager.get_peer_map().list_peers().await;
         let mut peer_infos = Vec::new();
         for peer in peers {
-            let mut peer_info = easytier_rpc::PeerInfo::default();
+            let mut peer_info = PeerInfo::default();
             peer_info.peer_id = peer.to_string();
 
             if let Some(conns) = self
diff --git a/easytier-core/src/rpc/lib.rs b/easytier-core/src/rpc/mod.rs
similarity index 100%
rename from easytier-core/src/rpc/lib.rs
rename to easytier-core/src/rpc/mod.rs
diff --git a/easytier-core/src/tests/mod.rs b/easytier-core/src/tests/mod.rs
index ed01a5e..984507b 100644
--- a/easytier-core/src/tests/mod.rs
+++ b/easytier-core/src/tests/mod.rs
@@ -124,7 +124,7 @@ pub fn enable_log() {
         .init();
 }
 
-fn check_route(ipv4: &str, dst_peer_id: uuid::Uuid, routes: Vec<easytier_rpc::Route>) {
+fn check_route(ipv4: &str, dst_peer_id: uuid::Uuid, routes: Vec<crate::rpc::Route>) {
     let mut found = false;
     for r in routes.iter() {
         if r.ipv4_addr == ipv4.to_string() {
diff --git a/easytier-core/src/tunnels/mod.rs b/easytier-core/src/tunnels/mod.rs
index 2f4db32..b064842 100644
--- a/easytier-core/src/tunnels/mod.rs
+++ b/easytier-core/src/tunnels/mod.rs
@@ -8,8 +8,8 @@ pub mod udp_tunnel;
 
 use std::{fmt::Debug, net::SocketAddr, pin::Pin, sync::Arc};
 
+use crate::rpc::TunnelInfo;
 use async_trait::async_trait;
-use easytier_rpc::TunnelInfo;
 use futures::{Sink, SinkExt, Stream};
 
 use thiserror::Error;
diff --git a/easytier-core/src/tunnels/tunnel_filter.rs b/easytier-core/src/tunnels/tunnel_filter.rs
index e1901b7..8209360 100644
--- a/easytier-core/src/tunnels/tunnel_filter.rs
+++ b/easytier-core/src/tunnels/tunnel_filter.rs
@@ -3,7 +3,7 @@ use std::{
     task::{Context, Poll},
 };
 
-use easytier_rpc::TunnelInfo;
+use crate::rpc::TunnelInfo;
 use futures::{Sink, SinkExt, Stream, StreamExt};
 
 use self::stats::Throughput;
diff --git a/easytier-core/src/tunnels/udp_tunnel.rs b/easytier-core/src/tunnels/udp_tunnel.rs
index 37d477a..ad1789e 100644
--- a/easytier-core/src/tunnels/udp_tunnel.rs
+++ b/easytier-core/src/tunnels/udp_tunnel.rs
@@ -2,7 +2,6 @@ use std::{fmt::Debug, pin::Pin, sync::Arc};
 
 use async_trait::async_trait;
 use dashmap::DashMap;
-use easytier_rpc::TunnelInfo;
 use futures::{stream::FuturesUnordered, SinkExt, StreamExt};
 use rkyv::{Archive, Deserialize, Serialize};
 use std::net::SocketAddr;
@@ -15,6 +14,7 @@ use tracing::Instrument;
 
 use crate::{
     common::rkyv_util::{self, encode_to_bytes},
+    rpc::TunnelInfo,
     tunnels::{build_url_from_socket_addr, close_tunnel, TunnelConnCounter, TunnelConnector},
 };