Windows驱动编程之NDIS(VPN)
一 引言
一篇有关Windows VPN代理技术分享，并不讨论VPN方案和隧道加密代理，而是NDIS小端数据包到应用层原理技术探讨分享。
个人Windows下用过两个OpenVpn驱动版本，tap-windows 5.0(Ndis 5.0)版本 <= win7，tap-windows 6.0(Ndis 6.0)版本 >= win8。
Jason Donenfeld并满意OpenVPN tap-Windows驱动，自研WireGuard Wintun代替tap-Windows的NDIS,后来减少r3r0上下文切换等优化，就有了WireGuard NT。
WirGuard口碑很不错，被Linux集成在了内核称艺术品。商用化方案越来越多使用WirGuard代替OpenVpn。r3也提供了高效开发，也不用关注驱动可以Go一把梭。
二 源码
OpenVpn TapWin下载地址：https://github.com/OpenVPN/tap-windows6
WireGuard Nt下载地址：https://git.zx2c4.com/wireguard-nt/
三 Tap-windows：
tap-windows5.0用Wdk7600编译，tap-windows6.0高版本wdk编译，用两套xp~win7 tap-5.0支持比较友好，tap-Win 5.0高版本系统会有问题，具体的可以看Github中Issues。
这里并不是讲OpenVpn它本身如何做隧道的，而是通过假设代理方案举例：
- 初始化tap驱动，注册小端生成虚拟网卡。
- 应用层设置路由表和虚拟网卡，指定IP路由到虚拟网卡。
- NDIS捕获完成IRP发送应用层，应用层拿到数据包Socket5或者私有代理。
- 代理回包应用层写回虚拟网卡。
Tap-windows5.0和6.0捕获数据包传输使用都是I/O，异步ReadFile/WriteFile MDL读写。这套方案做代理需要面对一些问题，NDIS提取网络层数据包应用层代理？
UDP处理稍微简单一些，TCP代理有点复杂，考虑TCP的每一个数据包(Wirshark抓到的一样)，包括握手挥手超时恢复等都要自己处理。
虽然有lWIP用来解决该问题，应用层帮你去维护阻塞控制、RTT、快速恢复转发等，最后通过IOCP或者Asio等框架转发链路数据至Server服务，BadVpn好像也是使的该方案。
数据包流动实现:
应用层：
- 初始化驱动以后，起线程异步读取发送ReadFile I/O：
for (i = 0; i < 8; i++)
{
readBytes = 0;
memset(&ol, 0, sizeof(ol));
ol.hEvent = g_ioEvent;
if (!ReadFile(hDevice, &rr, sizeof(rr), NULL, &ol))
{
if (GetLastError() != ERROR_IO_PENDING)
{
OutputDebugString(L"ReadFile Error!");
goto finish;
}
}
for (;;)
{
dwRes = WaitForMultipleObjects(
sizeof(events) / sizeof(events[0]),
events,
FALSE,
waitTimeout);
......
}
......
}
内核层:
- 驱动接收应用层发来的I/O，先把Read IRP Pending，AdapterCreate->CreateTapDevice->TapDeviceRead(IRP_MJ_READ),尝试拉取AdapterSendNetBufferLists捕获的数据包副本，如果有的话完成I/O反馈。
IoCsqInsertIrp(&adapter->PendingReadIrpQueue.CsqQueue, Irp, NULL);
tapProcessSendPacketQueue(adapter);
ntStatus = STATUS_PENDING;另一种写法Read I/O只负责Pendig IRP存入链表，Event来唤醒单独的内核线程尝试拉取副本完成IRP。
IoMarkIrpPending(irp);
InsertTailList(&g_pendedIoRequests, &irp->Tail.Overlay.ListEntry);
status = STATUS_PENDING;
KeSetEvent(&ThreadEvent, IO_NO_INCREMENT, FALSE);- AdapterSendNetBufferLists判断是否打开tap和已就绪，NdisDeviceStateD0是网络适配发出的电源状态，表示电源管理已准备就绪并且已激活,case里面过滤初始化和适配器不指示接收的情况。
` ` `
if(adapter->TapFileObject == NULL)
{
//
// Complete all NBLs and return if adapter not ready.
//
tapSendNetBufferListsComplete(
adapter,
NetBufferLists,
NDIS_STATUS_SUCCESS,
DispatchLevel
);
return;
}
if(!Adapter->LogicalMediaState)
{
status = NDIS_STATUS_MEDIA_DISCONNECTED;
}
else if(Adapter->CurrentPowerState != NdisDeviceStateD0)
{
status = NDIS_STATUS_LOW_POWER_STATE;
}
else if(Adapter->ResetInProgress)
{
status = NDIS_STATUS_RESET_IN_PROGRESS;
}
else
{
switch(Adapter->Locked.AdapterState)
{
case MiniportPausingState:
case MiniportPausedState:
status = NDIS_STATUS_PAUSED;
break;
case MiniportHaltedState:
status = NDIS_STATUS_INVALID_STATE;
break;
default:
status = NDIS_STATUS_SUCCESS;
break;
}
}
` ` `- 检测Net_Buf大小范围，大于等于以太网帧头大小(64byte)+IP头大小(20byte)，最大不能超以太网报头+MTU(MAX 1500)+VLAN。
// Minimum packet size is size of Ethernet plus IPv4 headers.
ASSERT(packetLength >= (ETHERNET_HEADER_SIZE + IP_HEADER_SIZE));
if(packetLength < (ETHERNET_HEADER_SIZE + IP_HEADER_SIZE))
{
return FALSE;
}
// Maximum size should be Ethernet header size plus MTU plus modest pad for
// VLAN tag.
ASSERT( packetLength <= (ETHERNET_HEADER_SIZE + VLAN_TAG_SIZE + Adapter->MtuSize));
if(packetLength > (ETHERNET_HEADER_SIZE + VLAN_TAG_SIZE + Adapter->MtuSize))
{
return FALSE;
}- 接下来对多种协议处理，拷贝完整的数据包到副本，插入PacketQueue传输队列。接下来和步骤1中完成IRP调用函数一致，IoCompleteRequest完成pending IRP，将数据反馈至应用层。
ETH_HEADR大小是 6 + 6 + 2 = 14byte
#define MACADDR_SIZE 6
typedef unsigned char MACADDR[MACADDR_SIZE];
typedef struct
{
MACADDR dest; /* destination eth addr */
MACADDR src; /* source ether addr */
USHORT proto; /* packet type ID field */
} ETH_HEADER, *PETH_HEADER;Proto这里处理类型如下：
#define NDIS_ETH_TYPE_IPV4 0x0800 // IPV4
#define NDIS_ETH_TYPE_ARP 0x0806 // ARP
#define NDIS_ETH_TYPE_IPV6 0x86dd // IPV6DHCP在哪里设置标志的？TAP_WIN_IOCTL_CONFIG_DHCP_MASQ
case TAP_WIN_IOCTL_CONFIG_DHCP_MASQ:
{
if(inBufLength >= sizeof(IPADDR)*4)
{
adapter->m_dhcp_enabled = FALSE;
adapter->m_dhcp_server_arp = FALSE;
adapter->m_dhcp_user_supplied_options_buffer_len = 0;
// Adapter IP addr / netmask
adapter->m_dhcp_addr =
((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[0];
adapter->m_dhcp_netmask =
((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[1];
// IP addr of DHCP masq server
adapter->m_dhcp_server_ip =
((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[2];
// Lease time in seconds
adapter->m_dhcp_lease_time =
((IPADDR*) (Irp->AssociatedIrp.SystemBuffer))[3];
GenerateRelatedMAC(
adapter->m_dhcp_server_mac,
adapter->CurrentAddress,
2
);
adapter->m_dhcp_enabled = TRUE;
adapter->m_dhcp_server_arp = TRUE;
CheckIfDhcpAndTunMode (adapter);
Irp->IoStatus.Information = 1; // Simple boolean value
DEBUGP (("[Boom] Configured DHCP MASQ.\n"));
}
else
{
NOTE_ERROR();
Irp->IoStatus.Status = ntStatus = STATUS_INVALID_PARAMETER;
}
}
break;AdapterSendNetBufferLists拷贝代码：
` ` `
// Allocate TAP packet memory
tapPacket = (PTAP_PACKET )NdisAllocateMemoryWithTagPriority(
Adapter->MiniportAdapterHandle,
TAP_PACKET_SIZE (packetLength+addHeaderSize),
TAP_PACKET_TAG,
NormalPoolPriority
);
// 提取Buf数据
packetData = NdisGetDataBuffer(NetBuffer,packetLength,tapPacket->m_Data+addHeaderSize,1,0);
// 拷贝数据到tapPacket+addHeaderSize后的位置，预留出来addHeaderSize
if(packetData != (tapPacket->m_Data+addHeaderSize))
{
// Packet data was contiguous and not yet copied to m_Data.
NdisMoveMemory(tapPacket->m_Data+addHeaderSize,packetData,packetLength);
}
// 填充addHeaderSize大小数据
if(addHeaderSize > 0)
{
// Add an 802.1Q header between the ethernet header and the payload
NdisMoveMemory(tapPacket->m_Data,tapPacket->m_Data+addHeaderSize,ETHERNET_HEADER_SIZE-2);
PETH_HEADER header = (PETH_HEADER)tapPacket->m_Data;
PETH_8021Q_HEADER tag = (PETH_8021Q_HEADER)(header+1);
header->proto = htons(0x8100);
USHORT tagValue = 0;
tagValue |= packetPriority.TagHeader.UserPriority<<13;
tagValue |= packetPriority.TagHeader.VlanId & 0xFFF;
tag->Tag = tagValue;
packetLength += addHeaderSize;
}
// DHCP的处理从数据链路层(ETH_HEADER) 到 网络层(IP_HDR) 到 UDPHDR(传输层) DHCP
......
const ETH_HEADER *eth = (ETH_HEADER *) tapPacket->m_Data;
const IPHDR *ip = (IPHDR *) (tapPacket->m_Data + sizeof (ETH_HEADER));
const UDPHDR *udp = (UDPHDR *) (tapPacket->m_Data + sizeof (ETH_HEADER) + sizeof (IPHDR));
......
else if (packetLength >= sizeof (ETH_HEADER) + sizeof (IPHDR) + sizeof (UDPHDR) + sizeof (DHCP)
&& eth->proto == htons (NDIS_ETH_TYPE_IPV4)
&& ip->version_len == 0x45 // IPv4, 20 byte header
&& ip->protocol == IPPROTO_UDP
&& udp->dest == htons (BOOTPS_PORT)
)
......
// 首先要从Ethernet->proto确认协议
ETH_HEADER *e;
e = (ETH_HEADER *) tapPacket->m_Data;
switch (ntohs (e->proto))
// ARP处理
if (packetLength != sizeof (ARP_PACKET))
{
goto no_queue;
}
ProcessARP (
Adapter,
(PARP_PACKET) tapPacket->m_Data,
Adapter->m_localIP,
Adapter->m_remoteNetwork,
Adapter->m_remoteNetmask,
Adapter->m_TapToUser.dest
);
// ipv4/ipv6处理
case NDIS_ETH_TYPE_IPV4:
// Make sure that packet is large enough to be IPv4.
if (packetLength < (ETHERNET_HEADER_SIZE + IP_HEADER_SIZE))
{
goto no_queue;
}
// Only accept directed packets, not broadcasts.
if (memcmp (e, &Adapter->m_TapToUser, ETHERNET_HEADER_SIZE))
{
goto no_queue;
}
// Packet looks like IPv4, queue it. :-)
tapPacket->m_SizeFlags |= TP_TUN;
break;
case NDIS_ETH_TYPE_IPV6:
// Make sure that packet is large enough to be IPv6.
if (packetLength < (ETHERNET_HEADER_SIZE + IPV6_HEADER_SIZE))
{
goto no_queue;
}
// Broadcasts and multicasts are handled specially
// (to be implemented)
// Neighbor discovery packets to fe80::8 are special
// OpenVPN sets this next-hop to signal "handled by tapdrv"
if ( HandleIPv6NeighborDiscovery(Adapter,tapPacket->m_Data,
packetLength) )
{
goto no_queue;
}
// Packet looks like IPv6, queue it. :-)
tapPacket->m_SizeFlags |= TP_TUN;
}
` ` `- ipv6处理时候会检测fe80::xxx，这类的本地单播不处理，有可能是自动配置地址，邻居发现等等。
static IPV6ADDR IPV6_NS_TARGET_MCAST =
{ 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x01, 0xff, 0x00, 0x00, 0x08 };
static IPV6ADDR IPV6_NS_TARGET_UNICAST =
{ 0xfe, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08 };
if ( memcmp( ipv6->daddr, IPV6_NS_TARGET_MCAST,
sizeof(IPV6ADDR) ) != 0 &&
memcmp( ipv6->daddr, IPV6_NS_TARGET_UNICAST,
sizeof(IPV6ADDR) ) != 0 )
{
return FALSE; // wrong target address
}
// ICMPv6 type+code must be 135/0 for NS
if ( icmpv6_ns->type != ICMPV6_TYPE_NS ||
icmpv6_ns->code != ICMPV6_CODE_0 )
{
return FALSE; // wrong ICMPv6 type
}
// 需要计算和填充checksum
icmpv6_csum = icmpv6_checksum (
(UCHAR*) &(na->icmpv6),
icmpv6_len,
na->ipv6.saddr,
na->ipv6.daddr
);最后调用tapProcessSendPacketQueue完成。
- 应用层拿到数据包以后代理后，代理回包如何处理(recv)？这就和Write操作有关dispatchTable[IRP_MJ_WRITE] = TapDeviceWrite;
......
......
// 拿到应用层传递过来的数据包
unsigned char* packetBuffer = (unsigned char *) Irp->AssociatedIrp.SystemBuffer;
ULONG packetLength = irpSp->Parameters.Write.Length;
PVOID packetPriority = 0;
DUMP_PACKET ("IRP_MJ_WRITE ETH",
packetBuffer,
packetLength);
// 8021Q处理
packetPriority = TapStrip8021Q(&packetBuffer, &packetLength);
// 发送包到虚拟网卡
ntStatus = TapSharedSendPacket(
adapter,
Irp,
packetBuffer,
packetLength,
packetPriority,
NULL,
0
);- TapSharedSendPacket-
// 小于60byte拷贝副本在MDL映射到NetBufferList
// Copy packet data to flat buffer.
......
if(PrefixLength > 0)
{
NdisMoveMemory(allocBuffer, PrefixData, PrefixLength);
}
NdisMoveMemory (allocBuffer + PrefixLength, PacketBuffer, PacketLength);
NdisZeroMemory(allocBuffer + fullLength, paddedLength - fullLength);
......
// 标记注入包
TAP_RX_NBL_FLAGS_CLEAR_ALL(netBufferList);
TAP_RX_NBL_FLAG_SET(netBufferList,TAP_RX_NBL_FLAGS_IS_INJECTED);
// 设置pending
IoMarkIrpPending(Irp);
IoSetCancelRoutine(Irp,NULL);
// Stash IRP pointer in NBL MiniportReserved[0] field.
netBufferList->MiniportReserved[0] = Irp;
netBufferList->MiniportReserved[1] = NULL;
NET_BUFFER_LIST_INFO(netBufferList, Ieee8021QNetBufferListInfo) = PacketPriority;
// Increment in-flight receive NBL count.
nblCount = NdisInterlockedIncrement(&Adapter->ReceiveNblInFlightCount);
ASSERT(nblCount > 0 );
//
// Indicate the packet
// -------------------
// This NBL contains the complete packet including Ethernet header and payload.
//
// 完整的数据调用ReceiveNet完成包
NdisMIndicateReceiveNetBufferLists(
Adapter->MiniportAdapterHandle,
netBufferList,
NDIS_DEFAULT_PORT_NUMBER,
1, // NumberOfNetBufferLists
0 // 清除所有标志
);
// 注意返回标志是Pending
return STATUS_PENDING;- tap 5.0 Recv NBL实现和6.0有些异同，直接发送：
if (!l_Adapter->m_tun && ((l_IrpSp->Parameters.Write.Length) >= ETHERNET_HEADER_SIZE))
{
......
NdisMEthIndicateReceive
(l_Adapter->m_MiniportAdapterHandle,
(NDIS_HANDLE)l_Adapter,
(unsigned char*)p_IRP->AssociatedIrp.SystemBuffer,
ETHERNET_HEADER_SIZE,
(unsigned char*)p_IRP->AssociatedIrp.SystemBuffer + ETHERNET_HEADER_SIZE,
l_IrpSp->Parameters.Write.Length - ETHERNET_HEADER_SIZE,
l_IrpSp->Parameters.Write.Length - ETHERNET_HEADER_SIZE);
NdisMEthIndicateReceiveComplete();
p_IRP->IoStatus.Status = l_Status = STATUS_SUCCESS;
......
}
else if (l_Adapter->m_tun && ((l_IrpSp->Parameters.Write.Length) >= IP_HEADER_SIZE))
{
......
if (IPH_GET_VER(((IPHDR*)p_IRP->AssociatedIrp.SystemBuffer)->version_len) == 6)
{
p_UserToTap = &l_Adapter->m_UserToTap_IPv6;
}
......
NdisMEthIndicateReceive
(l_Adapter->m_MiniportAdapterHandle,
(NDIS_HANDLE)l_Adapter,
(unsigned char*)p_UserToTap,
sizeof(ETH_HEADER),
(unsigned char*)p_IRP->AssociatedIrp.SystemBuffer,
l_IrpSp->Parameters.Write.Length,
l_IrpSp->Parameters.Write.Length);
NdisMEthIndicateReceiveComplete(l_Adapter->m_MiniportAdapterHandle);
p_IRP->IoStatus.Status = l_Status = STATUS_SUCCESS;
......
}NdisMIndicateReceiveNetBufferLists NDIS6.0或更高版本才可以使用，过程并不复杂，不过以前自己做NDIS轮子遇到不少麻烦，后来在做类似项目就直接用开源的驱动。
四 WireGuard NT:
这里并不是讲WirGuard NT本身如何做隧道的，而是通过假设代理方案举例：
- 初始化WireGuard驱动，注册小端生成虚拟网卡。
- 应用层设置路由表和虚拟网卡，指定IP路由到虚拟网卡。
- NDIS捕获数据包，内核WSK代理双向连接。
WireGuard NT梳理基于SendNetBufferLists/ReturnNetBufferLists WSK UDP处理主线
数据包流动实现:
应用层:
- 两个阶段：
第一个是通过WireGuardSetConfiguration-->DevIceIoControl WG_IOCTL_SET 发送 WIREGUARD_INTERFACE_HAS_PRIVATE_KEY,这个不做讨论。
第二个是通过WireGuardSetAdapterState-->DevIceIoControl WG_IOCTL_SET_ADAPTER_STATE 发送 WIREGUARD_ADAPTER_STATE_UP。
// WG_IOCTL_INTERFACE_HAS_LISTEN_PORT 初始化WSK
// WIREGUARD_INTERFACE_HAS_PRIVATE_KEY 初始化Key
// WireGuard首先要先发送key
struct
{
WIREGUARD_INTERFACE Interface;
WIREGUARD_PEER DemoServer;
WIREGUARD_ALLOWED_IP AllV4;
} Config = { .Interface = { .Flags = WIREGUARD_INTERFACE_HAS_PRIVATE_KEY, .PeersCount = 1 },
.DemoServer = { .Flags = WIREGUARD_PEER_HAS_PUBLIC_KEY | WIREGUARD_PEER_HAS_ENDPOINT,
.AllowedIPsCount = 1 },
.AllV4 = { .AddressFamily = AF_INET } };
// 通过发送Key初始化
WIREGUARD_SET_CONFIGURATION_FUNC WireGuardSetConfiguration;
_Use_decl_annotations_
BOOL WINAPI
WireGuardSetConfiguration(WIREGUARD_ADAPTER *Adapter, const WIREGUARD_INTERFACE *Config, DWORD Bytes)
{
HANDLE ControlFile = AdapterOpenDeviceObject(Adapter);
if (ControlFile == INVALID_HANDLE_VALUE)
return FALSE;
if (!DeviceIoControl(ControlFile, WG_IOCTL_SET, NULL, 0, (VOID *)Config, Bytes, &Bytes, NULL))
{
DWORD LastError = GetLastError();
CloseHandle(ControlFile);
SetLastError(LastError);
return FALSE;
}
CloseHandle(ControlFile);
return TRUE;
}
// WireGuardSetAdapterState把WIREGUARD_ADAPTER_STATE_UP参数传递到内核
Log(WIREGUARD_LOG_INFO, L"Setting configuration and adapter up");
if (!WireGuardSetConfiguration(Adapter, &Config.Interface, sizeof(Config)) ||
!WireGuardSetAdapterState(Adapter, WIREGUARD_ADAPTER_STATE_UP))
{
LastError = LogError(L"Failed to set configuration and adapter up", GetLastError());
goto cleanupAdapter;
}内核层:
- 不关注Key初始化，接收到WIREGUARD_ADAPTER_STATE_UP(1) = WG_IOCTL_ADAPTER_STATE_UP。
// WIREGUARD_ADAPTER_STATE_UP内核处理
case WG_IOCTL_SET_ADAPTER_STATE:
AdapterState(DeviceObject, Irp);
AdapterState{
case WG_IOCTL_ADAPTER_STATE_UP:
Irp->IoStatus.Status = Up(Wg);
}
WIREGUARD_ADAPTER_STATE_UP --> Up(Wg) --> SocketInit(Wg, Wg->IncomingPort);
// 上文说到也可以通过发送WG_IOCTL_INTERFACE_HAS_LISTEN_PORT来实现,他会执行以下代码
NTSTATUS Status;
if (IoctlInterface.Flags & WG_IOCTL_INTERFACE_HAS_LISTEN_PORT)
{
Status = SetListenPort(Wg, IoctlInterface.ListenPort);
if (!NT_SUCCESS(Status))
goto cleanupLock;
}
WG_IOCTL_INTERFACE_HAS_LISTEN_PORT --> SetListenPort() --> SocketInit()- SocketInit函数初始化WSK, 创建UDP Socket，WireGuard本身隧道加密传输基于UDP，Win下也不列外。
// 初始化固定大小的SendCtx
Status = ExInitializeLookasideListEx(
&SocketSendCtxCache, NULL, NULL, NonPagedPool, 0, sizeof(SOCKET_SEND_CTX), MEMORY_TAG, 0);
// 初始化/注册WSK_CLIENT_NPI
WSK_CLIENT_NPI WskClientNpi = { .Dispatch = &WskAppDispatchV1 };
Status = WskRegister(&WskClientNpi, &WskRegistration);
// 注册后要捕获WSK NPI(Network Programming Interface)
// WSK_INFINITE_WAIT就是要等待到WSK子系统准备好才可以
Status = WskCaptureProviderNPI(&WskRegistration, WSK_INFINITE_WAIT, &WskProviderNpi);
if (!NT_SUCCESS(Status))
goto cleanupWskRegister;
// 成功连接WSK子系统后，WSK_TRANSPORT_LIST_QUERY 检索可用的传输列表
Status = WskProviderNpi.Dispatch->WskControlClient(
WskProviderNpi.Client,
WSK_TRANSPORT_LIST_QUERY,
0,
NULL,
WskTransportsSize,
WskTransports,
&WskTransportsSize,
NULL);
for (SIZE_T i = 0, n = WskTransportsSize / sizeof(*WskTransports); i < n; ++i)
{
if (WskTransports[i].SocketType == SOCK_DGRAM && WskTransports[i].Protocol == IPPROTO_UDP)
{
if (WskTransports[i].AddressFamily == AF_UNSPEC)
{
WskHasIpv4Transport = TRUE;
WskHasIpv6Transport = TRUE;
}
else if (WskTransports[i].AddressFamily == AF_INET)
WskHasIpv4Transport = TRUE;
else if (WskTransports[i].AddressFamily == AF_INET6)
WskHasIpv6Transport = TRUE;
}
}
// 所有套接字自动启动回调, 使用的参数WSK_EVENT_RECEIVE_FROM参数。
WSK_EVENT_CALLBACK_CONTROL WskEventCallbackControl = { .NpiId = &NPI_WSK_INTERFACE_ID,
.EventMask = WSK_EVENT_RECEIVE_FROM };
Status = WskProviderNpi.Dispatch->WskControlClient(
WskProviderNpi.Client,
WSK_SET_STATIC_EVENT_CALLBACKS,
sizeof(WskEventCallbackControl),
&WskEventCallbackControl,
0,
NULL,
NULL,
NULL);
Status = NotifyRouteChange2(AF_INET, RouteNotification, &RoutingGenerationV4, FALSE, &RouteNotifierV4);
if (!NT_SUCCESS(Status))
goto cleanupWskProviderNPI;
Status = NotifyRouteChange2(AF_INET6, RouteNotification, &RoutingGenerationV6, FALSE, &RouteNotifierV6);
if (!NT_SUCCESS(Status))
goto cleanupRouteNotifierV4;
// 更具体的参数See Msdn: https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wsk/nc-wsk-pfn_wsk_control_client- 上面说到Init过程使用WSK_EVENT_RECEIVE_FROM参数，它对应的回调事件WskReceiveFromEvent。
// 如果List支持v4
if (WskHasIpv4Transport)
{
Status = CreateAndBindSocket(Wg, (SOCKADDR *)&Sa4, &New4);
if (!NT_SUCCESS(Status))
goto out;
}
// 如果List支持v6
if (WskHasIpv6Transport)
{
Sa6.sin6_port = Sa4.sin_port;
Status = CreateAndBindSocket(Wg, (SOCKADDR *)&Sa6, &New6);
if (!NT_SUCCESS(Status))
{
CloseSocket(New4);
New4 = NULL;
if (Status == STATUS_ADDRESS_ALREADY_EXISTS && !Port && Retries++ < 100)
goto retry;
goto out;
}
}- CreateAndBindSocket是WSK Create Bind的封装
// 默认是0.0.0.0和配置传递过来的Port
// 初始异步Event/Irp，Irp完成回调触发KeSetEvent Event
KeInitializeEvent(&Done, SynchronizationEvent, FALSE);
IoInitializeIrp(&I.Irp, sizeof(I.IrpBuffer), 1);
IoSetCompletionRoutine(&I.Irp, RaiseEventOnComplete, &Done, TRUE, TRUE, TRUE);
// Irp传递参数传递给了NPIDisPathch，WskReceiveFromEvent这个地方赋值了IPPROTO_UDP Receive回调函数
static CONST WSK_CLIENT_DATAGRAM_DISPATCH WskClientDatagramDispatch = { .WskReceiveFromEvent = Receive };
// 创建新UDP Socket：SOCK_DGRAM IPPROTO_UDP
Status = WskProviderNpi.Dispatch->WskSocket(
WskProviderNpi.Client,
Sa->sa_family,
SOCK_DGRAM,
IPPROTO_UDP,
WSK_FLAG_DATAGRAM_SOCKET,
Socket,
&WskClientDatagramDispatch,
Wg->SocketOwnerProcess,
NULL,
NULL,
&I.Irp);
ULONG True = TRUE;
if (Sa->sa_family == AF_INET)
{
// IP_PKTINFO允许启动/禁用v4 LPFN_WSARECVMSG(WSARecvMsg)返回数据包信息
Status = SetSockOpt(Sock, IPPROTO_IP, IP_PKTINFO, &True, sizeof(True));
if (!NT_SUCCESS(Status))
goto cleanupSocket;
}
else if (Sa->sa_family == AF_INET6)
{
Status = SetSockOpt(Sock, IPPROTO_IPV6, IPV6_V6ONLY, &True, sizeof(True));
if (!NT_SUCCESS(Status))
goto cleanupSocket;
Status = SetSockOpt(Sock, IPPROTO_IPV6, IPV6_PKTINFO, &True, sizeof(True));
if (!NT_SUCCESS(Status))
goto cleanupSocket;
}
Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Sock->Dispatch)->WskBind(Sock, Sa, 0, &I.Irp);
Status = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Sock->Dispatch)->WskGetLocalAddress(Sock, Sa, &I.Irp);- Up()执行SocketInit后，接下来DeviceStart调用PacketSendStagedPackets，当然这个函数在SendNetBufferList里面也会调用，函数会针对有key和无key分别处理。
......
Irql = RcuReadLock();
Keypair = NoiseKeypairGet(RcuDereference(NOISE_KEYPAIR, Peer->Keypairs.CurrentKeypair));
RcuReadUnlock(Irql);
if (!Keypair)
goto outNokey;
// 有key 数据生产消费
PeerGet(Keypair->Entry.Peer);
_Analysis_assume_(NET_BUFFER_LIST_FIRST_NB(Packets.Head)); /* Checked in SendNetBufferLists(). */
NET_BUFFER_LIST_KEYPAIR(Packets.Head) = Keypair;
PacketCreateData(Peer, Packets.Head);
outNokey:
// 无key
......
......
}- PacketCreateData处理：
Ret = QueueEnqueuePerDeviceAndPeer(&Wg->EncryptQueue, &Peer->TxQueue, &Wg->EncryptThreads, First);
if (Ret == STATUS_PIPE_BROKEN)
{// 失败处理
QueueEnqueuePerPeer(&Peer->Device->TxQueue, &Peer->TxSerialEntry, First, PACKET_STATE_DEAD);
MulticoreWorkQueueBump(&Wg->EncryptThreads);
}
if (NT_SUCCESS(Ret) || Ret == STATUS_PIPE_BROKEN)
return;- QueueEnqueuePerDeviceAndPeer 调用 QueueEnqueuePerDevice插入加密队列，PacketEncryptWorker线程处理NBL加密。
if (!QueueInsertPerPeer(PeerQueue, Nbl))
return STATUS_BUFFER_TOO_SMALL;
/* Then we queue it up in the device queue, which consumes the
* packet as soon as it can.
*/
// 排队消费数据包意思，就是加密处理掉这个数据包
if (!QueueEnqueuePerDevice(DeviceQueue, DeviceThreads, Nbl))
return STATUS_PIPE_BROKEN;- PacketEncryptWorker线程负责NBL加密，算法chacha20 poly1305e, See Rfc：https://www.rfc-editor.org/rfc/rfc7539。
......
EncryptPacket --> ChaCha20Poly1305EncryptMdl
// 加密失败包状态会改变
State = PACKET_STATE_DEAD;
......
QueueEnqueuePerPeer(&Peer->Device->TxQueue, &Peer->TxSerialEntry, First, State);
ProcessPerPeerWork(&Wg->TxQueue);
ProcessPerPeerWork{
PEER_SERIAL_ENTRY *Entry;
while ((Entry = PeerSerialDequeue(WorkQueue)) != NULL)
PeerSerialMaybeRetire(
WorkQueue,
Entry,
PacketPeerTxWork(CONTAINING_RECORD(Entry, WG_PEER, TxSerialEntry), PEER_XMIT_PACKETS_PER_ROUND));
}
PacketPeerTxWork{
// PacketCreateDataDone负责发送
if (State == PACKET_STATE_CRYPTED)
PacketCreateDataDone(Peer, First);
else
FreeSendNetBufferList(Peer->Device, First, 0);
}
PacketCreateDataDone {
if (NT_SUCCESS(SocketSendNblsToPeer(Peer, First, &IsKeepalive)) && !IsKeepalive)
TimersDataSent(Peer);
}
if (NT_SUCCESS(SocketSendNblsToPeer(Peer, First, &IsKeepalive)) && !IsKeepalive)
TimersDataSent(Peer);- SocketSendNblsToPeer负责WSK发送处理。
PFN_WSK_SEND_MESSAGES WskSendMessages = ((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskSendMessages;
#if NTDDI_VERSION == NTDDI_WIN7
if (NoWskSendMessages)
WskSendMessages = PolyfilledWskSendMessages;
#endif
Status = WskSendMessages(
Socket->Sock,
FirstWskBuf,
0,
(PSOCKADDR)&Peer->Endpoint.Addr,
(ULONG)WSA_CMSGDATA_ALIGN(Peer->Endpoint.Cmsg.cmsg_len) + WSA_CMSG_SPACE(0),
&Peer->Endpoint.Cmsg,
&Ctx->Irp);- SendNetBufferLists这里它也叫做生产者，就是数据包捕获源，这里不再关注细节了，它还是会调用PacketSendStagedPackets(步骤5)。
......
......
while (NetBufferListQueueLength(&Peer->StagedPacketQueue) > MAX_STAGED_PACKETS)
{
NET_BUFFER_LIST *NblToDiscard = NetBufferListDequeue(&Peer->StagedPacketQueue);
_Analysis_assume_(NblToDiscard); /* NetBufferListQueueLength() > MAX_STAGED_PACKETS implies
NetBufferListDequeue() returns a NBL. */
NET_BUFFER_LIST_STATUS(NblToDiscard) = NDIS_STATUS_FAILURE;
++Wg->Statistics.ifOutDiscards;
FreeSendNetBufferList(Wg, NblToDiscard, CompleteFlags | NDIS_SEND_COMPLETE_FLAGS_DISPATCH_LEVEL);
}
NetBufferListEnqueue(&Peer->StagedPacketQueue, Nbl);
KeReleaseSpinLock(&Peer->StagedPacketQueue.Lock, Irql);
PacketSendStagedPackets(Peer);
......
......- ReturnNetBufferLists里面处理比较简单，通过WSK回包。
for (NET_BUFFER_LIST *Nbl = First, *NextNbl; Nbl; Nbl = NextNbl)
{
NextNbl = NET_BUFFER_LIST_NEXT_NBL(Nbl);
NET_BUFFER_LIST_NEXT_NBL(Nbl) = NULL;
WSK_DATAGRAM_INDICATION *DatagramIndication = NET_BUFFER_LIST_DATAGRAM_INDICATION(Nbl);
SOCKET *Socket = (SOCKET *)DatagramIndication->Next;
DatagramIndication->Next = NULL;
((WSK_PROVIDER_DATAGRAM_DISPATCH *)Socket->Sock->Dispatch)->WskRelease(Socket->Sock, DatagramIndication);
MemFreeNetBufferList(Nbl);
ExReleaseRundownProtection(&Socket->ItemsInFlight);
}- 回包要经过PacketConsumeData-->PacketDecryptWorker线程解密，丢给PacketPeerRxWork调用NdisMIndicateReceiveNetBufferLists。
// PacketHandshakeRxWorker线程处理Recv调用NdisMIndicateReceiveNetBufferLists：
if (First)
NdisMIndicateReceiveNetBufferLists(First->SourceHandle, First, NDIS_DEFAULT_PORT_NUMBER, NumNbls, 0);- 关于Rcu使用，代码中以PeerGet()和PeerPut(Peer)一对，调用Put会调用回收函数KrefRelease()。另外初始化地方.InitializeHandlerEx = InitializeEx：
// 加密线程
Status = MulticoreWorkQueueInit(&Wg->EncryptThreads, PacketEncryptWorker);
if (!NT_SUCCESS(Status))
goto cleanupHandshakeRxQueue;
// 解密线程
Status = MulticoreWorkQueueInit(&Wg->DecryptThreads, PacketDecryptWorker);
if (!NT_SUCCESS(Status))
goto cleanupEncryptThreads;
// HANDSHAKE_TX_SEND的时候，调用PacketSendHandshakeInitiation
Status = MulticoreWorkQueueInit(&Wg->HandshakeTxThreads, PacketHandshakeTxWorker);
if (!NT_SUCCESS(Status))
goto cleanupDecryptThreads;
PacketSendHandshakeInitiation{
if (NoiseHandshakeCreateInitiation(&Packet, &Peer->Handshake))
{
CookieAddMacToPacket(&Packet, sizeof(Packet), Peer);
TimersAnyAuthenticatedPacketTraversal(Peer);
TimersAnyAuthenticatedPacketSent(Peer);
WriteNoFence64(&Peer->LastSentHandshake, KeQueryInterruptTime());
SocketSendBufferToPeer(Peer, &Packet, sizeof(Packet));
TimersHandshakeInitiated(Peer);
}
}
SocketSendBufferToPeer-->SocketResolvePeerEndpoint;
// PacketHandshakeRxWorker处理wsk recv
Status = MulticoreWorkQueueInit(&Wg->HandshakeRxThreads, PacketHandshakeRxWorker);
if (!NT_SUCCESS(Status))
goto cleanupHandshakeTxThreads;