本文档的Copyleft归yfydz所有，使用GPL发布，可以自由拷贝，转载，转载时请保持文档的完整性，严禁用于任何商业用途。
msn: 来源： 8. IPVS的数据包发送
IPVS连接中的数据包的发送方法是由ip_vs_bind_xmit()函数定义的,具体的发送数据包处理函数定义在net/ipv4/ipvs/ip_vs_xmit.c中。
8.1 NAT发送
NAT发送只发送请求方向的数据,因此是进行目的NAT
/* * NAT transmitter (only for outside-to-inside nat forwarding) * Not used for related ICMP */ int ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp) { struct rtable *rt; /* Route to the other host */ int mtu; struct iphdr *iph = skb->nh.iph;
EnterFunction(10);
/* check if it is a connection of no-client-port */ if (unlikely(cp->flags & IP_VS_CONN_F_NO_CPORT)) { // 如果连接标志了客户端端口为0,将当前skb中的端口填给连接 __u16 _pt, *p; p = skb_header_pointer(skb, iph->ihl*4, sizeof(_pt), &_pt); if (p == NULL) goto tx_error; // *p是源端口 ip_vs_conn_fill_cport(cp, *p); IP_VS_DBG(10, "filled cport=%d\n", ntohs(*p)); } // 查找路由,找不到的话发ICMP出错包 if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(iph->tos)))) goto tx_error_icmp;
/* MTU checking */ // 检查路由发出网卡的MTU,如果包长超过MTU又有DF标志,发送ICMP错误信息,而不进行分片操作 mtu = dst_mtu(&rt->u.dst); if ((skb->len > mtu) && (iph->frag_off&__constant_htons(IP_DF))) { ip_rt_put(rt); icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu)); IP_VS_DBG_RL_PKT(0, pp, skb, 0, "ip_vs_nat_xmit(): frag needed for"); goto tx_error; }
/* copy-on-write the packet before mangling it */ // 让skb包的IP头部分是可写的 if (!ip_vs_make_skb_writable(&skb, sizeof(struct iphdr))) goto tx_error_put;
// 扩充skb头部空间以容纳硬件MAC头数据 if (skb_cow(skb, rt->u.dst.dev->hard_header_len)) goto tx_error_put;
/* drop old route */ // 释放skb当前的路由cache dst_release(skb->dst); skb->dst = &rt->u.dst;
/* mangle the packet */ // 对上层协议(TCP/UDP...)进行目的NAT,因为要发送给实际的目的服务器 if (pp->dnat_handler && !pp->dnat_handler(&skb, pp, cp)) goto tx_error; // 修改目的地址为真实目的服务器地址 skb->nh.iph->daddr = cp->daddr; // 计算IP头校验和 ip_send_check(skb->nh.iph);
IP_VS_DBG_PKT(10, pp, skb, 0, "After DNAT");
/* FIXME: when application helper enlarges the packet and the length is larger than the MTU of outgoing device, there will be still MTU problem. */
/* Another hack: avoid icmp_send in ip_fragment */ // don't fragment标志 skb->local_df = 1; // 发送数据包,实际还是HOOK住netfilter的OUTPUT点,受OUTPUT规则限制 IP_VS_XMIT(skb, rt);
LeaveFunction(10); // 返回STOLEN告诉原来的hook点不用处理该包了 return NF_STOLEN;
tx_error_icmp: dst_link_failure(skb); tx_error: LeaveFunction(10); kfree_skb(skb); return NF_STOLEN; tx_error_put: ip_rt_put(rt); goto tx_error; }
8.2 TUNNEL发送
TUNNEL发送是把原来的IP部分再加在一个IPIP协议(4)头后发出去,新头的目的IP是真实目的服务器,源IP是真实客户端IP,该包是可以路由的,服务器的回应包将直接路由回去而不经过IPVS.
/* * IP Tunneling transmitter * * This function encapsulates the packet in a new IP packet, its * destination will be set to cp->daddr. Most code of this function * is taken from ipip.c. * * It is used in VS/TUN cluster. The load balancer selects a real * server from a cluster based on a scheduling algorithm, * encapsulates the request packet and forwards it to the selected * server. For example, all real servers are configured with * "ifconfig tunl0 <Virtual IP Address> up". When the server receives * the encapsulated packet, it will decapsulate the packet, processe * the request and return the response packets directly to the client * without passing the load balancer. This can greatly increase the * scalability of virtual server. * * Used for ANY protocol */ int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp) { struct rtable *rt; /* Route to the other host */ struct net_device *tdev; /* Device to other host */ struct iphdr *old_iph = skb->nh.iph; u8 tos = old_iph->tos; __be16 df = old_iph->frag_off; struct iphdr *iph; /* Our new IP header */ int max_headroom; /* The extra header space needed */ int mtu;
EnterFunction(10); // 只包装IP包，其他协议如ARP，IPX等不管 if (skb->protocol != __constant_htons(ETH_P_IP)) { IP_VS_DBG_RL("ip_vs_tunnel_xmit(): protocol error, " "ETH_P_IP: %d, skb protocol: %d\n", __constant_htons(ETH_P_IP), skb->protocol); goto tx_error; } // 根据连接信息找外出的路由cache if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(tos)))) goto tx_error_icmp;
// 数据包发出网卡 tdev = rt->u.dst.dev;
// 检查路径的MTU mtu = dst_mtu(&rt->u.dst) - sizeof(struct iphdr); if (mtu < 68) { ip_rt_put(rt); IP_VS_DBG_RL("ip_vs_tunnel_xmit(): mtu less than 68\n"); goto tx_error; } // 更新路由的MTU if (skb->dst) skb->dst->ops->update_pmtu(skb->dst, mtu);
// 检查don't fragement标志 df |= (old_iph->frag_off&__constant_htons(IP_DF));
if ((old_iph->frag_off&__constant_htons(IP_DF)) && mtu < ntohs(old_iph->tot_len)) { // 如果skb包长超过MTU又有DF标志,发送ICMP错误信息,而不进行分片操作 icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu)); ip_rt_put(rt); IP_VS_DBG_RL("ip_vs_tunnel_xmit(): frag needed\n"); goto tx_error; }
/* * Okay, now see if we can stuff it in the buffer as-is. */ // 计算需要添加的IP头的最大长度 max_headroom = LL_RESERVED_SPACE(tdev) + sizeof(struct iphdr);
if (skb_headroom(skb) < max_headroom || skb_cloned(skb) || skb_shared(skb)) { // 重新分配一个skb包,该skb头部足够大可容纳外部IP头空间 // 分配失败则不发送该包了 struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom); if (!new_skb) { ip_rt_put(rt); kfree_skb(skb); IP_VS_ERR_RL("ip_vs_tunnel_xmit(): no memory\n"); return NF_STOLEN; } // 将原来的skb释放掉 kfree_skb(skb); // 将skb指向新包,更新ip头指针 skb = new_skb; old_iph = skb->nh.iph; }
// skb->h是传输层头,现在要新加个IP头,原来的IP头就升级为传输层头 skb->h.raw = (void *) old_iph;
/* fix old IP header checksum */ // 计算老IP头的校验和 ip_send_check(old_iph);
// skb的data指针前移出IP头长度作为新IP头的起点 skb->nh.raw = skb_push(skb, sizeof(struct iphdr)); memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
/* drop old route */ // 更新路由cache dst_release(skb->dst); skb->dst = &rt->u.dst;
// 填写新IP头部信息 /* * Push down and install the IPIP header. */ iph = skb->nh.iph; iph->version = 4; iph->ihl = sizeof(struct iphdr)>>2; iph->frag_off = df; // 协议设置为IPIP, 值为4 iph->protocol = IPPROTO_IPIP; iph->tos = tos; iph->daddr = rt->rt_dst; iph->saddr = rt->rt_src; iph->ttl = old_iph->ttl; iph->tot_len = htons(skb->len); // 设置IP头中的ID值 ip_select_ident(iph, &rt->u.dst, NULL); // 计算IP头校验和 ip_send_check(iph);
/* Another hack: avoid icmp_send in ip_fragment */ // don't fragmemt skb->local_df = 1; // 发送新的skb包 IP_VS_XMIT(skb, rt);
LeaveFunction(10);
return NF_STOLEN;
tx_error_icmp: dst_link_failure(skb); tx_error: kfree_skb(skb); LeaveFunction(10); return NF_STOLEN; }
8.3 DR发送
DR发送是将原来的skb包中的目的MAC地址修改为目的服务器的MAC地址后直接发出,因此是不能路由的,IPVS均衡设备和目的服务器物理上必须在同一个二层子网。在DR模式下，IPVS和服务器都配置了相同的对外服务的VIP，服务器也配了自己的真实IP，不过服务器上配VIP的网卡属性中的NOARP信息是打开的，就是在该网卡上不响应ARP信息，但可以接收到达该VIP的数据包，这样外面请求包先是到IPVS均衡器，因为IPVS的VIP是响应ARP的，然后根据调度找一台服务器，用服务器的真实IP来确定路由，然后直接把包发出来，这时包中所有数据都没修改，因为目的服务器上VIP地址符合包中的目的地址，因此是可以接收该包的。
/* * Direct Routing transmitter * Used for ANY protocol */ int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp) { struct rtable *rt; /* Route to the other host */ struct iphdr *iph = skb->nh.iph; int mtu;
EnterFunction(10); // 根据连接指定的目的服务器找路由 if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(iph->tos)))) goto tx_error_icmp;
/* MTU checking */ // 检查MTU mtu = dst_mtu(&rt->u.dst); if ((iph->frag_off&__constant_htons(IP_DF)) && skb->len > mtu) { icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu)); ip_rt_put(rt); IP_VS_DBG_RL("ip_vs_dr_xmit(): frag needed\n"); goto tx_error; }
/* * Call ip_send_check because we are not sure it is called * after ip_defrag. Is copy-on-write needed? */ // 防止skb包是共用的,还被其他地方使用 if (unlikely((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)) { ip_rt_put(rt); return NF_STOLEN; } // 重新计算IP头校验和 ip_send_check(skb->nh.iph);
/* drop old route */ // 释放原来的路由 dst_release(skb->dst); // 指定新路由 skb->dst = &rt->u.dst;
/* Another hack: avoid icmp_send in ip_fragment */ skb->local_df = 1; // 直接发出了 IP_VS_XMIT(skb, rt);
LeaveFunction(10); return NF_STOLEN;
tx_error_icmp: dst_link_failure(skb); tx_error: kfree_skb(skb); LeaveFunction(10); return NF_STOLEN; }
8.4 NULL发送
啥也没干 /* * NULL transmitter (do nothing except return NF_ACCEPT) */ int ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp) { /* we do not touch skb and do not need pskb ptr */ return NF_ACCEPT; }
8.5 旁路发送
旁路模式,实际数据包不是给IPVS均衡器自己的,由IPVS进行转发 /* * Bypass transmitter * Let packets bypass the destination when the destination is not * available, it may be only used in transparent cache cluster. */ int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp) { struct rtable *rt; /* Route to the other host */ struct iphdr *iph = skb->nh.iph; u8 tos = iph->tos; int mtu; // 用当前IP包的目的地址作为查路由的key struct flowi fl = { .oif = 0, .nl_u = { .ip4_u = { .daddr = iph->daddr, .saddr = 0, .tos = RT_TOS(tos), } }, };
EnterFunction(10); // 查找当前数据包的目的IP地址对应的路由,而不是是IPVS连接的信息找路由 if (ip_route_output_key(&rt, &fl)) { IP_VS_DBG_RL("ip_vs_bypass_xmit(): ip_route_output error, " "dest: %u.%u.%u.%u\n", NIPQUAD(iph->daddr)); goto tx_error_icmp; }
// MTU检查 /* MTU checking */ mtu = dst_mtu(&rt->u.dst); if ((skb->len > mtu) && (iph->frag_off&__constant_htons(IP_DF))) { ip_rt_put(rt); icmp_send(skb, ICMP_DEST_UNREACH,ICMP_FRAG_NEEDED, htonl(mtu)); IP_VS_DBG_RL("ip_vs_bypass_xmit(): frag needed\n"); goto tx_error; }
/* * Call ip_send_check because we are not sure it is called * after ip_defrag. Is copy-on-write needed? */ // 防止skb包是共用的,还被其他地方使用 if (unlikely((skb = skb_share_check(skb, GFP_ATOMIC)) == NULL)) { ip_rt_put(rt); return NF_STOLEN; } // 计算IP头校验和 ip_send_check(skb->nh.iph);
/* drop old route */ // 释放老路由,更新路由 dst_release(skb->dst); skb->dst = &rt->u.dst;
/* Another hack: avoid icmp_send in ip_fragment */ skb->local_df = 1; // 发送 IP_VS_XMIT(skb, rt);
LeaveFunction(10); return NF_STOLEN;
tx_error_icmp: dst_link_failure(skb); tx_error: kfree_skb(skb); LeaveFunction(10); return NF_STOLEN; }
8.6 ICMP发送
发送各种ICMP错误信息包 /* * ICMP packet transmitter * called by the ip_vs_in_icmp */ int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, struct ip_vs_protocol *pp, int offset) { struct rtable *rt; /* Route to the other host */ int mtu; int rc;
EnterFunction(10);
/* The ICMP packet for VS/TUN, VS/DR and LOCALNODE will be forwarded directly here, because there is no need to translate address/port back */ if (IP_VS_FWD_METHOD(cp) != IP_VS_CONN_F_MASQ) { // TUNNEL和DR模式下直接调用连接的发送方法发送即可 if (cp->packet_xmit) rc = cp->packet_xmit(skb, cp, pp); else rc = NF_ACCEPT; /* do not touch skb anymore */ atomic_inc(&cp->in_pkts); goto out; }
/* * mangle and send the packet here (only for VS/NAT) */ // 根据连接的信息找路由 if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(skb->nh.iph->tos)))) goto tx_error_icmp;
/* MTU checking */ mtu = dst_mtu(&rt->u.dst); if ((skb->len > mtu) && (skb->nh.iph->frag_off&__constant_htons(IP_DF))) { ip_rt_put(rt); icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu)); IP_VS_DBG_RL("ip_vs_in_icmp(): frag needed\n"); goto tx_error; }
/* copy-on-write the packet before mangling it */ // 使skb包可写 if (!ip_vs_make_skb_writable(&skb, offset)) goto tx_error_put;
// 扩充skb头部空间以容纳硬件MAC头数据 if (skb_cow(skb, rt->u.dst.dev->hard_header_len)) goto tx_error_put;
// 路由更新 /* drop the old route when skb is not shared */ dst_release(skb->dst); skb->dst = &rt->u.dst;
// 对ICMP包进行地址转换 ip_vs_nat_icmp(skb, pp, cp, 0);
/* Another hack: avoid icmp_send in ip_fragment */ skb->local_df = 1;
IP_VS_XMIT(skb, rt);
rc = NF_STOLEN; goto out;
tx_error_icmp: dst_link_failure(skb); tx_error: dev_kfree_skb(skb); rc = NF_STOLEN; out: LeaveFunction(10); return rc; tx_error_put: ip_rt_put(rt); goto tx_error; }