From caa6cc58276e87de56057bb3bf964b7bec2027ee Mon Sep 17 00:00:00 2001 From: r-caamano Date: Mon, 26 Aug 2024 15:34:52 +0000 Subject: [PATCH 1/8] Initial dynamic PAT for TCP push and add outbound tracking as default for InternalInterfaces --- CHANGELOG.md | 6 ++ files/scripts/start_ebpf_controller.py | 2 +- files/scripts/start_ebpf_router.py | 2 +- files/scripts/start_ebpf_tunnel.py | 2 +- src/zfw.c | 9 +- src/zfw_monitor.c | 2 +- src/zfw_tc_ingress.c | 99 +++++++++++++++++++- src/zfw_tc_outbound_track.c | 119 ++++++++++++++++++++++++- 8 files changed, 230 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65ceccb..2c89773 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- +### +# [0.8.15] - 2024-08-26 +- Refactored all startup scripts to default InternalInterfaces to have outbound tracking enabled +- Refactored masquerade to use dynamic PAT vs static PAT + + ### # [0.8.14] - 2024-08-16 - Fixed issue where icmp type 3 tcp only accepting inbound for ports associated with local listening ports diff --git a/files/scripts/start_ebpf_controller.py b/files/scripts/start_ebpf_controller.py index 55a67e1..c80c34b 100755 --- a/files/scripts/start_ebpf_controller.py +++ b/files/scripts/start_ebpf_controller.py @@ -332,7 +332,7 @@ def set_local_rules(ip): else: outbound_passthrough_track[interface["Name"]] = False; else: - outbound_passthrough_track[interface["Name"]] = False; + outbound_passthrough_track[interface["Name"]] = True; if("PerInterfaceRules") in interface.keys(): if(interface["PerInterfaceRules"]): per_interface_rules[interface["Name"]] = True; diff --git a/files/scripts/start_ebpf_router.py b/files/scripts/start_ebpf_router.py index 5d89a12..31d97b0 100755 --- a/files/scripts/start_ebpf_router.py +++ b/files/scripts/start_ebpf_router.py @@ -279,7 +279,7 @@ def set_local_rules(resolver): else: per_interface_rules[interface["Name"]] = False; else: - per_interface_rules[interface["Name"]] = False; + per_interface_rules[interface["Name"]] = True; else: print('Mandatory key \"Name\" missing skipping internal interface entry!') diff --git a/files/scripts/start_ebpf_tunnel.py b/files/scripts/start_ebpf_tunnel.py index 29f0e9d..153acff 100755 --- a/files/scripts/start_ebpf_tunnel.py +++ b/files/scripts/start_ebpf_tunnel.py @@ -36,7 +36,7 @@ def tc_status(interface, direction): else: outbound_passthrough_track[interface["Name"]] = False; else: - outbound_passthrough_track[interface["Name"]] = False; + outbound_passthrough_track[interface["Name"]] = True; if("PerInterfaceRules") in interface.keys(): if(interface["PerInterfaceRules"]): per_interface_rules[interface["Name"]] = True; diff --git a/src/zfw.c b/src/zfw.c index bf73583..3942934 100644 --- a/src/zfw.c +++ b/src/zfw.c @@ -218,6 +218,7 @@ const char *egress6_map_path = "/sys/fs/bpf/tc/globals/zt_egress6_map"; const char *egress_count_map_path = "/sys/fs/bpf/tc/globals/egress_count_map"; const char *egress_count6_map_path = "/sys/fs/bpf/tc/globals/egress6_count_map"; const char *masquerade_map_path = "/sys/fs/bpf/tc/globals/masquerade_map"; +const char *masquerade_reverse_map_path = "/sys/fs/bpf/tc/globals/masquerade_reverse_map"; const char *icmp_masquerade_map_path = "/sys/fs/bpf/tc/globals/icmp_masquerade_map"; const char *icmp_echo_map_path = "/sys/fs/bpf/tc/globals/icmp_echo_map"; char doc[] = "zfw -- ebpf firewall configuration tool"; @@ -241,7 +242,7 @@ char *direction_string; char *masq_interface; char check_alt[IF_NAMESIZE]; -const char *argp_program_version = "0.8.14"; +const char *argp_program_version = "0.8.15"; struct ring_buffer *ring_buffer; __u32 if_list[MAX_IF_LIST_ENTRIES]; @@ -657,15 +658,15 @@ void disable_ebpf() disable = true; tc = true; interface_tc(); - const char *maps[36] = {tproxy_map_path, diag_map_path, if_map_path, count_map_path, + const char *maps[37] = {tproxy_map_path, diag_map_path, if_map_path, count_map_path, udp_map_path, matched_map_path, tcp_map_path, tun_map_path, if_tun_map_path, transp_map_path, rb_map_path, ddos_saddr_map_path, ddos_dport_map_path, syn_count_map_path, tp_ext_map_path, if_list_ext_map_path, range_map_path, wildcard_port_map_path, tproxy6_map_path, if6_map_path, count6_map_path, matched6_map_path, egress_range_map_path, egress_if_list_ext_map_path, egress_ext_map_path, egress_map_path, egress6_map_path, egress_count_map_path, egress_count6_map_path, egress_matched6_map_path, egress_matched_map_path, udp_ingress_map_path, tcp_ingress_map_path, - masquerade_map_path, icmp_masquerade_map_path, icmp_echo_map_path}; - for (int map_count = 0; map_count < 36; map_count++) + masquerade_map_path, icmp_masquerade_map_path, icmp_echo_map_path, masquerade_reverse_map_path}; + for (int map_count = 0; map_count < 37; map_count++) { int stat = remove(maps[map_count]); diff --git a/src/zfw_monitor.c b/src/zfw_monitor.c index 7079d22..33ee3c4 100644 --- a/src/zfw_monitor.c +++ b/src/zfw_monitor.c @@ -81,7 +81,7 @@ char check_alt[IF_NAMESIZE]; char doc[] = "zfw_monitor -- ebpf firewall monitor tool"; const char *rb_map_path = "/sys/fs/bpf/tc/globals/rb_map"; const char *tproxy_map_path = "/sys/fs/bpf/tc/globals/zt_tproxy_map"; -const char *argp_program_version = "0.8.14"; +const char *argp_program_version = "0.8.15"; union bpf_attr rb_map; int rb_fd = -1; diff --git a/src/zfw_tc_ingress.c b/src/zfw_tc_ingress.c index 9eb2fba..a25fd12 100644 --- a/src/zfw_tc_ingress.c +++ b/src/zfw_tc_ingress.c @@ -223,12 +223,29 @@ struct masq_key { __u16 dport; }; +/*Key to masquerade_reverse_map*/ +struct masq_reverse_key { + uint32_t ifindex; + union { + __u32 ip; + __u32 ip6[4]; + }__in46_u_src; + union { + __u32 ip; + __u32 ip6[4]; + }__in46_u_dest; + __u8 protocol; + __u16 sport; + __u16 dport; +}; + /*value to masquerade_map and icmp_masquerade_map*/ struct masq_value { union { __u32 ip; __u32 ip6[4]; }__in46_u_origin; + __u16 o_sport; }; /*Key to tun_map*/ @@ -610,6 +627,15 @@ struct { __uint(pinning, LIBBPF_PIN_BY_NAME); } masquerade_map SEC(".maps"); +/*stores reverse lookup table udp and tcp masquerade*/ +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(key_size, sizeof(struct masq_reverse_key)); + __uint(value_size,sizeof(struct masq_value)); + __uint(max_entries, BPF_MAX_SESSIONS * 2); + __uint(pinning, LIBBPF_PIN_BY_NAME); +} masquerade_reverse_map SEC(".maps"); + /*tracks icmp_echo masquerade*/ struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); @@ -698,6 +724,22 @@ static inline struct masq_value *get_masquerade(struct masq_key key){ return mv; } +/*Remove entry from masq state table*/ +static inline void del_masq(struct masq_key key){ + bpf_map_delete_elem(&masquerade_map, &key); +} + +/*Remove entry from reverse masq state table*/ +static inline void del_reverse_masq(struct masq_reverse_key key){ + bpf_map_delete_elem(&masquerade_reverse_map, &key); +} + +static inline struct masq_value *get_reverse_masquerade(struct masq_reverse_key key){ + struct masq_value *mv; + mv = bpf_map_lookup_elem(&masquerade_reverse_map, &key); + return mv; +} + static inline struct masq_value *get_icmp_masquerade(struct icmp_masq_key key){ struct masq_value *mv; mv = bpf_map_lookup_elem(&icmp_masquerade_map, &key); @@ -1852,7 +1894,8 @@ int bpf_sk_splice(struct __sk_buff *skb){ return TC_ACT_SHOT; } /*Calculate l4 Checksum*/ - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct tcphdr, check),local_ip4->ipaddr[0], iph->daddr, BPF_F_PSEUDO_HDR | 4); + int flags = BPF_F_PSEUDO_HDR; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct tcphdr, check), local_ip4->ipaddr[0] ,mv->__in46_u_origin.ip, flags | 4); iph = (struct iphdr *)(skb->data + sizeof(*eth)); if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; @@ -1869,6 +1912,24 @@ int bpf_sk_splice(struct __sk_buff *skb){ if ((unsigned long)(tcph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; } + tcph->dest = mv->o_sport; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct tcphdr, check), mk.sport , mv->o_sport, flags | 2); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tcph = (struct tcphdr *)((unsigned long)iph + sizeof(*iph)); + if ((unsigned long)(tcph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } } } tcp_state_key.__in46_u_dst.ip = tuple->ipv4.saddr; @@ -1901,6 +1962,24 @@ int bpf_sk_splice(struct __sk_buff *skb){ } } else if(tcph->rst){ + if(local_diag->masquerade){ + struct masq_reverse_key rk = {0}; + rk.dport = tcp_state_key.sport; + rk.sport = tcp_state_key.dport; + rk.ifindex = event.ifindex; + rk.__in46_u_dest.ip = tcp_state_key.__in46_u_src.ip; + rk.__in46_u_src.ip = tcp_state_key.__in46_u_dst.ip; + struct masq_value *rv = get_reverse_masquerade(rk); + if(rv){ + struct masq_key mk = {0}; + mk.dport = tcph->source; + mk.sport = rv->o_sport; + mk.__in46_u_dest.ip = iph->saddr; + mk.ifindex = event.ifindex; + del_masq(mk); + } + del_reverse_masq(rk); + } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); if(!tstate){ @@ -1913,6 +1992,24 @@ int bpf_sk_splice(struct __sk_buff *skb){ } else if(tcph->ack){ if((tstate->est) && (tstate->sfin == 1) && (tstate->cfin == 1) && (bpf_htonl(tcph->ack_seq) == (bpf_htonl(tstate->cfseq) + 1))){ + if(local_diag->masquerade){ + struct masq_reverse_key rk = {0}; + rk.dport = tcp_state_key.sport; + rk.sport = tcp_state_key.dport; + rk.ifindex = event.ifindex; + rk.__in46_u_dest.ip = tcp_state_key.__in46_u_src.ip; + rk.__in46_u_src.ip = tcp_state_key.__in46_u_dst.ip; + struct masq_value *rv = get_reverse_masquerade(rk); + if(rv){ + struct masq_key mk = {0}; + mk.dport = tcph->source; + mk.sport = rv->o_sport; + mk.__in46_u_dest.ip = iph->saddr; + mk.ifindex = event.ifindex; + del_masq(mk); + } + del_reverse_masq(rk); + } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); if(!tstate){ diff --git a/src/zfw_tc_outbound_track.c b/src/zfw_tc_outbound_track.c index ec4e66d..4a368b0 100644 --- a/src/zfw_tc_outbound_track.c +++ b/src/zfw_tc_outbound_track.c @@ -160,12 +160,29 @@ struct masq_key { __u16 dport; }; +/*Key to masquerade_reverse_map*/ +struct masq_reverse_key { + uint32_t ifindex; + union { + __u32 ip; + __u32 ip6[4]; + }__in46_u_src; + union { + __u32 ip; + __u32 ip6[4]; + }__in46_u_dest; + __u8 protocol; + __u16 sport; + __u16 dport; +}; + /*value to masquerade_map and icmp_masquerade_map*/ struct masq_value { union { __u32 ip; __u32 ip6[4]; }__in46_u_origin; + __u16 o_sport; }; /*value to diag_map*/ @@ -454,6 +471,15 @@ struct { __uint(pinning, LIBBPF_PIN_BY_NAME); } masquerade_map SEC(".maps"); +/*stores reverse lookup table udp and tcp masquerade*/ +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(key_size, sizeof(struct masq_reverse_key)); + __uint(value_size,sizeof(struct masq_value)); + __uint(max_entries, BPF_MAX_SESSIONS * 2); + __uint(pinning, LIBBPF_PIN_BY_NAME); +} masquerade_reverse_map SEC(".maps"); + struct { __uint(type, BPF_MAP_TYPE_LRU_HASH); __uint(key_size, sizeof(struct icmp_masq_key)); @@ -517,6 +543,26 @@ static inline void insert_masquerade(struct masq_value mv, struct masq_key key){ bpf_map_update_elem(&masquerade_map, &key, &mv,0); } +/*Remove entry from masq state table*/ +static inline void del_masq(struct masq_key key){ + bpf_map_delete_elem(&masquerade_map, &key); +} + +static inline void insert_reverse_masquerade(struct masq_value mv, struct masq_reverse_key key){ + bpf_map_update_elem(&masquerade_reverse_map, &key, &mv,0); +} + +/*Remove entry from reverse masq state table*/ +static inline void del_reverse_masq(struct masq_reverse_key key){ + bpf_map_delete_elem(&masquerade_reverse_map, &key); +} + +static inline struct masq_value *get_reverse_masquerade(struct masq_reverse_key key){ + struct masq_value *mv; + mv = bpf_map_lookup_elem(&masquerade_reverse_map, &key); + return mv; +} + static inline void insert_icmp_masquerade(struct masq_value mv, struct icmp_masq_key key){ bpf_map_update_elem(&icmp_masquerade_map, &key, &mv,0); } @@ -2254,13 +2300,33 @@ int bpf_sk_splice6(struct __sk_buff *skb){ tcp_state_key.sport = tuple->ipv4.sport; tcp_state_key.dport = tuple->ipv4.dport; if(local_diag->masquerade && local_ip4 && local_ip4->count){ + struct masq_reverse_key revk = {0}; + revk.__in46_u_src.ip = tuple->ipv4.saddr; + revk.__in46_u_dest.ip = tuple->ipv4.daddr; + revk.dport = tuple->ipv4.dport; + revk.sport = tuple->ipv4.sport; + revk.protocol = IPPROTO_TCP; + revk.ifindex = skb->ifindex; + __u16 rand_source_port = 0; + struct masq_value *revv = get_reverse_masquerade(revk); + if(revv){ + rand_source_port = revv->o_sport; + } + else{ + rand_source_port = bpf_htons(1024 + bpf_get_prandom_u32() % (65535 -1023)); + struct masq_value rev_new_val = {0}; + rev_new_val.o_sport = rand_source_port; + rev_new_val.__in46_u_origin.ip = 0; + insert_reverse_masquerade(rev_new_val,revk); + } __u32 l3_sum = bpf_csum_diff((__u32 *)&tuple->ipv4.saddr, sizeof(tuple->ipv4.saddr), (__u32 *)&local_ip4->ipaddr[0], sizeof(local_ip4->ipaddr[0]), 0); struct masq_value mv = {0}; mv.__in46_u_origin.ip = tuple->ipv4.saddr; + mv.o_sport = tuple->ipv4.sport; struct masq_key mk = {0}; mk.__in46_u_dest.ip = tuple->ipv4.daddr; mk.dport = tuple->ipv4.dport; - mk.sport = tuple->ipv4.sport; + mk.sport = rand_source_port; mk.ifindex = skb->ifindex; mk.protocol = IPPROTO_TCP; insert_masquerade(mv, mk); @@ -2284,7 +2350,8 @@ int bpf_sk_splice6(struct __sk_buff *skb){ return TC_ACT_SHOT; } /*Calculate l4 Checksum*/ - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct tcphdr, check), mv.__in46_u_origin.ip, iph->saddr, BPF_F_PSEUDO_HDR | 4); + int flags = BPF_F_PSEUDO_HDR; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct tcphdr, check), mv.__in46_u_origin.ip ,local_ip4->ipaddr[0], flags | 4); iph = (struct iphdr *)(skb->data + sizeof(*eth)); if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; @@ -2301,6 +2368,24 @@ int bpf_sk_splice6(struct __sk_buff *skb){ if ((unsigned long)(tcph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; } + tcph->source = rand_source_port; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct tcphdr, check), mv.o_sport , rand_source_port, flags | 2); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tcph = (struct tcphdr *)((unsigned long)iph + sizeof(*iph)); + if ((unsigned long)(tcph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } } unsigned long long tstamp = bpf_ktime_get_ns(); struct tcp_state *tstate; @@ -2339,6 +2424,21 @@ int bpf_sk_splice6(struct __sk_buff *skb){ else if(tcph->rst){ tstate = get_tcp(tcp_state_key); if(tstate){ + if(local_diag->masquerade){ + struct masq_reverse_key rk = {0}; + rk.dport = tcp_state_key.dport; + rk.sport = tcp_state_key.sport; + rk.ifindex = event.ifindex; + rk.__in46_u_dest.ip = tcp_state_key.__in46_u_dst.ip; + rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; + del_reverse_masq(rk); + struct masq_key mk = {0}; + mk.dport = tcph->dest; + mk.sport = tcph->source; + mk.__in46_u_dest.ip = iph->daddr; + mk.ifindex = event.ifindex; + del_masq(mk); + } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); if(!tstate){ @@ -2362,6 +2462,21 @@ int bpf_sk_splice6(struct __sk_buff *skb){ tstate->est = 1; } if((tstate->est) && (tstate->sfin == 1) && (tstate->cfin == 1) && (tstate->sfack) && (bpf_htonl(tcph->ack_seq) == (bpf_htonl(tstate->sfseq) + 1))){ + if(local_diag->masquerade){ + struct masq_reverse_key rk = {0}; + rk.dport = tcp_state_key.dport; + rk.sport = tcp_state_key.sport; + rk.ifindex = event.ifindex; + rk.__in46_u_dest.ip = tcp_state_key.__in46_u_dst.ip; + rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; + del_reverse_masq(rk); + struct masq_key mk = {0}; + mk.dport = tcph->dest; + mk.sport = tcph->source; + mk.__in46_u_dest.ip = iph->daddr; + mk.ifindex = event.ifindex; + del_masq(mk); + } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); if(!tstate){ From ffc822c3b8b96e4f48fc4655fd3f2bbe8eaa9cc7 Mon Sep 17 00:00:00 2001 From: r-caamano Date: Mon, 26 Aug 2024 15:52:06 +0000 Subject: [PATCH 2/8] Added protocol to key required to properly remove masquerade/reverse_masq map entries on tcp session termination events --- src/zfw_tc_ingress.c | 4 ++++ src/zfw_tc_outbound_track.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/zfw_tc_ingress.c b/src/zfw_tc_ingress.c index a25fd12..085bc8e 100644 --- a/src/zfw_tc_ingress.c +++ b/src/zfw_tc_ingress.c @@ -1969,6 +1969,7 @@ int bpf_sk_splice(struct __sk_buff *skb){ rk.ifindex = event.ifindex; rk.__in46_u_dest.ip = tcp_state_key.__in46_u_src.ip; rk.__in46_u_src.ip = tcp_state_key.__in46_u_dst.ip; + rk.protocol = IPPROTO_TCP; struct masq_value *rv = get_reverse_masquerade(rk); if(rv){ struct masq_key mk = {0}; @@ -1976,6 +1977,7 @@ int bpf_sk_splice(struct __sk_buff *skb){ mk.sport = rv->o_sport; mk.__in46_u_dest.ip = iph->saddr; mk.ifindex = event.ifindex; + mk.protocol = IPPROTO_TCP; del_masq(mk); } del_reverse_masq(rk); @@ -1999,6 +2001,7 @@ int bpf_sk_splice(struct __sk_buff *skb){ rk.ifindex = event.ifindex; rk.__in46_u_dest.ip = tcp_state_key.__in46_u_src.ip; rk.__in46_u_src.ip = tcp_state_key.__in46_u_dst.ip; + rk.protocol = IPPROTO_TCP; struct masq_value *rv = get_reverse_masquerade(rk); if(rv){ struct masq_key mk = {0}; @@ -2006,6 +2009,7 @@ int bpf_sk_splice(struct __sk_buff *skb){ mk.sport = rv->o_sport; mk.__in46_u_dest.ip = iph->saddr; mk.ifindex = event.ifindex; + mk.protocol = IPPROTO_TCP; del_masq(mk); } del_reverse_masq(rk); diff --git a/src/zfw_tc_outbound_track.c b/src/zfw_tc_outbound_track.c index 4a368b0..67ab0bf 100644 --- a/src/zfw_tc_outbound_track.c +++ b/src/zfw_tc_outbound_track.c @@ -2431,12 +2431,14 @@ int bpf_sk_splice6(struct __sk_buff *skb){ rk.ifindex = event.ifindex; rk.__in46_u_dest.ip = tcp_state_key.__in46_u_dst.ip; rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; + rk.protocol = IPPROTO_TCP; del_reverse_masq(rk); struct masq_key mk = {0}; mk.dport = tcph->dest; mk.sport = tcph->source; mk.__in46_u_dest.ip = iph->daddr; mk.ifindex = event.ifindex; + mk.protocol = IPPROTO_TCP; del_masq(mk); } del_tcp(tcp_state_key); @@ -2469,12 +2471,14 @@ int bpf_sk_splice6(struct __sk_buff *skb){ rk.ifindex = event.ifindex; rk.__in46_u_dest.ip = tcp_state_key.__in46_u_dst.ip; rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; + rk.protocol = IPPROTO_TCP; del_reverse_masq(rk); struct masq_key mk = {0}; mk.dport = tcph->dest; mk.sport = tcph->source; mk.__in46_u_dest.ip = iph->daddr; mk.ifindex = event.ifindex; + mk.protocol = IPPROTO_TCP; del_masq(mk); } del_tcp(tcp_state_key); From 323cbe36c15ff258715412135d344304b277bf2d Mon Sep 17 00:00:00 2001 From: r-caamano Date: Mon, 26 Aug 2024 18:48:46 +0000 Subject: [PATCH 3/8] added initial dynamic pat refactor for ipv4 udp --- src/zfw_tc_ingress.c | 49 ++++++++++++++++++++++++++++++++++++- src/zfw_tc_outbound_track.c | 43 ++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/src/zfw_tc_ingress.c b/src/zfw_tc_ingress.c index 085bc8e..cd1abec 100644 --- a/src/zfw_tc_ingress.c +++ b/src/zfw_tc_ingress.c @@ -2098,7 +2098,26 @@ int bpf_sk_splice(struct __sk_buff *skb){ return TC_ACT_SHOT; } /*Calculate l4 Checksum*/ - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check),local_ip4->ipaddr[0], iph->daddr, BPF_F_PSEUDO_HDR | 4); + int flags = BPF_F_PSEUDO_HDR; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check),local_ip4->ipaddr[0], iph->daddr, flags | 4); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + struct udphdr *udph = (struct udphdr *)((unsigned long)iph + sizeof(*iph)); + if ((unsigned long)(udph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + udph->dest = mv->o_sport; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mk.sport , mv->o_sport, flags | 2); iph = (struct iphdr *)(skb->data + sizeof(*eth)); if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; @@ -2122,6 +2141,34 @@ int bpf_sk_splice(struct __sk_buff *skb){ if(ustate){ /*if udp outbound state has been up for 30 seconds without traffic remove it from hashmap*/ if(tstamp > (ustate->tstamp + 30000000000)){ + if(local_diag->masquerade){ + struct iphdr *iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + struct udphdr *udph = (struct udphdr *)((unsigned long)iph + sizeof(*iph)); + if ((unsigned long)(udph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + struct masq_reverse_key rk = {0}; + rk.dport = udp_state_key.sport; + rk.sport = udp_state_key.dport; + rk.ifindex = event.ifindex; + rk.__in46_u_dest.ip = udp_state_key.__in46_u_src.ip; + rk.__in46_u_src.ip = udp_state_key.__in46_u_dst.ip; + rk.protocol = IPPROTO_UDP; + struct masq_value *rv = get_reverse_masquerade(rk); + if(rv){ + struct masq_key mk = {0}; + mk.dport = udph->source; + mk.sport = rv->o_sport; + mk.__in46_u_dest.ip = iph->saddr; + mk.ifindex = event.ifindex; + mk.protocol = IPPROTO_UDP; + del_masq(mk); + } + del_reverse_masq(rk); + } del_udp(udp_state_key); ustate = get_udp(udp_state_key); if(!ustate){ diff --git a/src/zfw_tc_outbound_track.c b/src/zfw_tc_outbound_track.c index 67ab0bf..78f2367 100644 --- a/src/zfw_tc_outbound_track.c +++ b/src/zfw_tc_outbound_track.c @@ -2526,13 +2526,33 @@ int bpf_sk_splice6(struct __sk_buff *skb){ udp_state_key.sport = tuple->ipv4.sport; udp_state_key.dport = tuple->ipv4.dport; if(local_diag->masquerade && local_ip4 && local_ip4->count){ + struct masq_reverse_key revk = {0}; + revk.__in46_u_src.ip = tuple->ipv4.saddr; + revk.__in46_u_dest.ip = tuple->ipv4.daddr; + revk.dport = tuple->ipv4.dport; + revk.sport = tuple->ipv4.sport; + revk.protocol = IPPROTO_UDP; + revk.ifindex = skb->ifindex; + __u16 rand_source_port = 0; + struct masq_value *revv = get_reverse_masquerade(revk); + if(revv){ + rand_source_port = revv->o_sport; + } + else{ + rand_source_port = bpf_htons(1024 + bpf_get_prandom_u32() % (65535 -1023)); + struct masq_value rev_new_val = {0}; + rev_new_val.o_sport = rand_source_port; + rev_new_val.__in46_u_origin.ip = 0; + insert_reverse_masquerade(rev_new_val,revk); + } __u32 l3_sum = bpf_csum_diff((__u32 *)&tuple->ipv4.saddr, sizeof(tuple->ipv4.saddr), (__u32 *)&local_ip4->ipaddr[0], sizeof(local_ip4->ipaddr[0]), 0); struct masq_value mv = {0}; mv.__in46_u_origin.ip = tuple->ipv4.saddr; + mv.o_sport = tuple->ipv4.sport; struct masq_key mk = {0}; mk.__in46_u_dest.ip = tuple->ipv4.daddr; mk.dport = tuple->ipv4.dport; - mk.sport = tuple->ipv4.sport; + mk.sport = rand_source_port; mk.ifindex = skb->ifindex; mk.protocol = IPPROTO_UDP; insert_masquerade(mv, mk); @@ -2552,7 +2572,8 @@ int bpf_sk_splice6(struct __sk_buff *skb){ return TC_ACT_SHOT; } /*Calculate l4 Checksum*/ - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mv.__in46_u_origin.ip, iph->saddr, BPF_F_PSEUDO_HDR | 4); + int flags = BPF_F_PSEUDO_HDR; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mv.__in46_u_origin.ip, iph->saddr, flags | 4); iph = (struct iphdr *)(skb->data + sizeof(*eth)); if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; @@ -2565,6 +2586,24 @@ int bpf_sk_splice6(struct __sk_buff *skb){ if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ return TC_ACT_SHOT; } + udph = (struct udphdr *)((unsigned long)iph + sizeof(*iph)); + if ((unsigned long)(udph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + udph->source = rand_source_port; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mv.o_sport , rand_source_port, flags | 2); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } } struct udp_state *ustate = get_udp(udp_state_key); if((!ustate) || (ustate->tstamp > (tstamp + 30000000000))){ From 993405d7a34f865d50e910626897305e1450aa30 Mon Sep 17 00:00:00 2001 From: r-caamano Date: Tue, 27 Aug 2024 17:26:38 +0000 Subject: [PATCH 4/8] Fixed incorrect change to per interface rules default for InternalInterfaces --- files/scripts/start_ebpf_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/scripts/start_ebpf_router.py b/files/scripts/start_ebpf_router.py index 31d97b0..a5df247 100755 --- a/files/scripts/start_ebpf_router.py +++ b/files/scripts/start_ebpf_router.py @@ -272,14 +272,14 @@ def set_local_rules(resolver): else: outbound_passthrough_track[interface["Name"]] = False; else: - outbound_passthrough_track[interface["Name"]] = False; + outbound_passthrough_track[interface["Name"]] = True; if("PerInterfaceRules") in interface.keys(): if(interface["PerInterfaceRules"]): per_interface_rules[interface["Name"]] = True; else: per_interface_rules[interface["Name"]] = False; else: - per_interface_rules[interface["Name"]] = True; + per_interface_rules[interface["Name"]] = False; else: print('Mandatory key \"Name\" missing skipping internal interface entry!') From 3b6b8c3b6f33f619c6b1674f0acf56284057d54b Mon Sep 17 00:00:00 2001 From: r-caamano Date: Wed, 28 Aug 2024 03:38:38 +0000 Subject: [PATCH 5/8] Added conditional to only recalc udp checksum on PAT if checksum is non-zero. Fixed incorrect comment in icmp section --- src/zfw_tc_ingress.c | 68 +++++++++++++++++++++---------------- src/zfw_tc_outbound_track.c | 64 +++++++++++++++++++--------------- 2 files changed, 74 insertions(+), 58 deletions(-) diff --git a/src/zfw_tc_ingress.c b/src/zfw_tc_ingress.c index cd1abec..1289354 100644 --- a/src/zfw_tc_ingress.c +++ b/src/zfw_tc_ingress.c @@ -1307,7 +1307,7 @@ int bpf_sk_splice(struct __sk_buff *skb){ unsigned long long tstamp = bpf_ktime_get_ns(); struct icmp_state *istate = get_icmp(icmp_state_key); if(istate){ - /*if icmp outbound state has been up for 30 seconds without traffic remove it from hashmap*/ + /*if icmp outbound state has been up for 15 seconds without traffic remove it from hashmap*/ if(tstamp > (istate->tstamp + 15000000000)){ del_icmp(icmp_state_key); istate = get_icmp(icmp_state_key); @@ -1680,7 +1680,7 @@ int bpf_sk_splice(struct __sk_buff *skb){ unsigned long long tstamp = bpf_ktime_get_ns(); struct icmp_state *istate = get_icmp(icmp_state_key); if(istate){ - /*if udp outbound state has been up for 30 seconds without traffic remove it from hashmap*/ + /*if udp outbound state has been up for 15 seconds without traffic remove it from hashmap*/ if(tstamp > (istate->tstamp + 15000000000)){ del_icmp(icmp_state_key); istate = get_icmp(icmp_state_key); @@ -2097,38 +2097,46 @@ int bpf_sk_splice(struct __sk_buff *skb){ if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ return TC_ACT_SHOT; } - /*Calculate l4 Checksum*/ - int flags = BPF_F_PSEUDO_HDR; - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check),local_ip4->ipaddr[0], iph->daddr, flags | 4); - iph = (struct iphdr *)(skb->data + sizeof(*eth)); - if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; - } - tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; - if(!tuple){ - return TC_ACT_SHOT; - } - tuple_len = sizeof(tuple->ipv4); - if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; - } struct udphdr *udph = (struct udphdr *)((unsigned long)iph + sizeof(*iph)); if ((unsigned long)(udph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; } - udph->dest = mv->o_sport; - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mk.sport , mv->o_sport, flags | 2); - iph = (struct iphdr *)(skb->data + sizeof(*eth)); - if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; - } - tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; - if(!tuple){ - return TC_ACT_SHOT; - } - tuple_len = sizeof(tuple->ipv4); - if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; + /*Calculate l4 Checksum*/ + if(udph->check != 0){ + int flags = BPF_F_PSEUDO_HDR; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check),local_ip4->ipaddr[0], iph->daddr, flags | 4); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + udph = (struct udphdr *)((unsigned long)iph + sizeof(*iph)); + if ((unsigned long)(udph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + udph->dest = mv->o_sport; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mk.sport , mv->o_sport, flags | 2); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + }else{ + udph->dest = mv->o_sport; } } } diff --git a/src/zfw_tc_outbound_track.c b/src/zfw_tc_outbound_track.c index 78f2367..f0ecdf4 100644 --- a/src/zfw_tc_outbound_track.c +++ b/src/zfw_tc_outbound_track.c @@ -2571,39 +2571,47 @@ int bpf_sk_splice6(struct __sk_buff *skb){ if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ return TC_ACT_SHOT; } - /*Calculate l4 Checksum*/ - int flags = BPF_F_PSEUDO_HDR; - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mv.__in46_u_origin.ip, iph->saddr, flags | 4); - iph = (struct iphdr *)(skb->data + sizeof(*eth)); - if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; - } - tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; - if(!tuple){ - return TC_ACT_SHOT; - } - tuple_len = sizeof(tuple->ipv4); - if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; - } udph = (struct udphdr *)((unsigned long)iph + sizeof(*iph)); if ((unsigned long)(udph + 1) > (unsigned long)skb->data_end){ return TC_ACT_SHOT; } - udph->source = rand_source_port; - bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mv.o_sport , rand_source_port, flags | 2); - iph = (struct iphdr *)(skb->data + sizeof(*eth)); - if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; - } - tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; - if(!tuple){ - return TC_ACT_SHOT; + /*Calculate l4 Checksum if checksum not equal to zero*/ + if(udph->check != 0){ + int flags = BPF_F_PSEUDO_HDR; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mv.__in46_u_origin.ip, iph->saddr, flags | 4); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + udph = (struct udphdr *)((unsigned long)iph + sizeof(*iph)); + if ((unsigned long)(udph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + udph->source = rand_source_port; + bpf_l4_csum_replace(skb, sizeof(struct ethhdr) + sizeof(struct iphdr) + offsetof(struct udphdr, check), mv.o_sport , rand_source_port, flags | 2); + iph = (struct iphdr *)(skb->data + sizeof(*eth)); + if ((unsigned long)(iph + 1) > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + tuple = (struct bpf_sock_tuple *)(void*)(long)&iph->saddr; + if(!tuple){ + return TC_ACT_SHOT; + } + tuple_len = sizeof(tuple->ipv4); + if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ + return TC_ACT_SHOT; + } + }else{ + udph->source = rand_source_port; } - tuple_len = sizeof(tuple->ipv4); - if ((unsigned long)tuple + tuple_len > (unsigned long)skb->data_end){ - return TC_ACT_SHOT; - } } struct udp_state *ustate = get_udp(udp_state_key); if((!ustate) || (ustate->tstamp > (tstamp + 30000000000))){ From 79147e6554c71d99ede54e6b5c4a65dbc1d78439 Mon Sep 17 00:00:00 2001 From: r-caamano Date: Wed, 28 Aug 2024 14:48:53 +0000 Subject: [PATCH 6/8] Added logging for IPv4 Masquerade, updated CHANGELOG --- CHANGELOG.md | 4 ++- src/zfw.c | 36 +++++++++++++++++++++ src/zfw_monitor.c | 36 +++++++++++++++++++++ src/zfw_tc_ingress.c | 51 +++++++++++++++++++++-------- src/zfw_tc_outbound_track.c | 64 +++++++++++++++++++++++++++++++++---- 5 files changed, 171 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c89773..5ed7296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,9 @@ All notable changes to this project will be documented in this file. The format ### # [0.8.15] - 2024-08-26 - Refactored all startup scripts to default InternalInterfaces to have outbound tracking enabled -- Refactored masquerade to use dynamic PAT vs static PAT +- Refactored IPv4 masquerade to use dynamic PAT vs static PAT and added RB logging +- Fixed issue where if IPv4 udp checksum was 0 masquerade erroneously attempted to recalculate the checksum + ### diff --git a/src/zfw.c b/src/zfw.c index 3942934..ded555a 100644 --- a/src/zfw.c +++ b/src/zfw.c @@ -93,6 +93,10 @@ #define CLIENT_INITIATED_ICMP_ECHO 29 #define IP6_HEADER_TOO_BIG 30 #define IPV6_TUPLE_TOO_BIG 31 +#define REVERSE_MASQUERADE_ENTRY_REMOVED 32 +#define MASQUERADE_ENTRY_REMOVED 33 +#define REVERSE_MASQUERADE_ENTRY_ADDED 34 +#define MASQUERADE_ENTRY_ADDED 35 bool ddos = false; bool add = false; @@ -3110,6 +3114,22 @@ static int process_events(void *ctx, void *data, size_t len) { state = "MATCHED_DROP_FILTER"; } + else if (code == REVERSE_MASQUERADE_ENTRY_ADDED) + { + state = "REVERSE_MASQUERADE_ENTRY_ADDED"; + } + else if (code == REVERSE_MASQUERADE_ENTRY_REMOVED) + { + state = "REVERSE_MASQUERADE_ENTRY_REMOVED"; + } + else if (code == MASQUERADE_ENTRY_ADDED) + { + state = "MASQUERADE_ENTRY_ADDED"; + } + else if (code == MASQUERADE_ENTRY_REMOVED) + { + state = "MASQUERADE_ENTRY_REMOVED"; + } if (state) { @@ -3448,6 +3468,22 @@ static int process_events(void *ctx, void *data, size_t len) { state = "MATCHED_DROP_FILTER"; } + else if (code == REVERSE_MASQUERADE_ENTRY_ADDED) + { + state = "REVERSE_MASQUERADE_ENTRY_ADDED"; + } + else if (code == REVERSE_MASQUERADE_ENTRY_REMOVED) + { + state = "REVERSE_MASQUERADE_ENTRY_REMOVED"; + } + else if (code == MASQUERADE_ENTRY_ADDED) + { + state = "MASQUERADE_ENTRY_ADDED"; + } + else if (code == MASQUERADE_ENTRY_REMOVED) + { + state = "MASQUERADE_ENTRY_REMOVED"; + } if (state) diff --git a/src/zfw_monitor.c b/src/zfw_monitor.c index 33ee3c4..9d9806d 100644 --- a/src/zfw_monitor.c +++ b/src/zfw_monitor.c @@ -70,6 +70,10 @@ #define CLIENT_INITIATED_ICMP_ECHO 29 #define IP6_HEADER_TOO_BIG 30 #define IPV6_TUPLE_TOO_BIG 31 +#define REVERSE_MASQUERADE_ENTRY_REMOVED 32 +#define MASQUERADE_ENTRY_REMOVED 33 +#define REVERSE_MASQUERADE_ENTRY_ADDED 34 +#define MASQUERADE_ENTRY_ADDED 35 bool logging = false; bool monitor = false; @@ -510,6 +514,22 @@ static int process_events(void *ctx, void *data, size_t len) { state = "MATCHED_DROP_FILTER"; } + else if (code == REVERSE_MASQUERADE_ENTRY_ADDED) + { + state = "REVERSE_MASQUERADE_ENTRY_ADDED"; + } + else if (code == REVERSE_MASQUERADE_ENTRY_REMOVED) + { + state = "REVERSE_MASQUERADE_ENTRY_REMOVED"; + } + else if (code == MASQUERADE_ENTRY_ADDED) + { + state = "MASQUERADE_ENTRY_ADDED"; + } + else if (code == MASQUERADE_ENTRY_REMOVED) + { + state = "MASQUERADE_ENTRY_REMOVED"; + } if (state) { @@ -848,6 +868,22 @@ static int process_events(void *ctx, void *data, size_t len) { state = "MATCHED_DROP_FILTER"; } + else if (code == REVERSE_MASQUERADE_ENTRY_ADDED) + { + state = "REVERSE_MASQUERADE_ENTRY_ADDED"; + } + else if (code == REVERSE_MASQUERADE_ENTRY_REMOVED) + { + state = "REVERSE_MASQUERADE_ENTRY_REMOVED"; + } + else if (code == MASQUERADE_ENTRY_ADDED) + { + state = "MASQUERADE_ENTRY_ADDED"; + } + else if (code == MASQUERADE_ENTRY_REMOVED) + { + state = "MASQUERADE_ENTRY_REMOVED"; + } if (state) diff --git a/src/zfw_tc_ingress.c b/src/zfw_tc_ingress.c index 1289354..97af5a4 100644 --- a/src/zfw_tc_ingress.c +++ b/src/zfw_tc_ingress.c @@ -78,6 +78,8 @@ #define ICMP_MATCHED_ACTIVE_STATE 28 #define IP6_HEADER_TOO_BIG 30 #define IPV6_TUPLE_TOO_BIG 31 +#define REVERSE_MASQUERADE_ENTRY_REMOVED 32 +#define MASQUERADE_ENTRY_REMOVED 33 #ifndef memcpy #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n)) #endif @@ -1964,11 +1966,11 @@ int bpf_sk_splice(struct __sk_buff *skb){ else if(tcph->rst){ if(local_diag->masquerade){ struct masq_reverse_key rk = {0}; - rk.dport = tcp_state_key.sport; - rk.sport = tcp_state_key.dport; + rk.dport = tcp_state_key.dport; + rk.sport = tcp_state_key.sport; rk.ifindex = event.ifindex; - rk.__in46_u_dest.ip = tcp_state_key.__in46_u_src.ip; - rk.__in46_u_src.ip = tcp_state_key.__in46_u_dst.ip; + rk.__in46_u_dest.ip = tcp_state_key.__in46_u_dst.ip; + rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; rk.protocol = IPPROTO_TCP; struct masq_value *rv = get_reverse_masquerade(rk); if(rv){ @@ -1979,8 +1981,16 @@ int bpf_sk_splice(struct __sk_buff *skb){ mk.ifindex = event.ifindex; mk.protocol = IPPROTO_TCP; del_masq(mk); + if(local_diag->verbose){ + event.tracking_code = MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_reverse_masq(rk); + if(local_diag->verbose){ + event.tracking_code = REVERSE_MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); @@ -1996,11 +2006,11 @@ int bpf_sk_splice(struct __sk_buff *skb){ if((tstate->est) && (tstate->sfin == 1) && (tstate->cfin == 1) && (bpf_htonl(tcph->ack_seq) == (bpf_htonl(tstate->cfseq) + 1))){ if(local_diag->masquerade){ struct masq_reverse_key rk = {0}; - rk.dport = tcp_state_key.sport; - rk.sport = tcp_state_key.dport; + rk.dport = tcp_state_key.dport; + rk.sport = tcp_state_key.sport; rk.ifindex = event.ifindex; - rk.__in46_u_dest.ip = tcp_state_key.__in46_u_src.ip; - rk.__in46_u_src.ip = tcp_state_key.__in46_u_dst.ip; + rk.__in46_u_dest.ip = tcp_state_key.__in46_u_dst.ip; + rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; rk.protocol = IPPROTO_TCP; struct masq_value *rv = get_reverse_masquerade(rk); if(rv){ @@ -2011,8 +2021,16 @@ int bpf_sk_splice(struct __sk_buff *skb){ mk.ifindex = event.ifindex; mk.protocol = IPPROTO_TCP; del_masq(mk); + if(local_diag->verbose){ + event.tracking_code = MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_reverse_masq(rk); + if(local_diag->verbose){ + event.tracking_code = REVERSE_MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); @@ -2022,7 +2040,6 @@ int bpf_sk_splice(struct __sk_buff *skb){ send_event(&event); } } - } else if((tstate->est) && (tstate->cfin == 1) && (bpf_htonl(tcph->ack_seq) == (bpf_htonl(tstate->cfseq) + 1))){ tstate->sfack = 1; @@ -2159,11 +2176,11 @@ int bpf_sk_splice(struct __sk_buff *skb){ return TC_ACT_SHOT; } struct masq_reverse_key rk = {0}; - rk.dport = udp_state_key.sport; - rk.sport = udp_state_key.dport; + rk.dport = udp_state_key.dport; + rk.sport = udp_state_key.sport; rk.ifindex = event.ifindex; - rk.__in46_u_dest.ip = udp_state_key.__in46_u_src.ip; - rk.__in46_u_src.ip = udp_state_key.__in46_u_dst.ip; + rk.__in46_u_dest.ip = udp_state_key.__in46_u_dst.ip; + rk.__in46_u_src.ip = udp_state_key.__in46_u_src.ip; rk.protocol = IPPROTO_UDP; struct masq_value *rv = get_reverse_masquerade(rk); if(rv){ @@ -2174,8 +2191,16 @@ int bpf_sk_splice(struct __sk_buff *skb){ mk.ifindex = event.ifindex; mk.protocol = IPPROTO_UDP; del_masq(mk); + if(local_diag->verbose){ + event.tracking_code = MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_reverse_masq(rk); + if(local_diag->verbose){ + event.tracking_code = REVERSE_MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_udp(udp_state_key); ustate = get_udp(udp_state_key); diff --git a/src/zfw_tc_outbound_track.c b/src/zfw_tc_outbound_track.c index f0ecdf4..226e6ff 100644 --- a/src/zfw_tc_outbound_track.c +++ b/src/zfw_tc_outbound_track.c @@ -63,6 +63,10 @@ #define CLIENT_INITIATED_ICMP_ECHO 29 #define IP6_HEADER_TOO_BIG 30 #define IPV6_TUPLE_TOO_BIG 31 +#define REVERSE_MASQUERADE_ENTRY_REMOVED 32 +#define MASQUERADE_ENTRY_REMOVED 33 +#define REVERSE_MASQUERADE_ENTRY_ADDED 34 +#define MASQUERADE_ENTRY_ADDED 35 #define memcpy(dest, src, n) __builtin_memcpy((dest), (src), (n)) struct bpf_event{ @@ -543,6 +547,12 @@ static inline void insert_masquerade(struct masq_value mv, struct masq_key key){ bpf_map_update_elem(&masquerade_map, &key, &mv,0); } +static inline struct masq_value *get_masquerade(struct masq_key key){ + struct masq_value *mv; + mv = bpf_map_lookup_elem(&masquerade_map, &key); + return mv; +} + /*Remove entry from masq state table*/ static inline void del_masq(struct masq_key key){ bpf_map_delete_elem(&masquerade_map, &key); @@ -2318,18 +2328,31 @@ int bpf_sk_splice6(struct __sk_buff *skb){ rev_new_val.o_sport = rand_source_port; rev_new_val.__in46_u_origin.ip = 0; insert_reverse_masquerade(rev_new_val,revk); + if(local_diag->verbose){ + event.tracking_code = REVERSE_MASQUERADE_ENTRY_ADDED; + send_event(&event); + } } __u32 l3_sum = bpf_csum_diff((__u32 *)&tuple->ipv4.saddr, sizeof(tuple->ipv4.saddr), (__u32 *)&local_ip4->ipaddr[0], sizeof(local_ip4->ipaddr[0]), 0); struct masq_value mv = {0}; - mv.__in46_u_origin.ip = tuple->ipv4.saddr; - mv.o_sport = tuple->ipv4.sport; struct masq_key mk = {0}; mk.__in46_u_dest.ip = tuple->ipv4.daddr; mk.dport = tuple->ipv4.dport; mk.sport = rand_source_port; mk.ifindex = skb->ifindex; mk.protocol = IPPROTO_TCP; - insert_masquerade(mv, mk); + struct masq_value *mvptr = get_masquerade(mk); + if(!mvptr){ + mv.__in46_u_origin.ip = tuple->ipv4.saddr; + mv.o_sport = tuple->ipv4.sport; + insert_masquerade(mv, mk); + if(local_diag->verbose){ + event.tracking_code = MASQUERADE_ENTRY_ADDED; + send_event(&event); + } + }else{ + mv = *mvptr; + } iph->saddr = local_ip4->ipaddr[0]; /*Calculate l3 Checksum*/ bpf_l3_csum_replace(skb, sizeof(struct ethhdr) + offsetof(struct iphdr, check), 0, l3_sum, 0); @@ -2433,6 +2456,10 @@ int bpf_sk_splice6(struct __sk_buff *skb){ rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; rk.protocol = IPPROTO_TCP; del_reverse_masq(rk); + if(local_diag->verbose){ + event.tracking_code = REVERSE_MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } struct masq_key mk = {0}; mk.dport = tcph->dest; mk.sport = tcph->source; @@ -2440,6 +2467,10 @@ int bpf_sk_splice6(struct __sk_buff *skb){ mk.ifindex = event.ifindex; mk.protocol = IPPROTO_TCP; del_masq(mk); + if(local_diag->verbose){ + event.tracking_code = MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); @@ -2473,6 +2504,10 @@ int bpf_sk_splice6(struct __sk_buff *skb){ rk.__in46_u_src.ip = tcp_state_key.__in46_u_src.ip; rk.protocol = IPPROTO_TCP; del_reverse_masq(rk); + if(local_diag->verbose){ + event.tracking_code = REVERSE_MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } struct masq_key mk = {0}; mk.dport = tcph->dest; mk.sport = tcph->source; @@ -2480,6 +2515,10 @@ int bpf_sk_splice6(struct __sk_buff *skb){ mk.ifindex = event.ifindex; mk.protocol = IPPROTO_TCP; del_masq(mk); + if(local_diag->verbose){ + event.tracking_code = MASQUERADE_ENTRY_REMOVED; + send_event(&event); + } } del_tcp(tcp_state_key); tstate = get_tcp(tcp_state_key); @@ -2544,18 +2583,31 @@ int bpf_sk_splice6(struct __sk_buff *skb){ rev_new_val.o_sport = rand_source_port; rev_new_val.__in46_u_origin.ip = 0; insert_reverse_masquerade(rev_new_val,revk); + if(local_diag->verbose){ + event.tracking_code = REVERSE_MASQUERADE_ENTRY_ADDED; + send_event(&event); + } } __u32 l3_sum = bpf_csum_diff((__u32 *)&tuple->ipv4.saddr, sizeof(tuple->ipv4.saddr), (__u32 *)&local_ip4->ipaddr[0], sizeof(local_ip4->ipaddr[0]), 0); struct masq_value mv = {0}; - mv.__in46_u_origin.ip = tuple->ipv4.saddr; - mv.o_sport = tuple->ipv4.sport; struct masq_key mk = {0}; mk.__in46_u_dest.ip = tuple->ipv4.daddr; mk.dport = tuple->ipv4.dport; mk.sport = rand_source_port; mk.ifindex = skb->ifindex; mk.protocol = IPPROTO_UDP; - insert_masquerade(mv, mk); + struct masq_value *mvptr = get_masquerade(mk); + if(!mvptr){ + mv.__in46_u_origin.ip = tuple->ipv4.saddr; + mv.o_sport = tuple->ipv4.sport; + insert_masquerade(mv, mk); + if(local_diag->verbose){ + event.tracking_code = MASQUERADE_ENTRY_ADDED; + send_event(&event); + } + }else{ + mv = *mvptr; + } iph->saddr = local_ip4->ipaddr[0]; /*Calculate l3 Checksum*/ bpf_l3_csum_replace(skb, sizeof(struct ethhdr) + offsetof(struct iphdr, check), 0, l3_sum, 0); From cef2afcd5b5ab9c1950d2a119dae878a12621393 Mon Sep 17 00:00:00 2001 From: r-caamano Date: Wed, 28 Aug 2024 16:03:43 +0000 Subject: [PATCH 7/8] Updated README and ebpf_config.json to show new default InternalInterfaces key values --- README.md | 24 +++++++++++------------- files/json/ebpf_config.json.sample | 2 +- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e2f40ed..8fec809 100644 --- a/README.md +++ b/README.md @@ -10,11 +10,12 @@ EBPF based masquerade capability for both IPv4/IPv6. ### Native EBPF based IPv4 and IPv6 Masquerade support -zfw can now provide native IPv4/IPv6 masquerade operation for outbound pass through connections which can be enabled via: +zfw can now provide native IPv4/IPv6 masquerade operation for outbound pass through connections which can be enabled via on WAN facing interface: ```sudo zfw -k, --masquerade ``` -This function requires that both ingress and egress TC filters are enabled on outbound interface. +This function requires that both ingress and egress TC filters are enabled on outbound interface. For IPv4 this is now using Dynamic PAT and IPv6 is using +static PAT. Note: When running on later kernels i.e. 6+ some older network hardware may not work with ebpf Dynamic PAT. ### Explicit Deny Rules This feature adds the ability to enter explicit deny rules by appending ```-d, --disable to the -I, --insert rule``` to both ingress and egress rules. Rule precedence is based on longest match prefix. If the prefix is the same then the precedence follows the order entry of the rules, which when listed will go from top to bottom for ports with in the same prefix e.g. @@ -97,8 +98,7 @@ The above should result in all outbound traffic except for arp and icmp to be dr will also be dropped unless ```sudo zfw -e ens33 is set```). ssh return traffic will also be allowed outbound unless ```ssh -x ens33 is set```. -In order to survive reboot you must have "OutboundPassThroughTrack": true which is default for ExternalInterfaces -but can also be explicitly set for InternalInterfaces. If per interface rules is not false then the egress rules would +If per interface rules is not false then the egress rules would need explicit -N for each rule in the same manner as ingress rules. i.e. set ```/opt/openziti/etc/ebpf_config.json``` as below changing interface name only @@ -107,7 +107,7 @@ i.e. set ```/opt/openziti/etc/ebpf_config.json``` as below changing interface na or equivalent InternalInterfaces config: -```{"InternalInterfaces":[{"Name":"ens33", "OutboundPassThroughTrack": true}],"ExternalInterfaces":[]}``` +```{"InternalInterfaces":[{"Name":"ens33"}],"ExternalInterfaces":[]}``` Then in executable script file ```/opt/openziti/bin/user/user_rules.sh``` ``` @@ -184,7 +184,7 @@ Rule Count: 1 - *Supports inbound ssh (Can be disabled via ```sudo zfw -x ```) (Care should be taken as this affects IPv4 as well) - Supports outbound stateful host connections (Inbound only if outbound initiated) - Supports outbound passthrough tracking. Sessions initiated from non-ebpf enabled and ebpf enabled internal interfaces out - through interface(s) defined as ExternalInterface or with "OutboundPassThroughTrack": true in /opt/openziti/etc/ebpf_config.json + through interface(s) defined as ExternalInterface (requires -N with -I unless "PerInterfaceRules": false) or InternalInterface in /opt/openziti/etc/ebpf_config.json or manually applied with sudo ```zfw -X -O /opt/openziti/zfw_outbound_track.o -z egress``` will allow stateful udp and tcp session traffic back in. - Support for inbound IPv6 filter destination rules. Currently only destination filtering is allowed. @@ -281,10 +281,9 @@ sudo cp /opt/openziti/etc/ebpf_config.json.sample /opt/openziti/etc/ebpf_config. sudo vi /opt/openziti/etc/ebpf_config.json ``` - Adding interfaces - Replace ens33 in line with:{"InternalInterfaces":[{"Name":"ens33" ,"OutboundPassThroughTrack": false, "PerInterfaceRules": false}], "ExternalInterfaces":[]} + Replace ens33 in line with:{"InternalInterfaces":[{"Name":"ens33"}], "ExternalInterfaces":[]} Replace with interface that you want to enable for ingress firewalling / openziti interception and - optionally ExternalInterfaces if running containers or other subtending devices (Described in more detail - later in this README.md). + optionally ExternalInterfaces if you want per interface rules -N with -I. ``` i.e. ens33 {"InternalInterfaces":[{"Name":"ens33"}], "ExternalInterfaces":[]} @@ -425,15 +424,14 @@ sudo zfw -X ens33 -O /opt/openziti/bin/zfw_tc_outbound_track.o --direction egres sudo vi /opt/openziti/etc/ebpf_config.json ``` ``` -{"InternalInterfaces":[{"Name":"ens37","OutboundPassThroughTrack": false, PerInterfaceRules: false}], +{"InternalInterfaces":[{"Name":"ens37","OutboundPassThroughTrack": true, PerInterfaceRules: false}], "ExternalInterfaces":[{"Name":"ens33", OutboundPassThroughTrack: true, PerInterfaceRules: true}]} ``` The above JSON sets up ens33 to be an internal interface (No outbound tracking) and ens33 as an external interface with outbound tracking (Default for External Interface). It also automatically adds runs the sudo zfw -P ens33 so ens33 (default for ExternalInterfaces) which requires -N to add inbound rules to it and will ignore rules where it is not in the interface list. Keys "OutboundPassThroughTrack" and "PerInterfaceRules" are shown with their default values, you only need to add them if you -want change the default operation for the interface type. Note: if ebpf is enabled on an interface before it has ip address assigned a rule assigned -with that interface name and -N it will not show up until at least one diag command is toggled or ebpf is disabled and re-enabled on it via -X, --set-tc-filter. +want change the default operation for the interface type. #### Single Interface config with ens33 facing lan local lan ``` @@ -470,7 +468,7 @@ source of the L2tpv3 tunnel on each zet host needs to be set to the ip address a interface which will be the first host address in the ZITI_DNS_IP_RANGE. In addition you will need to enable ebpf outbound tracking on the loopback interface. This can be setup vi /opt/openziti/etc/ebpf_config.json i.e. ``` -{"InternalInterfaces":[{"Name":"eth0", "OutboundPassThroughTrack": false, "PerInterfaceRules": false}, {"Name":"lo", "OutboundPassThroughTrack": true}],"ExternalInterfaces":[]} +{"InternalInterfaces":[{"Name":"eth0"}, {"Name":"lo", "OutboundPassThroughTrack": true}],"ExternalInterfaces":[]} ``` ### Ziti Edge Tunnel Bidirectional Transparency (zfw-tunnel only) diff --git a/files/json/ebpf_config.json.sample b/files/json/ebpf_config.json.sample index 39e6f8e..8852c5a 100644 --- a/files/json/ebpf_config.json.sample +++ b/files/json/ebpf_config.json.sample @@ -1,2 +1,2 @@ -{"InternalInterfaces":[{"Name":"ens33", "OutboundPassThroughTrack": false, "PerInterfaceRules": false}], +{"InternalInterfaces":[{"Name":"ens33", "OutboundPassThroughTrack": true, "PerInterfaceRules": false}], "ExternalInterfaces":[]} From b88ab99f3458dab3a12f64ec72fa8964f76f45da Mon Sep 17 00:00:00 2001 From: r-caamano Date: Wed, 28 Aug 2024 22:57:05 +0000 Subject: [PATCH 8/8] Fixed wording in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8fec809..fe63b9a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ EBPF based masquerade capability for both IPv4/IPv6. ### Native EBPF based IPv4 and IPv6 Masquerade support -zfw can now provide native IPv4/IPv6 masquerade operation for outbound pass through connections which can be enabled via on WAN facing interface: +zfw can now provide native IPv4/IPv6 masquerade operation for outbound pass through connections which can be enabled on a WAN facing interface: ```sudo zfw -k, --masquerade ```