https://stackoverflow.com/questions/47645859/meaning-of-modules-instances-in-proc-modules

Lsmod命令

基于manpage的说明， 这个命令的数据来源是 ： /proc/modules

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
[root@centos ~]# lsmod
Module                  Size  Used by
iptable_nat            12875  0
nf_conntrack_ipv4      19149  1
nf_defrag_ipv4         12729  1 nf_conntrack_ipv4
nf_nat_ipv4            14115  1 iptable_nat
nf_nat                 26583  1 nf_nat_ipv4
nf_conntrack          143360  3 nf_nat,nf_nat_ipv4,nf_conntrack_ipv4
iptable_filter         12810  0
nfit                   59735  0
libnvdimm             163620  1 nfit
iosf_mbi               15582  0
crc32_pclmul           13133  0
ghash_clmulni_intel    13273  0
aesni_intel           189456  0
ppdev                  17671  0
lrw                    13286  1 aesni_intel
gf128mul               15139  1 lrw
glue_helper            13990  1 aesni_intel
ablk_helper            13597  1 aesni_intel
cryptd                 21190  3 ghash_clmulni_intel,aesni_intel,ablk_helper
parport_pc             28205  0
parport                46395  2 ppdev,parport_pc
i2c_piix4              22401  0
pcspkr                 12718  0
ip_tables              27126  2 iptable_filter,iptable_nat
xfs                  1014152  1
libcrc32c              12644  3 xfs,nf_nat,nf_conntrack
crct10dif_pclmul       14307  0
crct10dif_common       12595  1 crct10dif_pclmul
crc32c_intel           22094  1
nvme                   32382  1
serio_raw              13434  0
ena                    96895  0
nvme_core              63547  3 nvme
sunrpc                366617  1

/proc/modules 的输出结果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
[root@centos ~]# cat /proc/modules
iptable_nat 12875 0 - Live 0xffffffffc0661000
nf_conntrack_ipv4 19149 1 - Live 0xffffffffc065b000
nf_defrag_ipv4 12729 1 nf_conntrack_ipv4, Live 0xffffffffc05fd000
nf_nat_ipv4 14115 1 iptable_nat, Live 0xffffffffc05f4000
nf_nat 26583 1 nf_nat_ipv4, Live 0xffffffffc0603000
nf_conntrack 143360 3 nf_conntrack_ipv4,nf_nat_ipv4,nf_nat, Live 0xffffffffc0634000
iptable_filter 12810 0 - Live 0xffffffffc05dd000
nfit 59735 0 - Live 0xffffffffc066d000
libnvdimm 163620 1 nfit, Live 0xffffffffc060b000
iosf_mbi 15582 0 - Live 0xffffffffc05ed000
crc32_pclmul 13133 0 - Live 0xffffffffc05e5000
ghash_clmulni_intel 13273 0 - Live 0xffffffffc059e000
aesni_intel 189456 0 - Live 0xffffffffc05ad000
ppdev 17671 0 - Live 0xffffffffc05a4000
lrw 13286 1 aesni_intel, Live 0xffffffffc0575000
gf128mul 15139 1 lrw, Live 0xffffffffc0590000
glue_helper 13990 1 aesni_intel, Live 0xffffffffc0570000
ablk_helper 13597 1 aesni_intel, Live 0xffffffffc0418000
cryptd 21190 3 ghash_clmulni_intel,aesni_intel,ablk_helper, Live 0xffffffffc0569000
parport_pc 28205 0 - Live 0xffffffffc0596000
parport 46395 2 ppdev,parport_pc, Live 0xffffffffc0583000
i2c_piix4 22401 0 - Live 0xffffffffc057c000
pcspkr 12718 0 - Live 0xffffffffc0466000
ip_tables 27126 2 iptable_nat,iptable_filter, Live 0xffffffffc045e000
xfs 1014152 1 - Live 0xffffffffc0470000
libcrc32c 12644 3 nf_nat,nf_conntrack,xfs, Live 0xffffffffc0459000
crct10dif_pclmul 14307 0 - Live 0xffffffffc0438000
crct10dif_common 12595 1 crct10dif_pclmul, Live 0xffffffffc046b000
crc32c_intel 22094 1 - Live 0xffffffffc041d000
nvme 32382 1 - Live 0xffffffffc040f000
serio_raw 13434 0 - Live 0xffffffffc0407000
ena 96895 0 - Live 0xffffffffc0440000
nvme_core 63547 3 nvme, Live 0xffffffffc0427000
sunrpc 366617 1 - Live 0xffffffffc03ac000

调试 Iptables 的方法

Kubernetes 里面大量的使用了iptables 规则的各种转换，可以使用iptables的trace功能进行追踪， 查看具体经过了哪些链表。

1
2
3
4
5
6
7
iptables -t raw -A PREROUTING -p tcp -s 172.31.47.174/32 -j TRACE
or
iptables -t raw -A PREROUTING -p tcp -d 172.31.71.167 --dport 8090 -j TRACE

modprobe ipt_LOG ip6t_LOG nfnetlink_log
modprobe nf_log_ipv4
xtables-monitor  --trace

完成之后就可以在 message 里面看到记录了。 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: raw:PREROUTING:policy:2 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: mangle:PREROUTING:policy:4 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: nat:PREROUTING:rule:1 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: nat:KUBE-SERVICES:return:45 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: nat:PREROUTING:rule:3 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: nat:PREROUTING:policy:4 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: mangle:FORWARD:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: filter:FORWARD:rule:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: filter:KUBE-PROXY-FIREWALL:return:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: filter:FORWARD:rule:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: filter:KUBE-FORWARD:return:4 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:24 2023] TRACE: filter:FORWARD:rule:3 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:KUBE-SERVICES:return:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:FORWARD:rule:4 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:KUBE-EXTERNAL-SERVICES:return:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:FORWARD:policy:5 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:POSTROUTING:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: nat:POSTROUTING:rule:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: nat:KUBE-POSTROUTING:rule:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: nat:POSTROUTING:rule:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: nat:AWS-SNAT-CHAIN-0:return:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: nat:POSTROUTING:policy:3 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=60 TOS=0x00 PREC=0x00 TTL=63 ID=26039 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415057 ACK=0 WINDOW=62727 RES=0x00 SYN URGP=0 OPT (020423010402080ADBFD1C1B0000000001030307)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: raw:PREROUTING:policy:2 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26040 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:PREROUTING:policy:4 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26040 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:FORWARD:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26040 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:FORWARD:rule:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26040 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:KUBE-FORWARD:rule:3 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26040 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:POSTROUTING:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26040 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: raw:PREROUTING:policy:2 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=135 TOS=0x00 PREC=0x00 TTL=64 ID=26041 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK PSH URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:PREROUTING:policy:4 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=135 TOS=0x00 PREC=0x00 TTL=64 ID=26041 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK PSH URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:FORWARD:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=135 TOS=0x00 PREC=0x00 TTL=63 ID=26041 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK PSH URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:FORWARD:rule:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=135 TOS=0x00 PREC=0x00 TTL=63 ID=26041 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK PSH URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:KUBE-FORWARD:rule:3 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=135 TOS=0x00 PREC=0x00 TTL=63 ID=26041 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK PSH URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:POSTROUTING:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=135 TOS=0x00 PREC=0x00 TTL=63 ID=26041 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415058 ACK=2994616626 WINDOW=491 RES=0x00 ACK PSH URGP=0 OPT (0101080ADBFD1E0F3D8DDFB5)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: raw:PREROUTING:policy:2 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26042 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:PREROUTING:policy:4 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26042 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:FORWARD:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26042 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:FORWARD:rule:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26042 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:KUBE-FORWARD:rule:3 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26042 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:POSTROUTING:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26042 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: raw:PREROUTING:policy:2 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26043 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK FIN URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:PREROUTING:policy:4 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26043 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK FIN URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:FORWARD:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26043 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK FIN URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:FORWARD:rule:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26043 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK FIN URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:KUBE-FORWARD:rule:3 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26043 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK FIN URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:POSTROUTING:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26043 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415141 ACK=2994616804 WINDOW=490 RES=0x00 ACK FIN URGP=0 OPT (0101080ADBFD1EFA3D8DE09F)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: raw:PREROUTING:policy:2 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26044 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415142 ACK=2994616805 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1FE43D8DE18A)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:PREROUTING:policy:4 IN=eth3 OUT= MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=64 ID=26044 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415142 ACK=2994616805 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1FE43D8DE18A)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:FORWARD:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26044 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415142 ACK=2994616805 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1FE43D8DE18A)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:FORWARD:rule:2 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26044 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415142 ACK=2994616805 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1FE43D8DE18A)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: filter:KUBE-FORWARD:rule:3 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26044 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415142 ACK=2994616805 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1FE43D8DE18A)
kern  :warn  : [Thu Sep 21 15:01:25 2023] TRACE: mangle:POSTROUTING:policy:1 IN=eth3 OUT=eni7b7c562198a MAC=02:19:e6:40:fc:c8:02:55:96:c9:21:66:08:00 SRC=172.31.47.174 DST=172.31.55.27 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=26044 DF PROTO=TCP SPT=22896 DPT=8090 SEQ=1763415142 ACK=2994616805 WINDOW=490 RES=0x00 ACK URGP=0 OPT (0101080ADBFD1FE43D8DE18A)

移除规则的命令： 

1
iptables -F PREROUTING -t raw

详细的命令参考这个Blog ： https://blog.csdn.net/qq_43684922/article/details/126815699

注： 这个功能非常清晰， 但是如果网络流量比较大会导致内核输出了太多的信息， 会卡， 谨慎写策略。

关于nf_conntrack的说明

refer: https://jusene.github.io/2020/10/29/conntrack/#%E8%83%8C%E6%99%AF
会遇到这样的日志: 

1
kernel: nf_conntrack: table full, dropping packet.

和这个模块的相关的参数可以在这里找到: https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt

通常情况下, 例如在 Kubernetes 中连接数以及并发比较高的情况, 可以使用这个:
nf_conntrack_buckets = 65535
这个参数指定了 hash 表的大小, 通常对于 4GB 以上内存大小的os, 设置为 65535. 如果需要追踪更多的连接, 应该增加.
我看到 archlinux 的默认值是: 

1
2
3
net.netfilter.nf_conntrack_buckets = 262144
net.netfilter.nf_conntrack_max = 262144
# max 应该 等于 Bucket * 4