AWS VPC CNI 소개
네트워크 기본 정보 확인
# CNI 정보 확인
kubectl describe daemonset aws-node --namespace kube-system | grep Image | cut -d "/" -f 2
# kube-proxy config 확인 : 모드 iptables 사용 >> ipvs 모드로 변경 해보자!
kubectl describe cm -n kube-system kube-proxy-config
...
mode: "iptables"
...
# 노드 IP 확인
aws ec2 describe-instances --query "Reservations[*].Instances[*].{PublicIPAdd:PublicIpAddress,PrivateIPAdd:PrivateIpAddress,InstanceName:Tags[?Key=='Name']|[0].Value,Status:State.Name}" --filters Name=instance-state-name,Values=running --output table
# 파드 IP 확인
kubectl get pod -n kube-system -o=custom-columns=NAME:.metadata.name,IP:.status.podIP,STATUS:.status.phase
# 파드 이름 확인
kubectl get pod -A -o name
# 파드 갯수 확인
kubectl get pod -A -o name | wc -l
노드에 네트워크 정보 확인
# CNI 정보 확인
for i in $N1 $N2 $N3; do echo ">> node $i <<"; ssh ec2-user@$i tree /var/log/aws-routed-eni; echo; done
ssh ec2-user@$N1 sudo cat /var/log/aws-routed-eni/plugin.log | jq
ssh ec2-user@$N1 sudo cat /var/log/aws-routed-eni/ipamd.log | jq
ssh ec2-user@$N1 sudo cat /var/log/aws-routed-eni/egress-v6-plugin.log | jq
ssh ec2-user@$N1 sudo cat /var/log/aws-routed-eni/ebpf-sdk.log | jq
ssh ec2-user@$N1 sudo cat /var/log/aws-routed-eni/network-policy-agent.log | jq
# 네트워크 정보 확인 : eniY는 pod network 네임스페이스와 veth pair
for i in $N1 $N2 $N3; do echo ">> node $i <<"; ssh ec2-user@$i sudo ip -br -c addr; echo; done
for i in $N1 $N2 $N3; do echo ">> node $i <<"; ssh ec2-user@$i sudo ip -c addr; echo; done
for i in $N1 $N2 $N3; do echo ">> node $i <<"; ssh ec2-user@$i sudo ip -c route; echo; done
ssh ec2-user@$N1 sudo iptables -t nat -S
ssh ec2-user@$N1 sudo iptables -t nat -L -n -v
노드에서 기본 네트워크 정보 확인
# coredns 파드 IP 정보 확인
kubectl get pod -n kube-system -l k8s-app=kube-dns -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
coredns-6777fcd775-57k77 1/1 Running 0 70m 192.168.1.142 ip-192-168-1-251.ap-northeast-2.compute.internal <none> <none>
coredns-6777fcd775-cvqsb 1/1 Running 0 70m 192.168.2.75 ip-192-168-2-34.ap-northeast-2.compute.internal <none> <none>
노드의 라우팅 정보 확인
➜ ~ for i in $N1 $N2 $N3; do echo ">> node $i <<"; ssh -i ~/.ssh/eks-demo ec2-user@$i sudo ip -c route; echo; done
>> node 3.35.234.190 <<
default via 192.168.3.1 dev ens5 proto dhcp src 192.168.3.46 metric 1024
192.168.0.2 via 192.168.3.1 dev ens5 proto dhcp src 192.168.3.46 metric 1024
192.168.3.0/24 dev ens5 proto kernel scope link src 192.168.3.46 metric 1024
192.168.3.1 dev ens5 proto dhcp scope link src 192.168.3.46 metric 1024
>> node 3.39.0.92 <<
default via 192.168.1.1 dev ens5 proto dhcp src 192.168.1.51 metric 1024
192.168.0.2 via 192.168.1.1 dev ens5 proto dhcp src 192.168.1.51 metric 1024
192.168.1.0/24 dev ens5 proto kernel scope link src 192.168.1.51 metric 1024
192.168.1.1 dev ens5 proto dhcp scope link src 192.168.1.51 metric 1024
192.168.1.17 dev enie50c6729a7f scope link
192.168.1.23 dev enicb6cd19a76e scope link
192.168.1.95 dev eni9f32e929b5b scope link
192.168.1.225 dev eni39975045e30 scope link
>> node 3.38.133.223 <<
default via 192.168.2.1 dev ens5 proto dhcp src 192.168.2.147 metric 1024
192.168.0.2 via 192.168.2.1 dev ens5 proto dhcp src 192.168.2.147 metric 1024
192.168.2.0/24 dev ens5 proto kernel scope link src 192.168.2.147 metric 1024
192.168.2.1 dev ens5 proto dhcp scope link src 192.168.2.147 metric 1024
2. AWS VPC CNI - 파드 기본 통신
[ec2-user@ip-192-168-1-51 ~]$ ip -br -c addr
lo UNKNOWN 127.0.0.1/8 ::1/128
ens5 UP 192.168.1.51/24 metric 1024 fe80::e8:14ff:fe85:b2df/64
enie50c6729a7f@if3 UP fe80::44a8:c1ff:fef9:cd12/64
eni9f32e929b5b@if3 UP fe80::209b:dbff:fe6f:7260/64
enicb6cd19a76e@if3 UP fe80::382b:e7ff:fea1:d724/64
eni39975045e30@if3 UP fe80::b88c:9eff:fed1:7a6e/64
ens6 UP 192.168.1.248/24 fe80::a4:2fff:fe65:9781/64
(admin:N/A) [root@operator-host .ssh]# kubectl get pod -n kube-system -o=custom-columns=NAME:.metadata.name,IP:.status.podIP,STATUS:.status.phase
NAME IP STATUS
aws-node-dk9kl 192.168.1.51 Running
aws-node-gqrbs 192.168.3.46 Running
aws-node-hvlz5 192.168.2.147 Running
coredns-9b5bc9468-g9hxl 192.168.1.95 Running
coredns-9b5bc9468-whh8w 192.168.1.17 Running
kube-proxy-l9cft 192.168.3.46 Running
kube-proxy-mbmt2 192.168.2.147 Running
kube-proxy-zq94c 192.168.1.51 Running
metrics-server-86bbfd75bb-t6v7q 192.168.1.23 Running
metrics-server-86bbfd75bb-vdclj 192.168.1.225 Running
테스트용 파드 생성 및 확인
# 워커 노드 - 모니터링
watch -d "ip link | egrep 'eth|eni' ;echo;echo "[ROUTE TABLE]"; route -n | grep eni"
# 작업용 EC2 - 파드 2개 생성
cat <<EOF | kubectl create -f -
apiVersion: v1
kind: Pod
metadata:
name: pod-1
labels:
app: pod
spec:
containers:
- name: netshoot-pod
image: nicolaka/netshoot
command: ["tail"]
args: ["-f", "/dev/null"]
terminationGracePeriodSeconds: 0
---
apiVersion: v1
kind: Pod
metadata:
name: pod-2
labels:
app: pod
spec:
containers:
- name: netshoot-pod
image: nicolaka/netshoot
command: ["tail"]
args: ["-f", "/dev/null"]
terminationGracePeriodSeconds: 0
EOF
# 파드 확인
kubectl get pod -o wide
(admin:N/A) [root@operator-host .ssh]# kubectl get pod -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod-1 0/1 ContainerCreating 0 7s <none> ip-192-168-3-46.ap-northeast-2.compute.internal <none> <none>
pod-2 0/1 ContainerCreating 0 7s <none> ip-192-168-2-147.ap-northeast-2.compute.internal <none> <none>
kubectl get pod -o=custom-columns=NAME:.metadata.name,IP:.status.podIP
NAME IP
pod-1 192.168.3.218
pod-2 192.168.2.215
파드간 통신
파드간 통신 흐름 : 별도의 오버레이(Overlay) 통신 기술 없이, VPC Native 하게 파드간 직접 통신이 가능하다.
- pod 내부에 route 는 virtual router 인 169.254.1.1 를 통해 통신한다.
pod-1 ~ ping 192.168.2.215
PING 192.168.2.215 (192.168.2.215) 56(84) bytes of data.
64 bytes from 192.168.2.215: icmp_seq=1 ttl=125 time=1.82 ms
64 bytes from 192.168.2.215: icmp_seq=2 ttl=125 time=1.47 ms
^C
--- 192.168.2.215 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1002ms
rtt min/avg/max/mdev = 1.473/1.648/1.824/0.175 ms
pod-1 ~ ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host proto kernel_lo
valid_lft forever preferred_lft forever
3: eth0@if3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9001 qdisc noqueue state UP group default
link/ether de:8d:5b:09:d1:ea brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 192.168.3.218/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::dc8d:5bff:fe09:d1ea/64 scope link proto kernel_ll
valid_lft forever preferred_lft forever
pod-1 ~ routt -n
zsh: command not found: routt
pod-1 ~ route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 169.254.1.1 0.0.0.0 UG 0 0 0 eth0
169.254.1.1 0.0.0.0 255.255.255.255 UH 0 0 0 eth0
파드 → 외부(인터넷) 통신
iptable 에 SNAT 을 통하여 노드의 eth0 IP로 변경되어서 외부와 통신됨
- VPC CNI 의 External source network address translation (
SNAT) 설정에 따라, 외부(인터넷) 통신 시 SNAT 하거나 혹은 SNAT 없이 통신을 할 수 있다. - 파드가 VPC peering, Transit VPC, or Direct Connect 등으로 연결된 곳과 통신 시 문제 발생 시 VPC CNI v1.8 이상으로 업데이트를 하자
# 작업용 EC2
# pod-1 Shell 실행
kubectl exec -it pod-2 -- zsh
# 아래부터는 pod-1 Shell 에서 실행 : 파드 통신 확인, 외부 통신 확인
----------------------------
ping -c 1 8.8.8.8 # 아래 tcpdump 시 실행
ping -i 0.1 8.8.8.8 # 아래 iptables 카운트 확인 시
curl ifconfig.co
...
# 모든 동작 확인 후 빠져나오기
exit
----------------------------
# 워커 노드 EC2
# TCPDUMP 확인
tcpdump -i any -nn icmp
tcpdump -i eth0 -nn icmp
tcpdump -i eth1 -nn icmp
tcpdump -i eniY -nn icmp
# 출력된 결과를 보고 어떻게 빠져나가는지 고민해보자!
ip rule
ip route show table main
iptables -L -n -v -t nat
iptables -t nat -S
# 파드가 외부와 통신시에는 아래 처럼 'AWS-SNAT-CHAIN-0, AWS-SNAT-CHAIN-1' 룰(rule)에 의해서 SNAT 되어서 외부와 통신!
# 참고로 뒤 IP는 eth0(ENI 첫번째)의 IP 주소이다
# --random-fully 동작 - 링크1 링크2
iptables -t nat -S | grep 'A AWS-SNAT-CHAIN'
-A AWS-SNAT-CHAIN-0 ! -d 192.168.0.0/16 -m comment --comment "AWS SNAT CHAIN" -j AWS-SNAT-CHAIN-1
-A AWS-SNAT-CHAIN-1 ! -o vlan+ -m comment --comment "AWS, SNAT" -m addrtype ! --dst-type LOCAL -j SNAT --to-source 192.168.1.64 --random-fully
## 아래 'mark 0x4000/0x4000' 매칭되지 않아서 RETURN 됨!
-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
-A KUBE-POSTROUTING -j MARK --set-xmark 0x4000/0x0
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE --random-fully
...
# 카운트 확인 시 AWS-SNAT-CHAIN-0, AWS-SNAT-CHAIN-1 에 매칭되어, 목적지가 192.168.0.0/16 아니고 외부 빠져나갈때 SNAT 192.168.1.64 변경되어 나간다!
iptables -t filter --zero; iptables -t nat --zero; iptables -t mangle --zero; iptables -t raw --zero
watch -d 'iptables -v --numeric --table nat --list AWS-SNAT-CHAIN-0; echo ; iptables -v --numeric --table nat --list AWS-SNAT-CHAIN-1; echo ; iptables -v --numeric --table nat --list KUBE-POSTROUTING'
conntrack -L -n |grep -v '169.254.169'
icmp 1 29 src=192.168.1.249 dst=8.8.8.8 type=8 code=0 id=12386 src=8.8.8.8 dst=192.168.1.64 type=0 code=0 id=8247 mark=128 use=1
# 다음 실습을 위해서 파드 삭제
kubectl delete pod pod-1 pod-2
5. 노드에 파드 생성 갯수 제한
인스턴스 타입 별 ENI 최대 갯수와 할당 가능한 최대 IP 갯수에 따라서 파드 배치 갯수가 결정됨
단, aws-node 와 kube-proxy 파드는 호스트의 IP를 사용함으로 최대 갯수에서 제외함
최대 파드 생성 갯수 : (Number of network interfaces for the instance type × (the number of IP addressess per network interface - 1)) + 2
- t3.medium : 15개
cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
labels:
app: nginx
spec:
replicas: 2
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:alpine
ports:
- containerPort: 80
EOF
kubectl scale deployment nginx-deployment --replicas=50
(admin:N/A) [root@operator-host ~]# k get pod | grep Pending
nginx-deployment-6c8cb99bb9-47f5g 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-9qwmz 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-9t9fk 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-ft8fp 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-kzmcj 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-pdss6 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-q4qdr 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-rc2pt 0/1 Pending 0 16s
nginx-deployment-6c8cb99bb9-tm297 0/1 Pending 0 16s
6. Service & AWS LoadBalancer Controller
서비스/파드 배포 테스트 with NLB
# 모니터링
watch -d kubectl get pod,svc,ep,endpointslices
# 디플로이먼트 & 서비스 생성
cat << EOF > echo-service-nlb.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy-echo
spec:
replicas: 2
selector:
matchLabels:
app: deploy-websrv
template:
metadata:
labels:
app: deploy-websrv
spec:
terminationGracePeriodSeconds: 0
containers:
- name: aews-websrv
image: k8s.gcr.io/echoserver:1.5
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
name: svc-nlb-ip-type
annotations:
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: "8080"
service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
spec:
ports:
- port: 80
targetPort: 8080
protocol: TCP
type: LoadBalancer
loadBalancerClass: service.k8s.aws/nlb
selector:
app: deploy-websrv
EOF
kubectl apply -f echo-service-nlb.yaml
# 확인
aws elbv2 describe-load-balancers --query 'LoadBalancers[*].State.Code' --output text
kubectl get deploy,pod
kubectl get svc,ep,ingressclassparams,targetgroupbindings
kubectl get targetgroupbindings -o json | jq
# 빠른 실습을 위해서 등록 취소 지연(드레이닝 간격) 수정 : 기본값 300초
echo-service-nlb.yaml 파일 IDE(VS code)에서 수정
..
apiVersion: v1
kind: Service
metadata:
name: svc-nlb-ip-type
annotations:
service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
service.beta.kubernetes.io/aws-load-balancer-healthcheck-port: "8080"
service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
service.beta.kubernetes.io/aws-load-balancer-target-group-attributes: deregistration_delay.timeout_seconds=60
...
kubectl apply -f echo-service-nlb.yaml
# AWS ELB(NLB) 정보 확인
aws elbv2 describe-load-balancers | jq
aws elbv2 describe-load-balancers --query 'LoadBalancers[*].State.Code' --output text
ALB_ARN=$(aws elbv2 describe-load-balancers --query 'LoadBalancers[?contains(LoadBalancerName, `k8s-default-svcnlbip`) == `true`].LoadBalancerArn' | jq -r '.[0]')
aws elbv2 describe-target-groups --load-balancer-arn $ALB_ARN | jq
TARGET_GROUP_ARN=$(aws elbv2 describe-target-groups --load-balancer-arn $ALB_ARN | jq -r '.TargetGroups[0].TargetGroupArn')
aws elbv2 describe-target-health --target-group-arn $TARGET_GROUP_ARN | jq
{
"TargetHealthDescriptions": [
{
"Target": {
"Id": "192.168.2.153",
"Port": 8080,
"AvailabilityZone": "ap-northeast-2b"
},
"HealthCheckPort": "8080",
"TargetHealth": {
"State": "initial",
"Reason": "Elb.RegistrationInProgress",
"Description": "Target registration is in progress"
}
},
...
# 웹 접속 주소 확인
kubectl get svc svc-nlb-ip-type -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' | awk '{ print "Pod Web URL = http://"$1 }'
# 파드 로깅 모니터링
kubectl logs -l app=deploy-websrv -f
kubectl stern -l app=deploy-websrv
# 분산 접속 확인
NLB=$(kubectl get svc svc-nlb-ip-type -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
curl -s $NLB
for i in {1..100}; do curl -s $NLB | grep Hostname ; done | sort | uniq -c | sort -nr
52 Hostname: deploy-echo-55456fc798-2w65p
48 Hostname: deploy-echo-55456fc798-cxl7z
# 지속적인 접속 시도 : 아래 상세 동작 확인 시 유용(패킷 덤프 등)
while true; do curl -s --connect-timeout 1 $NLB | egrep 'Hostname|client_address'; echo "----------" ; date "+%Y-%m-%d %H:%M:%S" ; sleep 1; done
- AWS NLB의 대상 그룹 확인 : IP를 확인해보자
- 파드 2개 → 1개 → 3개 설정 시 동작 : auto discovery ← 어떻게 가능할까?
# (신규 터미널) 모니터링
while true; do aws elbv2 describe-target-health --target-group-arn $TARGET_GROUP_ARN --output text; echo; done
# 작업용 EC2 - 파드 1개 설정
kubectl scale deployment deploy-echo --replicas=1
# 확인
kubectl get deploy,pod,svc,ep
NLB=$(kubectl get svc svc-nlb-ip-type -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
curl -s $NLB
for i in {1..100}; do curl -s --connect-timeout 1 $NLB | grep Hostname ; done | sort | uniq -c | sort -nr
# 파드 3개 설정
kubectl scale deployment deploy-echo --replicas=3
# 확인 : NLB 대상 타켓이 아직 initial 일 때 100번 반복 접속 시 어떻게 되는지 확인해보자!
kubectl get deploy,pod,svc,ep
NLB=$(kubectl get svc svc-nlb-ip-type -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
curl -s $NLB
for i in {1..100}; do curl -s --connect-timeout 1 $NLB | grep Hostname ; done | sort | uniq -c | sort -nr
#
kubectl describe deploy -n kube-system aws-load-balancer-controller | grep -i 'Service Account'
Service Account: aws-load-balancer-controller
# [AWS LB Ctrl] 클러스터 롤 바인딩 정보 확인
kubectl describe clusterrolebindings.rbac.authorization.k8s.io aws-load-balancer-controller-rolebinding
# [AWS LB Ctrl] 클러스터롤 확인
kubectl describe clusterroles.rbac.authorization.k8s.io aws-load-balancer-controller-role
리소스 삭제
kubectl delete deploy deploy-echo; kubectl delete svc svc-nlb-ip-type
7. Ingress
# 게임 파드와 Service, Ingress 배포
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Namespace
metadata:
name: game-2048
---
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: game-2048
name: deployment-2048
spec:
selector:
matchLabels:
app.kubernetes.io/name: app-2048
replicas: 2
template:
metadata:
labels:
app.kubernetes.io/name: app-2048
spec:
containers:
- image: public.ecr.aws/l6m2t8p7/docker-2048:latest
imagePullPolicy: Always
name: app-2048
ports:
- containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
namespace: game-2048
name: service-2048
spec:
ports:
- port: 80
targetPort: 80
protocol: TCP
type: NodePort
selector:
app.kubernetes.io/name: app-2048
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
namespace: game-2048
name: ingress-2048
annotations:
alb.ingress.kubernetes.io/scheme: internet-facing
alb.ingress.kubernetes.io/target-type: ip
spec:
ingressClassName: alb
rules:
- http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: service-2048
port:
number: 80
EOF
# 모니터링
watch -d kubectl get pod,ingress,svc,ep,endpointslices -n game-2048
# 생성 확인
kubectl get ingress,svc,ep,pod -n game-2048
kubectl get-all -n game-2048
kubectl get targetgroupbindings -n game-2048
# ALB 생성 확인
aws elbv2 describe-load-balancers --query 'LoadBalancers[?contains(LoadBalancerName, `k8s-game2048`) == `true`]' | jq
ALB_ARN=$(aws elbv2 describe-load-balancers --query 'LoadBalancers[?contains(LoadBalancerName, `k8s-game2048`) == `true`].LoadBalancerArn' | jq -r '.[0]')
aws elbv2 describe-target-groups --load-balancer-arn $ALB_ARN
TARGET_GROUP_ARN=$(aws elbv2 describe-target-groups --load-balancer-arn $ALB_ARN | jq -r '.TargetGroups[0].TargetGroupArn')
aws elbv2 describe-target-health --target-group-arn $TARGET_GROUP_ARN | jq
# Ingress 확인
kubectl describe ingress -n game-2048 ingress-2048
kubectl get ingress -n game-2048 ingress-2048 -o jsonpath="{.status.loadBalancer.ingress[*].hostname}{'\n'}"
# 게임 접속 : ALB 주소로 웹 접속
kubectl get ingress -n game-2048 ingress-2048 -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' | awk '{ print "Game URL = http://"$1 }'
# 파드 IP 확인
kubectl get pod -n game-2048 -owide