kubestr fio 테스트
fio 테스트
[global]
ioengine=libaio
direct=1
bs=4k
runtime=120
time_based=1
iodepth=16
numjobs=4
group_reporting
size=1g
rw=randread
[read]
kubestr fio -f fio-read.fio -s local-path --size 10G
PVC created kubestr-fio-pvc-lrn6s
Pod created kubestr-fio-pod-tsbdw
Running FIO test (fio-read.fio) on StorageClass (local-path) with a PVC of Size (10G)
Elapsed time- 2m33.026228946s
FIO test results:
FIO version - fio-3.36
Global options - ioengine=libaio verify= direct=1 gtod_reduce=
JobName:
blocksize= filesize= iodepth= rw=
read:
IOPS=3024.037598 BW(KiB/s)=12096
iops: min=2144 max=9001 avg=3025.845215
bw(KiB/s): min=8576 max=36007 avg=12103.715820
Disk stats (read/write):
nvme0n1: ios=362532/187 merge=0/9 ticks=6192083/4132 in_queue=6196215, util=96.232597%
- OK
AWS EBS Controller
EBS CSI Driver는 쿠버네티스에서 Amazon EBS 볼륨을 동적으로 프로비저닝하고 관리하는 CSI(Container Storage Interface) 기반의 드라이버이다.
(nana@myeks:N/A) [root@operator-host ~]# eksctl create addon --name aws-ebs-csi-driver --cluster ${CLUSTER_NAME} --service-account-role-arn arn:aws:iam::${ACCOUNT_ID}:role/AmazonEKS_EBS_CSI_DriverRole --force
2025-02-21 22:33:37 [ℹ] Kubernetes version "1.31" in use by cluster "myeks"
2025-02-21 22:33:37 [ℹ] IRSA is set for "aws-ebs-csi-driver" addon; will use this to configure IAM permissions
2025-02-21 22:33:37 [!] the recommended way to provide IAM permissions for "aws-ebs-csi-driver" addon is via pod identity associations; after addon creation is completed, run `eksctl utils migrate-to-pod-identity`
2025-02-21 22:33:37 [ℹ] using provided ServiceAccountRoleARN "arn:aws:iam::390844768149:role/AmazonEKS_EBS_CSI_DriverRole"
2025-02-21 22:33:37 [ℹ] creating addon: aws-ebs-csi-driver
(nana@myeks:N/A) [root@operator-host ~]# kubectl get sa -n kube-system ebs-csi-controller-sa -o yaml | head -5
Error from server (NotFound): serviceaccounts "ebs-csi-controller-sa" not found
AmazonEKS_EBS_CSI_DriverRole IAM 역할을 사용하여 애드온이 EBS 볼륨을 관리할 수 있도록 한다.
아직 만들어지고 있는 중이라서 서비스 계정이 없다고 나오게 된다.
(nana@myeks:N/A) [root@operator-host ~]# kubectl get deploy,ds -l=app.kubernetes.io/name=aws-ebs-csi-driver -n kube-system
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/ebs-csi-controller 2/2 2 2 35s
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
daemonset.apps/ebs-csi-node 3 3 3 3 3 kubernetes.io/os=linux 36s
daemonset.apps/ebs-csi-node-windows 0 0 0 0 0 kubernetes.io/os=windows 36s
(nana@myeks:N/A) [root@operator-host ~]# kubectl get pod -n kube-system -l 'app in (ebs-csi-controller,ebs-csi-node)'
NAME READY STATUS RESTARTS AGE
ebs-csi-controller-7f8f8cb84-f27l4 6/6 Running 0 36s
ebs-csi-controller-7f8f8cb84-jlsqp 6/6 Running 0 36s
ebs-csi-node-7bd5h 3/3 Running 0 37s
ebs-csi-node-dt2qr 3/3 Running 0 37s
ebs-csi-node-fj97t 3/3 Running 0 37s
(nana@myeks:N/A) [root@operator-host ~]# kubectl get pod -n kube-system -l app.kubernetes.io/component=csi-driver
NAME READY STATUS RESTARTS AGE
ebs-csi-controller-7f8f8cb84-f27l4 6/6 Running 0 37s
ebs-csi-controller-7f8f8cb84-jlsqp 6/6 Running 0 37s
ebs-csi-node-7bd5h 3/3 Running 0 38s
ebs-csi-node-dt2qr 3/3 Running 0 38s
ebs-csi-node-fj97t 3/3 Running 0 38s
(nana@myeks:N/A) [root@operator-host ~]# kubectl get pod -n kube-system -l app=ebs-csi-controller -o jsonpath='{.items[0].spec.containers[*].name}' ; echo
ebs-plugin csi-provisioner csi-attacher csi-snapshotter csi-resizer liveness-probe
(nana@myeks:N/A) [root@operator-host ~]# kubectl api-resources | grep -i csi
csidrivers storage.k8s.io/v1 false CSIDriver
csinodes storage.k8s.io/v1 false CSINode
csistoragecapacities storage.k8s.io/v1 true CSIStorageCapacity
(nana@myeks:N/A) [root@operator-host ~]# kubectl get csinodes
NAME DRIVERS AGE
ip-192-168-1-13.ap-northeast-2.compute.internal 1 5d1h
ip-192-168-2-229.ap-northeast-2.compute.internal 1 5d1h
ip-192-168-3-122.ap-northeast-2.compute.internal 1 5d1h
(nana@myeks:N/A) [root@operator-host ~]# kubectl describe csinodes
Name: ip-192-168-1-13.ap-northeast-2.compute.internal
Labels: <none>
Annotations: storage.alpha.kubernetes.io/migrated-plugins:
kubernetes.io/aws-ebs,kubernetes.io/azure-disk,kubernetes.io/azure-file,kubernetes.io/cinder,kubernetes.io/gce-pd,kubernetes.io/portworx-v...
CreationTimestamp: Sun, 16 Feb 2025 20:43:06 +0900
Spec:
Drivers:
ebs.csi.aws.com:
Node ID: i-0d6799dedb360590e
Allocatables:
Count: 25
Topology Keys: [kubernetes.io/os topology.ebs.csi.aws.com/zone topology.kubernetes.io/zone]
Events: <none>
Name: ip-192-168-2-229.ap-northeast-2.compute.internal
Labels: <none>
Annotations: storage.alpha.kubernetes.io/migrated-plugins:
kubernetes.io/aws-ebs,kubernetes.io/azure-disk,kubernetes.io/azure-file,kubernetes.io/cinder,kubernetes.io/gce-pd,kubernetes.io/portworx-v...
CreationTimestamp: Sun, 16 Feb 2025 20:43:00 +0900
Spec:
Drivers:
ebs.csi.aws.com:
Node ID: i-0ec89466e83491846
Allocatables:
Count: 25
Topology Keys: [kubernetes.io/os topology.ebs.csi.aws.com/zone topology.kubernetes.io/zone]
Events: <none>
Name: ip-192-168-3-122.ap-northeast-2.compute.internal
Labels: <none>
Annotations: storage.alpha.kubernetes.io/migrated-plugins:
kubernetes.io/aws-ebs,kubernetes.io/azure-disk,kubernetes.io/azure-file,kubernetes.io/cinder,kubernetes.io/gce-pd,kubernetes.io/portworx-v...
CreationTimestamp: Sun, 16 Feb 2025 20:43:01 +0900
Spec:
Drivers:
ebs.csi.aws.com:
Node ID: i-060b87d1d161c0e48
Allocatables:
Count: 25
Topology Keys: [kubernetes.io/os topology.ebs.csi.aws.com/zone topology.kubernetes.io/zone]
Events: <none>
각각의 EKS 노드에 EBS CSI Driver가 정상적으로 연결된 것을 확인할 수 있는데, 각 노드는 25개의 볼륨을 할당 가능하며 BS 볼륨이 특정 AZ에 배치될 수 있도록 topology.ebs.csi.aws.com/zone 정보가 설정된다.
(nana@myeks:N/A) [root@operator-host ~]# aws eks update-addon --cluster-name ${CLUSTER_NAME} --addon-name aws-ebs-csi-driver \
> --addon-version v1.39.0-eksbuild.1 --configuration-values '{
> "node": {
> "volumeAttachLimit": 31,
> "enableMetrics": true
> }
> }'
{
"update": {
"id": "a79dc9e7-8abc-38ad-9aaf-ea76635800ab",
"status": "InProgress",
"type": "AddonUpdate",
"params": [
{
"type": "AddonVersion",
"value": "v1.39.0-eksbuild.1"
},
{
"type": "ConfigurationValues",
"value": "{\n \"node\": {\n \"volumeAttachLimit\": 31,\n \"enableMetrics\": true\n }\n }"
}
],
"createdAt": "2025-02-21T22:50:59.641000+09:00",
"errors": []
}
}
(nana@myeks:N/A) [root@operator-host ~]# kubectl describe csinodes
...
Spec:
Drivers:
ebs.csi.aws.com:
Node ID: i-0ec89466e83491846
Allocatables:
Count: 31
pvc, pv 생성 테스트
pvc 생성
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: ebs-claim
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 4Gi
storageClassName: gp3
EOF
파드 생성
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: app
spec:
terminationGracePeriodSeconds: 3
containers:
- name: app
image: centos
command: ["/bin/sh"]
args: ["-c", "while true; do echo \$(date -u) >> /data/out.txt; sleep 5; done"]
volumeMounts:
- name: persistent-storage
mountPath: /data
volumes:
- name: persistent-storage
persistentVolumeClaim:
claimName: ebs-claim
EOF
생성 확인
(nana@myeks:N/A) [root@operator-host ~]# kubectl get pvc,pv,pod
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS VOLUMEATTRIBUTESCLASS AGE
persistentvolumeclaim/ebs-claim Bound pvc-555a8439-31fd-475a-bd6e-aa5704bf188b 4Gi RWO gp3 <unset> 27s
persistentvolumeclaim/kubestr-fio-pvc-zf8rq Bound pvc-18850c9e-88ed-48c4-97c2-b4a3dfc157f4 100Gi RWO local-path <unset> 36m
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS VOLUMEATTRIBUTESCLASS REASON AGE
persistentvolume/pvc-18850c9e-88ed-48c4-97c2-b4a3dfc157f4 100Gi RWO Delete Bound default/kubestr-fio-pvc-zf8rq local-path <unset> 36m
persistentvolume/pvc-555a8439-31fd-475a-bd6e-aa5704bf188b 4Gi RWO Delete Bound default/ebs-claim gp3 <unset> 4s
NAME READY STATUS RESTARTS AGE
pod/app 0/1 ContainerCreating 0 7s
pod/kubestr-fio-pod-gr6p7 1/1 Running 0 36m
(nana@myeks:N/A) [root@operator-host ~]# kubectl get VolumeAttachment
NAME ATTACHER PV NODE ATTACHED AGE
csi-a0facaf2ca346eaf8e31569d53b71e401e993d69b485336d2ce585120cf9dcfd ebs.csi.aws.com pvc-555a8439-31fd-475a-bd6e-aa5704bf188b ip-192-168-2-229.ap-northeast-2.compute.internal true 10s
(nana@myeks:N/A) [root@operator-host ~]# kubectl df-pv
INFO[2025-02-21T22:55:39+09:00] Either no volumes found in namespace/s: 'all' or the storage provisioner used for the volumes does not publish metrics to kubelet
# 조금 기다렸다가 다시 df-pv
(nana@myeks:N/A) [root@operator-host ~]# kubectl df-pv
PV NAME PVC NAME NAMESPACE NODE NAME POD NAME VOLUME MOUNT NAME SIZE USED AVAILABLE %USED IUSED IFREE %IUSED
pvc-555a8439-31fd-475a-bd6e-aa5704bf188b ebs-claim default ip-192-168-2-229.ap-northeast-2.compute.internal app persistent-storage 3Gi 60Mi 3Gi 1.50 4 2097148 0.00
[ec2-user@ip-192-168-3-122 ~]$ while true; do aws ec2 describe-volumes --filters Name=tag:ebs.csi.aws.com/cluster,Values=true --query "Volumes[].{VolumeId: VolumeId, VolumeType: VolumeType, InstanceId: Attachments[0].InstanceId, State: Attachments[0].State}" --output text; date; sleep 1; done
Fri Feb 21 13:55:12 UTC 2025
Fri Feb 21 13:55:14 UTC 2025
Fri Feb 21 13:55:16 UTC 2025
Fri Feb 21 13:55:19 UTC 2025
None None vol-04ade57551c3cdee0 gp3
Fri Feb 21 13:55:21 UTC 2025
None None vol-04ade57551c3cdee0 gp3
Fri Feb 21 13:55:23 UTC 2025
i-0ec89466e83491846 attached vol-04ade57551c3cdee0 gp3
Fri Feb 21 13:55:26 UTC 2025
(nana@myeks:N/A) [root@operator-host ~]# kubectl get node --label-columns=topology.ebs.csi.aws.com/zone,topology.k8s.aws/zone-id
NAME STATUS ROLES AGE VERSION ZONE ZONE-ID
ip-192-168-1-13.ap-northeast-2.compute.internal Ready <none> 5d2h v1.31.5-eks-5d632ec ap-northeast-2a apne2-az1
ip-192-168-2-229.ap-northeast-2.compute.internal Ready <none> 5d2h v1.31.5-eks-5d632ec ap-northeast-2b apne2-az2
ip-192-168-3-122.ap-northeast-2.compute.internal Ready <none> 5d2h v1.31.5-eks-5d632ec ap-northeast-2c apne2-az3
(nana@myeks:N/A) [root@operator-host ~]#
(nana@myeks:N/A) [root@operator-host ~]# kubectl describe node
Name: ip-192-168-1-13.ap-northeast-2.compute.internal
Roles: <none>
Labels: alpha.eksctl.io/cluster-name=myeks
alpha.eksctl.io/nodegroup-name=ng1
beta.kubernetes.io/arch=amd64
beta.kubernetes.io/instance-type=t3.medium
beta.kubernetes.io/os=linux
eks.amazonaws.com/capacityType=ON_DEMAND
eks.amazonaws.com/nodegroup=ng1
eks.amazonaws.com/nodegroup-image=ami-09aa6ccd2d0305078
eks.amazonaws.com/sourceLaunchTemplateId=lt-09e54fc3c4a0843b5
eks.amazonaws.com/sourceLaunchTemplateVersion=1
failure-domain.beta.kubernetes.io/region=ap-northeast-2
failure-domain.beta.kubernetes.io/zone=ap-northeast-2a
k8s.io/cloud-provider-aws=5553ae84a0d29114870f67bbabd07d44
kubernetes.io/arch=amd64
kubernetes.io/hostname=ip-192-168-1-13.ap-northeast-2.compute.internal
kubernetes.io/os=linux
node.kubernetes.io/instance-type=t3.medium
topology.ebs.csi.aws.com/zone=ap-northeast-2a
topology.k8s.aws/zone-id=apne2-az1
topology.kubernetes.io/region=ap-northeast-2
topology.kubernetes.io/zone=ap-northeast-2a
Annotations: alpha.kubernetes.io/provided-node-ip: 192.168.1.13
csi.volume.kubernetes.io/nodeid: {"ebs.csi.aws.com":"i-0d6799dedb360590e"}
node.alpha.kubernetes.io/ttl: 0
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Sun, 16 Feb 2025 20:43:06 +0900
Taints: <none>
Unschedulable: false
Lease:
HolderIdentity: ip-192-168-1-13.ap-northeast-2.compute.internal
AcquireTime: <unset>
RenewTime: Fri, 21 Feb 2025 23:33:18 +0900
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
MemoryPressure False Fri, 21 Feb 2025 23:30:14 +0900 Sun, 16 Feb 2025 20:43:05 +0900 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Fri, 21 Feb 2025 23:30:14 +0900 Sun, 16 Feb 2025 20:43:05 +0900 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Fri, 21 Feb 2025 23:30:14 +0900 Sun, 16 Feb 2025 20:43:05 +0900 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Fri, 21 Feb 2025 23:30:14 +0900 Sun, 16 Feb 2025 20:43:18 +0900 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.1.13
ExternalIP: 3.36.51.76
InternalDNS: ip-192-168-1-13.ap-northeast-2.compute.internal
Hostname: ip-192-168-1-13.ap-northeast-2.compute.internal
ExternalDNS: ec2-3-36-51-76.ap-northeast-2.compute.amazonaws.com
Capacity:
cpu: 2
ephemeral-storage: 125751276Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 3919536Ki
pods: 100
Allocatable:
cpu: 1930m
ephemeral-storage: 114818633946
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 3364528Ki
pods: 100
System Info:
Machine ID: ec2c51e08a14eb59df9e9435dfcf2229
System UUID: ec2c51e0-8a14-eb59-df9e-9435dfcf2229
Boot ID: de5cde2c-7797-4e32-950d-6e14d436e050
Kernel Version: 6.1.127-135.201.amzn2023.x86_64
OS Image: Amazon Linux 2023.6.20250203
Operating System: linux
Architecture: amd64
Container Runtime Version: containerd://1.7.25
Kubelet Version: v1.31.5-eks-5d632ec
Kube-Proxy Version: v1.31.5-eks-5d632ec
ProviderID: aws:///ap-northeast-2a/i-0d6799dedb360590e
Non-terminated Pods: (7 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age
--------- ---- ------------ ---------- --------------- ------------- ---
kube-system aws-node-r64zs 50m (2%) 0 (0%) 0 (0%) 0 (0%) 5d2h
kube-system ebs-csi-controller-7f8f8cb84-f27l4 60m (3%) 0 (0%) 240Mi (7%) 1536Mi (46%) 59m
kube-system ebs-csi-node-hnjrv 30m (1%) 0 (0%) 120Mi (3%) 768Mi (23%) 42m
kube-system external-dns-5dbdd664bc-v25xp 0 (0%) 0 (0%) 0 (0%) 0 (0%) 46h
kube-system kube-ops-view-657dbc6cd8-7kvtw 0 (0%) 0 (0%) 0 (0%) 0 (0%) 46h
kube-system kube-proxy-hh4tr 100m (5%) 0 (0%) 0 (0%) 0 (0%) 5d2h
local-path-storage local-path-provisioner-84967477f-7bg7g 0 (0%) 0 (0%) 0 (0%) 0 (0%) 3d21h
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 240m (12%) 0 (0%)
memory 360Mi (10%) 2304Mi (70%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
Events: <none>
Name: ip-192-168-2-229.ap-northeast-2.compute.internal
Roles: <none>
Labels: alpha.eksctl.io/cluster-name=myeks
alpha.eksctl.io/nodegroup-name=ng1
beta.kubernetes.io/arch=amd64
beta.kubernetes.io/instance-type=t3.medium
beta.kubernetes.io/os=linux
eks.amazonaws.com/capacityType=ON_DEMAND
eks.amazonaws.com/nodegroup=ng1
eks.amazonaws.com/nodegroup-image=ami-09aa6ccd2d0305078
eks.amazonaws.com/sourceLaunchTemplateId=lt-09e54fc3c4a0843b5
eks.amazonaws.com/sourceLaunchTemplateVersion=1
failure-domain.beta.kubernetes.io/region=ap-northeast-2
failure-domain.beta.kubernetes.io/zone=ap-northeast-2b
k8s.io/cloud-provider-aws=5553ae84a0d29114870f67bbabd07d44
kubernetes.io/arch=amd64
kubernetes.io/hostname=ip-192-168-2-229.ap-northeast-2.compute.internal
kubernetes.io/os=linux
node.kubernetes.io/instance-type=t3.medium
topology.ebs.csi.aws.com/zone=ap-northeast-2b
topology.k8s.aws/zone-id=apne2-az2
topology.kubernetes.io/region=ap-northeast-2
topology.kubernetes.io/zone=ap-northeast-2b
Annotations: alpha.kubernetes.io/provided-node-ip: 192.168.2.229
csi.volume.kubernetes.io/nodeid: {"ebs.csi.aws.com":"i-0ec89466e83491846"}
node.alpha.kubernetes.io/ttl: 0
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Sun, 16 Feb 2025 20:43:00 +0900
Taints: <none>
Unschedulable: false
Lease:
HolderIdentity: ip-192-168-2-229.ap-northeast-2.compute.internal
AcquireTime: <unset>
RenewTime: Fri, 21 Feb 2025 23:33:16 +0900
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
MemoryPressure False Fri, 21 Feb 2025 23:31:44 +0900 Sun, 16 Feb 2025 20:42:59 +0900 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Fri, 21 Feb 2025 23:31:44 +0900 Sun, 16 Feb 2025 20:42:59 +0900 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Fri, 21 Feb 2025 23:31:44 +0900 Sun, 16 Feb 2025 20:42:59 +0900 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Fri, 21 Feb 2025 23:31:44 +0900 Sun, 16 Feb 2025 20:43:10 +0900 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.2.229
ExternalIP: 43.203.132.94
InternalDNS: ip-192-168-2-229.ap-northeast-2.compute.internal
Hostname: ip-192-168-2-229.ap-northeast-2.compute.internal
ExternalDNS: ec2-43-203-132-94.ap-northeast-2.compute.amazonaws.com
Capacity:
cpu: 2
ephemeral-storage: 125751276Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 3919536Ki
pods: 100
Allocatable:
cpu: 1930m
ephemeral-storage: 114818633946
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 3364528Ki
pods: 100
System Info:
Machine ID: ec2975cab785ece83de533411578550f
System UUID: ec2975ca-b785-ece8-3de5-33411578550f
Boot ID: 359912a2-38ba-4d7c-8480-bf6417ad646f
Kernel Version: 6.1.127-135.201.amzn2023.x86_64
OS Image: Amazon Linux 2023.6.20250203
Operating System: linux
Architecture: amd64
Container Runtime Version: containerd://1.7.25
Kubelet Version: v1.31.5-eks-5d632ec
Kube-Proxy Version: v1.31.5-eks-5d632ec
ProviderID: aws:///ap-northeast-2b/i-0ec89466e83491846
Non-terminated Pods: (8 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age
--------- ---- ------------ ---------- --------------- ------------- ---
default app 0 (0%) 0 (0%) 0 (0%) 0 (0%) 38m
kube-system aws-load-balancer-controller-554fbd9d-qxc6v 0 (0%) 0 (0%) 0 (0%) 0 (0%) 46h
kube-system aws-node-z8sdh 50m (2%) 0 (0%) 0 (0%) 0 (0%) 5d2h
kube-system coredns-86f5954566-xbnc8 100m (5%) 0 (0%) 70Mi (2%) 170Mi (5%) 5d2h
kube-system ebs-csi-controller-7f8f8cb84-jlsqp 60m (3%) 0 (0%) 240Mi (7%) 1536Mi (46%) 59m
kube-system ebs-csi-node-9dld4 30m (1%) 0 (0%) 120Mi (3%) 768Mi (23%) 42m
kube-system kube-proxy-8sgcl 100m (5%) 0 (0%) 0 (0%) 0 (0%) 5d2h
kube-system metrics-server-6bf5998d9c-56skp 100m (5%) 0 (0%) 200Mi (6%) 400Mi (12%) 5d2h
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 440m (22%) 0 (0%)
memory 630Mi (19%) 2874Mi (87%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
Events: <none>
Name: ip-192-168-3-122.ap-northeast-2.compute.internal
Roles: <none>
Labels: alpha.eksctl.io/cluster-name=myeks
alpha.eksctl.io/nodegroup-name=ng1
beta.kubernetes.io/arch=amd64
beta.kubernetes.io/instance-type=t3.medium
beta.kubernetes.io/os=linux
eks.amazonaws.com/capacityType=ON_DEMAND
eks.amazonaws.com/nodegroup=ng1
eks.amazonaws.com/nodegroup-image=ami-09aa6ccd2d0305078
eks.amazonaws.com/sourceLaunchTemplateId=lt-09e54fc3c4a0843b5
eks.amazonaws.com/sourceLaunchTemplateVersion=1
failure-domain.beta.kubernetes.io/region=ap-northeast-2
failure-domain.beta.kubernetes.io/zone=ap-northeast-2c
k8s.io/cloud-provider-aws=5553ae84a0d29114870f67bbabd07d44
kubernetes.io/arch=amd64
kubernetes.io/hostname=ip-192-168-3-122.ap-northeast-2.compute.internal
kubernetes.io/os=linux
node.kubernetes.io/instance-type=t3.medium
topology.ebs.csi.aws.com/zone=ap-northeast-2c
topology.k8s.aws/zone-id=apne2-az3
topology.kubernetes.io/region=ap-northeast-2
topology.kubernetes.io/zone=ap-northeast-2c
Annotations: alpha.kubernetes.io/provided-node-ip: 192.168.3.122
csi.volume.kubernetes.io/nodeid: {"ebs.csi.aws.com":"i-060b87d1d161c0e48"}
node.alpha.kubernetes.io/ttl: 0
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Sun, 16 Feb 2025 20:43:01 +0900
Taints: <none>
Unschedulable: false
Lease:
HolderIdentity: ip-192-168-3-122.ap-northeast-2.compute.internal
AcquireTime: <unset>
RenewTime: Fri, 21 Feb 2025 23:33:12 +0900
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
MemoryPressure False Fri, 21 Feb 2025 23:30:02 +0900 Sun, 16 Feb 2025 20:43:00 +0900 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Fri, 21 Feb 2025 23:30:02 +0900 Sun, 16 Feb 2025 20:43:00 +0900 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Fri, 21 Feb 2025 23:30:02 +0900 Sun, 16 Feb 2025 20:43:00 +0900 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Fri, 21 Feb 2025 23:30:02 +0900 Sun, 16 Feb 2025 20:43:12 +0900 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.3.122
ExternalIP: 54.180.83.145
InternalDNS: ip-192-168-3-122.ap-northeast-2.compute.internal
Hostname: ip-192-168-3-122.ap-northeast-2.compute.internal
ExternalDNS: ec2-54-180-83-145.ap-northeast-2.compute.amazonaws.com
Capacity:
cpu: 2
ephemeral-storage: 125751276Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 3919536Ki
pods: 100
Allocatable:
cpu: 1930m
ephemeral-storage: 114818633946
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 3364528Ki
pods: 100
System Info:
Machine ID: ec2e23debc7133e222b82c4289eee47f
System UUID: ec2e23de-bc71-33e2-22b8-2c4289eee47f
Boot ID: d1b7846e-78e6-4f01-be08-74a5a6c76dbe
Kernel Version: 6.1.127-135.201.amzn2023.x86_64
OS Image: Amazon Linux 2023.6.20250203
Operating System: linux
Architecture: amd64
Container Runtime Version: containerd://1.7.25
Kubelet Version: v1.31.5-eks-5d632ec
Kube-Proxy Version: v1.31.5-eks-5d632ec
ProviderID: aws:///ap-northeast-2c/i-060b87d1d161c0e48
Non-terminated Pods: (7 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age
--------- ---- ------------ ---------- --------------- ------------- ---
default kubestr-fio-pod-gr6p7 0 (0%) 0 (0%) 0 (0%) 0 (0%) 74m
kube-system aws-load-balancer-controller-554fbd9d-bb2wm 0 (0%) 0 (0%) 0 (0%) 0 (0%) 46h
kube-system aws-node-2btth 50m (2%) 0 (0%) 0 (0%) 0 (0%) 5d2h
kube-system coredns-86f5954566-w24vz 100m (5%) 0 (0%) 70Mi (2%) 170Mi (5%) 5d2h
kube-system ebs-csi-node-k8t2k 30m (1%) 0 (0%) 120Mi (3%) 768Mi (23%) 42m
kube-system kube-proxy-4nlwd 100m (5%) 0 (0%) 0 (0%) 0 (0%) 5d2h
kube-system metrics-server-6bf5998d9c-h88hw 100m (5%) 0 (0%) 200Mi (6%) 400Mi (12%) 5d2h
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 380m (19%) 0 (0%)
memory 390Mi (11%) 1338Mi (40%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
Events: <none>
(nana@myeks:N/A) [root@operator-host ~]# kubectl exec app -- tail -f /data/out.txt
Fri Feb 21 14:33:16 UTC 2025
Fri Feb 21 14:33:21 UTC 2025
Fri Feb 21 14:33:26 UTC 2025
Fri Feb 21 14:33:31 UTC 2025
Fri Feb 21 14:33:36 UTC 2025
Fri Feb 21 14:33:41 UTC 2025
Fri Feb 21 14:33:46 UTC 2025
Fri Feb 21 14:33:51 UTC 2025
Fri Feb 21 14:33:56 UTC 2025
Fri Feb 21 14:34:01 UTC 2025
Fri Feb 21 14:34:06 UTC 2025
^C
(nana@myeks:N/A) [root@operator-host ~]# kubectl exec -it app -- sh -c 'df -hT --type=overlay'
Filesystem Type Size Used Avail Use% Mounted on
overlay overlay 120G 4.4G 116G 4% /
(nana@myeks:N/A) [root@operator-host ~]# kubectl exec -it app -- sh -c 'df -hT --type=xfs'
Filesystem Type Size Used Avail Use% Mounted on
/dev/nvme1n1 xfs 4.0G 61M 3.9G 2% /data
/dev/nvme0n1p1 xfs 120G 4.4G 116G 4% /etc/hosts
AWS EFS Controller
EFS는 NFS 기반의 공유 파일 스토리지로, 여러 인스턴스나 컨테이너에서 동시에 접근할 수 있다.
하지만 쿠버네티스 기본 드라이버로는 EFS를 사용할 수 없는데,EFS CSI Controller를 사용해서 쿠버네티스와 EFS를 연결하는 인터페이스로 동작하게 된다.
kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS STORAGECLASS
efs-pv 5Gi RWX Retain Available efs-sc
kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS
efs-claim Bound efs-pv 5Gi RWX efs-sc
kubectl get pods
NAME READY STATUS RESTARTS AGE
app1 1/1 Running 0 6s
app2 1/1 Running 0 6s
pvc를 생성하여 만들어준 pv와 바운드하여 파드를 배포한다.
두 파드는 /data 디렉터리에 EFS를 마운트하여 같은 스토리지를 공유하도록 설정돼있다.
kubectl get pods
NAME READY STATUS RESTARTS AGE
app1 1/1 Running 0 6s
app2 1/1 Running 0 6s
kubestr-fio-pod-gr6p7 1/1 Running 0 4h19m
kubectl exec -ti app1 -- sh -c "df -hT -t nfs4"
kubectl exec -ti app2 -- sh -c "df -hT -t nfs4"
Filesystem Type Size Used Available Use% Mounted on
127.0.0.1:/ nfs4 8.0E 0 8.0E 0% /data
tree /mnt/myefs
/mnt/myefs
├── out1.txt
├── out2.txt
└── test.txt
0 directories, 3 files
EFS 스토리지에 저장된 파일들을 확인함으로써 여러 파드가 같은 데이터를 공유하고 있다는 것을 알 수 있다.
Instance Store
EKS 클러스터에 새로운 노드 그룹을 추가하고 해당 노드에 원격 접속하여 디스크, 네트워크, 쿠버네티스 설정을 확인해본다.
ssh ec2-user@$N4 sudo nvme list
Node SN Model Namespace Usage Format FW Rev
---------------- -------------------- ---------------------------------------- --------- -------------------------- ---------------- --------
/dev/nvme0n1 vol026622c5300b95916 Amazon Elastic Block Store 1 32.21 GB / 32.21 GB 512 B + 0 B 1.0
/dev/nvme1n1 AWS21EF3B4C02F591522 Amazon EC2 NVMe Instance Storage 1 50.00 GB / 50.00 GB 512 B + 0 B 0
ssh ec2-user@$N4 sudo lsblk -e 7 -d
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
nvme0n1 259:0 0 30G 0 disk
nvme1n1 259:1 0 46.6G 0 disk /data
ssh ec2-user@$N4 sudo df -hT -t xfs
Filesystem Type Size Used Avail Use% Mounted on
/dev/nvme0n1p1 xfs 30G 3.5G 27G 12% /
/dev/nvme1n1 xfs 47G 365M 47G 1% /data
ssh ec2-user@$N4 sudo tree /data
/data
0 directories, 0 files
ssh ec2-user@$N4 sudo cat /etc/fstab
#
UUID=1dfdfe0d-276a-4d52-8572-ceb3b011d9ea / xfs defaults,noatime 1 1
/dev/nvme1n1 /data xfs defaults,noatime 0 2
NVMe 디스크가 정상적으로 인식되고, XFS로 포맷되어 있으며, /data에 마운트됨을 확인할 수 있다.
kubectl describe node -l disk=instancestore | grep Allocatable: -A7
Allocatable:
cpu: 1930m
ephemeral-storage: 27905944324
hugepages-1Gi: 0
hugepages-2Mi: 0
memory: 3068808Ki
pods: 110
System Info:
ssh ec2-user@$N4 sudo ps -ef | grep kubelet
root 3017 1 0 15:54 ? 00:00:04 /usr/bin/kubelet --config /etc/kubernetes/kubelet/kubelet-config.json --kubeconfig /var/lib/kubelet/kubeconfig --container-runtime-endpoint unix:///run/containerd/containerd.sock --image-credential-provider-config /etc/eks/image-credential-provider/config.json --image-credential-provider-bin-dir /etc/eks/image-credential-provider --node-ip=192.168.1.54 --pod-infra-container-image=602401143452.dkr.ecr.ap-northeast-2.amazonaws.com/eks/pause:3.5 --v=2 --hostname-override=ip-192-168-1-54.ap-northeast-2.compute.internal --cloud-provider=external --node-labels=eks.amazonaws.com/sourceLaunchTemplateVersion=1,alpha.eksctl.io/cluster-name=myeks,alpha.eksctl.io/nodegroup-name=ng2,disk=instancestore,eks.amazonaws.com/nodegroup-image=ami-0fa05db9e3c145f63,eks.amazonaws.com/capacityType=ON_DEMAND,eks.amazonaws.com/nodegroup=ng2,eks.amazonaws.com/sourceLaunchTemplateId=lt-01b2db7fd90cca5d0 --max-pods=29 --max-pods=110
root 3782 3711 0 15:54 ? 00:00:00 /bin/aws-ebs-csi-driver node --endpoint=unix:/csi/csi.sock --http-endpoint=0.0.0.0:3302 --csi-mount-point-prefix=/var/lib/kubelet/plugins/kubernetes.io/csi/ebs.csi.aws.com/ --volume-attach-limit=31 --logging-format=text --v=2
root 3828 3711 0 15:54 ? 00:00:00 /csi-node-driver-registrar --csi-address=/csi/csi.sock --kubelet-registration-path=/var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock --v=2
kubestr fio -f fio-read.fio -s local-path --size 10G --nodeselector disk=instancestore
PVC created kubestr-fio-pvc-fkg48
Pod created kubestr-fio-pod-pxw9j
Running FIO test (fio-read.fio) on StorageClass (local-path) with a PVC of Size (10G)
Elapsed time- 3m42.696294755s
FIO test results:
FIO version - fio-3.36
Global options - ioengine=libaio verify= direct=1 gtod_reduce=
JobName:
blocksize= filesize= iodepth= rw=
read:
IOPS=20308.748047 BW(KiB/s)=81234
iops: min=16210 max=93722 avg=20317.226562
bw(KiB/s): min=64840 max=374888 avg=81268.945312
Disk stats (read/write):
nvme1n1: ios=2433053/10 merge=0/3 ticks=7645486/14 in_queue=7645500, util=99.954948%
- OK