❯ aws eks update-cluster-config --region ap-northeast-2 --name $CLUSTER_NAME \
    --logging '{"clusterLogging":[{"types":["api","audit","authenticator","controllerManager","scheduler"],"enabled":true}]}'
    
{
    "update": {
        "id": "e749911e-7774-3b90-a3ed-d4b151c8ddcb",
        "status": "InProgress",
        "type": "LoggingUpdate",
        "params": [
            {
                "type": "ClusterLogging",
                "value": "{\"clusterLogging\":[{\"types\":[\"api\",\"audit\",\"authenticator\",\"controllerManager\",\"scheduler\"],\"enabled\":true}]}"
            }
        ],
        "createdAt": "2025-03-02T00:47:59.197000+09:00",
        "errors": []
    }
}

로그 그룹 (Log Group) # 여러 개의 로그 스트림 포함
 ├── 로그 스트림 (Log Stream) # 시간 순으로 저장된 로그 이벤트들의 모임
 │    ├── 로그 이벤트 (Log Event) # 실제 개별적인 메시지 포함
 │    ├── 로그 이벤트 (Log Event)
 │    ├── ...
 │
 ├── 로그 스트림 (Log Stream)
 │    ├── 로그 이벤트 (Log Event)
 │    ├── ...
 │
 ├── 로그 스트림 (Log Stream)
 │    ├── 로그 이벤트 (Log Event)
 │    ├── ...
❯ aws logs describe-log-groups | jq

{
  "logGroups": [
    {
      "logGroupName": "/aws/eks/eks-console-automode/cluster",
      "creationTime": 1738998815962,
      "metricFilterCount": 0,
      "arn": "arn:aws:logs:ap-northeast-2:xxx:log-group:/aws/eks/eks-console-automode/cluster:*",
      "storedBytes": 16658665,
      "logGroupClass": "STANDARD",
      "logGroupArn": "arn:aws:logs:ap-northeast-2:xxx:log-group:/aws/eks/eks-console-automode/cluster"
    },
    {
      "logGroupName": "/aws/eks/myeks/cluster",
      "creationTime": 1740844090723,
      "metricFilterCount": 0,
      "arn": "arn:aws:logs:ap-northeast-2:xxx:log-group:/aws/eks/myeks/cluster:*",
      "storedBytes": 0,
      "logGroupClass": "STANDARD",
      "logGroupArn": "arn:aws:logs:ap-northeast-2:xxx:log-group:/aws/eks/myeks/cluster"
    }
  ]
}

# follow 옵션을 주면 실시간 로그를 바로 출력할 수 있습니다.
❯ aws logs tail /aws/eks/$CLUSTER_NAME/cluster --follow

2025-03-01T15:48:11.801000+00:00 kube-apiserver-audit-bf7fa45ae808536e5c93358852add1a1 {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"1e533d1a-22ec-4ea6-be25-e8d61729c167","stage":"ResponseComplete","requestURI":"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/eks-coredns-autoscaler","verb":"get","user":{"username":"eks:coredns-autoscaler","groups":["system:authenticated"]},"sourceIPs":["10.0.180.168"],"userAgent":"controller/v0.0.0 (linux/amd64) kubernetes/$Format/leader-election","objectRef":{"resource":"leases","namespace":"kube-system","name":"eks-coredns-autoscaler","apiGroup":"coordination.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-03-01T09:34:31.582714Z","stageTimestamp":"2025-03-01T09:34:31.587464Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \"eks:coredns-autoscaler/kube-system\" of Role \"eks:coredns-autoscaler\" to User \"eks:coredns-autoscaler\""}}
2025-03-01T15:48:11.801000+00:00 kube-apiserver-audit-bf7fa45ae808536e5c93358852add1a1 {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"99780557-8244-477c-9ed4-373eb3c94726","stage":"ResponseComplete","requestURI":"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/kube-controller-manager?timeout=5s","verb":"get","user":{"username":"system:kube-controller-manager","groups":["system:authenticated"]},"sourceIPs":["10.0.180.168"],"userAgent":"kube-controller-manager/v1.31.6 (linux/amd64) kubernetes/7555883/leader-election","objectRef":{"resource":"leases","namespace":"kube-system","name":"kube-controller-manager","apiGroup":"coordination.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-03-01T09:34:31.794155Z","stageTimestamp":"2025-03-01T09:34:31.798149Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:kube-controller-manager\" of ClusterRole \"system:kube-controller-manager\" to User \"system:kube-controller-manager\""}}
...

❯ eksctl utils update-cluster-logging --cluster $CLUSTER_NAME --region ap-northeast-2 --disable-types all --approve

❯ aws logs delete-log-group --log-group-name /aws/eks/$CLUSTER_NAME/cluster

❯ kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/cloudwatch-namespace.yaml
namespace/amazon-cloudwatch created

❯ ClusterName=myeks                                       
❯ RegionName=ap-northeast-2                               
❯ FluentBitHttpPort='2020'                       
FluentBitReadFromHead='Off'
[[ ${FluentBitReadFromHead} = 'On' ]] && FluentBitReadFromTail='Off'|| FluentBitReadFromTail='On'
[[ -z ${FluentBitHttpPort} ]] && FluentBitHttpServer='Off' || FluentBitHttpServer='On'
kubectl create configmap fluent-bit-cluster-info \
--from-literal=cluster.name=${ClusterName} \
--from-literal=http.server=${FluentBitHttpServer} \
--from-literal=http.port=${FluentBitHttpPort} \
--from-literal=read.head=${FluentBitReadFromHead} \
--from-literal=read.tail=${FluentBitReadFromTail} \
--from-literal=logs.region=${RegionName} -n amazon-cloudwatch

❯ kubectl apply -f https://raw.githubusercontent.com/aws-samples/amazon-cloudwatch-container-insights/latest/k8s-deployment-manifest-templates/deployment-mode/daemonset/container-insights-monitoring/fluent-bit/fluent-bit.yaml
serviceaccount/fluent-bit created
clusterrole.rbac.authorization.k8s.io/fluent-bit-role created
clusterrolebinding.rbac.authorization.k8s.io/fluent-bit-role-binding created
configmap/fluent-bit-config created
daemonset.apps/fluent-bit created

# fluent-bit 설정파일 일부
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: amazon-cloudwatch
  labels:
    k8s-app: fluent-bit
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush                     5
        Grace                     30
        Log_Level                 error
        Daemon                    off
        Parsers_File              parsers.conf
        HTTP_Server               ${HTTP_SERVER}
        HTTP_Listen               0.0.0.0
        HTTP_Port                 ${HTTP_PORT}
        storage.path              /var/fluent-bit/state/flb-storage/
        storage.sync              normal
        storage.checksum          off
        storage.backlog.mem_limit 5M

    @INCLUDE application-log.conf
    @INCLUDE dataplane-log.conf
    @INCLUDE host-log.conf

  application-log.conf: |
    [INPUT]
        Name                tail
        Tag                 application.*
        Exclude_Path        /var/log/containers/cloudwatch-agent*, /var/log/containers/fluent-bit*, /var/log/containers/aws-node*, /var/log/containers/kube-proxy*, /var/log/containers/fluentd*
        Path                /var/log/containers/*.log
        multiline.parser    docker, cri
        DB                  /var/fluent-bit/state/flb_container.db
        Mem_Buf_Limit       50MB
        Skip_Long_Lines     On
        Refresh_Interval    10
        Rotate_Wait         30
        storage.type        filesystem
        Read_from_Head      ${READ_FROM_HEAD}

    [INPUT]
        Name                tail
        Tag                 application.*
        Path                /var/log/containers/fluent-bit*
        multiline.parser    docker, cri
        DB                  /var/fluent-bit/state/flb_log.db
        Mem_Buf_Limit       5MB
        Skip_Long_Lines     On
        Refresh_Interval    10
        Read_from_Head      ${READ_FROM_HEAD}

# Fluent Bit 에러 로그
CreateLogStream API responded with error='AccessDeniedException'
Failed to create log stream
Failed to send events

# CloudWatch로 로그 전송 권한이 포함된 IAM Role 생성
❯ eksctl create iamserviceaccount \
  --name fluent-bit \
  --namespace amazon-cloudwatch \
  --cluster $CLUSTER_NAME \
  --role-name $CLUSTER_NAME-fluent-bit-role \
  --attach-policy-arn arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy \
  --role-only \
  --approve
  
❯ kubectl annotate serviceaccount fluent-bit \
  -n amazon-cloudwatch \
  eks.amazonaws.com/role-arn=arn:aws:iam::941377114730:role/$CLUSTER_NAME-fluent-bit-role \
  --overwrite
serviceaccount/fluent-bit annotated

# fluent-bit daemonset 재시작