EKS 1.32 → 1.33 업그레이드 실전 가이드

1

aws eks update-cluster-version --name my-cluster --kubernetes-version 1.33

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# ECR 베이스 URL
ECR=602401143452.dkr.ecr.ap-northeast-2.amazonaws.com

# VPC CNI
kubectl set image daemonset/aws-node -n kube-system \
  aws-node=${ECR}/amazon-k8s-cni:v1.21.1

# CoreDNS
kubectl set image deployment/coredns -n kube-system \
  coredns=${ECR}/eks/coredns:v1.12.4-eksbuild.1

# kube-proxy
kubectl set image daemonset/kube-proxy -n kube-system \
  kube-proxy=${ECR}/eks/kube-proxy:v1.33.0-minimal-eksbuild.2

1
2

kubectl run test-pod --image=busybox --restart=Never --command -- sleep 30
kubectl get pod test-pod -o wide

NAME       READY   STATUS    RESTARTS   AGE   IP       NODE
test-pod   0/1     Pending   0          30s   <none>   <none>

1
2
3
4
5
6
7

# ConfigMap 설정
kubectl get configmap amazon-vpc-cni -n kube-system -o yaml
# enable-network-policy-controller: "false"

# DaemonSet 환경변수
kubectl get daemonset aws-node -n kube-system -o yaml | grep NETWORK_POLICY
# NETWORK_POLICY_ENFORCING_MODE: standard

1
2

# DaemonSet에서 충돌하는 env 제거
kubectl set env daemonset/aws-node -n kube-system NETWORK_POLICY_ENFORCING_MODE-

1
2
3

kubectl run test-pod2 --image=busybox --restart=Never --command -- sleep 30
kubectl get pod test-pod2 -o wide
# IP 할당 성공!

1
2
3
4

aws eks update-nodegroup-version \
  --cluster-name my-cluster \
  --nodegroup-name on-demand \
  --kubernetes-version 1.33

An error occurred (InvalidParameterException) when calling the UpdateNodegroupVersion operation:
Nodegroup can't be upgraded because maxSize (2) is equal to desiredSize (2).

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11

# maxSize 임시 증가
aws eks update-nodegroup-config \
  --cluster-name my-cluster \
  --nodegroup-name on-demand \
  --scaling-config minSize=1,maxSize=4,desiredSize=2

# 이후 업그레이드 재시도
aws eks update-nodegroup-version \
  --cluster-name my-cluster \
  --nodegroup-name on-demand \
  --kubernetes-version 1.33

1
2

kubectl patch ec2nodeclass karpenter --type=merge \
  -p '{"spec":{"amiSelectorTerms":[{"name":"amazon-eks-node-al2023-x86_64-standard-1.33*"}]}}'

1

kubectl logs -n karpenter -l app.kubernetes.io/name=karpenter --tail=100

ERROR controller.provisioner unable to determine if instance type
"t3.medium" is allowed by the NodePool, error: unsupported nodeClassRef
kind, no corresponding implementation found, expected one of
[]schema.GroupVersionKind{schema.GroupVersionKind{...}}

1
2
3
4
5
6
7
8
9

# CRD 수동 업데이트
KARPENTER_CRD="https://raw.githubusercontent.com/aws/karpenter/v1.5.0/pkg/apis/crds"

kubectl apply -f ${KARPENTER_CRD}/karpenter.sh_nodepools.yaml
kubectl apply -f ${KARPENTER_CRD}/karpenter.k8s.aws_ec2nodeclasses.yaml
kubectl apply -f ${KARPENTER_CRD}/karpenter.sh_nodeclaims.yaml

# Karpenter 재시작
kubectl rollout restart deployment/karpenter -n karpenter

1
2
3
4
5
6
7
8

agents:
  containers:
    agent:
      env:
        - name: DD_HOSTNAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName

1

kubectl logs -n datadog -l app=datadog -c agent --tail=50 | grep error

error pulling from collector "kubelet": couldn't fetch "podlist":
unable to unmarshal podlist... decode slice: expect [ or n, but found {

1
2
3
4

# Datadog Helm chart 업그레이드 (Agent 7.66.1+ 포함)
helm upgrade datadog datadog/datadog -n datadog \
  --reset-then-reuse-values \
  --version 3.163.0

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# 1. 클러스터 상태
aws eks describe-cluster --name ${CLUSTER} \
  --query 'cluster.{version:version,status:status}'

# 2. 노드 상태
kubectl get nodes -o wide

# 3. 핵심 워크로드 replicas 확인
kubectl get deploy -A \
  -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,REPLICAS:.spec.replicas \
  | grep -v "^kube-"

# 4. PDB 확인
kubectl get pdb -A

# 5. MNG maxSize 확인 (Surge 업그레이드용)
aws eks describe-nodegroup \
  --cluster-name ${CLUSTER} \
  --nodegroup-name ${NG} \
  --query 'nodegroup.scalingConfig'
# maxSize > desiredSize 필요!

# 6. VPC CNI Network Policy 설정 확인
kubectl get cm amazon-vpc-cni -n kube-system -o yaml \
  | grep enable-network-policy
kubectl get ds aws-node -n kube-system -o yaml \
  | grep -A1 NETWORK_POLICY_ENFORCING_MODE
# 불일치하면 env 제거 필요

1
2
3
4

aws eks update-cluster-version --name ${CLUSTER} --kubernetes-version 1.33

# 상태 확인 (UPDATING → ACTIVE)
watch "aws eks describe-cluster --name ${CLUSTER} --query 'cluster.status'"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# ECR 베이스 URL
ECR=602401143452.dkr.ecr.ap-northeast-2.amazonaws.com

# 1. VPC CNI 충돌 env 제거 (필수!)
kubectl set env daemonset/aws-node -n kube-system NETWORK_POLICY_ENFORCING_MODE-

# 2. VPC CNI
kubectl set image daemonset/aws-node -n kube-system \
  aws-node=${ECR}/amazon-k8s-cni:v1.21.1

# 3. CoreDNS
kubectl set image deployment/coredns -n kube-system \
  coredns=${ECR}/eks/coredns:v1.12.4-eksbuild.1

# 4. kube-proxy
kubectl set image daemonset/kube-proxy -n kube-system \
  kube-proxy=${ECR}/eks/kube-proxy:v1.33.0-minimal-eksbuild.2

# 5. Pod IP 할당 테스트
kubectl run test-pod --image=busybox --restart=Never --command -- sleep 30
kubectl get pod test-pod -o wide  # IP 할당 확인
kubectl delete pod test-pod

1
2
3
4
5
6
7

aws eks update-nodegroup-version \
  --cluster-name ${CLUSTER} \
  --nodegroup-name ${NG} \
  --kubernetes-version 1.33

# 상태 확인
watch "kubectl get nodes -o wide"

1
2
3
4
5
6

# AMI Selector 변경으로 Drift 트리거
kubectl patch ec2nodeclass karpenter --type=merge \
  -p '{"spec":{"amiSelectorTerms":[{"name":"amazon-eks-node-al2023-x86_64-standard-1.33*"}]}}'

# Drift 감지 및 노드 교체 확인
watch "kubectl get nodes -l karpenter.sh/nodepool=default -o wide"

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 1. 모든 노드 버전 확인
kubectl get nodes -o wide

# 2. Non-running Pod 확인
kubectl get pods -A | grep -v Running | grep -v Completed

# 3. 핵심 서비스 헬스체크
curl -s https://api.example.com/health

# 4. Datadog 에러율 확인 (업그레이드 전후 비교)
# - APM 대시보드에서 에러율 급증 여부 확인
# - 로그에서 새로운 에러 패턴 확인

gdoc_arrow_left_alt Liveness Probe가 서비스를 죽인 날 배포할 때마다 502가 터진다면, Pod 종료 전략을 의심하라 gdoc_arrow_right_alt