포스팅 목차
# 기존 실습 리소스 정리
ps0107@k8smaster1:~$ kubectl delete deploy nginx-nfs
deployment.extensions "nginx-nfs" deleted
ps0107@k8smaster1:~$ kubectl delete pvc pvc-one
persistentvolumeclaim "pvc-one" deleted
ps0107@k8smaster1:~$ kubectl delete pv pvvol-1
persistentvolume "pvvol-1" deleted# ResourceQuota 설정 해보기
- ResourceQouta를 기본 설정으로 생성해 보자. 그리고 PV,PVC를 생성하여 어떻게 변경되는지 확인해 보자
# 해당 namespace 안에서는 500 Mi 까지가 최대, 10개까지만 허용.
ps0107@k8smaster1:~$ vi storage-quota.yaml
apiVersion: v1
kind: ResourceQuota
metadata:
name: storagequota
spec:
hard:
persistentvolumeclaims: "10"
requests.storage: "500Mi"
# 테스트를 위한 small이라는 namespace 생성
ps0107@k8smaster1:~$ kubectl create namespace small
namespace/small created
# 생성된 small ns의 상세 보기
# resource quota, resource limits 가 현재 설정되어 있지 않음
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
No resource quota.
No resource limits.
# PV 생성
ps0107@k8smaster1:~$ kubectl -n small create -f PVol.yaml
persistentvolume/pvvol-1 created
# PVC 생성
ps0107@k8smaster1:~$ kubectl -n small create -f pvc.yaml
persistentvolumeclaim/pvc-one created
# small namespace 에 ResourceQuota 설정
ps0107@k8smaster1:~$ kubectl -n small create -f storage-quota.yaml
resourcequota/storagequota created
# 다시 small ns 자세히 보기로 확인
# Resource Quotas가 설정되어 있다.
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 1 10
requests.storage 200Mi 500Mi
No resource limits.
- deployment및 pod가 생성하여 어떻게 되는지 확인해 보자
# deployment를 생성하기 위해 테스트 yaml파일 생성
ps0107@k8smaster1:~$ vi nfs-pod.yaml
apiVersion: apps/v1beta1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
generation: 1
labels:
run: nginx
name: nginx-nfs
resourceVersion: "1411"
spec:
replicas: 1
selector:
matchLabels:
run: nginx
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
run: nginx
spec:
containers:
- image: nginx
imagePullPolicy: Always
name: nginx
volumeMounts:
- name: nfs-vol
mountPath: /opt
ports:
- containerPort: 80
protocol: TCP
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumes:
- name: nfs-vol
persistentVolumeClaim:
claimName: pvc-one
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
# deployment 생성
ps0107@k8smaster1:~$ kubectl -n small create -f nfs-pod.yaml
deployment.apps/nginx-nfs created
# 생성된 deployment 확인
ps0107@k8smaster1:~$ kubectl get deploy --namespace=small
NAME READY UP-TO-DATE AVAILABLE AGE
nginx-nfs 1/1 1 1 19s
# 생성된 deployment 상세 확인
ps0107@k8smaster1:~$ kubectl -n small describe deploy nginx-nfs
Name: nginx-nfs
Namespace: small
CreationTimestamp: Tue, 04 Feb 2020 01:59:35 +0000
Labels: run=nginx
Annotations: deployment.kubernetes.io/revision: 1
Selector: run=nginx
Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType: RollingUpdate
MinReadySeconds: 0
RollingUpdateStrategy: 1 max unavailable, 1 max surge
Pod Template:
Labels: run=nginx
Containers:
nginx:
Image: nginx
Port: 80/TCP
Host Port: 0/TCP
Environment: <none>
Mounts:
/opt from nfs-vol (rw)
Volumes:
nfs-vol:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: pvc-one
ReadOnly: false
Conditions:
Type Status Reason
---- ------ ------
Available True MinimumReplicasAvailable
Progressing True NewReplicaSetAvailable
OldReplicaSets: <none>
NewReplicaSet: nginx-nfs-86845f4d55 (1/1 replicas created)
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal ScalingReplicaSet 39s deployment-controller Scaled up replica set nginx-nfs-86845f4d55 to 1
# pod 확인
ps0107@k8smaster1:~$ kubectl -n small get pod
NAME READY STATUS RESTARTS AGE
nginx-nfs-86845f4d55-2srr8 1/1 Running 0 57s
# 생성된 pod 상세 확인
ps0107@k8smaster1:~$ kubectl -n small describe pod nginx-nfs-86845f4d55-2srr8
Name: nginx-nfs-86845f4d55-2srr8
Namespace: small
Priority: 0
Node: k8sworker1/10.146.0.4
Start Time: Tue, 04 Feb 2020 01:59:35 +0000
Labels: pod-template-hash=86845f4d55
run=nginx
Annotations: cni.projectcalico.org/podIP: 192.168.1.85/32
Status: Running
IP: 192.168.1.85
Controlled By: ReplicaSet/nginx-nfs-86845f4d55
Containers:
nginx:
Container ID: docker://21459d9797d5bfed49b1cdb10817e3236b73fd6089c5feaf299c627c6434d63f
Image: nginx
Image ID: docker-pullable://nginx@sha256:ad5552c786f128e389a0263104ae39f3d3c7895579d45ae716f528185b36bc6f
Port: 80/TCP
Host Port: 0/TCP
State: Running
Started: Tue, 04 Feb 2020 01:59:39 +0000
Ready: True
Restart Count: 0
Environment: <none>
Mounts:
/opt from nfs-vol (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-g9bxk (ro)
Conditions:
Type Status
Initialized True
Ready True
ContainersReady True
PodScheduled True
Volumes:
nfs-vol:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: pvc-one
ReadOnly: false
default-token-g9bxk:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-g9bxk
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Scheduled 71s default-scheduler Successfully assigned small/nginx-nfs-86845f4d55-2srr8 to k8sworker1
Normal Pulling 70s kubelet, k8sworker1 Pulling image "nginx"
Normal Pulled 67s kubelet, k8sworker1 Successfully pulled image "nginx"
Normal Created 67s kubelet, k8sworker1 Created container nginx
Normal Started 67s kubelet, k8sworker1 Started container nginx
# small namespace 확인
# resource quota 부분을 보면, 현재 1개 사용중이고, 200Mi 사용중인걸 확인 할 수 있다.
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 1 10
requests.storage 200Mi 500Mi
No resource limits.
- 이번엔 해당 host에서 /opt/sfw 안에 300M 파일을 생성하고, quota usage를 다시 확인해 본다
# 해당 host에서 /opt/sfw 안에 300M 파일을 생성하고, quota usage를 다시 본다.
ps0107@k8smaster1:~$ sudo dd if=/dev/zero of=/opt/sfw/bigfile bs=1M count=300
300+0 records in
300+0 records out
314572800 bytes (315 MB, 300 MiB) copied, 0.641641 s, 490 MB/s
# 기존 설정 그래도 보인다.
# nfs의 경우는 공유크기가 deployment에 대해 계산되지 않는다.
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 1 10
requests.storage 200Mi 500Mi
No resource limits.
ps0107@k8smaster1:~$ du -h /opt/
du: cannot read directory '/opt/containerd': Permission denied
4.0K/opt/containerd
301M/opt/sfw
101M/opt/cni/bin
101M/opt/cni
401M/opt/
- 이제 deployment가 할당량보다 더 많이 요청하면 어떻게 되는지 살펴본다.
# ----------------------------------------------
# 이제 deployment가 할당량보다 더 많이 요청하면 어떻게 되는지 살펴본다.
# ----------------------------------------------
# 삭제하기 위해 현재 deployment를 조회한다.
ps0107@k8smaster1:~$ kubectl -n small get deploy
NAME READY UP-TO-DATE AVAILABLE AGE
nginx-nfs 1/1 1 1 4m39s
# 기존 deployment 삭제
ps0107@k8smaster1:~$ kubectl -n small delete deploy nginx-nfs
deployment.extensions "nginx-nfs" deleted
# 기존 deployment 삭제하면서 pod 를 shutdown 시켰고,
# 다시 한번 small namespace를 확인해보면 storage가 보존(retain)되어 있음을 확인할수 있다.
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 1 10
requests.storage 200Mi 500Mi
No resource limits.
- 이번에는 pvc를 삭제 해보고, pv 정보를 확인해 본다.
# ----------------------------------------------------
# 이번에는 pvc를 삭제 해보고, pv 정보를 확인해 본다.
# ----------------------------------------------------
ps0107@k8smaster1:~$ kubectl -n small get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc-one Bound pvvol-1 1Gi RWX 9m58s
# pvc를 삭제
ps0107@k8smaster1:~$ kubectl -n small delete pvc pvc-one
persistentvolumeclaim "pvc-one" deleted
# pv 정보를 확인
# RECLAIMPOLICY, STATUS를 보면 Retain, Released 상태임을 알수 있다.
ps0107@k8smaster1:~$ kubectl -n small get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvvol-1 1Gi RWX Retain Released small/pvc-one 10m
- 이번엔 persistentVolumeReclaimPolicy 설정을 바꾸어 보며 테스트 해보자.
# ----------------------------------------------
# 이번엔 persistentVolumeReclaimPolicy 설정을 바꾸어 보자.
# ----------------------------------------------
# 현재 설정된 pv 객체에 대한 정보를 yaml형태로 보면 기본값이 Retain 임을 알수 있다.
ps0107@k8smaster1:~$ kubectl get pv/pvvol-1 -o yaml
apiVersion: v1
kind: PersistentVolume
metadata:
annotations:
pv.kubernetes.io/bound-by-controller: "yes"
creationTimestamp: "2020-02-04T01:54:52Z"
finalizers:
- kubernetes.io/pv-protection
name: pvvol-1
resourceVersion: "780598"
selfLink: /api/v1/persistentvolumes/pvvol-1
uid: a381b18a-da86-404a-90c8-215dac88d384
spec:
accessModes:
- ReadWriteMany
capacity:
storage: 1Gi
claimRef:
apiVersion: v1
kind: PersistentVolumeClaim
name: pvc-one
namespace: small
resourceVersion: "779741"
uid: 5d15e744-1810-4a82-9661-9930a92ed3d4
nfs:
path: /opt/sfw
server: k8smaster
persistentVolumeReclaimPolicy: Retain # <-- 기본 값은 Retain 임을 알수 있다.
volumeMode: Filesystem
status:
phase: Released
# 다시 생성하기 위해 object들을 삭제 한다.
# pv 삭제
ps0107@k8smaster1:~$ kubectl delete pv/pvvol-1
persistentvolume "pvvol-1" deleted
# yaml 파일에서 persistentVolumeReclaimPolicy 부분을 확인한다.
# 현재 Retain 으로 설정되어 있다.
ps0107@k8smaster1:~$ grep Retain PVol.yaml
persistentVolumeReclaimPolicy: Retain
# PV를 생성한다.
ps0107@k8smaster1:~$ kubectl create -f PVol.yaml
persistentvolume/pvvol-1 created
# persistentVolumeReclaimPolicy를 Delete 상태로 수정한다.
ps0107@k8smaster1:~$ kubectl patch pv pvvol-1 -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}'
persistentvolume/pvvol-1 patched
# patch된 pv 상태를 확인한다.
# RECLAIMPOLICY 항목이 Delete로 수정되어 있음을 확인
ps0107@k8smaster1:~$ kubectl get pv/pvvol-1
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvvol-1 1Gi RWX Delete Available 63s
# 현재 Resource Quota 설정을 확인 한다.
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 0 10
requests.storage 0 500Mi
No resource limits.
# pvc를 다시 생성해 본다.
# 실행중인 pod 가 없은 경우에도 resource usage를 기록하고 있다.
ps0107@k8smaster1:~$ kubectl -n small create -f pvc.yaml
persistentvolumeclaim/pvc-one created
# requests.storage 라인을 보면 확인 할수 있다
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 1 10
requests.storage 200Mi 500Mi
No resource limits.
# namespace에 있는 resourcequota를 삭제해보자.
# resourcequota를 조회 해본다.
ps0107@k8smaster1:~$ kubectl -n small get resourcequota
NAME CREATED AT
storagequota 2020-02-04T01:55:18Z
# 조회된 resourcequota를 삭제한다.
ps0107@k8smaster1:~$ kubectl -n small delete resourcequota storagequota
resourcequota "storagequota" deleted
# resourcequota 의 capacity를 100Mi로 수정한다.
ps0107@k8smaster1:~$ vi storage-quota.yaml
apiVersion: v1
kind: ResourceQuota
metadata:
name: storagequota
spec:
hard:
persistentvolumeclaims: "10"
requests.storage: "100Mi"
# 새로운 storage quota를 검증후 생성한다.
ps0107@k8smaster1:~$ kubectl -n small create -f storage-quota.yaml
resourcequota/storagequota created
# hard limit이 이미 초과 된걸로 확인된다.
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 1 10
requests.storage 200Mi 100Mi
No resource limits.
# deployment를 다시 생성해 본다.
ps0107@k8smaster1:~$ kubectl create -f nfs-pod.yaml -n small
deployment.apps/nginx-nfs created
# deployment 정보를 보면 error가 없다.
ps0107@k8smaster1:~$ kubectl -n small describe deploy/nginx-nfs
Name: nginx-nfs
Namespace: small
CreationTimestamp: Tue, 04 Feb 2020 02:11:27 +0000
Labels: run=nginx
Annotations: deployment.kubernetes.io/revision: 1
Selector: run=nginx
Replicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable
StrategyType: RollingUpdate
MinReadySeconds: 0
RollingUpdateStrategy: 1 max unavailable, 1 max surge
Pod Template:
Labels: run=nginx
Containers:
nginx:
Image: nginx
Port: 80/TCP
Host Port: 0/TCP
Environment: <none>
Mounts:
/opt from nfs-vol (rw)
Volumes:
nfs-vol:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: pvc-one
ReadOnly: false
Conditions:
Type Status Reason
---- ------ ------
Available True MinimumReplicasAvailable
Progressing True NewReplicaSetAvailable
OldReplicaSets: <none>
NewReplicaSet: nginx-nfs-86845f4d55 (1/1 replicas created)
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal ScalingReplicaSet 21s deployment-controller Scaled up replica set nginx-nfs-86845f4d55 to 1
# pod가 실제 running중인지 확인한다.
ps0107@k8smaster1:~$ kubectl -n small get po
NAME READY STATUS RESTARTS AGE
nginx-nfs-86845f4d55-krbll 1/1 Running 0 45s
# ----------------------------------------------
# 겉보기에는 하드 할당량이 설정되어 있었음에도 불구하고 더 많은 pod를 배치할 수 있었다.
# 스토리지 재확보가 가능한지 테스트해 보자. 먼저 deployment 및 pvc을 제거한다.
# ----------------------------------------------
# deployment 제거
ps0107@k8smaster1:~$ kubectl -n small delete deploy nginx-nfs
deployment.extensions "nginx-nfs" deleted
# pvc 제거
ps0107@k8smaster1:~$ kubectl -n small delete pvc/pvc-one
persistentvolumeclaim "pvc-one" deleted
# PV를 확인합니다. 제거 시도를 했지만 실패했음을 알 수 있습니다.
# 자세히 보면 NFS용 Delector Volume plugin의 부족과 관련된 오류가 나타납니다.
# 다른 스토리지 프로토콜들은 플러그인을 가지고 있다.
ps0107@k8smaster1:~$ kubectl -n small get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvvol-1 1Gi RWX Delete Failed small/pvc-one 6m51s
# pv 제거 후 deployment, pv, pvc 모두 제거 되었는지 확인한다.
ps0107@k8smaster1:~$ kubectl delete pv/pvvol-1
persistentvolume "pvvol-1" deleted
# PV를 만들기 위해 yaml 파일 생성
# persistentVolumeReclaimPolicy: Recycle로 수정
ps0107@k8smaster1:~$ vi PVol.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: pvvol-1
spec:
capacity:
storage: 1Gi
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Recycle
nfs:
path: /opt/sfw
server: k8smaster
readOnly: false
# LimitRange 설정을 위해 yaml 파일을 생성한다.
ps0107@k8smaster1:~$ vi low-resource-range.yaml
apiVersion: v1
kind: LimitRange
metadata:
name: low-resource-range
spec:
limits:
- default:
cpu: 1
memory: 500Mi
defaultRequest:
cpu: 0.5
memory: 100Mi
type: Container
# small namespace에 LimitRange 설정한다.
ps0107@k8smaster1:~$ kubectl -n small create -f low-resource-range.yaml
limitrange/low-resource-range created
# small namespace를 확인한다.
# Resource Quotas과 Resource Limits 설정을 둘다 볼수 있다
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 0 10
requests.storage 0 100Mi
Resource Limits
Type Resource Min Max Default Request Default Limit Max Limit/Request Ratio
---- -------- --- --- --------------- ------------- -----------------------
Container cpu - - 500m 1 -
Container memory - - 100Mi 500Mi -
# pv 생성
ps0107@k8smaster1:~$ kubectl -n small create -f PVol.yaml
persistentvolume/pvvol-1 created
# 생성된 pv 상태 확인
# RECLAIM POLICY가 Recycle로 생성 됨을 확인 할수 있다.
ps0107@k8smaster1:~$ kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvvol-1 1Gi RWX Recycle Available 6s
# 다시 pvc 생성을 시도해본다.Error가 발생한다.
# !!!!!! quota 는 resource limit 적용되는 경우에만 적용된다 !!!!!
ps0107@k8smaster1:~$ kubectl -n small create -f pvc.yaml
Error from server (Forbidden): error when creating "pvc.yaml":
persistentvolumeclaims "pvc-one" is forbidden: exceeded quota:
storagequota, requested: requests.storage=200Mi, used:
requests.storage=0, limited: requests.storage=100Mi
# resourcequota의 request.storage 를 500 Mi 로 수정한다.
ps0107@k8smaster1:~$ kubectl -n small edit resourcequotas
resourcequota/storagequota edited
# 다시 pvc 생성을 시도해본다.
# 이번에는 pvc가 잘 생성되었다.
ps0107@k8smaster1:~$ kubectl -n small create -f pvc.yaml
persistentvolumeclaim/pvc-one created
# deployment도 잘 생성이 된다.
ps0107@k8smaster1:~$ kubectl -n small create -f nfs-pod.yaml
deployment.apps/nginx-nfs created
# 이제 small namespace를 확인해본다.
ps0107@k8smaster1:~$ kubectl describe ns small
Name: small
Labels: <none>
Annotations: <none>
Status: Active
Resource Quotas
Name: storagequota
Resource Used Hard
-------- --- ---
persistentvolumeclaims 1 10
requests.storage 200Mi 500Mi
Resource Limits
Type Resource Min Max Default Request Default Limit Max Limit/Request Ratio
---- -------- --- --- --------------- ------------- -----------------------
Container memory - - 100Mi 500Mi -
Container cpu - - 500m 1 -
# deployment를 삭제한 후 PV, PVC 상태를 확인해보자
# deployment 삭제
ps0107@k8smaster1:~$ kubectl -n small delete deploy nginx-nfs
deployment.extensions "nginx-nfs" deleted
# pvc 상태 확인
ps0107@k8smaster1:~$ kubectl -n small get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pvc-one Bound pvvol-1 1Gi RWX 44s
# pv 상태 확인
# RECLAIM POLICY : Recycle
# STATUS : Bound
ps0107@k8smaster1:~$ kubectl -n small get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvvol-1 1Gi RWX Recycle Bound small/pvc-one 2m16s
# pvc를 삭제 한뒤 pv 상태를 확인 한다.
ps0107@k8smaster1:~$ kubectl -n small delete pvc pvc-one
persistentvolumeclaim "pvc-one" deleted
# pv 상태 확인
# RECLAIM POLICY : Recycle
# STATUS : Available
ps0107@k8smaster1:~$ kubectl -n small get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvvol-1 1Gi RWX Recycle Available 2m33s# 테스트가 끝났으니 리소스를 정리한다.
# 테스트가 끝났으니 resource를 정리한다.
ps0107@k8smaster1:~$ kubectl delete pv pvvol-1
persistentvolume "pvvol-1" deleted
