# Working with TLS
# 마스터 노드와 보조 노드 모두에서 kubelet 을 보자 .
# kube-apiserver 는 cerificates 과 authorization mode와 같은 보안 정보를 보여준다.
# kubelet 은 systemd 서비스이므로 아래와 같이 상태를 볼수 있다.
# 아래 CGroup 정보에 보면 설정 파일들이 어디에 있는지 알수 있다.
ps0107@k8smaster1:~$ systemctl status kubelet.service
● kubelet.service - kubelet: The Kubernetes Node Agent
Loaded: loaded (/lib/systemd/system/kubelet.service; enabled; vendor preset: enabled)
Drop-In: /etc/systemd/system/kubelet.service.d
└─10-kubeadm.conf
Active: active (running) since Tue 2020-01-28 08:29:18 UTC; 2 weeks 1 days ago
Docs: https://kubernetes.io/docs/home/
Main PID: 5279 (kubelet)
Tasks: 23
Memory: 54.6M
CPU: 10h 30min 49.915s
CGroup: /system.slice/kubelet.service
└─5279 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --cgroup-driver=cgroupfs --network-plugin=cni --pod-infra ....
# /var/lib/kubelet/config.yaml 안의 세팅을 보자.
# 살펴보면 kube-apiserver를 접근하기 위해 /etc/kubernetes/pki/ 디렉토리를 볼 수 있다.
# 끝부분을 좀더 보면 다른 pod spec 파일들을 찾을 수 있도록 디렉토리가 세팅되어 있다.
ps0107@k8smaster1:~$ sudo cat /var/lib/kubelet/config.yaml
address: 0.0.0.0
apiVersion: kubelet.config.k8s.io/v1beta1
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
cgroupDriver: cgroupfs
cgroupsPerQOS: true
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
configMapAndSecretChangeDetectionStrategy: Watch
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuCFSQuotaPeriod: 100ms
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kind: KubeletConfiguration
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeLeaseDurationSeconds: 40
nodeStatusReportFrequency: 1m0s
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
port: 10250
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
# 마스터 노드의 다른 에이전트들은 kube-apiserver와 상호작용하고 있다.
# 이러한 세팅을 하기 위해 만들어진 설정 파일들을 보자.
# yaml file들이 있고 file안에 cert 정보도 있다.
ps0107@k8smaster1:~$ sudo ls /etc/kubernetes/manifests/
etcd.yaml kube-apiserver.yaml kube-controller-manager.yaml kube-scheduler.yaml
ps0107@k8smaster1:~$ sudo cat /etc/kubernetes/manifests/kube-controller-manager.yaml
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: kube-controller-manager
tier: control-plane
name: kube-controller-manager
namespace: kube-system
spec:
containers:
- command:
- kube-controller-manager
- --allocate-node-cidrs=true
- --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf
- --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf
- --bind-address=127.0.0.1
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --cluster-cidr=192.168.0.0/16
- --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
- --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
- --controllers=*,bootstrapsigner,tokencleaner
- --kubeconfig=/etc/kubernetes/controller-manager.conf
- --leader-elect=true
- --node-cidr-mask-size=24
- --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
- --root-ca-file=/etc/kubernetes/pki/ca.crt
- --service-account-private-key-file=/etc/kubernetes/pki/sa.key
- --use-service-account-credentials=true
image: k8s.gcr.io/kube-controller-manager:v1.15.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 8
httpGet:
host: 127.0.0.1
path: /healthz
port: 10252
scheme: HTTP
initialDelaySeconds: 15
timeoutSeconds: 15
name: kube-controller-manager
resources:
requests:
cpu: 200m
volumeMounts:
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/ca-certificates
name: etc-ca-certificates
readOnly: true
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
name: flexvolume-dir
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/kubernetes/controller-manager.conf
name: kubeconfig
readOnly: true
- mountPath: /usr/local/share/ca-certificates
name: usr-local-share-ca-certificates
readOnly: true
- mountPath: /usr/share/ca-certificates
name: usr-share-ca-certificates
readOnly: true
hostNetwork: true
priorityClassName: system-cluster-critical
volumes:
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: ca-certs
- hostPath:
path: /etc/ca-certificates
type: DirectoryOrCreate
name: etc-ca-certificates
- hostPath:
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
type: DirectoryOrCreate
name: flexvolume-dir
- hostPath:
path: /etc/kubernetes/pki
type: DirectoryOrCreate
name: k8s-certs
- hostPath:
path: /etc/kubernetes/controller-manager.conf
type: FileOrCreate
name: kubeconfig
- hostPath:
path: /usr/local/share/ca-certificates
type: DirectoryOrCreate
name: usr-local-share-ca-certificates
- hostPath:
path: /usr/share/ca-certificates
type: DirectoryOrCreate
name: usr-share-ca-certificates
status: {}
# 토큰 사용은 구성요소 통신을 승인하는 데 중심이 된다. 토큰은 secrets 로 가진다.
# kube-system namespace의 secrets를 확인해 보자.
ps0107@k8smaster1:~$ kubectl -n kube-system get secrets
NAME TYPE DATA AGE
attachdetach-controller-token-q6gnc kubernetes.io/service-account-token 3 15d
bootstrap-signer-token-52n72 kubernetes.io/service-account-token 3 15d
bootstrap-token-jaeaqt bootstrap.kubernetes.io/token 6 15d
bootstrap-token-qa1m8y bootstrap.kubernetes.io/token 4 15d
calico-node-token-9d74h kubernetes.io/service-account-token 3 15d
certificate-controller-token-scw76 kubernetes.io/service-account-token 3 15d
clusterrole-aggregation-controller-token-bzb8m kubernetes.io/service-account-token 3 15d
coredns-token-cmpj6 kubernetes.io/service-account-token 3 15d
cronjob-controller-token-rdp76 kubernetes.io/service-account-token 3 15d
daemon-set-controller-token-zrfl2 kubernetes.io/service-account-token 3 15d
default-token-9xjr8 kubernetes.io/service-account-token 3 15d
deployment-controller-token-dghwg kubernetes.io/service-account-token 3 15d
disruption-controller-token-s5rdz kubernetes.io/service-account-token 3 15d
endpoint-controller-token-fk4gw kubernetes.io/service-account-token 3 15d
expand-controller-token-xqdz5 kubernetes.io/service-account-token 3 15d
generic-garbage-collector-token-gg8l7 kubernetes.io/service-account-token 3 15d
horizontal-pod-autoscaler-token-5xjpz kubernetes.io/service-account-token 3 15d
job-controller-token-ndn45 kubernetes.io/service-account-token 3 15d
kube-proxy-token-7jjl9 kubernetes.io/service-account-token 3 15d
kubeadm-certs Opaque 8 15d
metrics-server-token-6rpkb kubernetes.io/service-account-token 3 7d13h
namespace-controller-token-z7vnn kubernetes.io/service-account-token 3 15d
node-controller-token-jgg7f kubernetes.io/service-account-token 3 15d
persistent-volume-binder-token-7qksk kubernetes.io/service-account-token 3 15d
pod-garbage-collector-token-9bgvk kubernetes.io/service-account-token 3 15d
pv-protection-controller-token-7nbqk kubernetes.io/service-account-token 3 15d
pvc-protection-controller-token-zssxk kubernetes.io/service-account-token 3 15d
replicaset-controller-token-59szg kubernetes.io/service-account-token 3 15d
replication-controller-token-5j78s kubernetes.io/service-account-token 3 15d
resourcequota-controller-token-95qgb kubernetes.io/service-account-token 3 15d
service-account-controller-token-d4fmk kubernetes.io/service-account-token 3 15d
service-controller-token-wbbpt kubernetes.io/service-account-token 3 15d
statefulset-controller-token-hsk8b kubernetes.io/service-account-token 3 15d
tiller-token-6vv6j kubernetes.io/service-account-token 3 4d5h
token-cleaner-token-v8r9r kubernetes.io/service-account-token 3 15d
traefik-ingress-controller-token-8h7rs kubernetes.io/service-account-token 3 8d
ttl-controller-token-tjbl4 kubernetes.io/service-account-token 3 15d
# secrets 중에 하나를 살펴보자.
ps0107@k8smaster1:~$ kubectl -n kube-system get secrets certificate-controller-token-scw76 -o yaml
apiVersion: v1
data:
ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJd01ERXlPREE0TWprd09Wb1hEVE13TURFeU5UQTRNamt3T1Zvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTk9XCjZMdDkweTMvZDU2VEdIZFFWR25hZGtScXhPUERPYmplOVpUTkVIU3JWZUU3NWpPQ1A0RWdaTFVxcjBBWm5mSTIKQ0hSU3ZGdkdUT2Qza0o0bnRMS0RrT21VaWV4N2s0ZnRJM0lNc0RraDM4OTMzaGxMRDBBTzZzR3cvUnZpaVJ2Swo1N0RLVjRBaXFTTkRabXlpRFJHRFAvc3o1NllzTEdhQXJTUzVwcFYrQ2NKanhHeXV2UXNKZmZEekJsZ2FCakRoCmlWcDhrdTBNVUpFRktLbDNTNnhLZXRZRTRwZVpNcmlGZUpBb0pVZTZscmpYRWplVVN1a29FdmxmbGJIMGhWL3IKWk1ydkwwaGRKeWdIQjVnZWJnVHVCWDhIWkR4dDBFT1M5ZnBYNWxjYXRJR1g5NGVQdW9PcENYWDF1RVg4alpwYQpzZkpHSkRzdS9pTi9vMWZYcXU4Q0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFDa01VR0pPM2xycEppL2RKdmhZSkdwK29sMHQKQ3lUSVhrK2oyMDFhbmZKYkFiVllENStvb2FxUnRzNWFaa01RZ2loWjUyNGw2bGZ6N3NUNEE0M3VZL3lITmllRgpwb2RvdFFMWXlyRmRCcUoyaFhpcVFOQjRjNHloUXFkMG0rSTNwazJIMDNTOG85MWZ1VUJ3NWZSbDUxRUtIT3I1Ci84cldqdW55KzBRY1IvaEVLZlA5alV0NVZsbm4zMVJPMkhVeFUyTmlQTCtJK0dtWCtod3hzRFkyUk9TZ2xuWUsKcjNkaGU2RXpHQTZBcTJqMmtENTJteWFmNXNRY0NjeXg1cDlSNGdXcWxFTFU1VFZONldTWnVCT3AxQld6azI3NgpURFFRL0hVeHBGSTdZSEtTeVkza0xCb2JoSzViM0syS1RqQk45aEduMisxaHNETFM1R1ZRbkVQMlVTUT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
namespace: a3ViZS1zeXN0ZW0=
token: ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklpSjkuZXlKcGMzTWlPaUpyZFdKbGNtNWxkR1Z6TDNObGNuWnBZMlZoWTJOdmRXNTBJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5dVlXMWxjM0JoWTJVaU9pSnJkV0psTFhONWMzUmxiU0lzSW10MVltVnlibVYwWlhNdWFXOHZjMlZ5ZG1salpXRmpZMjkxYm5RdmMyVmpjbVYwTG01aGJXVWlPaUpqWlhKMGFXWnBZMkYwWlMxamIyNTBjbTlzYkdWeUxYUnZhMlZ1TFhOamR6YzJJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5elpYSjJhV05sTFdGalkyOTFiblF1Ym1GdFpTSTZJbU5sY25ScFptbGpZWFJsTFdOdmJuUnliMnhzWlhJaUxDSnJkV0psY201bGRHVnpMbWx2TDNObGNuWnBZMlZoWTJOdmRXNTBMM05sY25acFkyVXRZV05qYjNWdWRDNTFhV1FpT2lKall6RTVZVEV5TkMweU5HVmhMVFF4TTJVdFlURmlNUzAxWVRJd016VTBaR05oTVRJaUxDSnpkV0lpT2lKemVYTjBaVzA2YzJWeWRtbGpaV0ZqWTI5MWJuUTZhM1ZpWlMxemVYTjBaVzA2WTJWeWRHbG1hV05oZEdVdFkyOXVkSEp2Ykd4bGNpSjkuQnJJbnVnLVdLVV9QVlZPYzd4X2w4THB2QnpiUkFzVjdUVDFiZDg1S1hTRkUyN2JhNGVQdE83VldNQnlsSXlPTzN6bUpvWERaWk5pRWhYeERpNVpBRUJEVnRYRkx4YTNDYWtxWDJ5cmtqUnNBUEFjejZYQ3RBMDBmZFBBTHNqTjBXUXdibVJTUWZlYUMyZGpXWnJubVBQOXhpQVBlRzEtVHdpeFVmUVN5Q2kwb1pjdGd6MV9Cd0FRcWVtQ2pYUDlsOFpFRW5SbmNYWkE2VExWcUpmWDhwOC1zTkVBYUUtZlV5LW5uRDkxVmRrTlpJbW5sUURrSmZsNkJUcTBDMDFOV2doNkcyYlVkMzhLZlBuZzJsSl9PclBXQTRKUThzN3JoRTdVR2RMZlE0RkFMWWNIUll2SUVsUHNYdDNHeGJBS0xoWER3aDluLUlHOE40eTNIZzRQdlp3
kind: Secret
metadata:
annotations:
kubernetes.io/service-account.name: certificate-controller
kubernetes.io/service-account.uid: cc19a124-24ea-413e-a1b1-5a20354dca12
creationTimestamp: "2020-01-28T08:29:34Z"
name: certificate-controller-token-scw76
namespace: kube-system
resourceVersion: "230"
selfLink: /api/v1/namespaces/kube-system/secrets/certificate-controller-token-scw76
uid: c64afb36-0f19-4ccd-877c-fb36f660f07a
type: kubernetes.io/service-account-token
# kubectl config 명령은 view, update 파라메터를 사용할 수 있다.
# 업데이트할 때 오타가 클러스터에 대한 액세스를 제거하지 않을 수도 있다.
# 현재 구성 설정을 보자. 출력에서 키와 인증서들이 자동으로 수정된다.
ps0107@k8smaster1:~$ kubectl config view
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://k8smaster:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubernetes-admin
name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
user:
client-certificate-data: REDACTED. # <- 이부분 확인 (redacted : 수정된)
client-key-data: REDACTED # <- 이부분 확인
# 키 대신 admin의 암호 설정과 같은 옵션들을 보자.
ps0107@k8smaster1:~$ kubectl config set-credentials -h
Sets a user entry in kubeconfig
Specifying a name that already exists will merge new fields on top of existing values.
Client-certificate flags:
--client-certificate=certfile --client-key=keyfile
Bearer token flags:
--token=bearer_token
Basic auth flags:
--username=basic_user --password=basic_password
Bearer token and basic auth are mutually exclusive.
Examples:
# Set only the "client-key" field on the "cluster-admin"
# entry, without touching other values:
kubectl config set-credentials cluster-admin --client-key=~/.kube/admin.key
.....
# 이제 access configuration file을 변경하면서 다른 점들을 살펴보자.
# 미리 이전 파일을 백업 받아 둔다.
ps0107@k8smaster1:~$ cp ~/.kube/config ~/cluster-api-config
# kubectl틀과 kubeadm을 모두 사용하여 클러스터 및 보안 구성으로 작업할 수 있도록 탐색해보자.
# 다른 값 중에서 클러스터의 이름을 찾는다. kubeadm을 사용하기 위해서는 root로 되어야 한다.
ps0107@k8smaster1:~$ kubectl config <tab x 2>
current-context delete-context get-contexts set set-context unset view
delete-cluster get-clusters rename-context set-cluster set-credentials use-context
ps0107@k8smaster1:~$ sudo -i
root@k8smaster1:~# kubeadm token -h
This command manages bootstrap tokens. It is optional and needed only for advanced use cases.
In short, bootstrap tokens are used for establishing bidirectional trust between a client and a server.
A bootstrap token can be used when a client (for example a node that is about to join the cluster) needs
to trust the server it is talking to. Then a bootstrap token with the "signing" usage can be used.
bootstrap tokens can also function as a way to allow short-lived authentication to the API Server
(the token serves as a way for the API Server to trust the client), for example for doing the TLS Bootstrap.
What is a bootstrap token more exactly?
- It is a Secret in the kube-system namespace of type "bootstrap.kubernetes.io/token".
- A bootstrap token must be of the form "[a-z0-9]{6}.[a-z0-9]{16}". The former part is the public token ID,
while the latter is the Token Secret and it must be kept private at all circumstances!
- The name of the Secret must be named "bootstrap-token-(token-id)".
You can read more about bootstrap tokens here:
https://kubernetes.io/docs/admin/bootstrap-tokens/
Usage:
kubeadm token [flags]
kubeadm token [command]
Available Commands:
create Create bootstrap tokens on the server
delete Delete bootstrap tokens on the server
generate Generate and print a bootstrap token, but do not create it on the server
list List bootstrap tokens on the server
Flags:
--dry-run Whether to enable dry-run mode or not
-h, --help help for token
--kubeconfig string The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations can be searched for an existing kubeconfig file. (default "/etc/kubernetes/admin.conf")
Global Flags:
--log-file string If non-empty, use this log file
--log-file-max-size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
--rootfs string [EXPERIMENTAL] The path to the 'real' host root filesystem.
--skip-headers If true, avoid header prefixes in the log messages
--skip-log-headers If true, avoid headers when opening log files
-v, --v Level number for the log level verbosity
Use "kubeadm token [command] --help" for more information about a command.
root@k8smaster1:~# kubeadm config -h
There is a ConfigMap in the kube-system namespace called "kubeadm-config" that kubeadm uses to store internal configuration about the
cluster. kubeadm CLI v1.8.0+ automatically creates this ConfigMap with the config used with 'kubeadm init', but if you
initialized your cluster using kubeadm v1.7.x or lower, you must use the 'config upload' command to create this
ConfigMap. This is required so that 'kubeadm upgrade' can configure your upgraded cluster correctly.
Usage:
kubeadm config [flags]
kubeadm config [command]
Available Commands:
images Interact with container images used by kubeadm
migrate Read an older version of the kubeadm configuration API types from a file, and output the similar config object for the newer version
print Print configuration
view View the kubeadm configuration stored inside the cluster
Flags:
-h, --help help for config
--kubeconfig string The kubeconfig file to use when talking to the cluster. If the flag is not set, a set of standard locations can be searched for an existing kubeconfig file. (default "/etc/kubernetes/admin.conf")
Global Flags:
--log-file string If non-empty, use this log file
--log-file-max-size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
--rootfs string [EXPERIMENTAL] The path to the 'real' host root filesystem.
--skip-headers If true, avoid header prefixes in the log messages
--skip-log-headers If true, avoid headers when opening log files
-v, --v Level number for the log level verbosity
Use "kubeadm config [command] --help" for more information about a command.
# 클러스터의 기본 설정 세티등을 보자.
ps0107@k8smaster1:~$ sudo kubeadm config print init-defaults
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 1.2.3.4
bindPort: 6443
nodeRegistration:
criSocket: /var/run/dockershim.sock
name: k8smaster1
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: v1.14.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
scheduler: {}