部署dashboard之前，先确保traefik https方式部署成功，这样就可以通过 https 域名的方式访问dashboard，无需kube-proxy转发了。假设traefik-ingress https部署完成。

下载dashboard yaml文件

wget https://raw.githubusercontent.com/kubernetes/dashboard/master/src/deploy/alternative/kubernetes-dashboard.yaml

由于k8s开启了rbac认证，因此需要添加serviceaccount

[root@node-01 ~]# cat kubernetes-dashboard.yaml

Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");

you may not use this file except in compliance with the License.

You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software

distributed under the License is distributed on an "AS IS" BASIS,

WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

See the License for the specific language governing permissions and

limitations under the License.

------------------- Dashboard Secrets -------------------

apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kube-system
type: Opaque

---

apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-csrf
namespace: kube-system
type: Opaque
data:
csrf: ""

---

------------------- Dashboard Service Account -------------------

apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system

---

------------------- Dashboard Role & Role Binding -------------------

#kind: Role
#apiVersion: rbac.authorization.k8s.io/v1
#metadata:

name: kubernetes-dashboard-minimal

namespace: kube-system

#rules:

# Allow Dashboard to create 'kubernetes-dashboard-key-holder' secret.

#- apiGroups: [""]

resources: ["secrets"]

verbs: ["create"]

# Allow Dashboard to create 'kubernetes-dashboard-settings' config map.

#- apiGroups: [""]

resources: ["configmaps"]

verbs: ["create"]

# Allow Dashboard to get, update and delete Dashboard exclusive secrets.

#- apiGroups: [""]

resources: ["secrets"]

resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]

verbs: ["get", "update", "delete"]

# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.

#- apiGroups: [""]

resources: ["configmaps"]

resourceNames: ["kubernetes-dashboard-settings"]

verbs: ["get", "update"]

# Allow Dashboard to get metrics from heapster.

#- apiGroups: [""]

resources: ["services"]

resourceNames: ["heapster"]

verbs: ["proxy"]

#- apiGroups: [""]

resources: ["services/proxy"]

resourceNames: ["heapster", "http:heapster:", "https:heapster:"]

verbs: ["get"]

---

apiVersion: rbac.authorization.k8s.io/v1
#kind: RoleBinding
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-minimal
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io

kind: Role

kind: ClusterRole

name: kubernetes-dashboard-minimal

name: cluster-admin
subjects:

• kind: ServiceAccount
  name: kubernetes-dashboard
  namespace: kube-system

---

------------------- Dashboard Deployment -------------------

kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
spec:
containers:
- name: kubernetes-dashboard
image: k8s.gcr.io/kubernetes-dashboard-amd64:v1.10.1
ports:
- containerPort: 8443
protocol: TCP
args:
- --auto-generate-certificates
# Uncomment the following line to manually specify Kubernetes API server Host
# If not specified, Dashboard will attempt to auto discover the API server and connect
# to it. Uncomment only if the default does not work.
# - --apiserver-host=http://my-address:port
volumeMounts:
- name: kubernetes-dashboard-certs
mountPath: /certs
# Create on-disk volume to store exec logs
- mountPath: /tmp
name: tmp-volume
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
volumes:
- name: kubernetes-dashboard-certs
secret:
secretName: kubernetes-dashboard-certs
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard
# Comment the following tolerations if Dashboard must not be deployed on master
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule

---

------------------- Dashboard Service -------------------

kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
spec:
ports:
- port: 443
targetPort: 8443
selector:
k8s-app: kubernetes-dashboard

配置ingress

[root@node-01 ~]# cat kubernetes-dashboard-ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kube-ui
namespace: kube-system
spec:
rules:

• host: k8sui.ptengine.jp http: paths:
  • path: '/'
    backend:
    serviceName: kubernetes-dashboard
    servicePort: 443

添加本地host，测试。

1、使用kubernetes-dashboard-token的tocken登陆，先获取tocken，令牌方式登陆 

[root@node-01 ~]# kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kubernetes-dashboard-token|awk '{print $1}')|grep token:|awk '{print $2}'

2、创建一个名为admin的ServiceAccount并绑定名为cluster-admin的ClusterRole角色(该角色拥有集群最高权限)，使用下面的yaml文件创建admin用户并赋予他管理员权限，然后可以通过token登陆dashbaord。这种认证方式本质上是通过ServiceAccount的身份认证加上Bearer token请求API server的方式实现。

[root@node-01 ~]# cat admin-token.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: admin
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
subjects:

• kind: ServiceAccount
  name: admin

namespace: kube-system

apiVersion: v1
kind: ServiceAccount
metadata:
name: admin
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile

[root@node-01 ~]# kubectl create -f admin-token.yaml
[root@node-01 ~]# kubectl get secret -n kube-system | grep admin
admin-token-422fl kubernetes.io/service-account-token 3 17s

通过如下的命令来获取admin ServiceAccount的token：

[root@node-01 ~]# kubectl describe secret/admin-token-422fl -n kube-system
Name: admin-token-422fl
Namespace: kube-system
Labels:
Annotations: kubernetes.io/service-account.name: admin
kubernetes.io/service-account.uid: ec5caa59-7142-11e9-aa9a-fad20acb9b00

Type: kubernetes.io/service-account-token

Data

ca.crt: 1025 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi00MjJmbCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6ImVjNWNhYTU5LTcxNDItMTFlOS1hYTlhLWZhZDIwYWNiOWIwMCIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbiJ9.gXi0mToE0sct0soTeR_TLcDC5Xnr2xCZpvEn-VhE_hZX_QtzhqmgCcUy2wQmpjPoF6eku59dpQVp9WyBYY_rJaAY6HzB3Nzr3pZmDvNdj5Qe1QwxJadp38cqGs7Ao6EZg82wKoXqGI3481rU59BgbcbMeOO75d_e8iN7s64ErpJ25AAWIhfnNvHIJJUP0HoNU8uWbtrcCpceqm-gBY2-hKyqFH5dekMEdoz6GOH9w2xTYeF8Cl6d5xpQ8WcBJ60b7bSVV0PPlhVsswxkA0v95gDGj18rjrLoLJTc0rBOL4FwXOpMeyIO5y7HGXnHWWIL9gMInwoxGloxQJf7RWCRZw

如上，我们得到了该用户的token，dashboard登陆即可。

---

kubernetes 获取性能参数，默认使用 metric server 获取，通过修改kube-controller-manager.yaml，可以修改获取方式。此处介绍heapster。

1、 修改 /etc/kubernetes/manifests/kube-controller-manager.yaml 添加

--horizontal-pod-autoscaler-use-rest-clients=false

master节点需要全部修改。修改完后，重启kube-controller-manager，如果是kubeadm部署，容器会自动重启。

2、 部署的应用需要添加resource限制

resources:
limits:
cpu: 200m
memory: 30Mi
requests:
cpu: 100m
memory: 20Mi

3、 整合heapster 和 influxdb

在没有配置heapster和influxdb的情况下，pod的metric信息是无法获取到的，而早前版本K8S的HPA特性依赖的metric数据来源恰巧就是heapster和influxdb。heapster会在后面的版本中废弃。

准备yaml文件

# cat heapster-sa.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: heapster
namespace: kube-system

# cat heapster-rbac.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: heapster
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:heapster
subjects:

• kind: ServiceAccount
  name: heapster
  namespace: kube-system

# cat heapster-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: heapster
namespace: kube-system
spec:
replicas: 1
template:
metadata:
labels:
task: monitoring
k8s-app: heapster
spec:
serviceAccountName: heapster
containers:
- name: heapster
image: k8s.gcr.io/heapster-amd64:v1.4.2
imagePullPolicy: IfNotPresent
command:
- /heapster
- --source=kubernetes:https://kubernetes.default
- --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086

上面配置source和influxdb有问题，下面会修改。

# cat heapster-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
task: monitoring
kubernetes.io/cluster-service: 'true'
kubernetes.io/name: Heapster
name: heapster
namespace: kube-system
spec:
ports:

• port: 80
  targetPort: 8082
  selector:
  k8s-app: heapster

# cat influxdb-deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: monitoring-influxdb
namespace: kube-system
spec:
replicas: 1
template:
metadata:
labels:
task: monitoring
k8s-app: influxdb
spec:
containers:
- name: influxdb
image: k8s.gcr.io/heapster-influxdb-amd64:v1.3.3
volumeMounts:
- mountPath: /data
name: influxdb-storage
volumes:
- name: influxdb-storage
emptyDir: {}

# cat influxdb-service.yaml
apiVersion: v1
kind: Service
metadata:
labels:
task: monitoring
kubernetes.io/cluster-service: 'true'
kubernetes.io/name: monitoring-influxdb
name: monitoring-influxdb
namespace: kube-system
spec:
ports:

• port: 8086
  targetPort: 8086
  selector:
  k8s-app: influxdb

检查heapster日志

[root@node-01 hpa]# kubectl logs -f heapster-76b4794779-d2vph -n kube-system
I0508 06:16:51.944854 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
I0508 06:16:51.944890 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
E0508 06:17:05.003857 1 kubelet.go:231] error while getting containers from Kubelet: failed to get all container stats from Kubelet URL "http://172.19.8.114:10255/stats/container/": Post http://172.19.8.114:10255/stats/container/: dial tcp 172.19.8.114:10255: getsockopt: connection refused

通过kubectl top 命令也获取不到结果

[root@node-01 ~]# kubectl top pod
W0508 15:25:57.588871 8939 top_pod.go:259] Metrics not available for pod default/my-nginx-6785b88976-7rrll, age: 3h32m13.588851424s
error: Metrics not available for pod default/my-nginx-6785b88976-7rrll, age: 3h32m13.588851424s
[root@node-01 ~]# kubectl top node
error: metrics not available yet

解决办法：

#在heapster-deployment.yaml 清单文件中进行如下修改

• --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true
• --sink=influxdb:http://monitoring-influxdb.kube-system.svc.cluster.local:8086

然后删除heapster重建

kubectl delete -f heapster-deployment.yaml
kubectl apply -f heapster-deployment.yaml

继续 。。。。。发现新问题

遇到403错误

[root@node-01 hpa]# kubectl logs -f heapster-699c6b684d-8sj2q -n kube-system
I0508 06:20:33.630699 1 heapster.go:72] /heapster --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true --sink=influxdb:http://monitoring-influxdb.kube-system.svc.cluster.local:8086
I0508 06:20:33.630780 1 heapster.go:73] Heapster version v1.4.2
I0508 06:20:33.631200 1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default" and version v1
I0508 06:20:33.631235 1 configs.go:62] Using kubelet port 10250
I0508 06:20:33.657061 1 influxdb.go:278] created influxdb sink with options: host:monitoring-influxdb.kube-system.svc.cluster.local:8086 user:root db:k8s
I0508 06:20:33.657100 1 heapster.go:196] Starting with InfluxDB Sink
I0508 06:20:33.657111 1 heapster.go:196] Starting with Metric Sink
I0508 06:20:33.666165 1 heapster.go:106] Starting heapster on port 8082
I0508 06:20:38.888431 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
I0508 06:20:38.888461 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
I0508 06:20:54.158646 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
I0508 06:20:54.158676 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
E0508 06:21:05.018631 1 kubelet.go:231] error while getting containers from Kubelet: failed to get all container stats from Kubelet URL "https://172.19.8.113:10250/stats/container/": request failed - "403 Forbidden", response: "Forbidden (user=system:serviceaccount:kube-system:heapster, verb=create, resource=nodes, subresource=stats)"

解决办法：

查看ClusterRole: system:heapster的权限，发现的确没有针对Resource: nodes/stats 的create权限

[root@node-01 hpa]# kubectl describe clusterrole system:heapster
Name: system:heapster
Labels: kubernetes.io/bootstrapping=rbac-defaults
Annotations: rbac.authorization.kubernetes.io/autoupdate: true
PolicyRule:
Resources Non-Resource URLs Resource Names Verbs
--------- ----------------- -------------- -----
events [] [] [get list watch]
namespaces [] [] [get list watch]
nodes [] [] [get list watch]
pods [] [] [get list watch]
deployments.extensions [] [] [get list watch]

修改ClusterRole: system:heapster的权限

生成清单文件

kubectl get clusterrole system:heapster -o yaml > heapster_modify.yaml

修改文件，增加verbs:create权限，增加resources:nodes/stats

[root@node-01 hpa]# cat heapster_modify.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
creationTimestamp: "2019-05-06T06:24:10Z"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:heapster
resourceVersion: "50"
selfLink: /apis/rbac.authorization.k8s.io/v1/clusterroles/system%3Aheapster
uid: 8f773f30-6fc7-11e9-991a-fa982e6ff600
rules:

• apiGroups:
  • ""
    resources:
  • events
  • namespaces
  • nodes
  • pods
  • nodes/stats # 增加
    verbs:
  • create #增加
  • get
  • list
  • watch
• apiGroups:
  • extensions
    resources:
  • deployments
    verbs:
  • get
  • list
  • watch

执行

kubectl apply -f heapster_modify.yaml

删除heapster重新部署

kubectl delete -f heapster-deployment.yaml
kubectl apply -f heapster-deployment.yaml

再次检查heapster日志

[root@node-01 hpa]# kubectl logs -f heapster-699c6b684d-n2ggr -n kube-system
I0508 06:39:28.987133 1 heapster.go:72] /heapster --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true --sink=influxdb:http://monitoring-influxdb.kube-system.svc.cluster.local:8086
I0508 06:39:28.987229 1 heapster.go:73] Heapster version v1.4.2
I0508 06:39:28.987560 1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default" and version v1
I0508 06:39:28.987589 1 configs.go:62] Using kubelet port 10250
I0508 06:39:29.012055 1 influxdb.go:278] created influxdb sink with options: host:monitoring-influxdb.kube-system.svc.cluster.local:8086 user:root db:k8s
I0508 06:39:29.012098 1 heapster.go:196] Starting with InfluxDB Sink
I0508 06:39:29.012120 1 heapster.go:196] Starting with Metric Sink
I0508 06:39:29.021905 1 heapster.go:106] Starting heapster on port 8082
I0508 06:40:05.166962 1 influxdb.go:241] Created database "k8s" on influxDB server at "monitoring-influxdb.kube-system.svc.cluster.local:8086”
I0508 06:39:54.519349 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz
I0508 06:40:04.062180 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-bjksx
I0508 06:40:04.062246 1 handlers.go:215] No metrics for pod default/nginx-deployment-6d6fdc59f7-q4vjz

heapster默认30秒检查一次，因此需要等上30s才会收集到数据

[root@node-01 ~]# kubectl top nodes
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
node-01 305m 7% 2421Mi 31%
node-02 242m 6% 1906Mi 24%
node-03 224m 5% 1760Mi 22%
node-04 77m 1% 693Mi 8%
node-05 82m 2% 848Mi 10%
node-06 87m 2% 677Mi 8%
[root@node-01 ~]# kubectl top pods
NAME CPU(cores) MEMORY(bytes)
my-nginx-6785b88976-7rrll 0m 1Mi
nginx-deployment-6d6fdc59f7-bjksx 0m 1Mi
nginx-deployment-6d6fdc59f7-q4vjz 0m 1Mi

此时登录dashboard，可以单独内存、CPU信息。