ingress常用来做负载,常见的组合有
- ingress + nginx
- ingress + HAproxy
当前因个人项目,ingress-nginx用的比较多. 所以主要介绍前一种
ingress + nginx
Ingress 部署
第一步:安装插件
需要事先导入下载好的镜像,否则下载不下来
ingress镜像下载地址:
1 2 |
链接:https://pan.baidu.com/s/1_JG7nT81uh_Sy4hdpLgvvA 提取码:mint |
下载之后解压,先导入到你的所有节点docker中,再继续往下进行
官网提供的那个yaml本人亲测问题很多,建议用我提供这个
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 |
apiVersion: v1 kind: Namespace metadata: name: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: nginx-configuration namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: tcp-services namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- kind: ConfigMap apiVersion: v1 metadata: name: udp-services namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- apiVersion: v1 kind: ServiceAccount metadata: name: nginx-ingress-serviceaccount namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: nginx-ingress-clusterrole labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx rules: - apiGroups: - "" resources: - configmaps - endpoints - nodes - pods - secrets verbs: - list - watch - apiGroups: - "" resources: - nodes verbs: - get - apiGroups: - "" resources: - services verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - "extensions" - "networking.k8s.io" resources: - ingresses verbs: - get - list - watch - apiGroups: - "extensions" - "networking.k8s.io" resources: - ingresses/status verbs: - update --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role metadata: name: nginx-ingress-role namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx rules: - apiGroups: - "" resources: - configmaps - pods - secrets - namespaces verbs: - get - apiGroups: - "" resources: - configmaps resourceNames: # Defaults to "<election-id>-<ingress-class>" # Here: "<ingress-controller-leader>-<nginx>" # This has to be adapted if you change either parameter # when launching the nginx-ingress-controller. - "ingress-controller-leader-nginx" verbs: - get - update - apiGroups: - "" resources: - configmaps verbs: - create - apiGroups: - "" resources: - endpoints verbs: - get --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding metadata: name: nginx-ingress-role-nisa-binding namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: nginx-ingress-role subjects: - kind: ServiceAccount name: nginx-ingress-serviceaccount namespace: ingress-nginx --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: name: nginx-ingress-clusterrole-nisa-binding labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: nginx-ingress-clusterrole subjects: - kind: ServiceAccount name: nginx-ingress-serviceaccount namespace: ingress-nginx --- apiVersion: apps/v1 kind: Deployment metadata: name: nginx-ingress-controller namespace: ingress-nginx labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx spec: replicas: 1 selector: matchLabels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx template: metadata: labels: app.kubernetes.io/name: ingress-nginx app.kubernetes.io/part-of: ingress-nginx annotations: prometheus.io/port: "10254" prometheus.io/scrape: "true" spec: # wait up to five minutes for the drain of connections terminationGracePeriodSeconds: 300 serviceAccountName: nginx-ingress-serviceaccount containers: - name: nginx-ingress-controller image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0 args: - /nginx-ingress-controller - --configmap=$(POD_NAMESPACE)/nginx-configuration - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services - --udp-services-configmap=$(POD_NAMESPACE)/udp-services - --publish-service=$(POD_NAMESPACE)/ingress-nginx - --annotations-prefix=nginx.ingress.kubernetes.io securityContext: allowPrivilegeEscalation: true capabilities: drop: - ALL add: - NET_BIND_SERVICE # www-data -> 33 runAsUser: 33 env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace ports: - name: http containerPort: 80 - name: https containerPort: 443 livenessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 readinessProbe: failureThreshold: 3 httpGet: path: /healthz port: 10254 scheme: HTTP periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 lifecycle: preStop: exec: command: - /wait-shutdown --- |
插件安装成功后,查看
1 |
kubectl get pod -n ingress-nginx |
第二步:创建svc
当前service的类型为nodeport 这个根据自己需求。
1 2 3 |
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.30.0/deploy/static/provider/baremetal/service-nodeport.yaml #可以先下载service-nodeport.yaml下来再执行 |
创建完成后,查看
1 |
kubectl get svc -n ingress-nginx |
1 |
到此 ingress 部署完成--------------------------------------------------------- |
下来配置Ingress 代理访问
本篇文章只演示
- http
- https
- BasicAuth
- Nginx 进行重写
ingress工作原理图(个人理解,如有不足请指正)
1. Ingress HTTP 代理访问
根据上图理解,需要创建2组deployment (nginx-dm 和nginx-dm2) 和 2组 svc 分别是( (nginx-svc 和nginx-svc2) )
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
apiVersion: extensions/v1beta1 kind: Deployment metadata: name: nginx-dm spec: replicas: 2 template: metadata: labels: name: nginx spec: containers: - name: nginx image: hub.atshooter.com/k8s/nginx:v1.0 imagePullPolicy: IfNotPresent ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx-svc spec: ports: - port: 80 targetPort: 80 protocol: TCP selector: name: nginx --- apiVersion: extensions/v1beta1 kind: Ingress metadata: name: nginx-test spec: rules: - host: www.shooter.com http: paths: - path: / backend: serviceName: nginx-svc servicePort: 80 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
apiVersion: extensions/v1beta1 kind: Deployment metadata: name: nginx-dm2 spec: replicas: 2 template: metadata: labels: name: nginx-2 spec: containers: - name: nginx-2 image: hub.atshooter.com/k8s/nginx:v1.0 imagePullPolicy: IfNotPresent ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx-svc2 spec: ports: - port: 80 targetPort: 80 protocol: TCP selector: name: nginx-2 --- apiVersion: extensions/v1beta1 kind: Ingress metadata: name: nginx-test2 spec: rules: - host: www.wangmang.com http: paths: - path: / backend: serviceName: nginx-svc2 servicePort: 80 |
然后修改本地(宿主机)hosts映射到你的master节点,然后浏览器访问绑定的域名,端口为ingress-nginx svc所提供的http端口
1 2 |
http://www.shooter.com:30413 访问第一组pod http://www.wangmang.com:30413 访问第二组pod |
由于我2组pod都是同样镜像,所以看起来都一样,其实访问的是各自对应的svc下的pod
2.Ingress HTTPS 代理访问
1.创建证书,以及 cert 存储方式
1 2 3 4 5 |
mkdir -p /root/k8s/https && cd /root/k8s/https openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=nginxsvc/O=nginxsvc" kubectl create secret tls tls-secret --key tls.key --cert tls.crt |
创建ingress 和上面基本一致(这里只演示一组,太懒)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
apiVersion: extensions/v1beta1 kind: Deployment metadata: name: nginx-dm7 spec: replicas: 2 template: metadata: labels: name: nginx-7 spec: containers: - name: nginx-7 image: hub.atshooter.com/k8s/nginx:v1.0 imagePullPolicy: IfNotPresent ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx-svc7 spec: ports: - port: 80 targetPort: 80 protocol: TCP selector: name: nginx-7 --- apiVersion: extensions/v1beta1 kind: Ingress metadata: name: nginx-test-7 spec: tls: - hosts: - www.dadonggua.com secretName: tls-secret #secretName对应封装的时候创建的name rules: - host: www.dadonggua.com http: paths: - path: / backend: serviceName: nginx-svc7 servicePort: 80 |
创建完成后访问域名,注意这里域名后面加的端口是ingress-nginx svc 的https的端口
1 |
https://www.dadonggua.com:31883 |
3.Nginx 进行 BasicAuth
1 2 3 4 5 6 |
Nginx 进行 BasicAuth 基础认证 yum -y install httpd htpasswd -c auth foo #你的网页里用户名foo,密码回车后自己设置别忘了 kubectl create secret generic basic-auth --from-file=auth |
创建ingress 较上面略有改动(这里只演示一组,太懒)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
apiVersion: extensions/v1beta1 kind: Deployment metadata: name: nginx-dm8 spec: replicas: 2 template: metadata: labels: name: nginx-8 spec: containers: - name: nginx-8 image: hub.atshooter.com/k8s/nginx:v1.0 imagePullPolicy: IfNotPresent ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: nginx-svc8 spec: ports: - port: 80 targetPort: 80 protocol: TCP selector: name: nginx-8 --- apiVersion: extensions/v1beta1 kind: Ingress metadata: name: ingress-with-auth8 annotations: nginx.ingress.kubernetes.io/auth-type: basic nginx.ingress.kubernetes.io/auth-secret: basic-auth nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - foo' spec: rules: - host: www.bar.com http: paths: - path: / backend: serviceName: nginx-svc8 servicePort: 80 |
创建完成后,访问一下,注意这里是http访问模式
1 |
http://www.bar.com:30413 |
4.Nginx 进行重写
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
#必须重定向流量的目标URI 串 nginx.ingress.kubernetes.io/rewritetarget #指示位置部分是否仅可访问SSL(当Ingress包含证书时 默认为True) nginx.ingress.kubernetes.io/sslredirect #即使Ingress未启用TLS,也强制重定向到HTTPS nginx.ingress.kubernetes.io/forcessl-redirect #定义Controller必须重定向的应用程序根,如果它在'/'上下文中 nginx.ingress.kubernetes.io/approot #指示Ingress上定义的路径是否使用正则表达式 nginx.ingress.kubernetes.io/useregex |
deployment 和 svc 就不创建了 就直接创建ingress,让ingress直接把域名和关联service写入到ingress-nginx svc中然后重启,就可以通过域名访问了。
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
apiVersion: extensions/v1beta1 kind: Ingress metadata: name: nginx-test-rewrite-target annotations: nginx.ingress.kubernetes.io/rewrite-target: http://www.bar.com:30413/ spec: rules: - host: www.ooo.com http: paths: - path: / backend: serviceName: nginx-svc8 servicePort: 80 |
访问
1 |
http://www.ooo.com:30413 |
访问结果,应该会提示输入用户名密码
关于ingress文件解释(重点)
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
apiVersion: extensions/v1beta1 kind: Ingress metadata: name: nginx-test spec: rules: - host: www.shooter.com http: paths: - path: / backend: serviceName: nginx-svc servicePort: 80 重点:ingress controller 会感知所有namespace的ingress资源文件 用户执行 kubectl apply ingress.yaml文件后 k8s感知到变化就调用 ingress controller 把它翻译成nginx.conf 并写到ingress-nginx的pod中,pod的名字默认nginx-ingress-controller开头 |
进入到这个pod中我们查看部分配置文件
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 |
## start server www.ooo.com server { server_name www.ooo.com ; listen 80; set $proxy_upstream_name "-"; set $pass_access_scheme $scheme; set $pass_server_port $server_port; set $best_http_host $http_host; set $pass_port $pass_server_port; location ~* "^/" { set $namespace "default"; set $ingress_name "nginx-test-rewrite-target"; set $service_name "nginx-svc8"; set $service_port "80"; set $location_path "/"; rewrite_by_lua_block { lua_ingress.rewrite({ force_ssl_redirect = false, use_port_in_redirects = false, }) balancer.rewrite() plugins.run() } header_filter_by_lua_block { plugins.run() } body_filter_by_lua_block { } log_by_lua_block { balancer.log() monitor.call() plugins.run() } port_in_redirect off; set $balancer_ewma_score -1; set $proxy_upstream_name "default-nginx-svc8-80"; set $proxy_host $proxy_upstream_name; set $proxy_alternative_upstream_name ""; client_max_body_size 1m; proxy_set_header Host $best_http_host; # Pass the extracted client certificate to the backend # Allow websocket connections proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header X-Request-ID $req_id; proxy_set_header X-Real-IP $the_real_ip; proxy_set_header X-Forwarded-For $the_real_ip; proxy_set_header X-Forwarded-Host $best_http_host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Scheme $pass_access_scheme; # Pass the original X-Forwarded-For proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; # Custom headers to proxied server proxy_connect_timeout 5s; proxy_send_timeout 60s; proxy_read_timeout 60s; proxy_buffering off; proxy_buffer_size 4k; proxy_buffers 4 4k; proxy_request_buffering on; proxy_http_version 1.1; proxy_cookie_domain off; proxy_cookie_path off; # In case of errors try the next upstream server before returning an error proxy_next_upstream error timeout; proxy_next_upstream_timeout 0; proxy_next_upstream_tries 3; rewrite "(?i)/" http://www.bar.com:30413/ break; proxy_pass http://upstream_balancer; proxy_redirect off; } } ## end server www.ooo.com |
- 本文固定链接: https://www.yoyoask.com/?p=2227
- 转载请注明: shooter 于 SHOOTER 发表