Nginx
August 31, 2023
Three nginx controller in one cluster in YC
В этой статье рассмотрим пример создания 3 ингресс контроллеров в одном кластере.
Резервируем адреса для nginx controller.
export FOLDER=$(yc config get folder-id) export SUBNET=$(yc vpc subnet get subnet-a --format json | jq -r '.id') export SG=$(yc vpc sg get k8s-sg --format json | jq -r '.id')
yc vpc address create \ --folder-id $FOLDER \ --name nginx-ingress1 \ --description "Nginx ingress 1 IP cluster test" \ --labels test=nginx-ingress \ --external-ipv4 zone=ru-central1-a \ --async yc vpc address create \ --folder-id $FOLDER \ --name nginx-ingress2 \ --description "Nginx ingress 2 IP cluster test" \ --labels test=nginx-ingress \ --external-ipv4 zone=ru-central1-b \ --async yc vpc address create \ --folder-id $FOLDER \ --name nginx-ingress3 \ --description "Nginx ingress 3 IP cluster test" \ --labels test=nginx-ingress \ --external-ipv4 zone=ru-central1-c \ --async
export IP1=$(yc vpc address get nginx-ingress1 --format=json | jq -r ".external_ipv4_address" | jq -r ".address") export IP2=$(yc vpc address get nginx-ingress2 --format=json | jq -r ".external_ipv4_address" | jq -r ".address") export IP3=$(yc vpc address get nginx-ingress3 --format=json | jq -r ".external_ipv4_address" | jq -r ".address")
Создадим региональный кластер и нод группу для него: https://teletype.in/@cameda/ZQc2XiNvr3z
Установим 3 nginx ingress контроллера в разные ns.
#Установка первого nginx контроллера helm upgrade --install ingress-nginx1 ingress-nginx \ --repo https://kubernetes.github.io/ingress-nginx \ --namespace one --create-namespace \ --debug \ --set controller.ingressClass="nginx1" \ --set controller.ingressClassResource.name="nginx1" \ --set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-1" \ --set controller.ingressClassResource.enabled=true \ --set controller.ingressClassByName=true \ --set controller.service.loadBalancerIP=$IP1 #Установка второго nginx контроллера helm upgrade --install ingress-nginx2 ingress-nginx \ --repo https://kubernetes.github.io/ingress-nginx \ --namespace two --create-namespace \ --debug \ --set controller.ingressClass="nginx2" \ --set controller.ingressClassResource.name="nginx2" \ --set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-2" \ --set controller.ingressClassResource.enabled=true \ --set controller.ingressClassByName=true \ --set controller.service.loadBalancerIP=$IP2 #Установка третьего nginx контроллера helm upgrade --install ingress-nginx3 ingress-nginx \ --repo https://kubernetes.github.io/ingress-nginx \ --namespace three --create-namespace \ --debug \ --set controller.ingressClass="nginx3" \ --set controller.ingressClassResource.name="nginx3" \ --set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-3" \ --set controller.ingressClassResource.enabled=true \ --set controller.ingressClassByName=true \ --set controller.service.loadBalancerIP=$IP3
helm list -A NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION ingress-nginx1 one 1 2023-08-30 10:02:19.39627 +0300 MSK deployed ingress-nginx-4.7.1 1.8.1 ingress-nginx2 two 1 2023-08-30 10:02:43.895076 +0300 MSK deployed ingress-nginx-4.7.1 1.8.1 ingress-nginx3 three 1 2023-08-30 10:03:24.016176 +0300 MSK deployed ingress-nginx-4.7.1 1.8.1
kubectl get ingressclass NAME CONTROLLER PARAMETERS AGE nginx1 cameda1.tk/ingress-nginx-1 <none> 9m23s nginx2 cameda1.tk/ingress-nginx-2 <none> 8m58s nginx3 cameda1.tk/ingress-nginx-3 <none> 8m18s
kubectl get po -n one -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx1-controller-7476748b99-9l29g 1/1 Running 0 9m54s 10.90.1.171 cl1pqbv65ps0u8f1hts8-ymib <none> <none> kubectl get po -n two -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx2-controller-d9fb5d5dd-qfpjr 1/1 Running 0 9m37s 10.90.1.39 cl1pqbv65ps0u8f1hts8-ymib <none> <none> kubectl get po -n three -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx3-controller-669b98749c-tcf7g 1/1 Running 0 9m22s 10.90.0.88 cl1pqbv65ps0u8f1hts8-odub <none> <none>
kubectl -n one get svc -owide ingress-nginx1-controller NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR ingress-nginx1-controller LoadBalancer 10.91.131.184 51.250.72.47 80:31856/TCP,443:31014/TCP 17m app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx1,app.kubernetes.io/name=ingress-nginx kubectl -n two get svc -owide ingress-nginx2-controller NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR ingress-nginx2-controller LoadBalancer 10.91.44.167 158.160.71.215 80:31170/TCP,443:32479/TCP 18m app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx2,app.kubernetes.io/name=ingress-nginx kubectl -n three get svc -owide ingress-nginx3-controller NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR ingress-nginx3-controller LoadBalancer 10.91.20.212 51.250.44.15 80:30665/TCP,443:30719/TCP 19m app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx3,app.kubernetes.io/name=ingress-nginx
Создадим бакенд для примера.
Создадим конфигурационный файл и отправим его в каждый namespace.
Файл /etc/nginx/conf.d/default.conf
server {
listen 80 default_server;
listen [::]:80 default_server;
root /var/www/html/cameda1.tk;
index index.html index.htm index.nginx-debian.html;
server_name _;
location / {
try_files $uri $uri/ =404;
}
}Создадим 3 ConfigMap из этого файла.
kubectl create cm nginx-config --from-file default.conf -n one kubectl create cm nginx-config --from-file default.conf -n two kubectl create cm nginx-config --from-file default.conf -n three
Создадим файл index для каждого ns свой. Здесь пример для одного ns.
echo "One" > index.html #Создадим ConfigMap на основе файла. kubectl create cm index.html --from-file index.html -n one kubectl create cm index.html --from-file index.html -n two kubectl create cm index.html --from-file index.html -n three
Создадим деплоймент в каждом из namespace.
Положим спецификацию в файл deploy.yaml и используем команду:
kubectl create -f deploy.yaml
В спецификации меняем название namespace!!
apiVersion: apps/v1
kind: Deployment
metadata:
name: cameda-nginx
namespace: one
labels:
app: nginx
environment: prod
annotations:
author: cameda
spec:
replicas: 1
selector:
matchLabels:
app: nginx
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
type: RollingUpdate
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
- containerPort: 443
resources:
requests:
cpu: 300m
memory: 300Mi
limits:
memory: 400Mi
livenessProbe:
failureThreshold: 10
successThreshold: 1
httpGet:
path: /
port: 80
periodSeconds: 10
timeoutSeconds: 1
initialDelaySeconds: 5
readinessProbe:
failureThreshold: 3
successThreshold: 1
exec:
command:
- curl
- http://127.0.0.1:80
periodSeconds: 10
timeoutSeconds: 1
initialDelaySeconds: 7
volumeMounts:
- name: nginx-configmap
mountPath: /etc/nginx/conf.d
readOnly: true
- name: index
mountPath: /var/www/html/cameda1.tk
readOnly: true
restartPolicy: Always
dnsPolicy: ClusterFirst
hostname: nginx
subdomain: web
serviceAccountName: default
schedulerName: default-scheduler
terminationGracePeriodSeconds: 90
securityContext:
runAsUser: 0
runAsGroup: 0
fsGroup: 2000
tolerations:
- key: "cam"
operator: "Exists"
effect: "NoSchedule"
- key: "cam"
operator: "Exists"
effect: "NoExecute"
volumes:
- name: nginx-configmap
configMap:
name: nginx-config
- name: index
configMap:
name: index.htmlСоздадим три сервиса в каждом namespace.
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: service-nginx1
namespace: one
spec:
type: ClusterIP
selector:
app: nginx
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
EOF
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: service-nginx2
namespace: two
spec:
type: ClusterIP
selector:
app: nginx
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
EOF
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
name: service-nginx3
namespace: three
spec:
type: ClusterIP
selector:
app: nginx
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
EOFСоздадим 3 ingress в каждом namespace.
cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-ingress
namespace: one
annotations:
kubernetes.io/ingress.class: "nginx1"
spec:
ingressClassName: nginx1
rules:
- host: one.cameda1.tk
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: service-nginx1
port:
number: 80
EOF
cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-ingress
namespace: two
annotations:
kubernetes.io/ingress.class: "nginx2"
spec:
ingressClassName: nginx2
rules:
- host: two.cameda1.tk
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: service-nginx2
port:
number: 80
EOF
cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-ingress
namespace: three
annotations:
kubernetes.io/ingress.class: "nginx3"
spec:
ingressClassName: nginx3
rules:
- host: three.cameda1.tk
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: service-nginx3
port:
number: 80
EOFcurl http://one.cameda1.tk/ One curl http://two.cameda1.tk/ Two curl http://three.cameda1.tk/ Three