Nginx
August 31, 2023

Three nginx controller in one cluster in YC

В этой статье рассмотрим пример создания 3 ингресс контроллеров в одном кластере.

Резервируем адреса для nginx controller.

export FOLDER=$(yc config get folder-id)
export SUBNET=$(yc vpc subnet get subnet-a --format json | jq -r '.id')
export SG=$(yc vpc sg get k8s-sg --format json | jq -r '.id')
yc vpc address create \
  --folder-id $FOLDER \
  --name nginx-ingress1 \
  --description "Nginx ingress 1 IP cluster test" \
  --labels test=nginx-ingress \
  --external-ipv4 zone=ru-central1-a \
  --async
  
yc vpc address create \
  --folder-id $FOLDER \
  --name nginx-ingress2 \
  --description "Nginx ingress 2 IP cluster test" \
  --labels test=nginx-ingress \
  --external-ipv4 zone=ru-central1-b \
  --async
  
yc vpc address create \
  --folder-id $FOLDER \
  --name nginx-ingress3 \
  --description "Nginx ingress 3 IP cluster test" \
  --labels test=nginx-ingress \
  --external-ipv4 zone=ru-central1-c \
  --async
export IP1=$(yc vpc address get nginx-ingress1 --format=json | jq -r ".external_ipv4_address" | jq -r ".address")
export IP2=$(yc vpc address get nginx-ingress2 --format=json | jq -r ".external_ipv4_address" | jq -r ".address")
export IP3=$(yc vpc address get nginx-ingress3 --format=json | jq -r ".external_ipv4_address" | jq -r ".address")

Создадим региональный кластер и нод группу для него: https://teletype.in/@cameda/ZQc2XiNvr3z

Установим 3 nginx ingress контроллера в разные ns.

#Установка первого nginx контроллера
helm upgrade --install ingress-nginx1 ingress-nginx \
--repo https://kubernetes.github.io/ingress-nginx \
--namespace one --create-namespace \
--debug \
--set controller.ingressClass="nginx1" \
--set controller.ingressClassResource.name="nginx1" \
--set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-1" \
--set controller.ingressClassResource.enabled=true \
--set controller.ingressClassByName=true \
--set controller.service.loadBalancerIP=$IP1

#Установка второго nginx контроллера
helm upgrade --install ingress-nginx2 ingress-nginx \
--repo https://kubernetes.github.io/ingress-nginx \
--namespace two --create-namespace \
--debug \
--set controller.ingressClass="nginx2" \
--set controller.ingressClassResource.name="nginx2" \
--set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-2" \
--set controller.ingressClassResource.enabled=true \
--set controller.ingressClassByName=true \
--set controller.service.loadBalancerIP=$IP2

#Установка третьего nginx контроллера
helm upgrade --install ingress-nginx3 ingress-nginx \
--repo https://kubernetes.github.io/ingress-nginx \
--namespace three --create-namespace \
--debug \
--set controller.ingressClass="nginx3" \
--set controller.ingressClassResource.name="nginx3" \
--set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-3" \
--set controller.ingressClassResource.enabled=true \
--set controller.ingressClassByName=true \
--set controller.service.loadBalancerIP=$IP3

Что получилось.

helm list -A
NAME          	NAMESPACE	REVISION	UPDATED                             	STATUS  	CHART              	APP VERSION
ingress-nginx1	one      	1       	2023-08-30 10:02:19.39627 +0300 MSK 	deployed	ingress-nginx-4.7.1	1.8.1
ingress-nginx2	two      	1       	2023-08-30 10:02:43.895076 +0300 MSK	deployed	ingress-nginx-4.7.1	1.8.1
ingress-nginx3	three    	1       	2023-08-30 10:03:24.016176 +0300 MSK	deployed	ingress-nginx-4.7.1	1.8.1
kubectl get ingressclass
NAME     CONTROLLER                   PARAMETERS   AGE
nginx1   cameda1.tk/ingress-nginx-1   <none>       9m23s
nginx2   cameda1.tk/ingress-nginx-2   <none>       8m58s
nginx3   cameda1.tk/ingress-nginx-3   <none>       8m18s
kubectl get po -n one -owide
NAME                                         READY   STATUS    RESTARTS   AGE     IP            NODE                        NOMINATED NODE   READINESS GATES
ingress-nginx1-controller-7476748b99-9l29g   1/1     Running   0          9m54s   10.90.1.171   cl1pqbv65ps0u8f1hts8-ymib   <none>           <none>

kubectl get po -n two -owide
NAME                                        READY   STATUS    RESTARTS   AGE     IP           NODE                        NOMINATED NODE   READINESS GATES
ingress-nginx2-controller-d9fb5d5dd-qfpjr   1/1     Running   0          9m37s   10.90.1.39   cl1pqbv65ps0u8f1hts8-ymib   <none>           <none>

kubectl get po -n three -owide
NAME                                         READY   STATUS    RESTARTS   AGE     IP           NODE                        NOMINATED NODE   READINESS GATES
ingress-nginx3-controller-669b98749c-tcf7g   1/1     Running   0          9m22s   10.90.0.88   cl1pqbv65ps0u8f1hts8-odub   <none>           <none>
kubectl -n one get svc -owide ingress-nginx1-controller
NAME                        TYPE           CLUSTER-IP      EXTERNAL-IP    PORT(S)                      AGE   SELECTOR
ingress-nginx1-controller   LoadBalancer   10.91.131.184   51.250.72.47   80:31856/TCP,443:31014/TCP   17m   app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx1,app.kubernetes.io/name=ingress-nginx

kubectl -n two get svc -owide ingress-nginx2-controller
NAME                        TYPE           CLUSTER-IP     EXTERNAL-IP      PORT(S)                      AGE   SELECTOR
ingress-nginx2-controller   LoadBalancer   10.91.44.167   158.160.71.215   80:31170/TCP,443:32479/TCP   18m   app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx2,app.kubernetes.io/name=ingress-nginx

kubectl -n three get svc -owide ingress-nginx3-controller
NAME                        TYPE           CLUSTER-IP     EXTERNAL-IP    PORT(S)                      AGE   SELECTOR
ingress-nginx3-controller   LoadBalancer   10.91.20.212   51.250.44.15   80:30665/TCP,443:30719/TCP   19m   app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx3,app.kubernetes.io/name=ingress-nginx

Создадим бакенд для примера.

Создадим конфигурационный файл и отправим его в каждый namespace.

Файл /etc/nginx/conf.d/default.conf

server {
  listen 80 default_server;
  listen [::]:80 default_server;
  root /var/www/html/cameda1.tk;
  index index.html index.htm index.nginx-debian.html;
  server_name _;
  location / {
    try_files $uri $uri/ =404;
  }
}

Создадим 3 ConfigMap из этого файла.

kubectl create cm nginx-config --from-file default.conf -n one
kubectl create cm nginx-config --from-file default.conf -n two
kubectl create cm nginx-config --from-file default.conf -n three

Создадим файл index для каждого ns свой. Здесь пример для одного ns.

echo "One" > index.html

#Создадим ConfigMap на основе файла.
kubectl create cm index.html --from-file index.html -n one
kubectl create cm index.html --from-file index.html -n two
kubectl create cm index.html --from-file index.html -n three

Создадим деплоймент в каждом из namespace.

Положим спецификацию в файл deploy.yaml и используем команду:

kubectl create -f deploy.yaml

В спецификации меняем название namespace!!

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cameda-nginx
  namespace: one
  labels:
    app: nginx
    environment: prod
  annotations:
    author: cameda
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  strategy: 
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:latest
        imagePullPolicy: IfNotPresent
        ports: 
        - containerPort: 80
        - containerPort: 443  
        resources: 
          requests: 
            cpu: 300m
            memory: 300Mi 
          limits: 
            memory: 400Mi
        livenessProbe:
          failureThreshold: 10
          successThreshold: 1
          httpGet:
            path: /
            port: 80
          periodSeconds: 10
          timeoutSeconds: 1
          initialDelaySeconds: 5
        readinessProbe:
          failureThreshold: 3
          successThreshold: 1
          exec:
            command:
            - curl
            - http://127.0.0.1:80
          periodSeconds: 10
          timeoutSeconds: 1
          initialDelaySeconds: 7
        volumeMounts:
        - name: nginx-configmap
          mountPath: /etc/nginx/conf.d
          readOnly: true
        - name: index
          mountPath: /var/www/html/cameda1.tk
          readOnly: true
      restartPolicy: Always
      dnsPolicy: ClusterFirst
      hostname: nginx
      subdomain: web
      serviceAccountName: default
      schedulerName: default-scheduler
      terminationGracePeriodSeconds: 90
      securityContext:
        runAsUser: 0
        runAsGroup: 0
        fsGroup: 2000
      tolerations:
      - key: "cam"
        operator: "Exists"
        effect: "NoSchedule"
      - key: "cam"
        operator: "Exists"
        effect: "NoExecute"
      volumes:
      - name: nginx-configmap
        configMap:
          name: nginx-config
      - name: index
        configMap:
          name: index.html

Создадим три сервиса в каждом namespace.

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: service-nginx1
  namespace: one
spec:
  type: ClusterIP
  selector:
    app: nginx
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
EOF

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: service-nginx2
  namespace: two
spec:
  type: ClusterIP
  selector:
    app: nginx
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
EOF

cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Service
metadata:
  name: service-nginx3
  namespace: three
spec:
  type: ClusterIP
  selector:
    app: nginx
  ports:
    - name: http
      port: 80
      targetPort: 80
      protocol: TCP
EOF

Создадим 3 ingress в каждом namespace.

cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-ingress
  namespace: one
  annotations:
    kubernetes.io/ingress.class: "nginx1"
spec:
  ingressClassName: nginx1
  rules:
    - host: one.cameda1.tk
      http:
        paths:
        - path: /
          pathType: Prefix
          backend:
            service:
              name: service-nginx1
              port:
                number: 80
EOF

cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-ingress
  namespace: two
  annotations:
    kubernetes.io/ingress.class: "nginx2"
spec:
  ingressClassName: nginx2
  rules:
    - host: two.cameda1.tk
      http:
        paths:
        - path: /
          pathType: Prefix
          backend:
            service:
              name: service-nginx2
              port:
                number: 80
EOF

cat <<EOF | kubectl apply -f -
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-ingress
  namespace: three
  annotations:
    kubernetes.io/ingress.class: "nginx3"
spec:
  ingressClassName: nginx3
  rules:
    - host: three.cameda1.tk
      http:
        paths:
        - path: /
          pathType: Prefix
          backend:
            service:
              name: service-nginx3
              port:
                number: 80
EOF

Что в итоге получилось.

curl http://one.cameda1.tk/
One

curl http://two.cameda1.tk/
Two

curl http://three.cameda1.tk/
Three