Nginx
August 31, 2023
Three nginx controller in one cluster in YC
В этой статье рассмотрим пример создания 3 ингресс контроллеров в одном кластере.
Резервируем адреса для nginx controller.
export FOLDER=$(yc config get folder-id) export SUBNET=$(yc vpc subnet get subnet-a --format json | jq -r '.id') export SG=$(yc vpc sg get k8s-sg --format json | jq -r '.id')
yc vpc address create \ --folder-id $FOLDER \ --name nginx-ingress1 \ --description "Nginx ingress 1 IP cluster test" \ --labels test=nginx-ingress \ --external-ipv4 zone=ru-central1-a \ --async yc vpc address create \ --folder-id $FOLDER \ --name nginx-ingress2 \ --description "Nginx ingress 2 IP cluster test" \ --labels test=nginx-ingress \ --external-ipv4 zone=ru-central1-b \ --async yc vpc address create \ --folder-id $FOLDER \ --name nginx-ingress3 \ --description "Nginx ingress 3 IP cluster test" \ --labels test=nginx-ingress \ --external-ipv4 zone=ru-central1-c \ --async
export IP1=$(yc vpc address get nginx-ingress1 --format=json | jq -r ".external_ipv4_address" | jq -r ".address") export IP2=$(yc vpc address get nginx-ingress2 --format=json | jq -r ".external_ipv4_address" | jq -r ".address") export IP3=$(yc vpc address get nginx-ingress3 --format=json | jq -r ".external_ipv4_address" | jq -r ".address")
Создадим региональный кластер и нод группу для него: https://teletype.in/@cameda/ZQc2XiNvr3z
Установим 3 nginx ingress контроллера в разные ns.
#Установка первого nginx контроллера helm upgrade --install ingress-nginx1 ingress-nginx \ --repo https://kubernetes.github.io/ingress-nginx \ --namespace one --create-namespace \ --debug \ --set controller.ingressClass="nginx1" \ --set controller.ingressClassResource.name="nginx1" \ --set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-1" \ --set controller.ingressClassResource.enabled=true \ --set controller.ingressClassByName=true \ --set controller.service.loadBalancerIP=$IP1 #Установка второго nginx контроллера helm upgrade --install ingress-nginx2 ingress-nginx \ --repo https://kubernetes.github.io/ingress-nginx \ --namespace two --create-namespace \ --debug \ --set controller.ingressClass="nginx2" \ --set controller.ingressClassResource.name="nginx2" \ --set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-2" \ --set controller.ingressClassResource.enabled=true \ --set controller.ingressClassByName=true \ --set controller.service.loadBalancerIP=$IP2 #Установка третьего nginx контроллера helm upgrade --install ingress-nginx3 ingress-nginx \ --repo https://kubernetes.github.io/ingress-nginx \ --namespace three --create-namespace \ --debug \ --set controller.ingressClass="nginx3" \ --set controller.ingressClassResource.name="nginx3" \ --set controller.ingressClassResource.controllerValue="cameda1.tk/ingress-nginx-3" \ --set controller.ingressClassResource.enabled=true \ --set controller.ingressClassByName=true \ --set controller.service.loadBalancerIP=$IP3
helm list -A NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION ingress-nginx1 one 1 2023-08-30 10:02:19.39627 +0300 MSK deployed ingress-nginx-4.7.1 1.8.1 ingress-nginx2 two 1 2023-08-30 10:02:43.895076 +0300 MSK deployed ingress-nginx-4.7.1 1.8.1 ingress-nginx3 three 1 2023-08-30 10:03:24.016176 +0300 MSK deployed ingress-nginx-4.7.1 1.8.1
kubectl get ingressclass NAME CONTROLLER PARAMETERS AGE nginx1 cameda1.tk/ingress-nginx-1 <none> 9m23s nginx2 cameda1.tk/ingress-nginx-2 <none> 8m58s nginx3 cameda1.tk/ingress-nginx-3 <none> 8m18s
kubectl get po -n one -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx1-controller-7476748b99-9l29g 1/1 Running 0 9m54s 10.90.1.171 cl1pqbv65ps0u8f1hts8-ymib <none> <none> kubectl get po -n two -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx2-controller-d9fb5d5dd-qfpjr 1/1 Running 0 9m37s 10.90.1.39 cl1pqbv65ps0u8f1hts8-ymib <none> <none> kubectl get po -n three -owide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES ingress-nginx3-controller-669b98749c-tcf7g 1/1 Running 0 9m22s 10.90.0.88 cl1pqbv65ps0u8f1hts8-odub <none> <none>
kubectl -n one get svc -owide ingress-nginx1-controller NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR ingress-nginx1-controller LoadBalancer 10.91.131.184 51.250.72.47 80:31856/TCP,443:31014/TCP 17m app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx1,app.kubernetes.io/name=ingress-nginx kubectl -n two get svc -owide ingress-nginx2-controller NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR ingress-nginx2-controller LoadBalancer 10.91.44.167 158.160.71.215 80:31170/TCP,443:32479/TCP 18m app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx2,app.kubernetes.io/name=ingress-nginx kubectl -n three get svc -owide ingress-nginx3-controller NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR ingress-nginx3-controller LoadBalancer 10.91.20.212 51.250.44.15 80:30665/TCP,443:30719/TCP 19m app.kubernetes.io/component=controller,app.kubernetes.io/instance=ingress-nginx3,app.kubernetes.io/name=ingress-nginx
Создадим бакенд для примера.
Создадим конфигурационный файл и отправим его в каждый namespace.
Файл /etc/nginx/conf.d/default.conf
server { listen 80 default_server; listen [::]:80 default_server; root /var/www/html/cameda1.tk; index index.html index.htm index.nginx-debian.html; server_name _; location / { try_files $uri $uri/ =404; } }
Создадим 3 ConfigMap из этого файла.
kubectl create cm nginx-config --from-file default.conf -n one kubectl create cm nginx-config --from-file default.conf -n two kubectl create cm nginx-config --from-file default.conf -n three
Создадим файл index для каждого ns свой. Здесь пример для одного ns.
echo "One" > index.html #Создадим ConfigMap на основе файла. kubectl create cm index.html --from-file index.html -n one kubectl create cm index.html --from-file index.html -n two kubectl create cm index.html --from-file index.html -n three
Создадим деплоймент в каждом из namespace.
Положим спецификацию в файл deploy.yaml и используем команду:
kubectl create -f deploy.yaml
В спецификации меняем название namespace!!
apiVersion: apps/v1 kind: Deployment metadata: name: cameda-nginx namespace: one labels: app: nginx environment: prod annotations: author: cameda spec: replicas: 1 selector: matchLabels: app: nginx strategy: rollingUpdate: maxSurge: 1 maxUnavailable: 1 type: RollingUpdate template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx:latest imagePullPolicy: IfNotPresent ports: - containerPort: 80 - containerPort: 443 resources: requests: cpu: 300m memory: 300Mi limits: memory: 400Mi livenessProbe: failureThreshold: 10 successThreshold: 1 httpGet: path: / port: 80 periodSeconds: 10 timeoutSeconds: 1 initialDelaySeconds: 5 readinessProbe: failureThreshold: 3 successThreshold: 1 exec: command: - curl - http://127.0.0.1:80 periodSeconds: 10 timeoutSeconds: 1 initialDelaySeconds: 7 volumeMounts: - name: nginx-configmap mountPath: /etc/nginx/conf.d readOnly: true - name: index mountPath: /var/www/html/cameda1.tk readOnly: true restartPolicy: Always dnsPolicy: ClusterFirst hostname: nginx subdomain: web serviceAccountName: default schedulerName: default-scheduler terminationGracePeriodSeconds: 90 securityContext: runAsUser: 0 runAsGroup: 0 fsGroup: 2000 tolerations: - key: "cam" operator: "Exists" effect: "NoSchedule" - key: "cam" operator: "Exists" effect: "NoExecute" volumes: - name: nginx-configmap configMap: name: nginx-config - name: index configMap: name: index.html
Создадим три сервиса в каждом namespace.
cat <<EOF | kubectl apply -f - apiVersion: v1 kind: Service metadata: name: service-nginx1 namespace: one spec: type: ClusterIP selector: app: nginx ports: - name: http port: 80 targetPort: 80 protocol: TCP EOF cat <<EOF | kubectl apply -f - apiVersion: v1 kind: Service metadata: name: service-nginx2 namespace: two spec: type: ClusterIP selector: app: nginx ports: - name: http port: 80 targetPort: 80 protocol: TCP EOF cat <<EOF | kubectl apply -f - apiVersion: v1 kind: Service metadata: name: service-nginx3 namespace: three spec: type: ClusterIP selector: app: nginx ports: - name: http port: 80 targetPort: 80 protocol: TCP EOF
Создадим 3 ingress в каждом namespace.
cat <<EOF | kubectl apply -f - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx-ingress namespace: one annotations: kubernetes.io/ingress.class: "nginx1" spec: ingressClassName: nginx1 rules: - host: one.cameda1.tk http: paths: - path: / pathType: Prefix backend: service: name: service-nginx1 port: number: 80 EOF cat <<EOF | kubectl apply -f - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx-ingress namespace: two annotations: kubernetes.io/ingress.class: "nginx2" spec: ingressClassName: nginx2 rules: - host: two.cameda1.tk http: paths: - path: / pathType: Prefix backend: service: name: service-nginx2 port: number: 80 EOF cat <<EOF | kubectl apply -f - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx-ingress namespace: three annotations: kubernetes.io/ingress.class: "nginx3" spec: ingressClassName: nginx3 rules: - host: three.cameda1.tk http: paths: - path: / pathType: Prefix backend: service: name: service-nginx3 port: number: 80 EOF
curl http://one.cameda1.tk/ One curl http://two.cameda1.tk/ Two curl http://three.cameda1.tk/ Three