Skip to content

HAProxy to Kubernetes Connectivity

This document describes the network connectivity and load balancing configuration between HAProxy VMs in the DMZ zone and the Kubernetes cluster in the NON-DMZ zone for the JIO-Replica environment.

Architecture Overview

Application Specifications

SIA Dashboard

  • Service Type: HTTP/HTTPS
  • NodePort: 8101
  • Protocol: TCP (HTTP/HTTPS)
  • Namespace: dashboard-service
  • Health Check Path: /health or /actuator/health

SIM Service

  • Service Type: TCP (Custom Protocol)
  • NodePort: 9506
  • Protocol: TCP
  • Namespace: sim-service
  • Health Check: TCP check on port 9506

HAProxy Configuration

Global and Defaults Configuration

Create or update /etc/haproxy/haproxy.cfg on both HAProxy VMs:

# filepath: /etc/haproxy/haproxy.cfg
global
    log /dev/log local0
    log /dev/log local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

    # Default SSL material locations
    ca-base /etc/ssl/certs
    crt-base /etc/ssl/private

    # Security settings
    ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256
    ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets

defaults
    log     global
    mode    http
    option  httplog
    option  dontlognull
    timeout connect 5000
    timeout client  50000
    timeout server  50000
    errorfile 400 /etc/haproxy/errors/400.http
    errorfile 403 /etc/haproxy/errors/403.http
    errorfile 408 /etc/haproxy/errors/408.http
    errorfile 500 /etc/haproxy/errors/500.http
    errorfile 502 /etc/haproxy/errors/502.http
    errorfile 503 /etc/haproxy/errors/503.http
    errorfile 504 /etc/haproxy/errors/504.http

# HAProxy Statistics
listen stats
    bind *:8404
    stats enable
    stats uri /stats
    stats refresh 30s
    stats admin if TRUE

SIA Dashboard Configuration (HTTP/HTTPS)

HAProxy Configuration for SIA Dashboard
# filepath: /etc/haproxy/haproxy.cfg (continued)

#---------------------------------------------------------------------
# SIA Dashboard Frontend (HTTP)
#---------------------------------------------------------------------
frontend sia_dashboard_http
    bind *:80
    mode http
    option httplog
    option forwardfor
    
    # Redirect to HTTPS if needed
    # redirect scheme https code 301 if !{ ssl_fc }
    
    default_backend sia_dashboard_k8s

#---------------------------------------------------------------------
# SIA Dashboard Frontend (HTTPS)
#---------------------------------------------------------------------
frontend sia_dashboard_https
    bind *:443 ssl crt /etc/haproxy/certs/sia-dashboard.pem
    mode http
    option httplog
    option forwardfor
    http-request add-header X-Forwarded-Proto https
    
    default_backend sia_dashboard_k8s

#---------------------------------------------------------------------
# SIA Dashboard Backend (K8s NodePort)
#---------------------------------------------------------------------
backend sia_dashboard_k8s
    mode http
    balance roundrobin
    option httpchk GET /health
    http-check expect status 200
    
    # All K8s worker nodes
    server k8s-worker-1 10.63.121.149:8101 check inter 5s fall 3 rise 2
    server k8s-worker-2 10.63.121.152:8101 check inter 5s fall 3 rise 2
    server k8s-worker-3 10.63.121.157:8101 check inter 5s fall 3 rise 2
    server k8s-worker-4 10.63.121.158:8101 check inter 5s fall 3 rise 2
    server k8s-worker-5 10.63.121.153:8101 check inter 5s fall 3 rise 2
    server k8s-worker-6 10.63.121.159:8101 check inter 5s fall 3 rise 2

SIM Service Configuration (TCP)

HAProxy Configuration for SIM Service
# filepath: /etc/haproxy/haproxy.cfg (continued)

#---------------------------------------------------------------------
# SIM Service Frontend (TCP)
#---------------------------------------------------------------------
frontend sim_service_tcp
    bind *:9506
    mode tcp
    option tcplog
    
    default_backend sim_service_k8s

#---------------------------------------------------------------------
# SIM Service Backend (K8s NodePort)
#---------------------------------------------------------------------
backend sim_service_k8s
    mode tcp
    balance roundrobin
    option tcp-check
    tcp-check connect port 9506
    
    # All K8s worker nodes
    server k8s-worker-1 10.63.121.149:9506 check inter 5s fall 3 rise 2
    server k8s-worker-2 10.63.121.152:9506 check inter 5s fall 3 rise 2
    server k8s-worker-3 10.63.121.157:9506 check inter 5s fall 3 rise 2
    server k8s-worker-4 10.63.121.158:9506 check inter 5s fall 3 rise 2
    server k8s-worker-5 10.63.121.153:9506 check inter 5s fall 3 rise 2
    server k8s-worker-6 10.63.121.159:9506 check inter 5s fall 3 rise 2

Deployment Steps

  1. Update HAProxy Configuration: Copy the above HAProxy configuration snippets into the /etc/haproxy/haproxy.cfg file on both HAProxy VMs.
  2. Restart HAProxy Service: Apply the new configuration by restarting the HAProxy service
  3. Verify Connectivity: Test connectivity to the SIA Dashboard and SIM Service through HAProxy to ensure proper routing and load balancing.
  4. Monitor HAProxy Stats: Access the HAProxy stats page at http://<HAProxy-VM-IP>:8404/stats to monitor the health and performance of the backends.
  5. Adjust as Needed: Based on monitoring data, adjust load balancing algorithms or health check parameters as necessary to optimize performance.
  6. Documentation: Update internal documentation to reflect the new HAProxy to Kubernetes connectivity setup for future reference.

SIA-Dashboard-Service NodePort

# filepath: sia-dashboard-nodeport.yaml
apiVersion: v1
kind: Service
metadata:
  name: sia-dashboard
  namespace: dashboard-service
spec:
  type: NodePort
  selector:
    app: sia-dashboard
  ports:
    - name: http
      protocol: TCP
      port: 8101
      targetPort: 8101
      nodePort: 8101

SIM-Service NodePort

# filepath: kubernetes/sim-service-nodeport.yaml
apiVersion: v1
kind: Service
metadata:
  name: sim-service
  namespace: sim-service
spec:
  type: NodePort
  selector:
    app: sim-service
  ports:
    - name: tcp
      protocol: TCP
      port: 9506
      targetPort: 9506
      nodePort: 9506

NodePort Range

Ensure that ports 8101 and 9506 are within the Kubernetes NodePort range (default: 30000-32767). If not, you may need to adjust the service-node-port-range parameter in the kube-apiserver configuration or use ports within the default range.

HAProxy Service Management Commands

# Backup existing configuration
sudo cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.backup.$(date +%Y%m%d)

# Edit configuration
sudo vi /etc/haproxy/haproxy.cfg

# Validate configuration
sudo haproxy -c -f /etc/haproxy/haproxy.cfg

# Restart HAProxy
sudo systemctl restart haproxy

# Enable on boot
sudo systemctl enable haproxy

# Check status
sudo systemctl status haproxy

Configure SSL Certificates (for HTTPS)

# Create certificate directory
sudo mkdir -p /etc/haproxy/certs

# Combine certificate and key into PEM format
sudo cat /path/to/certificate.crt /path/to/private.key > /etc/haproxy/certs/sia-dashboard.pem

# Set permissions
sudo chmod 600 /etc/haproxy/certs/sia-dashboard.pem
sudo chown haproxy:haproxy /etc/haproxy/certs/sia-dashboard.pem

Self-Signed Certificate

For testing, you can generate a self-signed certificate: bash sudo openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/haproxy/certs/sia-dashboard.key -out /etc/haproxy/certs/sia-dashboard.crt sudo cat /etc/haproxy/certs/sia-dashboard.crt /etc/haproxy/certs/sia-dashboard.key > /etc/haproxy/certs/sia-dashboard.pem

Firewall Rules

Ensure that the following firewall rules are in place to allow traffic between HAProxy VMs and Kubernetes worker nodes:

  • Allow inbound traffic on ports 80 and 443 (for SIA Dashboard) to HAProxy VMs
  • Allow inbound traffic on port 9506 (for SIM Service) to HAProxy VMs
  • Allow outbound traffic from HAProxy VMs to Kubernetes worker nodes on ports 8101 and 9506
Source Destination Port Protocol Description
10.64.232.193 (HAProxy-1) 10.63.121.149-159 (K8s Workers) 8101 TCP SIA Dashboard
10.64.232.194 (HAProxy-2) 10.63.121.149-159 (K8s Workers) 8101 TCP SIA Dashboard
10.64.232.193 (HAProxy-1) 10.63.121.149-159 (K8s Workers) 9506 TCP SIM Service
10.64.232.194 (HAProxy-2) 10.63.121.149-159 (K8s Workers) 9506 TCP SIM Service

Verify Connectivity from HAProxy

# Test SIA Dashboard connectivity
for ip in 10.63.121.149 10.63.121.152 10.63.121.157 10.63.121.158 10.63.121.153 10.63.121.159; do
  echo "Testing $ip:8101"
  nc -zv $ip 8101
done

# Test SIM Service connectivity
for ip in 10.63.121.149 10.63.121.152 10.63.121.157 10.63.121.158 10.63.121.153 10.63.121.159; do
  echo "Testing $ip:9506"
  nc -zv $ip 9506
done

Testing and Verification

Access HAProxy statistics page: 

http://10.64.232.193:8404/stats
http://10.64.232.194:8404/stats

Test SIA Dashboard

# Test TCP connectivity
nc -zv 10.64.232.193 9506

# Test with telnet
telnet 10.64.232.193 9506

Verify Kubernetes Services

# Check SIA Dashboard service
kubectl get svc sia-dashboard -n dashboard-service
kubectl describe svc sia-dashboard -n dashboard-service

# Check SIM Service
kubectl get svc sim-service -n sim-service
kubectl describe svc sim-service -n sim-service

# Test from within the cluster
kubectl run test-pod --rm -i --tty --image=nicolaka/netshoot -- /bin/bash
# Inside the pod:
curl http://sia-dashboard.dashboard-service.svc.cluster.local:8101/health
nc -zv sim-service.sim-service.svc.cluster.local 9506

Troubleshooting

Connection Refused

Connection Issues

If you encounter connection refused errors:

Check Kubernetes Services:

kubectl get svc -n dashboard-service
kubectl get svc -n sim-service
kubectl get endpoints -n dashboard-service
kubectl get endpoints -n sim-service

Verify pods are running: 

kubectl get pods -n dashboard-service
kubectl get pods -n sim-service

Check NodePort is listening: 

# On any K8s worker node
sudo netstat -tlnp | grep 8101
sudo netstat -tlnp | grep 9506

HAProxy Backend Down 

# On HAProxy VMs
sudo tail -f /var/log/haproxy.log

# Check systemd journal
sudo journalctl -u haproxy -f