chore: add test for EG cert rotation

internal/xds/server/runner/runner.go

@@ -88,10 +88,17 @@
 	}
 	r.Logger.Info("loaded TLS certificate and key")
 
-	r.grpc = grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
-		MinTime:             15 * time.Second,
-		PermitWithoutStream: true,
-	}))
+	r.grpc = grpc.NewServer(
+		grpc.Creds(credentials.NewTLS(tlsConfig)),
+		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime:             15 * time.Second,
+			PermitWithoutStream: true,
+		}),
+		grpc.KeepaliveParams(keepalive.ServerParameters{
+			Time:    60 * time.Second,
+			Timeout: 20 * time.Second,
+		}),
+	)
 
 	r.cache = cache.NewSnapshotCache(true, r.Logger)
 	registerServer(serverv3.NewServer(ctx, r.cache, r.cache), r.grpc)

test/e2e/base/manifests.yaml

@@ -603,3 +603,18 @@ spec:
         - name: static-file-server
           image: envoyproxy/gateway-static-file-server
           imagePullPolicy: IfNotPresent
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: envoy-gateway-ext-lb
+  namespace: envoy-gateway-system
+spec:
+  selector:
+    control-plane: envoy-gateway
+  ports:
+    - name: grpc
+      port: 18000
+      protocol: TCP
+      targetPort: 18000
+  type: LoadBalancer

test/e2e/testdata/certificate-rotation.yaml

@@ -0,0 +1,16 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: http-for-cert-rotation
+  namespace: gateway-conformance-infra
+spec:
+  parentRefs:
+  - name: same-namespace
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /cert-rotation
+    backendRefs:
+    - name: infra-backend-v1
+      port: 8080

test/e2e/tests/certificate_rotation.go

@@ -0,0 +1,173 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build e2e
+
+package tests
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"testing"
+	"time"
+
+	discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/gateway-api/conformance/utils/http"
+	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
+	"sigs.k8s.io/gateway-api/conformance/utils/suite"
+	"sigs.k8s.io/gateway-api/conformance/utils/tlog"
+
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+	"github.com/envoyproxy/gateway/internal/crypto"
+	"github.com/envoyproxy/gateway/internal/envoygateway/config"
+	provider "github.com/envoyproxy/gateway/internal/provider/kubernetes"
+)
+
+func init() {
+	ConformanceTests = append(ConformanceTests, CertificateRotationTest)
+}
+
+var CertificateRotationTest = suite.ConformanceTest{
+	ShortName:   "CertificateRotation",
+	Description: "Rotate Control Plane Certificates",
+	Manifests:   []string{},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+		t.Run("Envoy Gateway uses new TLS credentials after rotation", func(t *testing.T) {
+			envoyGatewayNS := "envoy-gateway-system"
+			EnvoyGatewayLBSVC := "envoy-gateway-ext-lb"
+			EnvoyCertificateSecret := "envoy"
+			EnvoyGatewayXDSPort := 18000
+			var envoyGatewayAddr string
+
+			ctx := context.Background()
+			envoyGatewaySvc := &corev1.Service{}
+			err := suite.Client.Get(ctx, types.NamespacedName{Namespace: envoyGatewayNS, Name: EnvoyGatewayLBSVC}, envoyGatewaySvc)
+			require.NoError(t, err)
+			require.Len(t, envoyGatewaySvc.Status.LoadBalancer.Ingress, 1)
+			require.NotEmpty(t, envoyGatewaySvc.Status.LoadBalancer.Ingress[0].IP)
+
+			if IPFamily == "ipv6" {
+				envoyGatewayAddr = fmt.Sprintf("[%s]:%d", envoyGatewaySvc.Status.LoadBalancer.Ingress[0].IP, EnvoyGatewayXDSPort)
+			} else {
+				envoyGatewayAddr = fmt.Sprintf("%s:%d", envoyGatewaySvc.Status.LoadBalancer.Ingress[0].IP, EnvoyGatewayXDSPort)
+			}
+
+			// get the current envoy TLS credentials
+			certNN := types.NamespacedName{Namespace: envoyGatewayNS, Name: EnvoyCertificateSecret}
+			crt, key, ca, err := GetTLSSecret(suite.Client, certNN)
+			require.NoError(t, err)
+
+			// create a gRPC client with envoy's TLS credentials
+			tlsConfig, err := tlsClientConfig(crt, key, ca)
+			require.NoError(t, err)
+			conn, err := grpc.NewClient(envoyGatewayAddr,
+				grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
+			require.NoError(t, err)
+
+			// Connect to Envoy Gateway's XDS endpoint with Envoy TLS credentials
+			streamClient, err := discovery.NewAggregatedDiscoveryServiceClient(conn).
+				StreamAggregatedResources(ctx)
+			require.NoError(t, err)
+			require.NotNil(t, streamClient)
+			err = conn.Close()
+			require.NoError(t, err)
+
+			// rotate certs and apply them similar to how EG certgen works
+			certs, err := crypto.GenerateCerts(&config.Server{
+				EnvoyGateway: &egv1a1.EnvoyGateway{
+					EnvoyGatewaySpec: egv1a1.EnvoyGatewaySpec{
+						Provider: &egv1a1.EnvoyGatewayProvider{
+							Type: egv1a1.ProviderTypeKubernetes,
+						},
+						Gateway: egv1a1.DefaultGateway(),
+					},
+				},
+				Namespace: envoyGatewayNS,
+				DNSDomain: "cluster.local",
+			})
+			require.NoError(t, err)
+			secrets := provider.CertsToSecret(envoyGatewayNS, certs)
+			_, err = provider.CreateOrUpdateSecrets(ctx, suite.Client, secrets, true)
+			require.NoError(t, err)
+
+			// Wait for connection with new credentials to succeed
+			http.AwaitConvergence(
+				t,
+				1,
+				suite.TimeoutConfig.NamespacesMustBeReady,
+				func(_ time.Duration) bool {
+					// create gRPC client with envoy's TLS credentials
+					tlsConfig, err = tlsClientConfig(certs.EnvoyCertificate, certs.EnvoyPrivateKey,
+						certs.CACertificate)
+					require.NoError(t, err)
+					conn, err = grpc.NewClient(envoyGatewayAddr,
+						grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)))
+					require.NoError(t, err)
+
+					// Connect to Envoy Gateway's XDS endpoint with Envoy TLS credentials
+					streamClient, err = discovery.NewAggregatedDiscoveryServiceClient(conn).
+						StreamAggregatedResources(ctx)
+					if err != nil {
+						tlog.Logf(t, "failed to connect to Envoy Gateway with new tls credentials: %v", err)
+						err = conn.Close()
+						require.NoError(t, err)
+						time.Sleep(1 * time.Second)
+						return false
+					}
+
+					tlog.Logf(t, "Connected to Envoy Gateway with new tls credentials")
+					err = conn.Close()
+					require.NoError(t, err)
+					return true
+				})
+
+			// Apply a new config and confirm that it's programmed successfully on proxies
+			suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, "testdata/certificate-rotation.yaml", false)
+			ns := "gateway-conformance-infra"
+			routeNN := types.NamespacedName{Name: "http-for-cert-rotation", Namespace: ns}
+			gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
+			gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
+			kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{ns})
+
+			expected := http.ExpectedResponse{
+				Request: http.Request{
+					Path: "/cert-rotation",
+				},
+				Response: http.Response{
+					StatusCode: 200,
+				},
+				Namespace: ns,
+			}
+
+			http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expected)
+		})
+	},
+}
+
+func tlsClientConfig(certPem []byte, keyPem []byte, caPem []byte) (*tls.Config, error) {
+	cert, err := tls.X509KeyPair(certPem, keyPem)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected error creating cert: %w", err)
+	}
+
+	certPool := x509.NewCertPool()
+	if !certPool.AppendCertsFromPEM(caPem) {
+		return nil, fmt.Errorf("unexpected error adding trusted CA: %w", err)
+	}
+
+	return &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      certPool,
+		ServerName:   "envoy-gateway.envoy-gateway-system.svc.cluster.local",
+		MinVersion:   tls.VersionTLS13,
+	}, nil
+}

test/e2e/tests/client_mtls.go

@@ -68,7 +68,7 @@ var ClientMTLSTest = suite.ConformanceTest{
 
 			// This test uses the same key/cert pair as both a client cert and server cert
 			// Both backend and client treat the self-signed cert as a trusted CA
-			cPem, keyPem, err := GetTLSSecret(suite.Client, certNN)
+			cPem, keyPem, _, err := GetTLSSecret(suite.Client, certNN)
 			if err != nil {
 				t.Fatalf("unexpected error finding TLS secret: %v", err)
 			}
@@ -107,7 +107,7 @@ var ClientMTLSTest = suite.ConformanceTest{
 			req := http.MakeRequest(t, &expected, gwAddr, "HTTPS", "https")
 
 			// added but not used, as these are required by test utils when for SNI to be added
-			cPem, keyPem, err := GetTLSSecret(suite.Client, certNN)
+			cPem, keyPem, _, err := GetTLSSecret(suite.Client, certNN)
 			if err != nil {
 				t.Fatalf("unexpected error finding TLS secret: %v", err)
 			}
@@ -172,21 +172,25 @@ func WaitForConsistentMTLSResponse(t *testing.T, r roundtripper.RoundTripper, re
 }
 
 // GetTLSSecret fetches the named Secret and converts both cert and key to []byte
-func GetTLSSecret(client client.Client, secretName types.NamespacedName) ([]byte, []byte, error) {
-	var cert, key []byte
+func GetTLSSecret(client client.Client, secretName types.NamespacedName) ([]byte, []byte, []byte, error) {
+	var cert, key, ca []byte
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 
 	secret := &corev1.Secret{}
 	err := client.Get(ctx, secretName, secret)
 	if err != nil {
-		return cert, key, fmt.Errorf("error fetching TLS Secret: %w", err)
+		return cert, key, nil, fmt.Errorf("error fetching TLS Secret: %w", err)
 	}
 	cert = secret.Data["tls.crt"]
 	key = secret.Data["tls.key"]
 
-	return cert, key, nil
+	if secret.Data["ca.crt"] != nil {
+		ca = secret.Data["ca.crt"]
+	}
+
+	return cert, key, ca, nil
 }
 
 func dialWithTLSVersion(t *testing.T, gwAddr string, baseTLSConfig *tls.Config, version uint16, expectedError bool) {

test/e2e/tests/tlsroute_with_backend.go

@@ -91,7 +91,7 @@ func testTLSRouteWithBackend(t *testing.T, suite *suite.ConformanceTestSuite, ro
 
 	// This test uses the same key/cert pair as both a client cert and server cert
 	// Both backend and client treat the self-signed cert as a trusted CA
-	cPem, keyPem, err := GetTLSSecret(suite.Client, certNN)
+	cPem, keyPem, _, err := GetTLSSecret(suite.Client, certNN)
 	if err != nil {
 		t.Fatalf("unexpected error finding TLS secret: %v", err)
 	}