@@ -3,11 +3,14 @@ package images

 import (

 	"fmt"

 	"net/http"

+	"strconv"

+	"strings"

 	"time"

 	g "github.com/onsi/ginkgo"

 	o "github.com/onsi/gomega"

+	kapi "k8s.io/kubernetes/pkg/api"

 	"k8s.io/kubernetes/pkg/util/wait"

 	e2e "k8s.io/kubernetes/test/e2e/framework"

@@ -32,6 +35,9 @@ var _ = g.Describe("[networking][router] openshift routers", func() {

 	g.Describe("The HAProxy router", func() {

 		g.It("should serve the correct routes when scoped to a single namespace and label set", func() {

 			oc.SetOutputDir(exutil.TestContext.OutputDir)

+			ns := oc.KubeFramework().Namespace.Name

+			execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")

+			defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, kapi.NewDeleteOptions(1)) }()

 			g.By(fmt.Sprintf("creating a scoped router from a config file %q", configPath))

@@ -54,28 +60,25 @@ var _ = g.Describe("[networking][router] openshift routers", func() {

 			g.By("waiting for the healthz endpoint to respond")

 			healthzURI := fmt.Sprintf("http://%s:1936/healthz", routerIP)

-			err = waitForRouterOKResponse(healthzURI, routerIP, 2*time.Minute)

+			err = waitForRouterOKResponseExec(ns, execPodName, healthzURI, routerIP, 60*2)

 			o.Expect(err).NotTo(o.HaveOccurred())

 			g.By("waiting for the valid route to respond")

-			err = waitForRouterOKResponse(routerURL, "first.example.com", 2*time.Minute)

+			err = waitForRouterOKResponseExec(ns, execPodName, routerURL, "first.example.com", 60*2)

 			o.Expect(err).NotTo(o.HaveOccurred())

 			for _, host := range []string{"second.example.com", "third.example.com"} {

 				g.By(fmt.Sprintf("checking that %s does not match a route", host))

-				req, err := requestViaReverseProxy("GET", routerURL, host)

-				o.Expect(err).NotTo(o.HaveOccurred())

-				resp, err := http.DefaultClient.Do(req)

+				err = expectRouteStatusCodeExec(ns, execPodName, routerURL, host, http.StatusServiceUnavailable)

 				o.Expect(err).NotTo(o.HaveOccurred())

-				resp.Body.Close()

-				if resp.StatusCode != http.StatusServiceUnavailable {

-					e2e.Failf("should have had a 503 status code for %s", host)

-				}

 			}

 		})

+

 		g.It("should override the route host with a custom value", func() {

 			oc.SetOutputDir(exutil.TestContext.OutputDir)

 			ns := oc.KubeFramework().Namespace.Name

+			execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")

+			defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, kapi.NewDeleteOptions(1)) }()

 			g.By(fmt.Sprintf("creating a scoped router from a config file %q", configPath))

@@ -100,68 +103,91 @@ var _ = g.Describe("[networking][router] openshift routers", func() {

 			g.By("waiting for the healthz endpoint to respond")

 			healthzURI := fmt.Sprintf("http://%s:1936/healthz", routerIP)

-			err = waitForRouterOKResponse(healthzURI, routerIP, 2*time.Minute)

+			err = waitForRouterOKResponseExec(ns, execPodName, healthzURI, routerIP, 60*2)

 			o.Expect(err).NotTo(o.HaveOccurred())

+			oc.KubeFramework().Client.Pods(ns)

+

 			g.By("waiting for the valid route to respond")

-			err = waitForRouterOKResponse(routerURL, fmt.Sprintf(pattern, "route-1", ns), 2*time.Minute)

+			err = waitForRouterOKResponseExec(ns, execPodName, routerURL, fmt.Sprintf(pattern, "route-1", ns), 60*2)

 			o.Expect(err).NotTo(o.HaveOccurred())

 			g.By("checking that the stored domain name does not match a route")

 			host := "first.example.com"

-			req, err := requestViaReverseProxy("GET", routerURL, host)

-			o.Expect(err).NotTo(o.HaveOccurred())

-			resp, err := http.DefaultClient.Do(req)

+			err = expectRouteStatusCodeExec(ns, execPodName, routerURL, host, http.StatusServiceUnavailable)

 			o.Expect(err).NotTo(o.HaveOccurred())

-			resp.Body.Close()

-			if resp.StatusCode != http.StatusServiceUnavailable {

-				e2e.Failf("should have had a 503 status code for %s", host)

-			}

 			for _, host := range []string{"route-1", "route-2"} {

 				host = fmt.Sprintf(pattern, host, ns)

-				g.By(fmt.Sprintf("checking that %s does not match a route", host))

-				req, err := requestViaReverseProxy("GET", routerURL, host)

-				o.Expect(err).NotTo(o.HaveOccurred())

-				resp, err := http.DefaultClient.Do(req)

+				g.By(fmt.Sprintf("checking that %s matches a route", host))

+				err = expectRouteStatusCodeExec(ns, execPodName, routerURL, host, http.StatusOK)

 				o.Expect(err).NotTo(o.HaveOccurred())

-				resp.Body.Close()

-				if resp.StatusCode != http.StatusOK {

-					e2e.Failf("should have had a 200 status code for %s", host)

-				}

 			}

 		})

 	})

 })

-func waitForRouterOKResponse(url, host string, timeout time.Duration) error {

-	return wait.Poll(time.Second, timeout, func() (bool, error) {

-		req, err := requestViaReverseProxy("GET", url, host)

-		if err != nil {

-			return false, err

-		}

-		resp, err := http.DefaultClient.Do(req)

-		if err != nil {

-			return false, nil

-		}

-		resp.Body.Close()

-		if resp.StatusCode == http.StatusServiceUnavailable {

-			// not ready yet

-			return false, nil

-		}

-		if resp.StatusCode != http.StatusOK {

-			e2e.Logf("unexpected response: %#v", resp.StatusCode)

-			return false, nil

-		}

-		return true, nil

-	})

+func waitForRouterOKResponseExec(ns, execPodName, url, host string, timeoutSeconds int) error {

+	cmd := fmt.Sprintf(`

+		set -e

+		for i in $(seq 1 %d); do

+			code=$( curl -s -o /dev/null -w '%%{http_code}\n' --header 'Host: %s' %q )

+			echo $code

+			if [[ $code -eq 200 ]]; then

+				exit 0

+			fi

+			if [[ $code -ne 503 ]]; then

+				exit 1

+			fi

+			sleep 1

+		done

+		`, timeoutSeconds, host, url)

+	output, err := e2e.RunHostCmd(ns, execPodName, cmd)

+	if err != nil {

+		return fmt.Errorf("host command failed: %v\n%s", err, output)

+	}

+	lines := strings.Split(strings.TrimSpace(output), "\n")

+	if lines[len(lines)-1] != "200" {

+		return fmt.Errorf("last response from server was not 200:\n%s", output)

+	}

+	return nil

+}

+

+func expectRouteStatusCodeRepeatedExec(ns, execPodName, url, host string, statusCode int, times int) error {

+	cmd := fmt.Sprintf(`

+		set -e

+		for i in $(seq 1 %d); do

+			code=$( curl -s -o /dev/null -w '%%{http_code}\n' --header 'Host: %s' %q )

+			echo $code

+			if [[ $code -ne %d ]]; then

+				exit 1

+			fi

+		done

+		`, times, host, url, statusCode)

+	output, err := e2e.RunHostCmd(ns, execPodName, cmd)

+	if err != nil {

+		return fmt.Errorf("host command failed: %v\n%s", err, output)

+	}

+	return nil

+}

+

+func expectRouteStatusCodeExec(ns, execPodName, url, host string, statusCode int) error {

+	cmd := fmt.Sprintf("curl -s -o /dev/null -w '%%{http_code}' --header 'Host: %s' %q", host, url)

+	output, err := e2e.RunHostCmd(ns, execPodName, cmd)

+	if err != nil {

+		return fmt.Errorf("host command failed: %v\n%s", err, output)

+	}

+	if output != strconv.Itoa(statusCode) {

+		return fmt.Errorf("last response from server was not %d: %s", statusCode, output)

+	}

+	return nil

 }

-func requestViaReverseProxy(method, url, host string) (*http.Request, error) {

-	req, err := http.NewRequest(method, url, nil)

+func getAuthenticatedRouteURLViaPod(ns, execPodName, url, host, user, pass string) (string, error) {

+	cmd := fmt.Sprintf("curl -s -u %s:%s --header 'Host: %s' %q", user, pass, host, url)

+	output, err := e2e.RunHostCmd(ns, execPodName, cmd)

 	if err != nil {

-		return nil, err

+		return "", fmt.Errorf("host command failed: %v\n%s", err, output)

 	}

-	req.Host = host

-	return req, nil

+	return output, nil

 }

@@ -3,7 +3,6 @@ package images

 import (

 	"encoding/csv"

 	"fmt"

-	"io/ioutil"

 	"net/http"

 	"strconv"

 	"strings"

@@ -12,6 +11,7 @@ import (

 	g "github.com/onsi/ginkgo"

 	o "github.com/onsi/gomega"

+	kapi "k8s.io/kubernetes/pkg/api"

 	"k8s.io/kubernetes/pkg/util/wait"

 	e2e "k8s.io/kubernetes/test/e2e/framework"

@@ -35,8 +35,10 @@ var _ = g.Describe("[networking][router] weighted openshift router", func() {

 	g.Describe("The HAProxy router", func() {

 		g.It("should appropriately serve a route that points to two services", func() {

-

 			oc.SetOutputDir(exutil.TestContext.OutputDir)

+			ns := oc.KubeFramework().Namespace.Name

+			execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")

+			defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, kapi.NewDeleteOptions(1)) }()

 			g.By(fmt.Sprintf("creating a weighted router from a config file %q", configPath))

@@ -60,34 +62,30 @@ var _ = g.Describe("[networking][router] weighted openshift router", func() {

 			g.By("waiting for the healthz endpoint to respond")

 			healthzURI := fmt.Sprintf("http://%s:1936/healthz", routerIP)

-			err = waitForRouterOKResponse(healthzURI, routerIP, 2*time.Minute)

+			err = waitForRouterOKResponseExec(ns, execPodName, healthzURI, routerIP, 60*2)

 			o.Expect(err).NotTo(o.HaveOccurred())

 			host := "weighted.example.com"

-			g.By(fmt.Sprintf("checking that 10 requests go through successfully"))

-			for i := 1; i <= 100; i++ {

-				err = waitForRouterOKResponse(routerURL, "weighted.example.com", 1*time.Minute)

-				o.Expect(err).NotTo(o.HaveOccurred())

-			}

-			g.By(fmt.Sprintf("checking that stats can be obtained successfully"))

-			statsURL := fmt.Sprintf("http://%s:1936/;csv", routerIP)

-			req, err := requestViaReverseProxy("GET", statsURL, host)

-			req.SetBasicAuth("admin", "password")

-			resp, err := http.DefaultClient.Do(req)

+			times := 100

+			g.By(fmt.Sprintf("checking that %d requests go through successfully", times))

+			// wait for the request to stabilize

+			err = waitForRouterOKResponseExec(ns, execPodName, routerURL, "weighted.example.com", 60*2)

 			o.Expect(err).NotTo(o.HaveOccurred())

-			if resp.StatusCode != http.StatusOK {

-				e2e.Failf("unexpected response: %#v", resp.StatusCode)

-			}

-

-			g.By(fmt.Sprintf("checking that weights are respected by the router"))

-			stats, err := ioutil.ReadAll(resp.Body)

-			defer resp.Body.Close()

-			trafficValues, err := parseStats(string(stats), "weightedroute", 7)

+			// all requests should now succeed

+			err = expectRouteStatusCodeRepeatedExec(ns, execPodName, routerURL, "weighted.example.com", http.StatusOK, times)

 			o.Expect(err).NotTo(o.HaveOccurred())

-			if len(trafficValues) != 2 {

-				e2e.Failf("Expected 2 weighted backends for incoming traffic, found %d", len(trafficValues))

-			}

+			g.By(fmt.Sprintf("checking that there are two weighted backends in the router stats"))

+			var trafficValues []string

+			err = wait.PollImmediate(100*time.Millisecond, 2*time.Minute, func() (bool, error) {

+				statsURL := fmt.Sprintf("http://%s:1936/;csv", routerIP)

+				stats, err := getAuthenticatedRouteURLViaPod(ns, execPodName, statsURL, host, "admin", "password")

+				o.Expect(err).NotTo(o.HaveOccurred())

+				trafficValues, err = parseStats(stats, "weightedroute", 7)

+				o.Expect(err).NotTo(o.HaveOccurred())

+				return len(trafficValues) == 2, nil

+			})

+			o.Expect(err).NotTo(o.HaveOccurred())

 			trafficEP1, err := strconv.Atoi(trafficValues[0])

 			o.Expect(err).NotTo(o.HaveOccurred())

@@ -101,12 +99,8 @@ var _ = g.Describe("[networking][router] weighted openshift router", func() {

 			g.By(fmt.Sprintf("checking that zero weights are also respected by the router"))

 			host = "zeroweight.example.com"

-			req, _ = requestViaReverseProxy("GET", routerURL, host)

-			resp, err = http.DefaultClient.Do(req)

+			err = expectRouteStatusCodeExec(ns, execPodName, routerURL, host, http.StatusServiceUnavailable)

 			o.Expect(err).NotTo(o.HaveOccurred())

-			if resp.StatusCode != http.StatusServiceUnavailable {

-				e2e.Failf("Expected zero weighted route to return a 503, but got %v", resp.StatusCode)

-			}

 		})

 	})

 })

@@ -33,6 +33,7 @@ import (

 	"k8s.io/kubernetes/pkg/util/sets"

 	"k8s.io/kubernetes/pkg/util/uuid"

 	"k8s.io/kubernetes/pkg/util/wait"

+	"k8s.io/kubernetes/test/e2e/framework"

 	buildapi "github.com/openshift/origin/pkg/build/api"

 	"github.com/openshift/origin/pkg/client"

@@ -1216,3 +1217,43 @@ func GetPodForImage(dockerImageReference string) *kapi.Pod {

 		Image: dockerImageReference,

 	})

 }

+

+// CreateExecPodOrFail creates a simple busybox pod in a sleep loop used as a

+// vessel for kubectl exec commands.

+// Returns the name of the created pod.

+// TODO: expose upstream

+func CreateExecPodOrFail(client kcoreclient.CoreInterface, ns, name string) string {

+	framework.Logf("Creating new exec pod")

+	execPod := framework.NewHostExecPodSpec(ns, name)

+	created, err := client.Pods(ns).Create(execPod)

+	o.Expect(err).NotTo(o.HaveOccurred())

+	err = wait.PollImmediate(framework.Poll, 5*time.Minute, func() (bool, error) {

+		retrievedPod, err := client.Pods(execPod.Namespace).Get(created.Name)

+		if err != nil {

+			return false, nil

+		}

+		return retrievedPod.Status.Phase == kapi.PodRunning, nil

+	})

+	o.Expect(err).NotTo(o.HaveOccurred())

+	return created.Name

+}

+

+// CreateExecPodOnNode launches a exec pod in the given namespace and node

+// waits until it's Running, created pod name would be returned

+// TODO: expose upstream

+func CreateExecPodOnNode(client kcoreclient.CoreInterface, ns, nodeName, name string) string {

+	framework.Logf("Creating exec pod %q in namespace %q", name, ns)

+	execPod := framework.NewHostExecPodSpec(ns, name)

+	execPod.Spec.NodeName = nodeName

+	created, err := client.Pods(ns).Create(execPod)

+	o.Expect(err).NotTo(o.HaveOccurred())

+	err = wait.PollImmediate(framework.Poll, 5*time.Minute, func() (bool, error) {

+		retrievedPod, err := client.Pods(execPod.Namespace).Get(created.Name)

+		if err != nil {

+			return false, nil

+		}

+		return retrievedPod.Status.Phase == kapi.PodRunning, nil

+	})

+	o.Expect(err).NotTo(o.HaveOccurred())

+	return created.Name

+}