Browse code

builder: increase git check timeout exponentially

The check will either succeed or the build will timeout

Cesar Wong authored on 2016/06/01 20:57:00
Showing 4 changed files
... ...
@@ -48,7 +48,7 @@ func NewDockerBuilder(dockerClient DockerClient, buildsClient client.BuildInterf
48 48
 		build:        build,
49 49
 		gitClient:    gitClient,
50 50
 		tar:          tar.New(),
51
-		urlTimeout:   urlCheckTimeout,
51
+		urlTimeout:   initialURLCheckTimeout,
52 52
 		client:       buildsClient,
53 53
 		cgLimits:     cgLimits,
54 54
 	}
... ...
@@ -21,10 +21,15 @@ import (
21 21
 )
22 22
 
23 23
 const (
24
-	// urlCheckTimeout is the timeout used to check the source URL
25
-	// If fetching the URL exceeds the timeout, then the build will
26
-	// not proceed further and stop
27
-	urlCheckTimeout = 16 * time.Second
24
+	// initialURLCheckTimeout is the initial timeout used to check the
25
+	// source URL.  If fetching the URL exceeds the timeout, then a longer
26
+	// timeout will be tried until the fetch either succeeds or the build
27
+	// itself times out.
28
+	initialURLCheckTimeout = 16 * time.Second
29
+
30
+	// timeoutIncrementFactor is the factor to use when increasing
31
+	// the timeout after each unsuccessful try
32
+	timeoutIncrementFactor = 4
28 33
 )
29 34
 
30 35
 type gitAuthError string
... ...
@@ -104,8 +109,7 @@ func fetchSource(dockerClient DockerClient, dir string, build *api.Build, urlTim
104 104
 // remote repository failed to authenticate.
105 105
 // Since this is calling the 'git' binary, the proxy settings should be
106 106
 // available for this command.
107
-func checkRemoteGit(gitClient GitClient, url string, timeout time.Duration) error {
108
-	glog.V(4).Infof("git ls-remote --heads %s", url)
107
+func checkRemoteGit(gitClient GitClient, url string, initialTimeout time.Duration) error {
109 108
 
110 109
 	var (
111 110
 		out    string
... ...
@@ -113,26 +117,34 @@ func checkRemoteGit(gitClient GitClient, url string, timeout time.Duration) erro
113 113
 		err    error
114 114
 	)
115 115
 
116
-	out, errOut, err = gitClient.TimedListRemote(timeout, url, "--heads")
117
-	if _, ok := err.(*git.TimeoutError); err != nil && ok {
118
-		return fmt.Errorf("timeout while waiting for remote repository %q", url)
119
-	}
120
-
121
-	if len(out) != 0 {
122
-		glog.V(4).Infof(out)
123
-	}
124
-	if len(errOut) != 0 {
125
-		glog.V(4).Infof(errOut)
116
+	timeout := initialTimeout
117
+	for {
118
+		glog.V(4).Infof("git ls-remote --heads %s", url)
119
+		out, errOut, err = gitClient.TimedListRemote(timeout, url, "--heads")
120
+		if len(out) != 0 {
121
+			glog.V(4).Infof(out)
122
+		}
123
+		if len(errOut) != 0 {
124
+			glog.V(4).Infof(errOut)
125
+		}
126
+		if err != nil {
127
+			if _, ok := err.(*git.TimeoutError); ok {
128
+				timeout = timeout * timeoutIncrementFactor
129
+				glog.Infof("WARNING: timed out waiting for git server, will wait %s", timeout)
130
+				continue
131
+			}
132
+		}
133
+		break
126 134
 	}
127
-
128
-	combinedOut := out + errOut
129
-	switch {
130
-	case strings.Contains(combinedOut, "Authentication failed"):
131
-		return gitAuthError(url)
132
-	case strings.Contains(combinedOut, "not found"):
133
-		return gitNotFoundError(url)
135
+	if err != nil {
136
+		combinedOut := out + errOut
137
+		switch {
138
+		case strings.Contains(combinedOut, "Authentication failed"):
139
+			return gitAuthError(url)
140
+		case strings.Contains(combinedOut, "not found"):
141
+			return gitNotFoundError(url)
142
+		}
134 143
 	}
135
-
136 144
 	return err
137 145
 }
138 146
 
... ...
@@ -32,16 +32,6 @@ func TestCheckRemoteGit(t *testing.T) {
32 32
 		t.Errorf("expected gitAuthError, got %q", v)
33 33
 	}
34 34
 
35
-	t0 := time.Now()
36
-	err = checkRemoteGit(gitRepo, "https://254.254.254.254/foo/bar", 4*time.Second)
37
-	t1 := time.Now()
38
-	if err == nil || (err != nil && !strings.Contains(fmt.Sprintf("%s", err), "timeout")) {
39
-		t.Errorf("expected timeout error, got %q", err)
40
-	}
41
-	if t1.Sub(t0) > 5*time.Second {
42
-		t.Errorf("expected timeout in 4 seconds, it took %v", t1.Sub(t0))
43
-	}
44
-
45 35
 	err = checkRemoteGit(gitRepo, "https://github.com/openshift/origin", 10*time.Second)
46 36
 	if err != nil {
47 37
 		t.Errorf("unexpected error %q", err)
... ...
@@ -115,7 +115,7 @@ func (s *S2IBuilder) Build() error {
115 115
 	download := &downloader{
116 116
 		s:       s,
117 117
 		in:      os.Stdin,
118
-		timeout: urlCheckTimeout,
118
+		timeout: initialURLCheckTimeout,
119 119
 
120 120
 		dir:        srcDir,
121 121
 		contextDir: contextDir,