Browse code

Rsync fixes

Cesar Wong authored on 2015/10/28 06:28:39
Showing 6 changed files
... ...
@@ -11,7 +11,7 @@ import (
11 11
 )
12 12
 
13 13
 // copyStrategies is an ordered list of copyStrategy objects that behaves as a single
14
-// strategy.
14
+// strategy. If a strategy fails with a setup error, it continues on to the next strategy.
15 15
 type copyStrategies []copyStrategy
16 16
 
17 17
 // ensure copyStrategies implements the copyStrategy interface
... ...
@@ -1,6 +1,7 @@
1 1
 package rsync
2 2
 
3 3
 import (
4
+	"bytes"
4 5
 	"errors"
5 6
 	"io"
6 7
 	"strings"
... ...
@@ -12,10 +13,10 @@ import (
12 12
 	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
13 13
 )
14 14
 
15
-var (
16
-	testRsyncCommand = []string{"rsync", "--version"}
17
-)
18
-
15
+// rsyncStrategy implements the rsync copy strategy
16
+// The rsync strategy calls the local rsync command directly and passes the OpenShift
17
+// CLI rsh command as the remote shell command for rsync. It requires that rsync be
18
+// present in both the client machine and the remote container.
19 19
 type rsyncStrategy struct {
20 20
 	Flags          []string
21 21
 	RshCommand     string
... ...
@@ -64,15 +65,16 @@ func (r *rsyncStrategy) Copy(source, destination *pathSpec, out, errOut io.Write
64 64
 	glog.V(3).Infof("Copying files with rsync")
65 65
 	cmd := append([]string{"rsync"}, r.Flags...)
66 66
 	cmd = append(cmd, "-e", r.RshCommand, source.RsyncPath(), destination.RsyncPath())
67
-	err := r.LocalExecutor.Execute(cmd, nil, out, errOut)
67
+	errBuf := &bytes.Buffer{}
68
+	err := r.LocalExecutor.Execute(cmd, nil, out, errBuf)
68 69
 	if isExitError(err) {
69 70
 		// Determine whether rsync is present in the pod container
70
-		testRsyncErr := executeWithLogging(r.RemoteExecutor, testRsyncCommand)
71
+		testRsyncErr := checkRsync(r.RemoteExecutor)
71 72
 		if testRsyncErr != nil {
72
-			glog.V(4).Infof("error testing whether rsync is available: %v", testRsyncErr)
73 73
 			return strategySetupError("rsync not available in container")
74 74
 		}
75 75
 	}
76
+	io.Copy(errOut, errBuf)
76 77
 	return err
77 78
 }
78 79
 
... ...
@@ -44,6 +44,16 @@ var (
44 44
 	random = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
45 45
 )
46 46
 
47
+// rsyncDaemonStrategy implements the rsync-daemon strategy.
48
+// The rsync-daemon strategy uses the rsync command on the container to
49
+// to start rsync in daemon mode. It listens on a randomly selected port.
50
+// The container's port is then forwarded to the client machine so it's
51
+// accessible by the local rsync command. The local rsync command is invoked
52
+// to copy to (or from) an rsync URL using the local port. Once the copy
53
+// is finished, the port-forward is terminated, and the daemon on the
54
+// container is killed. This strategy requires thar rsync be present in
55
+// both the remote container and the local machine. It also requires that
56
+// the container allow executing a shell 'sh', cat, printf, and kill commands.
47 57
 type rsyncDaemonStrategy struct {
48 58
 	Flags          []string
49 59
 	RemoteExecutor executor
... ...
@@ -146,6 +156,10 @@ func (s *rsyncDaemonStrategy) startRemoteDaemon() error {
146 146
 	err = s.RemoteExecutor.Execute([]string{"sh"}, cmdIn, cmdOut, cmdErr)
147 147
 	if err != nil {
148 148
 		glog.V(1).Infof("Error starting rsync daemon: %v. Out: %s, Err: %s\n", err, cmdOut.String(), cmdErr.String())
149
+		// Determine whether rsync is present in the container
150
+		if checkRsync(s.RemoteExecutor) != nil {
151
+			return strategySetupError("rsync not available in container")
152
+		}
149 153
 		return err
150 154
 	}
151 155
 	s.daemonPort = port
... ...
@@ -210,18 +224,16 @@ func (s *rsyncDaemonStrategy) Copy(source, destination *pathSpec, out, errOut io
210 210
 	if err != nil {
211 211
 		if isExitError(err) {
212 212
 			return strategySetupError(fmt.Sprintf("cannot start remote rsync daemon: %v", err))
213
-		} else {
214
-			return err
215 213
 		}
214
+		return err
216 215
 	}
217 216
 	defer s.killRemoteDaemon()
218 217
 	err = s.startPortForward()
219 218
 	if err != nil {
220 219
 		if isExitError(err) {
221 220
 			return strategySetupError(fmt.Sprintf("cannot start port-forward: %v", err))
222
-		} else {
223
-			return err
224 221
 		}
222
+		return err
225 223
 	}
226 224
 	defer s.stopPortForward()
227 225
 
... ...
@@ -1,11 +1,13 @@
1 1
 package rsync
2 2
 
3 3
 import (
4
+	"bytes"
4 5
 	"errors"
5 6
 	"fmt"
6 7
 	"io"
7 8
 	"io/ioutil"
8 9
 	"os"
10
+	"path"
9 11
 	"path/filepath"
10 12
 	"strings"
11 13
 
... ...
@@ -17,6 +19,12 @@ import (
17 17
 	"github.com/openshift/origin/pkg/cmd/util/clientcmd"
18 18
 )
19 19
 
20
+// tarStrategy implements the tar copy strategy.
21
+// The tar strategy consists of creating a tar of the file contents to copy
22
+// and then streaming them to/from the container to the destination to a tar
23
+// command waiting for STDIN input. If the --delete flag is specified, the
24
+// contents of the destination directory are first cleared before the copy.
25
+// The tar strategy requires that the remote container contain the tar command.
20 26
 type tarStrategy struct {
21 27
 	Quiet          bool
22 28
 	Delete         bool
... ...
@@ -94,8 +102,13 @@ func (r *tarStrategy) Copy(source, destination *pathSpec, out, errOut io.Writer)
94 94
 		}
95 95
 	} else {
96 96
 		glog.V(4).Infof("Creating local tar file %s from remote path %s", tmp.Name(), source.Path)
97
-		err = tarRemote(r.RemoteExecutor, source.Path, tmp, errOut)
97
+		errBuf := &bytes.Buffer{}
98
+		err = tarRemote(r.RemoteExecutor, source.Path, tmp, errBuf)
98 99
 		if err != nil {
100
+			if checkTar(r.RemoteExecutor) != nil {
101
+				return strategySetupError("tar not available in container")
102
+			}
103
+			io.Copy(errOut, errBuf)
99 104
 			return fmt.Errorf("error creating remote tar of source directory: %v", err)
100 105
 		}
101 106
 	}
... ...
@@ -113,10 +126,17 @@ func (r *tarStrategy) Copy(source, destination *pathSpec, out, errOut io.Writer)
113 113
 	// Extract tar
114 114
 	if destination.Local() {
115 115
 		glog.V(4).Infof("Untarring temp file %s to local directory %s", tmp.Name(), destination.Path)
116
-		err = untarLocal(r.Tar, destination.Path, tmp)
116
+		err = untarLocal(r.Tar, destination.Path, tmp, r.Quiet, out)
117 117
 	} else {
118 118
 		glog.V(4).Infof("Untarring temp file %s to remote directory %s", tmp.Name(), destination.Path)
119
-		err = untarRemote(r.RemoteExecutor, destination.Path, r.Quiet, tmp, out, errOut)
119
+		errBuf := &bytes.Buffer{}
120
+		err = untarRemote(r.RemoteExecutor, destination.Path, r.Quiet, tmp, out, errBuf)
121
+		if err != nil {
122
+			if checkTar(r.RemoteExecutor) != nil {
123
+				return strategySetupError("tar not available in container")
124
+			}
125
+			io.Copy(errOut, errBuf)
126
+		}
120 127
 	}
121 128
 	if err != nil {
122 129
 		return fmt.Errorf("error extracting tar at destination directory: %v", err)
... ...
@@ -144,7 +164,12 @@ func (r *tarStrategy) String() string {
144 144
 
145 145
 func tarRemote(exec executor, sourceDir string, out, errOut io.Writer) error {
146 146
 	glog.V(4).Infof("Tarring %s remotely", sourceDir)
147
-	cmd := []string{"tar", "-C", sourceDir, "-c", "."}
147
+	var cmd []string
148
+	if strings.HasSuffix(sourceDir, "/") {
149
+		cmd = []string{"tar", "-C", sourceDir, "-c", "."}
150
+	} else {
151
+		cmd = []string{"tar", "-C", path.Dir(sourceDir), "-c", path.Base(sourceDir)}
152
+	}
148 153
 	glog.V(4).Infof("Remote tar command: %s", strings.Join(cmd, " "))
149 154
 	return exec.Execute(cmd, nil, out, errOut)
150 155
 }
... ...
@@ -162,9 +187,12 @@ func tarLocal(tar tar.Tar, sourceDir string, w io.Writer) error {
162 162
 	return tar.CreateTarStream(sourceDir, includeParent, w)
163 163
 }
164 164
 
165
-func untarLocal(tar tar.Tar, destinationDir string, r io.Reader) error {
165
+func untarLocal(tar tar.Tar, destinationDir string, r io.Reader, quiet bool, logger io.Writer) error {
166 166
 	glog.V(4).Infof("Extracting tar locally to %s", destinationDir)
167
-	return tar.ExtractTarStream(destinationDir, r)
167
+	if quiet {
168
+		return tar.ExtractTarStream(destinationDir, r)
169
+	}
170
+	return tar.ExtractTarStreamWithLogging(destinationDir, r, logger)
168 171
 }
169 172
 
170 173
 func untarRemote(exec executor, destinationDir string, quiet bool, in io.Reader, out, errOut io.Writer) error {
... ...
@@ -36,7 +36,7 @@ for the copy.`
36 36
   $ %[1]s POD:/remote/dir/ ./local/dir`
37 37
 
38 38
 	noRsyncUnixWarning    = "rsync command not found in path. Please use your package manager to install it."
39
-	noRsyncWindowsWarning = "rsync command not found in path. Download cwRsync for windows and add it to your path."
39
+	noRsyncWindowsWarning = "rsync command not found in path. Download cwRsync for Windows and add it to your PATH."
40 40
 )
41 41
 
42 42
 // copyStrategy
... ...
@@ -11,6 +11,11 @@ import (
11 11
 	"github.com/spf13/cobra"
12 12
 )
13 13
 
14
+var (
15
+	testRsyncCommand = []string{"rsync", "--version"}
16
+	testTarCommand   = []string{"tar", "--version"}
17
+)
18
+
14 19
 // executeWithLogging will execute a command and log its output
15 20
 func executeWithLogging(e executor, cmd []string) error {
16 21
 	w := &bytes.Buffer{}
... ...
@@ -62,3 +67,11 @@ func isExitError(err error) bool {
62 62
 	_, exitErr := err.(*exec.ExitError)
63 63
 	return exitErr
64 64
 }
65
+
66
+func checkRsync(e executor) error {
67
+	return executeWithLogging(e, testRsyncCommand)
68
+}
69
+
70
+func checkTar(e executor) error {
71
+	return executeWithLogging(e, testTarCommand)
72
+}