Browse code

Implement a simple Git server for hosting repos

Supports auto linking a build config to a git repo
mirror, as well as integrating automatic push.

Clayton Coleman authored on 2015/05/01 12:25:58
Showing 15 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,13 @@
0
+#
1
+# This is an example Git server for OpenShift Origin.
2
+#
3
+# The standard name for this image is openshift/origin-gitserver
4
+#
5
+FROM openshift/origin
6
+
7
+ADD hooks/ /var/lib/git-hooks/
8
+RUN ln -s /usr/bin/openshift /usr/bin/openshift-gitserver && \
9
+    mkdir -p /var/lib/git
10
+VOLUME /var/lib/git
11
+
12
+ENTRYPOINT ["/usr/bin/openshift-gitserver"]
0 13
new file mode 100644
... ...
@@ -0,0 +1,16 @@
0
+Configurable Git Server
1
+=======================
2
+
3
+This example provides automatic mirroring of Git repositories, intended
4
+for use within a container or Kubernetes pod. It can clone repositories
5
+from remote systems on startup as well as remotely register hooks. It
6
+can automatically initialize and receive Git directories on push.
7
+
8
+In the more advanced modes, it can integrate with an OpenShift server to
9
+automatically perform actions when new repositories are created, like
10
+reading the build configs in the current namespace and performing
11
+automatic mirroring of their input, and creating new build-configs when
12
+content is pushed.
13
+
14
+The Dockerfile built by this example is published as
15
+openshift/origin-gitserver
0 16
\ No newline at end of file
1 17
new file mode 100755
... ...
@@ -0,0 +1,48 @@
0
+#!/bin/bash
1
+#
2
+# detect-language returns a string indicating the image repository to use as a base
3
+# image for this source repository. Return "docker.io/*/*" for Docker images, a two
4
+# segment entry for a local image repository, or a single segment name to search
5
+# in the current namespace. Set a tag to qualify the version - e.g. "ruby:1.9.3",
6
+# "nodejs:0.10".
7
+#
8
+
9
+set -o errexit
10
+set -o nounset
11
+set -o pipefail
12
+
13
+function has {
14
+  [[ -n $(git ls-tree --full-name --name-only HEAD ${@:1}) ]]
15
+}
16
+function key {
17
+  git config --local --get "${1}"
18
+}
19
+
20
+prefix=${PREFIX:-openshift/}
21
+
22
+if has Gemfile; then
23
+  echo "${prefix}ruby"
24
+  exit 0
25
+fi
26
+
27
+if has requirements.txt; then
28
+  echo "${prefix}python"
29
+  exit 0
30
+fi
31
+
32
+if has package.json app.json; then
33
+  echo "${prefix}nodejs"
34
+  exit 0
35
+fi
36
+
37
+if has '*.go'; then
38
+  echo "${prefix}golang"
39
+  exit 0
40
+fi
41
+
42
+if has index.php; then
43
+  echo "${prefix}php"
44
+  exit 0
45
+fi
46
+
47
+exit 1
0 48
\ No newline at end of file
1 49
new file mode 100755
... ...
@@ -0,0 +1,54 @@
0
+#!/bin/bash
1
+
2
+set -o errexit
3
+set -o nounset
4
+set -o pipefail
5
+
6
+function key {
7
+  git config --local --get "${1}"
8
+}
9
+function addkey {
10
+  git config --local --add "${1}" "${2}"
11
+}
12
+
13
+function detect {
14
+  if detected=$(key openshift.io.detect); then
15
+    exit 0
16
+  fi
17
+  if ! url=$(key gitserver.self.url); then
18
+    echo "detect: no self url set"
19
+    exit 0
20
+  fi
21
+
22
+  # TODO: make it easier to find the build config name created
23
+  # by osc new-app
24
+  name=$(basename "${url}")
25
+  name="${name%.*}"
26
+
27
+  if ! lang=$($(dirname $0)/detect-language); then
28
+    exit 0
29
+  fi
30
+  echo "detect: found language ${lang} for ${name}"
31
+
32
+  if ! osc=$(which osc); then
33
+    echo "detect: osc is not installed"
34
+    addkey openshift.io.detect 1
35
+    exit 0
36
+  fi
37
+  osc new-app "${lang}~${url}"
38
+  if webhook=$(osc start-build --list-webhooks="generic" "${name}" | head -n 1); then
39
+    addkey openshift.io.webhook "${webhook}"
40
+  fi
41
+  addkey openshift.io.detect 1
42
+}
43
+
44
+cat > /tmp/postreceived
45
+
46
+detect
47
+
48
+if webhook=$(key openshift.io.webhook); then
49
+  # TODO: print output from the server about the hook status
50
+  osc start-build --from-webhook="${webhook}"
51
+  # TODO: follow logs
52
+  echo "build: started"
53
+fi
0 54
new file mode 100644
... ...
@@ -0,0 +1,23 @@
0
+package main
1
+
2
+import (
3
+	"fmt"
4
+	"log"
5
+	"os"
6
+
7
+	"github.com/openshift/origin/pkg/gitserver"
8
+)
9
+
10
+func main() {
11
+	if len(os.Args) != 1 {
12
+		fmt.Printf(`git-server - Expose Git repositories to the network
13
+
14
+%[1]s`, gitserver.EnvironmentHelp)
15
+		os.Exit(0)
16
+	}
17
+	config, err := gitserver.NewEnviromentConfig()
18
+	if err != nil {
19
+		log.Fatal(err)
20
+	}
21
+	log.Fatal(gitserver.Start(config))
22
+}
0 23
new file mode 100644
... ...
@@ -0,0 +1,74 @@
0
+apiVersion: v1beta3
1
+kind: List
2
+items:
3
+- apiVersion: v1beta3
4
+  kind: DeploymentConfig
5
+  metadata:
6
+    name: gitserver
7
+  spec:
8
+    triggers:
9
+    - type: ConfigChange
10
+    replicas: 1
11
+    selector:
12
+      run-container: gitserver
13
+    template:
14
+      metadata:
15
+        labels:
16
+          run-container: gitserver
17
+      spec:
18
+        containers:
19
+        - name: gitserver
20
+          image: openshift/origin-gitserver
21
+          ports:
22
+          - containerPort: 8080
23
+          env:
24
+          - name: PUBLIC_URL
25
+            value: "http://gitserver.myproject.local:8080" # TODO: this needs to be resolved from env
26
+          - name: GIT_HOME
27
+            value: /var/lib/git
28
+          - name: HOOK_PATH
29
+            value: /var/lib/git-hooks
30
+          - name: ALLOW_GIT_PUSH
31
+            value: "yes"
32
+          - name: ALLOW_GIT_HOOKS
33
+            value: "yes"
34
+          - name: ALLOW_LAZY_CREATE
35
+            value: "yes"
36
+          - name: AUTOLINK_CONFIG
37
+            value: /var/lib/gitsecrets/admin.kubeconfig # TODO: use the service account secret
38
+          - name: AUTOLINK_NAMESPACE
39
+            #value: # TODO: use env generation
40
+          - name: AUTOLINK_HOOK
41
+            value:
42
+          - name: REQUIRE_GIT_AUTH
43
+            #value: user:password # if set, authentication is required to push to this server
44
+          #- name: GIT_INITIAL_CLONE_1
45
+          #  value:
46
+          - name: OPENSHIFTCONFIG
47
+            value: /var/lib/gitsecrets/admin.kubeconfig # TODO: use the service account secret
48
+          volumeMounts:
49
+          - name: config
50
+            mountPath: /var/lib/gitsecrets/
51
+            readOnly: true
52
+        volumes:
53
+        - name: config
54
+          secret:
55
+            secretName: gitserver-config
56
+- apiVersion: v1beta3
57
+  kind: Secret
58
+  metadata:
59
+    name: gitserver-config
60
+  spec:
61
+    data:
62
+      # Needs to be populated
63
+      admin.kubeconfig:
64
+- apiVersion: v1beta3
65
+  kind: Service
66
+  metadata:
67
+    name: gitserver
68
+  spec:
69
+    selector:
70
+      run-container: gitserver
71
+    ports:
72
+    - port: 8080
73
+      targetPort: 8080
... ...
@@ -59,6 +59,7 @@ image openshift/origin-docker-registry       images/dockerregistry
59 59
 # images that depend on openshift/origin
60 60
 image openshift/origin-deployer              images/deployer
61 61
 image openshift/origin-docker-builder        images/builder/docker/docker-builder
62
+image openshift/origin-gitserver             examples/gitserver
62 63
 image openshift/origin-sti-builder           images/builder/docker/sti-builder
63 64
 # extra images (not part of infrastructure)
64 65
 image openshift/hello-openshift              examples/hello-openshift
... ...
@@ -55,6 +55,7 @@ readonly OPENSHIFT_BINARY_SYMLINKS=(
55 55
   openshift-deploy
56 56
   openshift-sti-build
57 57
   openshift-docker-build
58
+  openshift-gitserver
58 59
   osc
59 60
   osadm
60 61
 )
... ...
@@ -1,3 +1,9 @@
1
+#
2
+# This is the integrated OpenShift Origin Docker registry. It is configured to
3
+# publish metadata to OpenShift to provide automatic management of images on push.
4
+#
5
+# The standard name for this image is openshift/origin-docker-registry
6
+#
1 7
 FROM openshift/origin-base
2 8
 
3 9
 ADD config.yml /config.yml
4 10
new file mode 100644
... ...
@@ -0,0 +1,61 @@
0
+package gitserver
1
+
2
+import (
3
+	"fmt"
4
+	"log"
5
+	"net/url"
6
+
7
+	cmdutil "github.com/GoogleCloudPlatform/kubernetes/pkg/kubectl/cmd/util"
8
+	"github.com/spf13/cobra"
9
+
10
+	"github.com/openshift/origin/pkg/gitserver"
11
+	"github.com/openshift/origin/pkg/gitserver/autobuild"
12
+)
13
+
14
+const longCommandDesc = `
15
+Start a Git server
16
+
17
+This command launches a Git HTTP/HTTPS server that supports push and pull, mirroring,
18
+and automatic creation of OpenShift applications on push.
19
+
20
+%[1]s
21
+`
22
+
23
+// NewCommandGitServer launches a Git server
24
+func NewCommandGitServer(name string) *cobra.Command {
25
+	cmd := &cobra.Command{
26
+		Use:   name,
27
+		Short: "Start a Git server",
28
+		Long:  fmt.Sprintf(longCommandDesc, gitserver.EnvironmentHelp),
29
+		Run: func(c *cobra.Command, args []string) {
30
+			err := RunGitServer()
31
+			cmdutil.CheckErr(err)
32
+		},
33
+	}
34
+
35
+	return cmd
36
+}
37
+
38
+func RunGitServer() error {
39
+	config, err := gitserver.NewEnviromentConfig()
40
+	if err != nil {
41
+		return err
42
+	}
43
+	link, err := autobuild.NewAutoLinkBuildsFromEnvironment()
44
+	switch {
45
+	case err == autobuild.ErrNotEnabled:
46
+	case err != nil:
47
+		log.Fatal(err)
48
+	default:
49
+		link.LinkFn = func(name string) *url.URL { return gitserver.RepositoryURL(config, name, nil) }
50
+		clones, err := link.Link()
51
+		if err != nil {
52
+			log.Printf("error: %v", err)
53
+			break
54
+		}
55
+		for name, v := range clones {
56
+			config.InitialClones[name] = v
57
+		}
58
+	}
59
+	return gitserver.Start(config)
60
+}
... ...
@@ -20,6 +20,7 @@ import (
20 20
 	"github.com/openshift/origin/pkg/cmd/flagtypes"
21 21
 	"github.com/openshift/origin/pkg/cmd/infra/builder"
22 22
 	"github.com/openshift/origin/pkg/cmd/infra/deployer"
23
+	"github.com/openshift/origin/pkg/cmd/infra/gitserver"
23 24
 	"github.com/openshift/origin/pkg/cmd/infra/router"
24 25
 	"github.com/openshift/origin/pkg/cmd/server/start"
25 26
 	"github.com/openshift/origin/pkg/cmd/templates"
... ...
@@ -59,6 +60,8 @@ func CommandFor(basename string) *cobra.Command {
59 59
 		cmd = builder.NewCommandSTIBuilder(basename)
60 60
 	case "openshift-docker-build":
61 61
 		cmd = builder.NewCommandDockerBuilder(basename)
62
+	case "openshift-gitserver":
63
+		cmd = gitserver.NewCommandGitServer(basename)
62 64
 	case "osc":
63 65
 		cmd = cli.NewCommandCLI(basename, basename)
64 66
 	case "osadm":
... ...
@@ -104,6 +107,7 @@ func NewCommandOpenShift() *cobra.Command {
104 104
 		deployer.NewCommandDeployer("deploy"),
105 105
 		builder.NewCommandSTIBuilder("sti-build"),
106 106
 		builder.NewCommandDockerBuilder("docker-build"),
107
+		gitserver.NewCommandGitServer("git-server"),
107 108
 	)
108 109
 	root.AddCommand(infra)
109 110
 
110 111
new file mode 100644
... ...
@@ -0,0 +1,194 @@
0
+package autobuild
1
+
2
+import (
3
+	"fmt"
4
+	"net/url"
5
+	"os"
6
+	"path/filepath"
7
+
8
+	kapi "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
9
+	kclient "github.com/GoogleCloudPlatform/kubernetes/pkg/client"
10
+	kclientcmd "github.com/GoogleCloudPlatform/kubernetes/pkg/client/clientcmd"
11
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
12
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
13
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/errors"
14
+
15
+	buildapi "github.com/openshift/origin/pkg/build/api"
16
+	"github.com/openshift/origin/pkg/client"
17
+	"github.com/openshift/origin/pkg/generate/git"
18
+	"github.com/openshift/origin/pkg/gitserver"
19
+)
20
+
21
+type AutoLinkBuilds struct {
22
+	Namespaces []string
23
+	Builders   []kapi.ObjectReference
24
+	Client     client.BuildConfigsNamespacer
25
+
26
+	CurrentNamespace string
27
+
28
+	PostReceiveHook string
29
+
30
+	LinkFn func(name string) *url.URL
31
+}
32
+
33
+var ErrNotEnabled = fmt.Errorf("not enabled")
34
+
35
+func NewAutoLinkBuildsFromEnvironment() (*AutoLinkBuilds, error) {
36
+	config := &AutoLinkBuilds{}
37
+
38
+	file := os.Getenv("AUTOLINK_CONFIG")
39
+	if len(file) == 0 {
40
+		return nil, ErrNotEnabled
41
+	}
42
+	clientConfig, namespace, err := clientFromConfig(file)
43
+	if err != nil {
44
+		return nil, err
45
+	}
46
+	client, err := client.New(clientConfig)
47
+	if err != nil {
48
+		return nil, err
49
+	}
50
+	config.Client = client
51
+
52
+	if value := os.Getenv("AUTOLINK_NAMESPACE"); len(value) > 0 {
53
+		namespace = value
54
+	}
55
+	if len(namespace) == 0 {
56
+		return nil, ErrNotEnabled
57
+	}
58
+
59
+	if value := os.Getenv("AUTOLINK_HOOK"); len(value) > 0 {
60
+		abs, err := filepath.Abs(value)
61
+		if err != nil {
62
+			return nil, err
63
+		}
64
+		if _, err := os.Stat(abs); err != nil {
65
+			return nil, err
66
+		}
67
+		config.PostReceiveHook = abs
68
+	}
69
+
70
+	config.Namespaces = []string{namespace}
71
+	config.CurrentNamespace = namespace
72
+	return config, nil
73
+}
74
+
75
+func clientFromConfig(path string) (*kclient.Config, string, error) {
76
+	rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: path}
77
+	credentials, err := rules.Load()
78
+	if err != nil {
79
+		return nil, "", fmt.Errorf("the provided credentials %q could not be loaded: %v", path, err)
80
+	}
81
+	cfg := kclientcmd.NewDefaultClientConfig(*credentials, &kclientcmd.ConfigOverrides{})
82
+	config, err := cfg.ClientConfig()
83
+	if err != nil {
84
+		return nil, "", fmt.Errorf("the provided credentials %q could not be used: %v", path, err)
85
+	}
86
+	namespace, _ := cfg.Namespace()
87
+	return config, namespace, nil
88
+}
89
+
90
+func (a *AutoLinkBuilds) Link() (map[string]gitserver.Clone, error) {
91
+	errs := []error{}
92
+	builders := []*buildapi.BuildConfig{}
93
+	for _, namespace := range a.Namespaces {
94
+		list, err := a.Client.BuildConfigs(namespace).List(labels.Everything(), fields.Everything())
95
+		if err != nil {
96
+			errs = append(errs, err)
97
+			continue
98
+		}
99
+		for i := range list.Items {
100
+			builders = append(builders, &list.Items[i])
101
+		}
102
+	}
103
+	for _, b := range a.Builders {
104
+		if hasItem(builders, b) {
105
+			continue
106
+		}
107
+		config, err := a.Client.BuildConfigs(b.Namespace).Get(b.Name)
108
+		if err != nil {
109
+			errs = append(errs, err)
110
+			continue
111
+		}
112
+		builders = append(builders, config)
113
+	}
114
+
115
+	hooks := make(map[string]string)
116
+	if len(a.PostReceiveHook) > 0 {
117
+		hooks["post-receive"] = a.PostReceiveHook
118
+	}
119
+
120
+	clones := make(map[string]gitserver.Clone)
121
+	for _, builder := range builders {
122
+		source := builder.Parameters.Source.Git
123
+		if source == nil {
124
+			continue
125
+		}
126
+		if builder.Annotations == nil {
127
+			builder.Annotations = make(map[string]string)
128
+		}
129
+
130
+		// calculate the origin URL
131
+		uri := source.URI
132
+		if value, ok := builder.Annotations["git.openshift.io/origin-url"]; ok {
133
+			uri = value
134
+		}
135
+		if len(uri) == 0 {
136
+			continue
137
+		}
138
+		origin, err := git.ParseRepository(uri)
139
+		if err != nil {
140
+			errs = append(errs, err)
141
+			continue
142
+		}
143
+
144
+		// calculate the local repository name and self URL
145
+		name := builder.Name
146
+		if a.CurrentNamespace != builder.Namespace {
147
+			name = fmt.Sprintf("%s.%s", builder.Namespace, name)
148
+		}
149
+		name = fmt.Sprintf("%s.git", name)
150
+		self := a.LinkFn(name)
151
+		if self == nil {
152
+			errs = append(errs, fmt.Errorf("no self URL available, can't update %s", name))
153
+			continue
154
+		}
155
+
156
+		// we can't clone from ourself
157
+		if self.Host == origin.Host {
158
+			continue
159
+		}
160
+
161
+		// update the existing builder
162
+		changed := false
163
+		if builder.Annotations["git.openshift.io/origin-url"] != origin.String() {
164
+			builder.Annotations["git.openshift.io/origin-url"] = origin.String()
165
+			changed = true
166
+		}
167
+		if source.URI != self.String() {
168
+			source.URI = self.String()
169
+			changed = true
170
+		}
171
+		if changed {
172
+			if _, err := a.Client.BuildConfigs(builder.Namespace).Update(builder); err != nil {
173
+				errs = append(errs, err)
174
+				continue
175
+			}
176
+		}
177
+
178
+		clones[name] = gitserver.Clone{
179
+			URL:   *origin,
180
+			Hooks: hooks,
181
+		}
182
+	}
183
+	return clones, errors.NewAggregate(errs)
184
+}
185
+
186
+func hasItem(items []*buildapi.BuildConfig, item kapi.ObjectReference) bool {
187
+	for _, c := range items {
188
+		if c.Namespace == item.Namespace && c.Name == item.Name {
189
+			return true
190
+		}
191
+	}
192
+	return false
193
+}
0 194
new file mode 100644
... ...
@@ -0,0 +1,298 @@
0
+// Package gitserver provides a smart Git HTTP server that can also set and
1
+// remove hooks. The server is lightweight (<7M compiled with a ~2M footprint)
2
+// and can mirror remote repositories in a containerized environment.
3
+package gitserver
4
+
5
+import (
6
+	"fmt"
7
+	"log"
8
+	"net/http"
9
+	"net/url"
10
+	"os"
11
+	"os/exec"
12
+	"path/filepath"
13
+	"regexp"
14
+	"strings"
15
+
16
+	"github.com/AaronO/go-git-http"
17
+	"github.com/AaronO/go-git-http/auth"
18
+	"github.com/GoogleCloudPlatform/kubernetes/pkg/healthz"
19
+	"github.com/prometheus/client_golang/prometheus"
20
+
21
+	"github.com/openshift/origin/pkg/generate/git"
22
+)
23
+
24
+const (
25
+	initialClonePrefix = "GIT_INITIAL_CLONE_"
26
+	EnvironmentHelp    = `Supported environment variables:
27
+GIT_HOME
28
+  directory containing Git repositories; defaults to current directory
29
+PUBLIC_URL
30
+  the url of this server for constructing URLs that point to this repository
31
+GIT_PATH
32
+  path to Git binary; defaults to location of 'git' in PATH
33
+HOOK_PATH
34
+  path to a directory containing hooks for all repositories; if not set no global hooks will be used
35
+ALLOW_GIT_PUSH
36
+  if 'no', pushes will be not be accepted; defaults to true
37
+ALLOW_GIT_HOOKS
38
+  if 'no', hooks cannot be read or set; defaults to true
39
+ALLOW_LAZY_CREATE
40
+  if 'no', repositories will not automatically be initialized on push; defaults to true
41
+REQUIRE_GIT_AUTH
42
+  a user/password combination required to access the repo of the form "<user>:<password>"; defaults to none
43
+GIT_FORCE_CLEAN
44
+  if 'yes', any initial repository directories will be deleted prior to start; defaults to no
45
+  WARNING: this is destructive and you will lose any data you have already pushed
46
+GIT_INITIAL_CLONE_*=<url>[;<name>]
47
+  each environment variable in this pattern will be cloned when the process starts; failures will be logged
48
+  <name> must be [A-Z0-9_\-\.], the cloned directory name will be lowercased. If the name is invalid the
49
+  process will halt. If the repository already exists on disk, it will be updated from the remote.
50
+`
51
+)
52
+
53
+var (
54
+	invalidCloneNameChars = regexp.MustCompile("[^a-zA-Z0-9_\\-\\.]")
55
+	reservedNames         = map[string]struct{}{"_": {}}
56
+
57
+	eventCounter = prometheus.NewCounterVec(
58
+		prometheus.CounterOpts{
59
+			Name: "git_event_count",
60
+			Help: "Counter of events broken out for each repository and type",
61
+		},
62
+		[]string{"repository", "type"},
63
+	)
64
+)
65
+
66
+func init() {
67
+	prometheus.MustRegister(eventCounter)
68
+}
69
+
70
+// Config represents the configuration to use for running the server
71
+type Config struct {
72
+	Home      string
73
+	GitBinary string
74
+	URL       *url.URL
75
+
76
+	AllowHooks      bool
77
+	AllowPush       bool
78
+	AllowLazyCreate bool
79
+
80
+	HookDirectory string
81
+	MaxHookBytes  int64
82
+
83
+	Listen string
84
+
85
+	AuthenticatorFn func(http http.Handler) http.Handler
86
+
87
+	CleanBeforeClone bool
88
+	InitialClones    map[string]Clone
89
+}
90
+
91
+// Clone is a repository to clone
92
+type Clone struct {
93
+	URL   url.URL
94
+	Hooks map[string]string
95
+}
96
+
97
+// NewDefaultConfig returns a default server config.
98
+func NewDefaultConfig() *Config {
99
+	return &Config{
100
+		Home:         "",
101
+		GitBinary:    "git",
102
+		Listen:       ":8080",
103
+		MaxHookBytes: 50 * 1024,
104
+	}
105
+}
106
+
107
+// NewEnviromentConfig sets up the initial config from environment variables
108
+func NewEnviromentConfig() (*Config, error) {
109
+	config := NewDefaultConfig()
110
+
111
+	home := os.Getenv("GIT_HOME")
112
+	if len(home) == 0 {
113
+		return nil, fmt.Errorf("GIT_HOME is required")
114
+	}
115
+	abs, err := filepath.Abs(home)
116
+	if err != nil {
117
+		return nil, fmt.Errorf("Can't make %q absolute: %v", home, err)
118
+	}
119
+	if stat, err := os.Stat(abs); err != nil || !stat.IsDir() {
120
+		return nil, fmt.Errorf("GIT_HOME must be an existing directory: %v", err)
121
+	}
122
+	config.Home = home
123
+
124
+	if publicURL := os.Getenv("PUBLIC_URL"); len(publicURL) > 0 {
125
+		valid, err := url.Parse(publicURL)
126
+		if err != nil {
127
+			return nil, fmt.Errorf("PUBLIC_URL must be a valid URL: %v", err)
128
+		}
129
+		config.URL = valid
130
+	}
131
+
132
+	gitpath := os.Getenv("GIT_PATH")
133
+	if len(gitpath) == 0 {
134
+		path, err := exec.LookPath("git")
135
+		if err != nil {
136
+			return nil, fmt.Errorf("could not find 'git' in PATH; specify GIT_PATH or set your PATH")
137
+		}
138
+		gitpath = path
139
+	}
140
+	config.GitBinary = gitpath
141
+
142
+	config.AllowPush = os.Getenv("ALLOW_GIT_PUSH") != "no"
143
+	config.AllowHooks = os.Getenv("ALLOW_GIT_HOOKS") != "no"
144
+	config.AllowLazyCreate = os.Getenv("ALLOW_LAZY_CREATE") != "no"
145
+
146
+	if hookpath := os.Getenv("HOOK_PATH"); len(hookpath) != 0 {
147
+		path, err := filepath.Abs(hookpath)
148
+		if err != nil {
149
+			return nil, fmt.Errorf("HOOK_PATH was set but cannot be made absolute: %v", err)
150
+		}
151
+		if stat, err := os.Stat(path); err != nil || !stat.IsDir() {
152
+			return nil, fmt.Errorf("HOOK_PATH must be an existing directory if set: %v", err)
153
+		}
154
+		config.HookDirectory = path
155
+	}
156
+
157
+	if value := os.Getenv("REQUIRE_GIT_AUTH"); len(value) > 0 {
158
+		parts := strings.Split(value, ":")
159
+		if len(parts) != 2 {
160
+			return nil, fmt.Errorf("REQUIRE_GIT_AUTH must be a username and password separated by a ':'")
161
+		}
162
+		username, password := parts[0], parts[1]
163
+		config.AuthenticatorFn = auth.Authenticator(func(info auth.AuthInfo) (bool, error) {
164
+			if info.Push && !config.AllowPush {
165
+				return false, nil
166
+			}
167
+			if info.Username != username || info.Password != password {
168
+				return false, nil
169
+			}
170
+			return true, nil
171
+		})
172
+	}
173
+
174
+	if value := os.Getenv("GIT_LISTEN"); len(value) > 0 {
175
+		config.Listen = value
176
+	}
177
+
178
+	config.CleanBeforeClone = os.Getenv("GIT_FORCE_CLEAN") == "yes"
179
+
180
+	clones := make(map[string]Clone)
181
+	for _, env := range os.Environ() {
182
+		if !strings.HasPrefix(env, initialClonePrefix) {
183
+			continue
184
+		}
185
+		parts := strings.SplitN(env, "=", 2)
186
+		if len(parts) != 2 {
187
+			continue
188
+		}
189
+		key, value := parts[0], parts[1]
190
+		part := key[len(initialClonePrefix):]
191
+		if len(part) == 0 {
192
+			continue
193
+		}
194
+		if len(value) == 0 {
195
+			return nil, fmt.Errorf("%s must not have an empty value", key)
196
+		}
197
+
198
+		defaultName := strings.Replace(strings.ToLower(part), "_", "-", -1)
199
+		values := strings.Split(value, ";")
200
+
201
+		var uri, name string
202
+		switch len(values) {
203
+		case 1:
204
+			uri, name = values[0], ""
205
+		case 2:
206
+			uri, name = values[0], values[1]
207
+			if len(name) == 0 {
208
+				return nil, fmt.Errorf("%s name may not be empty", key)
209
+			}
210
+		default:
211
+			return nil, fmt.Errorf("%s may only have two segments (<url> or <url>;<name>)", key)
212
+		}
213
+
214
+		url, err := git.ParseRepository(uri)
215
+		if err != nil {
216
+			return nil, fmt.Errorf("%s is not a valid repository URI: %v", key, err)
217
+		}
218
+		switch url.Scheme {
219
+		case "http", "https", "git", "ssh":
220
+		default:
221
+			return nil, fmt.Errorf("%s %q must be a http, https, git, or ssh URL", key, uri)
222
+		}
223
+
224
+		if len(name) == 0 {
225
+			if n, ok := git.NameFromRepositoryURL(url); ok {
226
+				name = n + ".git"
227
+			}
228
+		}
229
+		if len(name) == 0 {
230
+			name = defaultName + ".git"
231
+		}
232
+
233
+		if invalidCloneNameChars.MatchString(name) {
234
+			return nil, fmt.Errorf("%s name %q must be only letters, numbers, dashes, or underscores", key, name)
235
+		}
236
+		if _, ok := reservedNames[name]; ok {
237
+			return nil, fmt.Errorf("%s name %q is reserved (%v)", key, name, reservedNames)
238
+		}
239
+
240
+		clones[name] = Clone{
241
+			URL: *url,
242
+		}
243
+	}
244
+	config.InitialClones = clones
245
+
246
+	return config, nil
247
+}
248
+
249
+func handler(config *Config) http.Handler {
250
+	git := githttp.New(config.Home)
251
+	git.GitBinPath = config.GitBinary
252
+	git.UploadPack = config.AllowPush
253
+	git.ReceivePack = config.AllowPush
254
+	git.EventHandler = func(ev githttp.Event) {
255
+		path := ev.Dir
256
+		if strings.HasPrefix(path, config.Home+"/") {
257
+			path = path[len(config.Home)+1:]
258
+		}
259
+		eventCounter.WithLabelValues(path, ev.Type.String()).Inc()
260
+	}
261
+	handler := http.Handler(git)
262
+
263
+	if config.AllowLazyCreate {
264
+		handler = lazyInitRepositoryHandler(config, handler)
265
+	}
266
+
267
+	if config.AuthenticatorFn != nil {
268
+		handler = config.AuthenticatorFn(handler)
269
+	}
270
+	return handler
271
+}
272
+
273
+func Start(config *Config) error {
274
+	if err := clone(config); err != nil {
275
+		return err
276
+	}
277
+	handler := handler(config)
278
+
279
+	ops := http.NewServeMux()
280
+	if config.AllowHooks {
281
+		ops.Handle("/hooks/", prometheus.InstrumentHandler("hooks", http.StripPrefix("/hooks", hooksHandler(config))))
282
+	}
283
+	/*ops.Handle("/reflect/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
284
+		defer r.Body.Close()
285
+		fmt.Fprintf(os.Stdout, "%s %s\n", r.Method, r.URL)
286
+		io.Copy(os.Stdout, r.Body)
287
+	}))*/
288
+	ops.Handle("/metrics", prometheus.UninstrumentedHandler())
289
+	healthz.InstallHandler(ops)
290
+
291
+	mux := http.NewServeMux()
292
+	mux.Handle("/", prometheus.InstrumentHandler("git", handler))
293
+	mux.Handle("/_/", http.StripPrefix("/_", ops))
294
+
295
+	log.Printf("Serving %s on %s", config.Home, config.Listen)
296
+	return http.ListenAndServe(config.Listen, mux)
297
+}
0 298
new file mode 100644
... ...
@@ -0,0 +1,89 @@
0
+package gitserver
1
+
2
+import (
3
+	"fmt"
4
+	"io"
5
+	"log"
6
+	"net/http"
7
+	"os"
8
+	"path/filepath"
9
+	"strings"
10
+)
11
+
12
+func hooksHandler(config *Config) http.Handler {
13
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
14
+		segments := strings.Split(r.URL.Path[1:], "/")
15
+		for _, s := range segments {
16
+			if len(s) == 0 || s == "." || s == ".." {
17
+				http.NotFound(w, r)
18
+				return
19
+			}
20
+		}
21
+		if !config.AllowPush {
22
+			http.Error(w, "Forbidden", http.StatusForbidden)
23
+			return
24
+		}
25
+
26
+		switch len(segments) {
27
+		case 2:
28
+			path := filepath.Join(config.Home, segments[0], "hooks", segments[1])
29
+			if segments[0] == "hooks" {
30
+				path = filepath.Join(config.HookDirectory, segments[1])
31
+			}
32
+
33
+			switch r.Method {
34
+			// TODO: support HEAD or prevent GET for security
35
+			case "GET":
36
+				w.Header().Set("Content-Type", "text/plain")
37
+				http.ServeFile(w, r, path)
38
+
39
+			case "DELETE":
40
+				if err := os.Remove(path); err != nil {
41
+					log.Printf("error: attempted to remove %s: %v", path, err)
42
+				}
43
+				w.WriteHeader(http.StatusNoContent)
44
+
45
+			case "PUT":
46
+				if stat, err := os.Stat(path); err == nil {
47
+					if stat.IsDir() || (stat.Mode()&0111) == 0 {
48
+						http.Error(w, fmt.Errorf("only executable hooks can be changed: %v", stat).Error(), http.StatusInternalServerError)
49
+						return
50
+					}
51
+					// unsymlink and overwrite
52
+					if (stat.Mode() & os.ModeSymlink) != 0 {
53
+						os.Remove(path)
54
+					}
55
+				}
56
+				f, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0750)
57
+				if err != nil {
58
+					http.Error(w, fmt.Errorf("unable to open hook file: %v", err).Error(), http.StatusInternalServerError)
59
+					return
60
+				}
61
+				defer f.Close()
62
+				max := config.MaxHookBytes + 1
63
+				body := io.LimitReader(r.Body, max)
64
+				buf := make([]byte, max)
65
+				n, err := io.ReadFull(body, buf)
66
+				if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
67
+					http.Error(w, fmt.Errorf("unable to read hook: %v", err).Error(), http.StatusInternalServerError)
68
+					return
69
+				}
70
+				if int64(n) == max {
71
+					http.Error(w, fmt.Errorf("hook was too long, truncated to %d bytes", config.MaxHookBytes).Error(), 422)
72
+				} else {
73
+					w.WriteHeader(http.StatusOK)
74
+				}
75
+				if _, err := f.Write(buf[:n]); err != nil {
76
+					http.Error(w, fmt.Errorf("unable to write hook: %v", err).Error(), http.StatusInternalServerError)
77
+					return
78
+				}
79
+
80
+			default:
81
+				http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
82
+			}
83
+
84
+		default:
85
+			http.NotFound(w, r)
86
+		}
87
+	})
88
+}
0 89
new file mode 100644
... ...
@@ -0,0 +1,213 @@
0
+package gitserver
1
+
2
+import (
3
+	"fmt"
4
+	"io/ioutil"
5
+	"log"
6
+	"net/http"
7
+	"net/url"
8
+	"os"
9
+	"os/exec"
10
+	"path/filepath"
11
+	"regexp"
12
+	"strings"
13
+
14
+	"github.com/openshift/origin/pkg/generate/git"
15
+)
16
+
17
+var lazyInitMatch = regexp.MustCompile("^/([^\\/]+?)/info/refs$")
18
+
19
+// lazyInitRepositoryHandler creates a handler that will initialize a Git repository
20
+// if it does not yet exist.
21
+func lazyInitRepositoryHandler(config *Config, handler http.Handler) http.Handler {
22
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
23
+		if r.Method != "GET" {
24
+			handler.ServeHTTP(w, r)
25
+			return
26
+		}
27
+		match := lazyInitMatch.FindStringSubmatch(r.URL.Path)
28
+		if match == nil {
29
+			handler.ServeHTTP(w, r)
30
+			return
31
+		}
32
+		name := match[1]
33
+		if name == "." || name == ".." {
34
+			handler.ServeHTTP(w, r)
35
+			return
36
+		}
37
+		path := filepath.Join(config.Home, name)
38
+		_, err := os.Stat(path)
39
+		if !os.IsNotExist(err) {
40
+			handler.ServeHTTP(w, r)
41
+			return
42
+		}
43
+
44
+		self := RepositoryURL(config, name, r)
45
+		log.Printf("Lazily initializing bare repository %s", self.String())
46
+
47
+		defaultHooks, err := loadHooks(config.HookDirectory)
48
+		if err != nil {
49
+			log.Printf("error: unable to load default hooks: %v", err)
50
+			http.Error(w, fmt.Sprintf("unable to initialize repository: %v", err), http.StatusInternalServerError)
51
+			return
52
+		}
53
+
54
+		// TODO: capture init hook output for Git
55
+		if _, err := newRepository(config, path, defaultHooks, self, nil); err != nil {
56
+			log.Printf("error: unable to initialize repo %s: %v", path, err)
57
+			http.Error(w, fmt.Sprintf("unable to initialize repository: %v", err), http.StatusInternalServerError)
58
+			os.RemoveAll(path)
59
+			return
60
+		}
61
+		eventCounter.WithLabelValues(name, "init").Inc()
62
+
63
+		handler.ServeHTTP(w, r)
64
+	})
65
+}
66
+
67
+// RepositoryURL creates the public URL for the named git repo. If both config.URL and
68
+// request are nil, the returned URL will be nil.
69
+func RepositoryURL(config *Config, name string, r *http.Request) *url.URL {
70
+	var url url.URL
71
+	switch {
72
+	case config.URL != nil:
73
+		url = *config.URL
74
+	case r != nil:
75
+		url = *r.URL
76
+		url.Host = r.Host
77
+		url.Scheme = "http"
78
+	default:
79
+		return nil
80
+	}
81
+	url.Path = "/" + name
82
+	url.RawQuery = ""
83
+	url.Fragment = ""
84
+	return &url
85
+}
86
+
87
+func newRepository(config *Config, path string, hooks map[string]string, self *url.URL, origin *url.URL) ([]byte, error) {
88
+	var out []byte
89
+	repo := git.NewRepositoryForBinary(config.GitBinary)
90
+
91
+	if origin != nil {
92
+		if err := repo.CloneMirror(path, origin.String()); err != nil {
93
+			return out, err
94
+		}
95
+	} else {
96
+		if err := repo.Init(path, true); err != nil {
97
+			return out, err
98
+		}
99
+	}
100
+
101
+	if self != nil {
102
+		if err := repo.AddLocalConfig(path, "gitserver.self.url", self.String()); err != nil {
103
+			return out, err
104
+		}
105
+	}
106
+
107
+	// remove all sample hooks, ignore errors here
108
+	if files, err := ioutil.ReadDir(filepath.Join(path, "hooks")); err == nil {
109
+		for _, file := range files {
110
+			os.Remove(filepath.Join(path, "hooks", file.Name()))
111
+		}
112
+	}
113
+
114
+	for name, hook := range hooks {
115
+		dest := filepath.Join(path, "hooks", name)
116
+		if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
117
+			return out, err
118
+		}
119
+		if err := os.Symlink(hook, dest); err != nil {
120
+			return out, err
121
+		}
122
+	}
123
+
124
+	if initHook, ok := hooks["init"]; ok {
125
+		cmd := exec.Command(initHook)
126
+		cmd.Dir = path
127
+		result, err := cmd.CombinedOutput()
128
+		if err != nil {
129
+			return out, fmt.Errorf("init hook failed: %v\n%s", err, string(result))
130
+		}
131
+		out = result
132
+	}
133
+
134
+	return out, nil
135
+}
136
+
137
+// clone clones the provided git repositories
138
+func clone(config *Config) error {
139
+	defaultHooks, err := loadHooks(config.HookDirectory)
140
+	if err != nil {
141
+		return err
142
+	}
143
+
144
+	errs := []error{}
145
+	for name, v := range config.InitialClones {
146
+		hooks := mergeHooks(defaultHooks, v.Hooks)
147
+		url := v.URL
148
+		url.Fragment = ""
149
+		path := filepath.Join(config.Home, name)
150
+		ok, err := git.IsBareRoot(path)
151
+		if err != nil {
152
+			errs = append(errs, err)
153
+			continue
154
+		}
155
+		if ok {
156
+			if !config.CleanBeforeClone {
157
+				continue
158
+			}
159
+			log.Printf("Removing %s", path)
160
+			if err := os.RemoveAll(path); err != nil {
161
+				errs = append(errs, err)
162
+				continue
163
+			}
164
+		}
165
+		log.Printf("Cloning %s into %s", url.String(), path)
166
+
167
+		self := RepositoryURL(config, name, nil)
168
+		if _, err := newRepository(config, path, hooks, self, &url); err != nil {
169
+			// TODO: tear this directory down
170
+			errs = append(errs, err)
171
+			continue
172
+		}
173
+	}
174
+	if len(errs) > 0 {
175
+		s := []string{}
176
+		for _, err := range errs {
177
+			s = append(s, err.Error())
178
+		}
179
+		return fmt.Errorf("Initial clone failed:\n* %s", strings.Join(s, "\n* "))
180
+	}
181
+	return nil
182
+}
183
+
184
+func loadHooks(path string) (map[string]string, error) {
185
+	hooks := make(map[string]string)
186
+	if len(path) == 0 {
187
+		return hooks, nil
188
+	}
189
+	files, err := ioutil.ReadDir(path)
190
+	if err != nil {
191
+		return nil, err
192
+	}
193
+	for _, file := range files {
194
+		if file.IsDir() || (file.Mode().Perm()&0111) == 0 {
195
+			continue
196
+		}
197
+		hook := filepath.Join(path, file.Name())
198
+		name := filepath.Base(hook)
199
+		hooks[name] = hook
200
+	}
201
+	return hooks, nil
202
+}
203
+
204
+func mergeHooks(hooks ...map[string]string) map[string]string {
205
+	hook := make(map[string]string)
206
+	for _, m := range hooks {
207
+		for k, v := range m {
208
+			hook[k] = v
209
+		}
210
+	}
211
+	return hook
212
+}