Browse code

Generate plugins list through https requests

The devstack plugins list can be generated through web requests in
environments (such as the proposal slave) that lack copies of all
the relevant git repositories.

One downside to this is that there is no way of getting the last
modification time of the plugin.

Change-Id: I2c5c9282a8ad80014cad171a4dfbdc8f26044cd1

Clint Adams authored on 2016/02/19 04:46:35
Showing 3 changed files
... ...
@@ -14,6 +14,6 @@ The following are plugins that a script has found in the openstack/
14 14
 namespace, which includes but is not limited to official OpenStack
15 15
 projects.
16 16
 
17
-+------------------+------------------------------------------------------------+------------+
18
-|Plugin Name       |URL                                                         |Date        |
19
-+------------------+------------------------------------------------------------+------------+
17
++------------------+-------------------------------------------------------------------------+
18
+|Plugin Name       |URL                                                                      |
19
++------------------+-------------------------------------------------------------------------+
20 20
new file mode 100644
... ...
@@ -0,0 +1,59 @@
0
+#! /usr/bin/env python
1
+
2
+# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+# not use this file except in compliance with the License. You may obtain
6
+# a copy of the License at
7
+#
8
+#    http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+# License for the specific language governing permissions and limitations
14
+# under the License.
15
+
16
+# This script is intended to be run as part of a periodic proposal bot
17
+# job in OpenStack infrastructure.
18
+#
19
+# In order to function correctly, the environment in which the
20
+# script runs must have
21
+#   * network access to the review.openstack.org Gerrit API
22
+#     working directory
23
+#   * network access to https://git.openstack.org/cgit
24
+
25
+import json
26
+import requests
27
+
28
+url = 'https://review.openstack.org/projects/'
29
+
30
+# This is what a project looks like
31
+'''
32
+  "openstack-attic/akanda": {
33
+    "id": "openstack-attic%2Fakanda",
34
+    "state": "READ_ONLY"
35
+  },
36
+'''
37
+
38
+def is_in_openstack_namespace(proj):
39
+    return proj.startswith('openstack/')
40
+
41
+# Rather than returning a 404 for a nonexistent file, cgit delivers a
42
+# 0-byte response to a GET request.  It also does not provide a
43
+# Content-Length in a HEAD response, so the way we tell if a file exists
44
+# is to check the length of the entire GET response body.
45
+def has_devstack_plugin(proj):
46
+    r = requests.get("https://git.openstack.org/cgit/%s/plain/devstack/plugin.sh" % proj)
47
+    if len(r.text) > 0:
48
+        return True
49
+    else:
50
+        False
51
+
52
+r = requests.get(url)
53
+projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:])))
54
+
55
+found_plugins = filter(has_devstack_plugin, projects)
56
+
57
+for project in found_plugins:
58
+    print project[10:]
... ...
@@ -19,11 +19,19 @@
19 19
 #
20 20
 # In order to function correctly, the environment in which the
21 21
 # script runs must have
22
+#   * a writable doc/source directory relative to the current
23
+#     working directory
24
+#   AND ( (
22 25
 #   * git
23 26
 #   * all git repos meant to be searched for plugins cloned and
24 27
 #     at the desired level of up-to-datedness
25
-#   * a writable doc/source directory relative to the current
28
+#   * the environment variable git_dir pointing to the location
29
+#   * of said git repositories
30
+#   ) OR (
31
+#   * network access to the review.openstack.org Gerrit API
26 32
 #     working directory
33
+#   * network access to https://git.openstack.org/cgit
34
+#   ))
27 35
 #
28 36
 # If a file named data/devstack-plugins-registry.header or
29 37
 # data/devstack-plugins-registry.footer is found relative to the
... ...
@@ -35,25 +43,38 @@ declare -A plugins
35 35
 
36 36
 test -r data/devstack-plugins-registry.header && cat data/devstack-plugins-registry.header
37 37
 
38
-pushd ${git_dir:-/opt/openstack} >/dev/null
39
-for i in *; do
40
-    pushd ${i} >/dev/null
41
-    if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
42
-        test -n "$output" && plugins[$i]=${output}
43
-    fi
38
+if test -n "$git_dir"; then
39
+    pushd ${git_dir} >/dev/null
40
+    for i in *; do
41
+        pushd ${i} >/dev/null
42
+        if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
43
+            test -n "$output" && plugins[$i]=${output}
44
+        fi
45
+        popd >/dev/null
46
+    done
44 47
     popd >/dev/null
45
-done
46
-popd >/dev/null
47
-
48
-sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
49
-
50
-for k in "${sorted_plugins[@]}"; do
51
-    project=${k:0:18}
52
-    giturl="git://git.openstack.org/openstack/${k:0:26}"
53
-    pdate="${plugins[$k]}"
54
-    printf "|%-18s|%-60s|%-12s|\n" "${project}" "${giturl}" "${pdate}"
55
-    printf "+------------------+------------------------------------------------------------+------------+\n"
56
-done
48
+
49
+    sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
50
+
51
+    for k in "${sorted_plugins[@]}"; do
52
+        project=${k:0:18}
53
+        giturl="git://git.openstack.org/openstack/${k:0:26}"
54
+        pdate="${plugins[$k]}"
55
+        printf "|%-18s|%-60s (%-10s)|\n" "${project}" "${giturl}" "${pdate}"
56
+        printf "+------------------+-------------------------------------------------------------------------+\n"
57
+    done
58
+
59
+else
60
+    sorted_plugins=$(python tools/generate-devstack-plugins-list.py)
61
+
62
+    for k in ${sorted_plugins}; do
63
+        project=${k:0:18}
64
+        giturl="git://git.openstack.org/openstack/${k:0:26}"
65
+        printf "|%-18s|%-73s|\n" "${project}" "${giturl}"
66
+        printf "+------------------+-------------------------------------------------------------------------+\n"
67
+    done
68
+
69
+fi
57 70
 
58 71
 test -r data/devstack-plugins-registry.footer && cat data/devstack-plugins-registry.footer
59 72
 ) > doc/source/plugin-registry.rst