Browse code

Cloud init patches to support azure provisioning

Change-Id: Ib60836b789dab89665abcb4f5babe74961e957b9
Reviewed-on: http://photon-jenkins.eng.vmware.com:8082/4848
Tested-by: gerrit-photon <photon-checkins@vmware.com>
Reviewed-by: Anish Swaminathan <anishs@vmware.com>

suezzelur authored on 2018/03/01 11:38:48
Showing 6 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,149 @@
0
+diff -rup cloud-init-0.7.9/cloudinit/sources/DataSourceAzure.py cloud-init-0.7.9-1/cloudinit/sources/DataSourceAzure.py
1
+--- cloud-init-0.7.9/cloudinit/sources/DataSourceAzure.py	2016-12-23 08:37:45.000000000 -0800
2
+@@ -23,12 +23,18 @@ LOG = logging.getLogger(__name__)
3
+ 
4
+ DS_NAME = 'Azure'
5
+ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"}
6
+-AGENT_START = ['service', 'walinuxagent', 'start']
7
++AGENT_START = ['systemctl', 'start', 'waagent']
8
+ AGENT_START_BUILTIN = "__builtin__"
9
+ BOUNCE_COMMAND = [
10
+     'sh', '-xc',
11
+     "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"
12
+ ]
13
++
14
++BOUNCE_COMMAND_PHOTON = [
15
++    'sh', '-xc',
16
++    ("i=$interface; x=0; ifconfig $i down || x=$?; "
17
++     "ifconfig $i up || x=$?; exit $x")
18
++]
19
+ # azure systems will always have a resource disk, and 66-azure-ephemeral.rules
20
+ # ensures that it gets linked to this path.
21
+ RESOURCE_DISK_PATH = '/dev/disk/cloud/azure_resource'
22
+@@ -40,7 +46,7 @@ BUILTIN_DS_CONFIG = {
23
+     'hostname_bounce': {
24
+         'interface': 'eth0',
25
+         'policy': True,
26
+-        'command': BOUNCE_COMMAND,
27
++        'command': 'builtin',
28
+         'hostname_command': 'hostname',
29
+     },
30
+     'disk_aliases': {'ephemeral0': RESOURCE_DISK_PATH},
31
+@@ -154,7 +160,7 @@ class DataSourceAzureNet(sources.DataSou
32
+ 
33
+             missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
34
+                                     func=wait_for_files,
35
+-                                    args=(fp_files,))
36
++                                    args=(fp_files,180))
37
+         if len(missing):
38
+             LOG.warn("Did not find files, but going on: %s", missing)
39
+ 
40
+@@ -362,7 +368,9 @@ def perform_hostname_bounce(hostname, cf
41
+     env['old_hostname'] = prev_hostname
42
+ 
43
+     if command == "builtin":
44
+-        command = BOUNCE_COMMAND
45
++        LOG.debug(
46
++                "Skipping network bounce: ifupdown utils aren't present.")
47
++        return
48
+ 
49
+     LOG.debug("pubhname: publishing hostname [%s]", msg)
50
+     shell = not isinstance(command, (list, tuple))
51
+@@ -396,7 +404,7 @@ def pubkeys_from_crt_files(flist):
52
+     return pubkeys
53
+ 
54
+ 
55
+-def wait_for_files(flist, maxwait=60, naplen=.5, log_pre=""):
56
++def wait_for_files(flist, maxwait, naplen=.5, log_pre=""):
57
+     need = set(flist)
58
+     waited = 0
59
+     while True:
60
+ 
61
+diff -rup cloud-init-0.7.9/cloudinit/sources/helpers/azure.py cloud-init-0.7.9-1/cloudinit/sources/helpers/azure.py
62
+--- cloud-init-0.7.9/cloudinit/sources/helpers/azure.py	2016-12-23 08:37:45.000000000 -0800
63
+@@ -8,16 +8,18 @@ import socket
64
+ import struct
65
+ import tempfile
66
+ import time
67
++import configobj
68
+ 
69
+ from cloudinit import stages
70
+ from contextlib import contextmanager
71
+ from xml.etree import ElementTree
72
+ 
73
+ from cloudinit import util
74
++from io import StringIO
75
+ 
76
+ 
77
+ LOG = logging.getLogger(__name__)
78
+-
79
++NETWORKD_LEASES_DIR = '/run/systemd/netif/leases'
80
+ 
81
+ @contextmanager
82
+ def cd(newdir):
83
+@@ -264,6 +266,32 @@ class WALinuxAgentShim(object):
84
+         return dhcp_options
85
+ 
86
+     @staticmethod
87
++    def networkd_parse_lease(content):
88
++        """Parse a systemd lease file content as in /run/systemd/netif/leases/
89
++        Parse this (almost) ini style file even though it says:
90
++          # This is private data. Do not parse.
91
++        Simply return a dictionary of key/values."""
92
++
93
++        return dict(configobj.ConfigObj(StringIO(content), list_values=False))
94
++
95
++    @staticmethod
96
++    def networkd_load_leases(leases_d=None):
97
++        """Return a dictionary of dictionaries representing each lease
98
++        found in lease_d.i
99
++        The top level key will be the filename, which is typically the ifindex."""
100
++
101
++        if leases_d is None:
102
++            leases_d = NETWORKD_LEASES_DIR
103
++
104
++        ret = {}
105
++        if not os.path.isdir(leases_d):
106
++            return ret
107
++        for lfile in os.listdir(leases_d):
108
++            ret[lfile] = WALinuxAgentShim.networkd_parse_lease(
109
++                util.load_file(os.path.join(leases_d, lfile)))
110
++        return ret
111
++
112
++    @staticmethod
113
+     def _get_value_from_dhcpoptions(dhcp_options):
114
+         if dhcp_options is None:
115
+             return None
116
+@@ -277,13 +305,28 @@ class WALinuxAgentShim(object):
117
+         return _value
118
+ 
119
+     @staticmethod
120
++    def networkd_get_option_from_leases(keyname, leases_d=None):
121
++        if leases_d is None:
122
++            leases_d = NETWORKD_LEASES_DIR
123
++        leases = WALinuxAgentShim.networkd_load_leases(leases_d=leases_d)
124
++        for ifindex, data in sorted(leases.items()):
125
++            if data.get(keyname):
126
++                return data[keyname]
127
++        return None
128
++
129
++    @staticmethod
130
++    def _networkd_get_value_from_leases(leases_d=None):
131
++        return WALinuxAgentShim.networkd_get_option_from_leases(
132
++            'OPTION_245', leases_d=leases_d)
133
++
134
++    @staticmethod
135
+     def find_endpoint(fallback_lease_file=None):
136
+         LOG.debug('Finding Azure endpoint...')
137
+         value = None
138
+         # Option-245 stored in /run/cloud-init/dhclient.hooks/<ifc>.json
139
+         # a dhclient exit hook that calls cloud-init-dhclient-hook
140
+-        dhcp_options = WALinuxAgentShim._load_dhclient_json()
141
+-        value = WALinuxAgentShim._get_value_from_dhcpoptions(dhcp_options)
142
++        value = WALinuxAgentShim._networkd_get_value_from_leases()
143
++        LOG.debug('networkd value from lease %s', value)
144
+         if value is None:
145
+             # Fallback and check the leases file if unsuccessful
146
+             LOG.debug("Unable to find endpoint in dhclient logs. "
... ...
@@ -2,7 +2,7 @@
2 2
 
3 3
 Name:           cloud-init
4 4
 Version:        0.7.9
5
-Release:        13%{?dist}
5
+Release:        14%{?dist}
6 6
 Summary:        Cloud instance init scripts
7 7
 Group:          System Environment/Base
8 8
 License:        GPLv3
... ...
@@ -21,6 +21,7 @@ Patch5:         datasource-guestinfo.patch
21 21
 Patch6:         systemd-service-changes.patch
22 22
 Patch7:         makecheck.patch
23 23
 Patch8:         systemd-resolved-config.patch
24
+Patch9:         cloud-init-azureds.patch
24 25
 
25 26
 BuildRequires:  python3
26 27
 BuildRequires:  python3-libs
... ...
@@ -66,6 +67,7 @@ ssh keys and to let the user run various scripts.
66 66
 %patch6 -p1
67 67
 %patch7 -p1
68 68
 %patch8 -p1
69
+%patch9 -p1
69 70
 
70 71
 find systemd -name cloud*.service | xargs sed -i s/StandardOutput=journal+console/StandardOutput=journal/g
71 72
 
... ...
@@ -136,6 +138,8 @@ rm -rf $RPM_BUILD_ROOT
136 136
 
137 137
 
138 138
 %changelog
139
+*   Wed Feb 28 2018 Anish Swaminathan <anishs@vmware.com> 0.7.9-14
140
+-   Add support for systemd constructs for azure DS
139 141
 *   Mon Oct 16 2017 Vinay Kulkarni <kulakrniv@vmware.com> 0.7.9-13
140 142
 -   Support configuration of systemd resolved.conf
141 143
 *   Wed Sep 20 2017 Alexey Makhalov <amakhalov@vmware.com> 0.7.9-12
... ...
@@ -19,8 +19,9 @@ diff -rup cloud-init-0.7.9/systemd/cloud-init.service cloud-init-0.7.9-new/syste
19 19
  Wants=cloud-init-local.service
20 20
  Wants=sshd-keygen.service
21 21
  Wants=sshd.service
22
-@@ -9,9 +8,7 @@ After=networking.service
22
+@@ -9,9 +8,8 @@ After=networking.service
23 23
  Before=network-online.target
24
++After=systemd-networkd-wait-online.service
24 25
  Before=sshd-keygen.service
25 26
  Before=sshd.service
26 27
 -Before=sysinit.target
... ...
@@ -1,7 +1,7 @@
1 1
 Summary:        Sudo
2 2
 Name:           sudo
3 3
 Version:        1.8.20p2
4
-Release:        4%{?dist}
4
+Release:        5%{?dist}
5 5
 License:        ISC
6 6
 URL:            https://www.sudo.ws/
7 7
 Group:          System Environment/Security
... ...
@@ -11,6 +11,7 @@ Source0:        http://www.sudo.ws/sudo/dist/%{name}-%{version}.tar.gz
11 11
 %define sha1    sudo=7aa187518735312a82c5fcb3d253ed700cb8c68e
12 12
 BuildRequires:  man-db
13 13
 BuildRequires:  Linux-PAM-devel
14
+BuildRequires:  sed
14 15
 Requires:       Linux-PAM
15 16
 Requires:       shadow
16 17
 
... ...
@@ -40,10 +41,9 @@ make install DESTDIR=%{buildroot}
40 40
 install -v -dm755 %{buildroot}/%{_docdir}/%{name}-%{version}
41 41
 find %{buildroot}/%{_libdir} -name '*.la' -delete
42 42
 find %{buildroot}/%{_libdir} -name '*.so~' -delete
43
-cat >> %{buildroot}/etc/sudoers << EOF
44
-%wheel ALL=(ALL) ALL
45
-%sudo   ALL=(ALL) ALL
46
-EOF
43
+sed -i '/#includedir.*/i \
44
+%wheel ALL=(ALL) ALL \
45
+%sudo   ALL=(ALL) ALL' %{buildroot}/etc/sudoers
47 46
 install -vdm755 %{buildroot}/etc/pam.d
48 47
 cat > %{buildroot}/etc/pam.d/sudo << EOF
49 48
 #%%PAM-1.0
... ...
@@ -90,6 +90,8 @@ rm -rf %{buildroot}/*
90 90
 %exclude  /etc/sudoers.dist
91 91
 
92 92
 %changelog
93
+*   Thu Mar 01 2018 Anish Swaminathan <anishs@vmware.com> 1.8.20p2-5
94
+-   Move includedir sudoers.d to end of sudoers file
93 95
 *   Tue Oct 10 2017 Alexey Makhalov <amakhalov@vmware.com> 1.8.20p2-4
94 96
 -   No direct toybox dependency, shadow depends on toybox
95 97
 *   Mon Sep 18 2017 Alexey Makhalov <amakhalov@vmware.com> 1.8.20p2-3
... ...
@@ -8,9 +8,6 @@ ln -s ../docker.service docker.service
8 8
 ln -s ../waagent.service waagent.service
9 9
 ln -s ../sshd-keygen.service sshd-keygen.service
10 10
 
11
-#Disable cloud-init
12
-rm -rf /etc/systemd/system/cloud-init.target.wants
13
-
14 11
 # Remove ssh host keys and add script to regenerate them at boot time.
15 12
 
16 13
 rm -f /etc/ssh/ssh_host_*
... ...
@@ -58,4 +55,4 @@ sed -i 's/$photon_cmdline $systemd_cmdline/init=\/lib\/systemd\/systemd loglevel
58 58
 rm /boot/system.map*
59 59
 
60 60
 waagent -force -deprovision+user
61
-export HISTSIZE=0
62 61
\ No newline at end of file
62
+export HISTSIZE=0
... ...
@@ -5,8 +5,7 @@
5 5
 # when a 'default' entry is found it will reference the 'default_user'
6 6
 # from the distro configuration specified below
7 7
 users:
8
-   - name: root
9
-     lock-passwd: true
8
+   - default
10 9
 
11 10
 # If this is set, 'root' will not be able to ssh in and they 
12 11
 # will get a message to login instead as a different user
... ...
@@ -31,11 +30,13 @@ cloud_init_modules:
31 31
 # - seed_random
32 32
  - bootcmd
33 33
  - write-files
34
-# - growpart
35
-# - resizefs
34
+ - growpart
35
+ - resizefs
36
+ - disk_setup
37
+ - mounts
36 38
  - set_hostname
37 39
  - update_hostname
38
-# - update_etc_hosts
40
+ - update_etc_hosts
39 41
 # - ca-certs
40 42
 # - rsyslog
41 43
  - users-groups
... ...
@@ -46,8 +47,6 @@ cloud_config_modules:
46 46
 # Emit the cloud config ready event
47 47
 # this can be used by upstart jobs for 'start on cloud-config'.
48 48
 # - emit_upstart
49
-# - disk_setup
50
-# - mounts
51 49
  - ssh-import-id
52 50
 # - set-passwords
53 51
  - package-update-upgrade-install
... ...
@@ -79,12 +78,18 @@ cloud_final_modules:
79 79
 # System and/or distro specific settings
80 80
 # (not accessible to handlers/transforms)
81 81
 system_info:
82
+   default_user:
83
+      name: photon
84
+      lock_passwd: true
85
+      gecos: Azure User
86
+      groups: [wheel, adm, systemd-journal]
87
+      sudo: ["ALL=(ALL) NOPASSWD:ALL"]
88
+      shell: /bin/bash
82 89
    # This will affect which distro class gets used
83 90
    distro: photon
84 91
    # Other config here will be given to the distro class and/or path classes
85 92
    paths:
86 93
       cloud_dir: /var/lib/cloud/
87 94
       templates_dir: /etc/cloud/templates/
88
-      upstart_dir: /etc/init/
89 95
 
90 96
    ssh_svcname: ssh