1. devstack
  2. Commit 36ddea31a257b38aa22d3232f4c4389c851f5456
Browse code

Add devstack base job for zuul v3

This should be managed in the devstack repo, since it's a base job to
run devstack.

Change-Id: Iffe54fbccbccd68db08f79a1b51dd7f76dbff408
Depends-On: Ie2119f24360d56690ffd772b95a9ea6b98dd4a39

Monty Taylor authored on 2017/10/03 00:05:17
Showing 41 changed files
.zuul.yaml
History View file @ 36ddea3
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,85 @@
0 
+- nodeset:
1 
+    name: openstack-single-node
2 
+    nodes:
3 
+      - name: controller
4 
+        label: ubuntu-xenial
5 
+    groups:
6 
+      - name: tempest
7 
+        nodes:
8 
+          - controller
9 
+
10 
+- nodeset:
11 
+    name: openstack-two-node
12 
+    nodes:
13 
+      - name: controller
14 
+        label: ubuntu-xenial
15 
+      - name: compute1
16 
+        label: ubuntu-xenial
17 
+    groups:
18 
+      - name: tempest
19 
+        nodes:
20 
+          - controller
21 
+      - name: compute
22 
+        nodes:
23 
+          - controller
24 
+          - compute1
25 
+
26 
+- job:
27 
+    name: devstack
28 
+    parent: multinode
29 
+    description: Base devstack job
30 
+    nodeset: openstack-single-node
31 
+    required-projects:
32 
+      - openstack-dev/devstack
33 
+      - openstack/cinder
34 
+      - openstack/glance
35 
+      - openstack/keystone
36 
+      - openstack/neutron
37 
+      - openstack/nova
38 
+      - openstack/requirements
39 
+      - openstack/swift
40 
+    timeout: 7200
41 
+    vars:
42 
+      devstack_localrc:
43 
+        DATABASE_PASSWORD: secretdatabase
44 
+        RABBIT_PASSWORD: secretrabbit
45 
+        ADMIN_PASSWORD: secretadmin
46 
+        SERVICE_PASSWORD: secretservice
47 
+        NETWORK_GATEWAY: 10.1.0.1
48 
+        Q_USE_DEBUG_COMMAND: True
49 
+        FIXED_RANGE: 10.1.0.0/20
50 
+        IPV4_ADDRS_SAFE_TO_USE: 10.1.0.0/20
51 
+        FLOATING_RANGE: 172.24.5.0/24
52 
+        PUBLIC_NETWORK_GATEWAY: 172.24.5.1
53 
+        FLOATING_HOST_PREFIX: 172.24.4
54 
+        FLOATING_HOST_MASK: 23
55 
+        SWIFT_REPLICAS: 1
56 
+        SWIFT_START_ALL_SERVICES: False
57 
+        LOGFILE: /opt/stack/logs/devstacklog.txt
58 
+        LOG_COLOR: False
59 
+        VERBOSE: True
60 
+        NETWORK_GATEWAY: 10.1.0.1
61 
+        NOVNC_FROM_PACKAGE: True
62 
+        ERROR_ON_CLONE: True
63 
+        # NOTE(dims): etcd 3.x is not available in debian/ubuntu
64 
+        # etc. As a stop gap measure, devstack uses wget to download
65 
+        # from the location below for all the CI jobs.
66 
+        ETCD_DOWNLOAD_URL: "http://tarballs.openstack.org/etcd/"
67 
+      devstack_services:
68 
+        horizon: False
69 
+        tempest: False
70 
+    pre-run: playbooks/pre
71 
+    post-run: playbooks/post
72 
+
73 
+
74 
+- project:
75 
+    name: openstack-dev/devstack
76 
+    check:
77 
+      jobs:
78 
+        - devstack:
79 
+            files:
80 
+              - ^playbooks/pre
81 
+              - ^playbooks/post
82 
+              - ^playbooks/devstack
83 
+              - ^roles/
84 
+              - .zuul.yaml
playbooks/devstack.yaml
History View file @ 36ddea3
0 85 
new file mode 100644
... ... 
@@ -0,0 +1,3 @@
0 
+- hosts: all
1 
+  roles:
2 
+    - run-devstack
playbooks/post.yaml
History View file @ 36ddea3
0 3 
new file mode 100644
... ... 
@@ -0,0 +1,4 @@
0 
+- hosts: all
1 
+  roles:
2 
+    - export-devstack-journal
3 
+    - fetch-devstack-log-dir
playbooks/pre.yaml
History View file @ 36ddea3
0 4 
new file mode 100644
... ... 
@@ -0,0 +1,22 @@
0 
+- hosts: all
1 
+  roles:
2 
+    - configure-swap
3 
+    - setup-stack-user
4 
+    - setup-tempest-user
5 
+    - setup-devstack-source-dirs
6 
+    - setup-devstack-log-dir
7 
+    - setup-devstack-cache
8 
+    - start-fresh-logging
9 
+    - write-devstack-local-conf
10 
+  # TODO(jeblair): remove when configure-mirrors is fixed
11 
+  tasks:
12 
+    - name: Hack mirror_info
13 
+      shell:
14 
+        _raw_params: |
15 
+          mkdir /etc/ci
16 
+          cat << "EOF" > /etc/ci/mirror_info.sh
17 
+          export NODEPOOL_UCA_MIRROR=http://mirror.dfw.rax.openstack.org/ubuntu-cloud-archive
18 
+          EOF
19 
+      args:
20 
+        executable: /bin/bash
21 
+      become: true
roles/configure-swap/README.rst
History View file @ 36ddea3
0 22 
new file mode 100644
... ... 
@@ -0,0 +1,11 @@
0 
+Configure a swap partition
1 
+
2 
+Creates a swap partition on the ephemeral block device (the rest of which
3 
+will be mounted on /opt).
4 
+
5 
+**Role Variables**
6 
+
7 
+.. zuul:rolevar:: configure_swap_size
8 
+   :default: 8192
9 
+
10 
+   The size of the swap partition, in MiB.
roles/configure-swap/defaults/main.yaml
History View file @ 36ddea3
0 11 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+configure_swap_size: 8192
roles/configure-swap/tasks/ephemeral.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,110 @@
0 
+# Configure attached ephemeral devices for storage and swap
1 
+
2 
+- assert:
3 
+    that:
4 
+      - "ephemeral_device is defined"
5 
+
6 
+- name: Set partition names
7 
+  set_fact:
8 
+    swap_partition: "{{ ephemeral_device}}1"
9 
+    opt_partition: "{{ ephemeral_device}}2"
10 
+
11 
+- name: Ensure ephemeral device is unmounted
12 
+  become: yes
13 
+  mount:
14 
+    name: "{{ ephemeral_device }}"
15 
+    state: unmounted
16 
+
17 
+- name: Get existing partitions
18 
+  become: yes
19 
+  parted:
20 
+    device: "{{ ephemeral_device }}"
21 
+    unit: MiB
22 
+  register: ephemeral_partitions
23 
+
24 
+- name: Remove any existing partitions
25 
+  become: yes
26 
+  parted:
27 
+    device: "{{ ephemeral_device }}"
28 
+    number: "{{ item.num }}"
29 
+    state: absent
30 
+  with_items:
31 
+    - "{{ ephemeral_partitions.partitions }}"
32 
+
33 
+- name: Create new disk label
34 
+  become: yes
35 
+  parted:
36 
+    label: msdos
37 
+    device: "{{ ephemeral_device }}"
38 
+
39 
+- name: Create swap partition
40 
+  become: yes
41 
+  parted:
42 
+    device: "{{ ephemeral_device }}"
43 
+    number: 1
44 
+    state: present
45 
+    part_start: '0%'
46 
+    part_end: "{{ configure_swap_size }}MiB"
47 
+
48 
+- name: Create opt partition
49 
+  become: yes
50 
+  parted:
51 
+    device: "{{ ephemeral_device }}"
52 
+    number: 2
53 
+    state: present
54 
+    part_start: "{{ configure_swap_size }}MiB"
55 
+    part_end: "100%"
56 
+
57 
+- name: Make swap on partition
58 
+  become: yes
59 
+  command: "mkswap {{ swap_partition }}"
60 
+
61 
+- name: Write swap to fstab
62 
+  become: yes
63 
+  mount:
64 
+    path: none
65 
+    src: "{{ swap_partition }}"
66 
+    fstype: swap
67 
+    opts: sw
68 
+    passno: 0
69 
+    dump: 0
70 
+    state: present
71 
+
72 
+# XXX: does "parted" plugin ensure the partition is available
73 
+# before moving on?  No udev settles here ...
74 
+
75 
+- name: Add all swap
76 
+  become: yes
77 
+  command: swapon -a
78 
+
79 
+- name: Create /opt filesystem
80 
+  become: yes
81 
+  filesystem:
82 
+    fstype: ext4
83 
+    dev: "{{ opt_partition }}"
84 
+
85 
+# Rackspace at least does not have enough room for two devstack
86 
+# installs on the primary partition.  We copy in the existing /opt to
87 
+# the new partition on the ephemeral device, and then overmount /opt
88 
+# to there for the test runs.
89 
+#
90 
+# NOTE(ianw): the existing "mount" touches fstab.  There is currently (Sep2017)
91 
+# work in [1] to split mount & fstab into separate parts, but for now we bundle
92 
+# it into an atomic shell command
93 
+# [1] https://github.com/ansible/ansible/pull/27174
94 
+- name: Copy old /opt
95 
+  become: yes
96 
+  shell: |
97 
+    mount {{ opt_partition }} /mnt
98 
+    find /opt/ -mindepth 1 -maxdepth 1 -exec mv {} /mnt/ \;
99 
+    umount /mnt
100 
+
101 
+# This overmounts any existing /opt
102 
+- name: Add opt to fstab and mount
103 
+  become: yes
104 
+  mount:
105 
+    path: /opt
106 
+    src: "{{ opt_partition }}"
107 
+    fstype: ext4
108 
+    opts: noatime
109 
+    state: mounted
roles/configure-swap/tasks/main.yaml
History View file @ 36ddea3
0 110 
new file mode 100644
... ... 
@@ -0,0 +1,63 @@
0 
+# On RAX hosts, we have a small root partition and a large,
1 
+# unallocated ephemeral device attached at /dev/xvde
2 
+- name: Set ephemeral device if /dev/xvde exists
3 
+  when: ansible_devices["xvde"] is defined
4 
+  set_fact:
5 
+    ephemeral_device: "/dev/xvde"
6 
+
7 
+# On other providers, we have a device called "ephemeral0".
8 
+#
9 
+# NOTE(ianw): Once [1] is in our ansible (2.4 era?), we can figure
10 
+# this out more directly by walking the device labels in the facts
11 
+#
12 
+# [1] https://github.com/ansible/ansible/commit/d46dd99f47c0ee5081d15bc5b741e9096d8bfd3e
13 
+- name: Set ephemeral device by label
14 
+  when: ephemeral_device is undefined
15 
+  block:
16 
+    - name: Get ephemeral0 device node
17 
+      command: /sbin/blkid -L ephemeral0
18 
+      register: ephemeral0
19 
+      # If this doesn't exist, returns !0
20 
+      ignore_errors: yes
21 
+      changed_when: False
22 
+
23 
+    - name: Set ephemeral device if LABEL exists
24 
+      when: "ephemeral0.rc == 0"
25 
+      set_fact:
26 
+        ephemeral_device: "{{ ephemeral0.stdout }}"
27 
+
28 
+# If we have ephemeral storage and we don't appear to have setup swap,
29 
+# we will create a swap and move /opt to a large data partition there.
30 
+- include: ephemeral.yaml
31 
+  static: no
32 
+  when:
33 
+    - ephemeral_device is defined
34 
+    - ansible_memory_mb['swap']['total'] | int + 10 <= configure_swap_size
35 
+
36 
+# If no ephemeral device and no swap, then we will setup some swap
37 
+# space on the root device to ensure all hosts a consistent memory
38 
+# environment.
39 
+- include: root.yaml
40 
+  static: no
41 
+  when:
42 
+    - ephemeral_device is undefined
43 
+    - ansible_memory_mb['swap']['total'] | int + 10 <= configure_swap_size
44 
+
45 
+# ensure a standard level of swappiness.  Some platforms
46 
+# (rax+centos7) come with swappiness of 0 (presumably because the
47 
+# vm doesn't come with swap setup ... but we just did that above),
48 
+# which depending on the kernel version can lead to the OOM killer
49 
+# kicking in on some processes despite swap being available;
50 
+# particularly things like mysql which have very high ratio of
51 
+# anonymous-memory to file-backed mappings.
52 
+#
53 
+# This sets swappiness low; we really don't want to be relying on
54 
+# cloud I/O based swap during our runs if we can help it
55 
+- name: Set swappiness
56 
+  become: yes
57 
+  sysctl:
58 
+    name: vm.swappiness
59 
+    value: 30
60 
+    state: present
61 
+
62 
+- debug:  var=ephemeral_device
roles/configure-swap/tasks/root.yaml
History View file @ 36ddea3
0 63 
new file mode 100644
... ... 
@@ -0,0 +1,63 @@
0 
+# If no ephemeral devices are available, use root filesystem
1 
+
2 
+- name: Calculate required swap
3 
+  set_fact:
4 
+    swap_required: "{{ configure_swap_size - ansible_memory_mb['swap']['total'] | int }}"
5 
+
6 
+- block:
7 
+    - name: Get root filesystem
8 
+      shell: df --output='fstype' /root | tail -1
9 
+      register: root_fs
10 
+
11 
+    - name: Save root filesystem
12 
+      set_fact:
13 
+        root_filesystem: "{{ root_fs.stdout }}"
14 
+
15 
+    - debug: var=root_filesystem
16 
+
17 
+# Note, we don't use a sparse device to avoid wedging when disk space
18 
+# and memory are both unavailable.
19 
+
20 
+# Cannot fallocate on filesystems like XFS, so use slower dd
21 
+- name: Create swap backing file for non-EXT fs
22 
+  when: '"ext" not in root_filesystem'
23 
+  become: yes
24 
+  command: dd if=/dev/zero of=/root/swapfile bs=1M count={{ swap_required }}
25 
+  args:
26 
+    creates: /root/swapfile
27 
+
28 
+- name: Create sparse swap backing file for EXT fs
29 
+  when: '"ext" in root_filesystem'
30 
+  become: yes
31 
+  command: fallocate -l {{ swap_required }}M /root/swapfile
32 
+  args:
33 
+    creates: /root/swapfile
34 
+
35 
+- name: Ensure swapfile perms
36 
+  become: yes
37 
+  file:
38 
+    path: /root/swapfile
39 
+    owner: root
40 
+    group: root
41 
+    mode: 0600
42 
+
43 
+- name: Make swapfile
44 
+  become: yes
45 
+  command: mkswap /root/swapfile
46 
+
47 
+- name: Write swap to fstab
48 
+  become: yes
49 
+  mount:
50 
+    path: none
51 
+    src: /root/swapfile
52 
+    fstype: swap
53 
+    opts: sw
54 
+    passno: 0
55 
+    dump: 0
56 
+    state: present
57 
+
58 
+- name: Add all swap
59 
+  become: yes
60 
+  command: swapon -a
61 
+
62 
+- debug: var=swap_required
roles/export-devstack-journal/README.rst
History View file @ 36ddea3
0 63 
new file mode 100644
... ... 
@@ -0,0 +1,15 @@
0 
+Export journal files from devstack services
1 
+
2 
+Export the systemd journal for every devstack service in native
3 
+journal format as well as text.  Also, export a syslog-style file with
4 
+kernal and sudo messages.
5 
+
6 
+Writes the output to the ``logs/`` subdirectory of
7 
+``devstack_base_dir``.
8 
+
9 
+**Role Variables**
10 
+
11 
+.. zuul:rolevar:: devstack_base_dir
12 
+   :default: /opt/stack
13 
+
14 
+   The devstack base directory.
roles/export-devstack-journal/defaults/main.yaml
History View file @ 36ddea3
0 15 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+devstack_base_dir: /opt/stack
roles/export-devstack-journal/tasks/main.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,29 @@
0 
+# TODO: convert this to ansible
1 
+- name: Export journal files
2 
+  become: true
3 
+  shell:
4 
+    cmd: |
5 
+      u=""
6 
+      name=""
7 
+      for u in `systemctl list-unit-files | grep devstack | awk '{print $1}'`; do
8 
+        name=$(echo $u | sed 's/devstack@/screen-/' | sed 's/\.service//')
9 
+        journalctl -o short-precise --unit $u | tee {{ devstack_base_dir }}/logs/$name.txt > /dev/null
10 
+      done
11 
+
12 
+      # Export the journal in export format to make it downloadable
13 
+      # for later searching. It can then be rewritten to a journal native
14 
+      # format locally using systemd-journal-remote. This makes a class of
15 
+      # debugging much easier. We don't do the native conversion here as
16 
+      # some distros do not package that tooling.
17 
+      journalctl -u 'devstack@*' -o export | \
18 
+          xz --threads=0 - > {{ devstack_base_dir }}/logs/devstack.journal.xz
19 
+
20 
+      # The journal contains everything running under systemd, we'll
21 
+      # build an old school version of the syslog with just the
22 
+      # kernel and sudo messages.
23 
+      journalctl \
24 
+          -t kernel \
25 
+          -t sudo \
26 
+          --no-pager \
27 
+          --since="$(cat {{ devstack_base_dir }}/log-start-timestamp.txt)" \
28 
+        | tee {{ devstack_base_dir }}/logs/syslog.txt > /dev/null
roles/fetch-devstack-log-dir/README.rst
History View file @ 36ddea3
0 29 
new file mode 100644
... ... 
@@ -0,0 +1,10 @@
0 
+Fetch content from the devstack log directory
1 
+
2 
+Copy logs from every host back to the zuul executor.
3 
+
4 
+**Role Variables**
5 
+
6 
+.. zuul:rolevar:: devstack_base_dir
7 
+   :default: /opt/stack
8 
+
9 
+   The devstack base directory.
roles/fetch-devstack-log-dir/defaults/main.yaml
History View file @ 36ddea3
0 10 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+devstack_base_dir: /opt/stack
roles/fetch-devstack-log-dir/tasks/main.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,5 @@
0 
+- name: Collect devstack logs
1 
+  synchronize:
2 
+    dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
3 
+    mode: pull
4 
+    src: "{{ devstack_base_dir }}/logs"
roles/run-devstack/README.rst
History View file @ 36ddea3
0 5 
new file mode 100644
... ... 
@@ -0,0 +1,8 @@
0 
+Run devstack
1 
+
2 
+**Role Variables**
3 
+
4 
+.. zuul:rolevar:: devstack_base_dir
5 
+   :default: /opt/stack
6 
+
7 
+   The devstack base directory.
roles/run-devstack/defaults/main.yaml
History View file @ 36ddea3
0 8 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+devstack_base_dir: /opt/stack
roles/run-devstack/tasks/main.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,6 @@
0 
+- name: Run devstack
1 
+  command: ./stack.sh
2 
+  args:
3 
+    chdir: "{{devstack_base_dir}}/devstack"
4 
+  become: true
5 
+  become_user: stack
roles/setup-devstack-cache/README.rst
History View file @ 36ddea3
0 6 
new file mode 100644
... ... 
@@ -0,0 +1,15 @@
0 
+Set up the devstack cache directory
1 
+
2 
+If the node has a cache of devstack image files, copy it into place.
3 
+
4 
+**Role Variables**
5 
+
6 
+.. zuul:rolevar:: devstack_base_dir
7 
+   :default: /opt/stack
8 
+
9 
+   The devstack base directory.
10 
+
11 
+.. zuul:rolevar:: devstack_cache_dir
12 
+   :default: /opt/cache
13 
+
14 
+   The directory with the cached files.
roles/setup-devstack-cache/defaults/main.yaml
History View file @ 36ddea3
0 15 
new file mode 100644
... ... 
@@ -0,0 +1,2 @@
0 
+devstack_base_dir: /opt/stack
1 
+devstack_cache_dir: /opt/cache
roles/setup-devstack-cache/tasks/main.yaml
History View file @ 36ddea3
0 2 
new file mode 100644
... ... 
@@ -0,0 +1,14 @@
0 
+- name: Copy cached devstack files
1 
+  # This uses hard links to avoid using extra space.
2 
+  command: "find {{ devstack_cache_dir }}/files -mindepth 1 -maxdepth 1 -exec cp -l {} {{ devstack_base_dir }}/devstack/files/ ;"
3 
+  become: true
4 
+
5 
+- name: Set ownership of cached files
6 
+  file:
7 
+    path: '{{ devstack_base_dir }}/devstack/files'
8 
+    state: directory
9 
+    recurse: true
10 
+    owner: stack
11 
+    group: stack
12 
+    mode: a+r
13 
+  become: yes
roles/setup-devstack-log-dir/README.rst
History View file @ 36ddea3
0 14 
new file mode 100644
... ... 
@@ -0,0 +1,11 @@
0 
+Set up the devstack log directory
1 
+
2 
+Create a log directory on the ephemeral disk partition to save space
3 
+on the root device.
4 
+
5 
+**Role Variables**
6 
+
7 
+.. zuul:rolevar:: devstack_base_dir
8 
+   :default: /opt/stack
9 
+
10 
+   The devstack base directory.
roles/setup-devstack-log-dir/defaults/main.yaml
History View file @ 36ddea3
0 11 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+devstack_base_dir: /opt/stack
roles/setup-devstack-log-dir/tasks/main.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,5 @@
0 
+- name: Create logs directory
1 
+  file:
2 
+    path: '{{ devstack_base_dir }}/logs'
3 
+    state: directory
4 
+  become: yes
roles/setup-devstack-source-dirs/README.rst
History View file @ 36ddea3
0 5 
new file mode 100644
... ... 
@@ -0,0 +1,11 @@
0 
+Set up the devstack source directories
1 
+
2 
+Ensure that the base directory exists, and then move the source repos
3 
+into it.
4 
+
5 
+**Role Variables**
6 
+
7 
+.. zuul:rolevar:: devstack_base_dir
8 
+   :default: /opt/stack
9 
+
10 
+   The devstack base directory.
roles/setup-devstack-source-dirs/defaults/main.yaml
History View file @ 36ddea3
0 11 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+devstack_base_dir: /opt/stack
roles/setup-devstack-source-dirs/tasks/main.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,22 @@
0 
+- name: Find all source repos used by this job
1 
+  find:
2 
+    paths:
3 
+      - src/git.openstack.org/openstack
4 
+      - src/git.openstack.org/openstack-dev
5 
+      - src/git.openstack.org/openstack-infra
6 
+    file_type: directory
7 
+  register: found_repos
8 
+
9 
+- name: Copy Zuul repos into devstack working directory
10 
+  command: rsync -a {{ item.path }} {{ devstack_base_dir }}
11 
+  with_items: '{{ found_repos.files }}'
12 
+  become: yes
13 
+
14 
+- name: Set ownership of repos
15 
+  file:
16 
+    path: '{{ devstack_base_dir }}'
17 
+    state: directory
18 
+    recurse: true
19 
+    owner: stack
20 
+    group: stack
21 
+  become: yes
roles/setup-stack-user/README.rst
History View file @ 36ddea3
0 22 
new file mode 100644
... ... 
@@ -0,0 +1,16 @@
0 
+Set up the `stack` user
1 
+
2 
+Create the stack user, set up its home directory, and allow it to
3 
+sudo.
4 
+
5 
+**Role Variables**
6 
+
7 
+.. zuul:rolevar:: devstack_base_dir
8 
+   :default: /opt/stack
9 
+
10 
+   The devstack base directory.
11 
+
12 
+.. zuul:rolevar:: devstack_stack_home_dir
13 
+   :default: {{ devstack_base_dir }}
14 
+
15 
+   The home directory for the stack user.
roles/setup-stack-user/defaults/main.yaml
History View file @ 36ddea3
0 16 
new file mode 100644
... ... 
@@ -0,0 +1,2 @@
0 
+devstack_base_dir: /opt/stack
1 
+devstack_stack_home_dir: '{{ devstack_base_dir }}'
roles/setup-stack-user/files/50_stack_sh
History View file @ 36ddea3
0 2 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+stack ALL=(root) NOPASSWD:ALL
roles/setup-stack-user/tasks/main.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,45 @@
0 
+- name: Create stack group
1 
+  group:
2 
+    name: stack
3 
+  become: yes
4 
+
5 
+# NOTE(andreaf) Create a user home_dir is not safe via
6 
+# the user module since it will fail if the containing
7 
+# folder does not exists. If the folder does exists and
8 
+# it's empty, the skeleton is setup and ownership set.
9 
+- name: Create the stack user home folder
10 
+  file:
11 
+    path: '{{ devstack_stack_home_dir }}'
12 
+    state: directory
13 
+  become: yes
14 
+
15 
+- name: Create stack user
16 
+  user:
17 
+    name: stack
18 
+    shell: /bin/bash
19 
+    home: '{{ devstack_stack_home_dir }}'
20 
+    group: stack
21 
+  become: yes
22 
+
23 
+- name: Set stack user home directory permissions
24 
+  file:
25 
+    path: '{{ devstack_stack_home_dir }}'
26 
+    mode: 0755
27 
+  become: yes
28 
+
29 
+- name: Copy 50_stack_sh file to /etc/sudoers.d
30 
+  copy:
31 
+    src: 50_stack_sh
32 
+    dest: /etc/sudoers.d
33 
+    mode: 0440
34 
+    owner: root
35 
+    group: root
36 
+  become: yes
37 
+
38 
+- name: Create new/.cache folder within BASE
39 
+  file:
40 
+    path: '{{ devstack_stack_home_dir }}/.cache'
41 
+    state: directory
42 
+    owner: stack
43 
+    group: stack
44 
+  become: yes
roles/setup-tempest-user/README.rst
History View file @ 36ddea3
0 45 
new file mode 100644
... ... 
@@ -0,0 +1,10 @@
0 
+Set up the `tempest` user
1 
+
2 
+Create the tempest user and allow it to sudo.
3 
+
4 
+**Role Variables**
5 
+
6 
+.. zuul:rolevar:: devstack_base_dir
7 
+   :default: /opt/stack
8 
+
9 
+   The devstack base directory.
roles/setup-tempest-user/files/51_tempest_sh
History View file @ 36ddea3
0 10 
new file mode 100644
... ... 
@@ -0,0 +1,3 @@
0 
+tempest ALL=(root) NOPASSWD:/sbin/ip
1 
+tempest ALL=(root) NOPASSWD:/sbin/iptables
2 
+tempest ALL=(root) NOPASSWD:/usr/bin/ovsdb-client
roles/setup-tempest-user/tasks/main.yaml
History View file @ 36ddea3
0 3 
new file mode 100644
... ... 
@@ -0,0 +1,20 @@
0 
+- name: Create tempest group
1 
+  group:
2 
+    name: tempest
3 
+  become: yes
4 
+
5 
+- name: Create tempest user
6 
+  user:
7 
+    name: tempest
8 
+    shell: /bin/bash
9 
+    group: tempest
10 
+  become: yes
11 
+
12 
+- name: Copy 51_tempest_sh to /etc/sudoers.d
13 
+  copy:
14 
+    src: 51_tempest_sh
15 
+    dest: /etc/sudoers.d
16 
+    owner: root
17 
+    group: root
18 
+    mode: 0440
19 
+  become: yes
roles/start-fresh-logging/README.rst
History View file @ 36ddea3
0 20 
new file mode 100644
... ... 
@@ -0,0 +1,11 @@
0 
+Restart logging on all hosts
1 
+
2 
+Restart syslog so that the system logs only include output from the
3 
+job.
4 
+
5 
+**Role Variables**
6 
+
7 
+.. zuul:rolevar:: devstack_base_dir
8 
+   :default: /opt/stack
9 
+
10 
+   The devstack base directory.
roles/start-fresh-logging/defaults/main.yaml
History View file @ 36ddea3
0 11 
new file mode 100644
... ... 
@@ -0,0 +1 @@
0 
+devstack_base_dir: /opt/stack
roles/start-fresh-logging/tasks/main.yaml
History View file @ 36ddea3
0 1 
new file mode 100644
... ... 
@@ -0,0 +1,56 @@
0 
+- name: Check for /bin/journalctl file
1 
+  command: which journalctl
2 
+  changed_when: False
3 
+  failed_when: False
4 
+  register: which_out
5 
+
6 
+- block:
7 
+    - name: Get current date
8 
+      command: date +"%Y-%m-%d %H:%M:%S"
9 
+      register: date_out
10 
+
11 
+    - name: Copy current date to log-start-timestamp.txt
12 
+      copy:
13 
+        dest: "{{ devstack_base_dir }}/log-start-timestamp.txt"
14 
+        content: "{{ date_out.stdout }}"
15 
+  when: which_out.rc == 0
16 
+  become: yes
17 
+
18 
+- block:
19 
+    - name: Stop rsyslog
20 
+      service: name=rsyslog state=stopped
21 
+
22 
+    - name: Save syslog file prior to devstack run
23 
+      command: mv /var/log/syslog /var/log/syslog-pre-devstack
24 
+
25 
+    - name: Save kern.log file prior to devstack run
26 
+      command: mv /var/log/kern.log /var/log/kern_log-pre-devstack
27 
+
28 
+    - name: Recreate syslog file
29 
+      file: name=/var/log/syslog state=touch
30 
+
31 
+    - name: Recreate syslog file owner and group
32 
+      command: chown /var/log/syslog --ref /var/log/syslog-pre-devstack
33 
+
34 
+    - name: Recreate syslog file permissions
35 
+      command: chmod /var/log/syslog --ref /var/log/syslog-pre-devstack
36 
+
37 
+    - name: Add read permissions to all on syslog file
38 
+      file: name=/var/log/syslog mode=a+r
39 
+
40 
+    - name: Recreate kern.log file
41 
+      file: name=/var/log/kern.log state=touch
42 
+
43 
+    - name: Recreate kern.log file owner and group
44 
+      command: chown /var/log/kern.log --ref /var/log/kern_log-pre-devstack
45 
+
46 
+    - name: Recreate kern.log file permissions
47 
+      command: chmod /var/log/kern.log --ref /var/log/kern_log-pre-devstack
48 
+
49 
+    - name: Add read permissions to all on kern.log file
50 
+      file: name=/var/log/kern.log mode=a+r
51 
+
52 
+    - name: Start rsyslog
53 
+      service: name=rsyslog state=started
54 
+  when: which_out.rc == 1
55 
+  become: yes
roles/write-devstack-local-conf/README.rst
History View file @ 36ddea3
0 56 
new file mode 100644
... ... 
@@ -0,0 +1,63 @@
0 
+Write the local.conf file for use by devstack
1 
+
2 
+**Role Variables**
3 
+
4 
+.. zuul:rolevar:: devstack_base_dir
5 
+   :default: /opt/stack
6 
+
7 
+   The devstack base directory.
8 
+
9 
+.. zuul:rolevar:: devstack_local_conf_path
10 
+   :default: {{ devstack_base_dir }}/devstack/local.conf
11 
+
12 
+   The path of the local.conf file.
13 
+
14 
+.. zuul:rolevar:: devstack_localrc
15 
+   :type: dict
16 
+
17 
+   A dictionary of variables that should be written to the localrc
18 
+   section of local.conf.  The values (which are strings) may contain
19 
+   bash shell variables, and will be ordered so that variables used by
20 
+   later entries appear first.
21 
+
22 
+.. zuul:rolevar:: devstack_local_conf
23 
+   :type: dict
24 
+
25 
+   A complex argument consisting of nested dictionaries which combine
26 
+   to form the meta-sections of the local_conf file.  The top level is
27 
+   a dictionary of phases, followed by dictionaries of filenames, then
28 
+   sections, which finally contain key-value pairs for the INI file
29 
+   entries in those sections.
30 
+
31 
+   The keys in this dictionary are the devstack phases.
32 
+
33 
+   .. zuul:rolevar:: [phase]
34 
+      :type: dict
35 
+
36 
+      The keys in this dictionary are the filenames for this phase.
37 
+
38 
+      .. zuul:rolevar:: [filename]
39 
+         :type: dict
40 
+
41 
+         The keys in this dictionary are the INI sections in this file.
42 
+
43 
+         .. zuul:rolevar:: [section]
44 
+            :type: dict
45 
+
46 
+            This is a dictionary of key-value pairs which comprise
47 
+            this section of the INI file.
48 
+
49 
+.. zuul:rolevar:: devstack_services
50 
+   :type: dict
51 
+
52 
+   A dictionary mapping service names to boolean values.  If the
53 
+   boolean value is ``false``, a ``disable_service`` line will be
54 
+   emitted for the service name.  If it is ``true``, then
55 
+   ``enable_service`` will be emitted.  All other values are ignored.
56 
+
57 
+.. zuul:rolevar:: devstack_plugins
58 
+   :type: dict
59 
+
60 
+   A dictionary mapping a plugin name to a git repo location.  If the
61 
+   location is a non-empty string, then an ``enable_plugin`` line will
62 
+   be emmitted for the plugin name.
roles/write-devstack-local-conf/defaults/main.yaml
History View file @ 36ddea3
0 63 
new file mode 100644
... ... 
@@ -0,0 +1,2 @@
0 
+devstack_base_dir: /opt/stack
1 
+devstack_local_conf_path: "{{ devstack_base_dir }}/devstack/local.conf"
roles/write-devstack-local-conf/library/devstack_local_conf.py
History View file @ 36ddea3
0 2 
new file mode 100644
... ... 
@@ -0,0 +1,185 @@
0 
+# Copyright (C) 2017 Red Hat, Inc.
1 
+#
2 
+# Licensed under the Apache License, Version 2.0 (the "License");
3 
+# you may not use this file except in compliance with the License.
4 
+# You may obtain a copy of the License at
5 
+#
6 
+#    http://www.apache.org/licenses/LICENSE-2.0
7 
+#
8 
+# Unless required by applicable law or agreed to in writing, software
9 
+# distributed under the License is distributed on an "AS IS" BASIS,
10 
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
11 
+# implied.
12 
+#
13 
+# See the License for the specific language governing permissions and
14 
+# limitations under the License.
15 
+
16 
+import re
17 
+
18 
+
19 
+class VarGraph(object):
20 
+    # This is based on the JobGraph from Zuul.
21 
+
22 
+    def __init__(self, vars):
23 
+        self.vars = {}
24 
+        self._varnames = set()
25 
+        self._dependencies = {}  # dependent_var_name -> set(parent_var_names)
26 
+        for k, v in vars.items():
27 
+            self._varnames.add(k)
28 
+        for k, v in vars.items():
29 
+            self._addVar(k, str(v))
30 
+
31 
+    bash_var_re = re.compile(r'\$\{?(\w+)')
32 
+    def getDependencies(self, value):
33 
+        return self.bash_var_re.findall(value)
34 
+
35 
+    def _addVar(self, key, value):
36 
+        if key in self.vars:
37 
+            raise Exception("Variable {} already added".format(key))
38 
+        self.vars[key] = value
39 
+        # Append the dependency information
40 
+        self._dependencies.setdefault(key, set())
41 
+        try:
42 
+            for dependency in self.getDependencies(value):
43 
+                if dependency == key:
44 
+                    # A variable is allowed to reference itself; no
45 
+                    # dependency link needed in that case.
46 
+                    continue
47 
+                if dependency not in self._varnames:
48 
+                    # It's not necessary to create a link for an
49 
+                    # external variable.
50 
+                    continue
51 
+                # Make sure a circular dependency is never created
52 
+                ancestor_vars = self._getParentVarNamesRecursively(
53 
+                    dependency, soft=True)
54 
+                ancestor_vars.add(dependency)
55 
+                if any((key == anc_var) for anc_var in ancestor_vars):
56 
+                    raise Exception("Dependency cycle detected in var {}".
57 
+                                    format(key))
58 
+                self._dependencies[key].add(dependency)
59 
+        except Exception:
60 
+            del self.vars[key]
61 
+            del self._dependencies[key]
62 
+            raise
63 
+
64 
+    def getVars(self):
65 
+        ret = []
66 
+        keys = sorted(self.vars.keys())
67 
+        seen = set()
68 
+        for key in keys:
69 
+            dependencies = self.getDependentVarsRecursively(key)
70 
+            for var in dependencies + [key]:
71 
+                if var not in seen:
72 
+                    ret.append((var, self.vars[var]))
73 
+                    seen.add(var)
74 
+        return ret
75 
+
76 
+    def getDependentVarsRecursively(self, parent_var):
77 
+        dependent_vars = []
78 
+
79 
+        current_dependent_vars = self._dependencies[parent_var]
80 
+        for current_var in current_dependent_vars:
81 
+            if current_var not in dependent_vars:
82 
+                dependent_vars.append(current_var)
83 
+            for dep in self.getDependentVarsRecursively(current_var):
84 
+                if dep not in dependent_vars:
85 
+                    dependent_vars.append(dep)
86 
+        return dependent_vars
87 
+
88 
+    def _getParentVarNamesRecursively(self, dependent_var, soft=False):
89 
+        all_parent_vars = set()
90 
+        vars_to_iterate = set([dependent_var])
91 
+        while len(vars_to_iterate) > 0:
92 
+            current_var = vars_to_iterate.pop()
93 
+            current_parent_vars = self._dependencies.get(current_var)
94 
+            if current_parent_vars is None:
95 
+                if soft:
96 
+                    current_parent_vars = set()
97 
+                else:
98 
+                    raise Exception("Dependent var {} not found: ".format(
99 
+                                    dependent_var))
100 
+            new_parent_vars = current_parent_vars - all_parent_vars
101 
+            vars_to_iterate |= new_parent_vars
102 
+            all_parent_vars |= new_parent_vars
103 
+        return all_parent_vars
104 
+
105 
+
106 
+class LocalConf(object):
107 
+
108 
+    def __init__(self, localrc, localconf, services, plugins):
109 
+        self.localrc = []
110 
+        self.meta_sections = {}
111 
+        if plugins:
112 
+            self.handle_plugins(plugins)
113 
+        if services:
114 
+            self.handle_services(services)
115 
+        if localrc:
116 
+            self.handle_localrc(localrc)
117 
+        if localconf:
118 
+            self.handle_localconf(localconf)
119 
+
120 
+    def handle_plugins(self, plugins):
121 
+        for k, v in plugins.items():
122 
+            if v:
123 
+                self.localrc.append('enable_plugin {} {}'.format(k, v))
124 
+
125 
+    def handle_services(self, services):
126 
+        for k, v in services.items():
127 
+            if v is False:
128 
+                self.localrc.append('disable_service {}'.format(k))
129 
+            elif v is True:
130 
+                self.localrc.append('enable_service {}'.format(k))
131 
+
132 
+    def handle_localrc(self, localrc):
133 
+        vg = VarGraph(localrc)
134 
+        for k, v in vg.getVars():
135 
+            self.localrc.append('{}={}'.format(k, v))
136 
+
137 
+    def handle_localconf(self, localconf):
138 
+        for phase, phase_data in localconf.items():
139 
+            for fn, fn_data in phase_data.items():
140 
+                ms_name = '[[{}|{}]]'.format(phase, fn)
141 
+                ms_data = []
142 
+                for section, section_data in fn_data.items():
143 
+                    ms_data.append('[{}]'.format(section))
144 
+                    for k, v in section_data.items():
145 
+                        ms_data.append('{} = {}'.format(k, v))
146 
+                    ms_data.append('')
147 
+                self.meta_sections[ms_name] = ms_data
148 
+
149 
+    def write(self, path):
150 
+        with open(path, 'w') as f:
151 
+            f.write('[[local|localrc]]\n')
152 
+            f.write('\n'.join(self.localrc))
153 
+            f.write('\n\n')
154 
+            for section, lines in self.meta_sections.items():
155 
+                f.write('{}\n'.format(section))
156 
+                f.write('\n'.join(lines))
157 
+
158 
+
159 
+def main():
160 
+    module = AnsibleModule(
161 
+        argument_spec=dict(
162 
+            plugins=dict(type='dict'),
163 
+            services=dict(type='dict'),
164 
+            localrc=dict(type='dict'),
165 
+            local_conf=dict(type='dict'),
166 
+            path=dict(type='str'),
167 
+        )
168 
+    )
169 
+
170 
+    p = module.params
171 
+    lc = LocalConf(p.get('localrc'),
172 
+                   p.get('local_conf'),
173 
+                   p.get('services'),
174 
+                   p.get('plugins'))
175 
+    lc.write(p['path'])
176 
+
177 
+    module.exit_json()
178 
+
179 
+
180 
+from ansible.module_utils.basic import *  # noqa
181 
+from ansible.module_utils.basic import AnsibleModule
182 
+
183 
+if __name__ == '__main__':
184 
+    main()
roles/write-devstack-local-conf/tasks/main.yaml
History View file @ 36ddea3
0 185 
new file mode 100644
... ... 
@@ -0,0 +1,9 @@
0 
+- name: Write a job-specific local_conf file
1 
+  become: true
2 
+  become_user: stack
3 
+  devstack_local_conf:
4 
+    path: "{{ devstack_local_conf_path }}"
5 
+    plugins: "{{ devstack_plugins|default(omit) }}"
6 
+    services: "{{ devstack_services|default(omit) }}"
7 
+    localrc: "{{ devstack_localrc|default(omit) }}"
8 
+    local_conf: "{{ devstack_local_conf|default(omit) }}"