1. docker
  2. Commit a2ffa637ceb694b1e7a21972a9482a0c9c1676c8
Browse code

testing infrastructure, PR #2195: Add aggregated docker-ci email report

Daniel Mizyrycki authored on 2013/10/17 05:26:39
Showing 3 changed files
hack/infrastructure/docker-ci/report/Dockerfile
History View file @ a2ffa63
1 1 
new file mode 100644
... ... 
@@ -0,0 +1,28 @@
0 
+# VERSION:        0.22
1 
+# DOCKER-VERSION  0.6.3
2 
+# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
3 
+# DESCRIPTION:    Generate docker-ci daily report
4 
+# COMMENTS:       The build process is initiated by deployment.py
5 
+                  Report configuration is passed through ./credentials.json at
6 
+#                 deployment time.
7 
+# TO_BUILD:       docker build -t report .
8 
+# TO_DEPLOY:      docker run report
9 
+
10 
+from ubuntu:12.04
11 
+maintainer Daniel Mizyrycki <daniel@dotcloud.com>
12 
+
13 
+env PYTHONPATH /report
14 
+
15 
+
16 
+# Add report dependencies
17 
+run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > \
18 
+    /etc/apt/sources.list
19 
+run apt-get update; apt-get install -y python2.7 python-pip ssh rsync
20 
+
21 
+# Set San Francisco timezone
22 
+run echo "America/Los_Angeles" >/etc/timezone
23 
+run dpkg-reconfigure --frontend noninteractive tzdata
24 
+
25 
+# Add report code and set default container command
26 
+add . /report
27 
+cmd "/report/report.py"
hack/infrastructure/docker-ci/report/deployment.py
History View file @ a2ffa63
0 28 
new file mode 100755
... ... 
@@ -0,0 +1,130 @@
0 
+#!/usr/bin/env python
1 
+
2 
+'''Deploy docker-ci report container on Digital Ocean.
3 
+Usage:
4 
+    export CONFIG_JSON='
5 
+        { "DROPLET_NAME":        "Digital_Ocean_dropplet_name",
6 
+          "DO_CLIENT_ID":        "Digital_Ocean_client_id",
7 
+          "DO_API_KEY":          "Digital_Ocean_api_key",
8 
+          "DOCKER_KEY_ID":       "Digital_Ocean_ssh_key_id",
9 
+          "DOCKER_CI_KEY_PATH":  "docker-ci_private_key_path",
10 
+          "DOCKER_CI_PUB":       "$(cat docker-ci_ssh_public_key.pub)",
11 
+          "DOCKER_CI_ADDRESS"    "user@docker-ci_fqdn_server",
12 
+          "SMTP_USER":           "SMTP_server_user",
13 
+          "SMTP_PWD":            "SMTP_server_password",
14 
+          "EMAIL_SENDER":        "Buildbot_mailing_sender",
15 
+          "EMAIL_RCP":           "Buildbot_mailing_receipient" }'
16 
+    python deployment.py
17 
+'''
18 
+
19 
+import re, json, requests, base64
20 
+from fabric import api
21 
+from fabric.api import cd, run, put, sudo
22 
+from os import environ as env
23 
+from time import sleep
24 
+from datetime import datetime
25 
+
26 
+# Populate environment variables
27 
+CONFIG = json.loads(env['CONFIG_JSON'])
28 
+for key in CONFIG:
29 
+    env[key] = CONFIG[key]
30 
+
31 
+# Load DOCKER_CI_KEY
32 
+env['DOCKER_CI_KEY'] = open(env['DOCKER_CI_KEY_PATH']).read()
33 
+
34 
+DROPLET_NAME = env.get('DROPLET_NAME','report')
35 
+TIMEOUT = 120            # Seconds before timeout droplet creation
36 
+IMAGE_ID = 894856        # Docker on Ubuntu 13.04
37 
+REGION_ID = 4            # New York 2
38 
+SIZE_ID = 66             # memory 512MB
39 
+DO_IMAGE_USER = 'root'   # Image user on Digital Ocean
40 
+API_URL = 'https://api.digitalocean.com/'
41 
+
42 
+
43 
+class digital_ocean():
44 
+
45 
+    def __init__(self, key, client):
46 
+        '''Set default API parameters'''
47 
+        self.key = key
48 
+        self.client = client
49 
+        self.api_url = API_URL
50 
+
51 
+    def api(self, cmd_path, api_arg={}):
52 
+        '''Make api call'''
53 
+        api_arg.update({'api_key':self.key, 'client_id':self.client})
54 
+        resp = requests.get(self.api_url + cmd_path, params=api_arg).text
55 
+        resp = json.loads(resp)
56 
+        if resp['status'] != 'OK':
57 
+            raise Exception(resp['error_message'])
58 
+        return resp
59 
+
60 
+    def droplet_data(self, name):
61 
+        '''Get droplet data'''
62 
+        data = self.api('droplets')
63 
+        data = [droplet for droplet in data['droplets']
64 
+            if droplet['name'] == name]
65 
+        return data[0] if data else {}
66 
+
67 
+def json_fmt(data):
68 
+    '''Format json output'''
69 
+    return json.dumps(data, sort_keys = True, indent = 2)
70 
+
71 
+
72 
+do = digital_ocean(env['DO_API_KEY'], env['DO_CLIENT_ID'])
73 
+
74 
+# Get DROPLET_NAME data
75 
+data = do.droplet_data(DROPLET_NAME)
76 
+
77 
+# Stop processing if DROPLET_NAME exists on Digital Ocean
78 
+if data:
79 
+    print ('Droplet: {} already deployed. Not further processing.'
80 
+        .format(DROPLET_NAME))
81 
+    exit(1)
82 
+
83 
+# Create droplet
84 
+do.api('droplets/new', {'name':DROPLET_NAME, 'region_id':REGION_ID,
85 
+    'image_id':IMAGE_ID, 'size_id':SIZE_ID,
86 
+    'ssh_key_ids':[env['DOCKER_KEY_ID']]})
87 
+
88 
+# Wait for droplet to be created.
89 
+start_time = datetime.now()
90 
+while (data.get('status','') != 'active' and (
91 
+ datetime.now()-start_time).seconds < TIMEOUT):
92 
+    data = do.droplet_data(DROPLET_NAME)
93 
+    print data['status']
94 
+    sleep(3)
95 
+
96 
+# Wait for the machine to boot
97 
+sleep(15)
98 
+
99 
+# Get droplet IP
100 
+ip = str(data['ip_address'])
101 
+print 'droplet: {}    ip: {}'.format(DROPLET_NAME, ip)
102 
+
103 
+api.env.host_string = ip
104 
+api.env.user = DO_IMAGE_USER
105 
+api.env.key_filename = env['DOCKER_CI_KEY_PATH']
106 
+
107 
+# Correct timezone
108 
+sudo('echo "America/Los_Angeles" >/etc/timezone')
109 
+sudo('dpkg-reconfigure --frontend noninteractive tzdata')
110 
+
111 
+# Load JSON_CONFIG environment for Dockerfile
112 
+CONFIG_JSON= base64.b64encode(
113 
+    '{{"DOCKER_CI_PUB":     "{DOCKER_CI_PUB}",'
114 
+    '  "DOCKER_CI_KEY":     "{DOCKER_CI_KEY}",'
115 
+    '  "DOCKER_CI_ADDRESS": "{DOCKER_CI_ADDRESS}",'
116 
+    '  "SMTP_USER":         "{SMTP_USER}",'
117 
+    '  "SMTP_PWD":          "{SMTP_PWD}",'
118 
+    '  "EMAIL_SENDER":      "{EMAIL_SENDER}",'
119 
+    '  "EMAIL_RCP":         "{EMAIL_RCP}"}}'.format(**env))
120 
+
121 
+run('mkdir -p /data/report')
122 
+put('./', '/data/report')
123 
+with cd('/data/report'):
124 
+    run('chmod 700 report.py')
125 
+    run('echo "{}" > credentials.json'.format(CONFIG_JSON))
126 
+    run('docker build -t report .')
127 
+    run('rm credentials.json')
128 
+    run("echo -e '30 09 * * * /usr/bin/docker run report\n' |"
129 
+        " /usr/bin/crontab -")
hack/infrastructure/docker-ci/report/report.py
History View file @ a2ffa63
0 130 
new file mode 100755
... ... 
@@ -0,0 +1,145 @@
0 
+#!/usr/bin/python
1 
+
2 
+'''CONFIG_JSON is a json encoded string base64 environment variable. It is used
3 
+to clone docker-ci database, generate docker-ci report and submit it by email.
4 
+CONFIG_JSON data comes from the file /report/credentials.json inserted in this
5 
+container by deployment.py:
6 
+
7 
+{ "DOCKER_CI_PUB":       "$(cat docker-ci_ssh_public_key.pub)",
8 
+  "DOCKER_CI_KEY":       "$(cat docker-ci_ssh_private_key.key)",
9 
+  "DOCKER_CI_ADDRESS":   "user@docker-ci_fqdn_server",
10 
+  "SMTP_USER":           "SMTP_server_user",
11 
+  "SMTP_PWD":            "SMTP_server_password",
12 
+  "EMAIL_SENDER":        "Buildbot_mailing_sender",
13 
+  "EMAIL_RCP":           "Buildbot_mailing_receipient" }  '''
14 
+
15 
+import os, re, json, sqlite3, datetime, base64
16 
+import smtplib
17 
+from datetime import timedelta
18 
+from subprocess import call
19 
+from os import environ as env
20 
+
21 
+TODAY = datetime.date.today()
22 
+
23 
+# Load credentials to the environment
24 
+env['CONFIG_JSON'] = base64.b64decode(open('/report/credentials.json').read())
25 
+
26 
+# Remove SSH private key as it needs more processing
27 
+CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','',
28 
+    env['CONFIG_JSON'], flags=re.DOTALL))
29 
+
30 
+# Populate environment variables
31 
+for key in CONFIG:
32 
+    env[key] = CONFIG[key]
33 
+
34 
+# Load SSH private key
35 
+env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1',
36 
+    env['CONFIG_JSON'],flags=re.DOTALL)
37 
+
38 
+# Prevent rsync to validate host on first connection to docker-ci
39 
+os.makedirs('/root/.ssh')
40 
+open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY'])
41 
+os.chmod('/root/.ssh/id_rsa',0600)
42 
+open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n')
43 
+
44 
+
45 
+# Sync buildbot database from docker-ci
46 
+call('rsync {}:/data/buildbot/master/state.sqlite .'.format(
47 
+    env['DOCKER_CI_ADDRESS']), shell=True)
48 
+
49 
+class SQL:
50 
+    def __init__(self, database_name):
51 
+        sql = sqlite3.connect(database_name)
52 
+        # Use column names as keys for fetchall rows
53 
+        sql.row_factory = sqlite3.Row
54 
+        sql = sql.cursor()
55 
+        self.sql = sql
56 
+
57 
+    def query(self,query_statement):
58 
+        return self.sql.execute(query_statement).fetchall()
59 
+
60 
+sql = SQL("state.sqlite")
61 
+
62 
+
63 
+class Report():
64 
+
65 
+    def __init__(self,period='',date=''):
66 
+        self.data = []
67 
+        self.period = 'date' if not period else period
68 
+        self.date = str(TODAY) if not date else date
69 
+        self.compute()
70 
+
71 
+    def compute(self):
72 
+        '''Compute report'''
73 
+        if self.period == 'week':
74 
+            self.week_report(self.date)
75 
+        else:
76 
+            self.date_report(self.date)
77 
+
78 
+
79 
+    def date_report(self,date):
80 
+        '''Create a date test report'''
81 
+        builds = []
82 
+        # Get a queryset with all builds from date
83 
+        rows = sql.query('SELECT * FROM builds JOIN buildrequests'
84 
+            ' WHERE builds.brid=buildrequests.id and'
85 
+            ' date(start_time, "unixepoch", "localtime") = "{0}"'
86 
+            ' GROUP BY number'.format(date))
87 
+        build_names = sorted(set([row['buildername'] for row in rows]))
88 
+        # Create a report build line for a given build
89 
+        for build_name in build_names:
90 
+            tried = len([row['buildername']
91 
+                for row in rows if row['buildername'] == build_name])
92 
+            fail_tests = [row['buildername'] for row in rows if (
93 
+                row['buildername'] == build_name and row['results'] != 0)]
94 
+            fail = len(fail_tests)
95 
+            fail_details = ''
96 
+            fail_pct = int(100.0*fail/tried) if  tried != 0 else 100
97 
+            builds.append({'name': build_name, 'tried': tried, 'fail': fail,
98 
+                'fail_pct': fail_pct, 'fail_details':fail_details})
99 
+        if builds:
100 
+            self.data.append({'date': date, 'builds': builds})
101 
+
102 
+
103 
+    def week_report(self,date):
104 
+        '''Add the week's date test reports to report.data'''
105 
+        date = datetime.datetime.strptime(date,'%Y-%m-%d').date()
106 
+        last_monday = date - datetime.timedelta(days=date.weekday())
107 
+        week_dates = [last_monday + timedelta(days=x) for x in range(7,-1,-1)]
108 
+        for date in week_dates:
109 
+            self.date_report(str(date))
110 
+
111 
+    def render_text(self):
112 
+        '''Return rendered report in text format'''
113 
+        retval = ''
114 
+        fail_tests = {}
115 
+        for builds in self.data:
116 
+            retval += 'Test date: {0}\n'.format(builds['date'],retval)
117 
+            table = ''
118 
+            for build in builds['builds']:
119 
+                table += ('Build {name:15}   Tried: {tried:4}   '
120 
+                    ' Failures: {fail:4} ({fail_pct}%)\n'.format(**build))
121 
+                if build['name'] in fail_tests:
122 
+                    fail_tests[build['name']] += build['fail_details']
123 
+                else:
124 
+                    fail_tests[build['name']] = build['fail_details']
125 
+            retval += '{0}\n'.format(table)
126 
+            retval += '\n    Builds failing'
127 
+            for fail_name in fail_tests:
128 
+                retval += '\n' + fail_name + '\n'
129 
+                for (fail_id,fail_url,rn_tests,nr_errors,log_errors,
130 
+                 tracelog_errors) in fail_tests[fail_name]:
131 
+                    retval += fail_url + '\n'
132 
+            retval += '\n\n'
133 
+        return retval
134 
+
135 
+
136 
+# Send email
137 
+smtp_from = env['EMAIL_SENDER']
138 
+subject = '[docker-ci] Daily report for {}'.format(str(TODAY))
139 
+msg = "From: {}\r\nTo: {}\r\nSubject: {}\r\n\r\n".format(
140 
+    smtp_from, env['EMAIL_RCP'], subject)
141 
+msg = msg + Report('week').render_text()
142 
+server = smtplib.SMTP_SSL('smtp.mailgun.org')
143 
+server.login(env['SMTP_USER'], env['SMTP_PWD'])
144 
+server.sendmail(smtp_from, env['EMAIL_RCP'], msg)