diff options
Diffstat (limited to 'scripts/contrib/oe-build-perf-report-email.py')
| -rwxr-xr-x | scripts/contrib/oe-build-perf-report-email.py | 154 |
1 files changed, 6 insertions, 148 deletions
diff --git a/scripts/contrib/oe-build-perf-report-email.py b/scripts/contrib/oe-build-perf-report-email.py index de3862c897..c900720f6e 100755 --- a/scripts/contrib/oe-build-perf-report-email.py +++ b/scripts/contrib/oe-build-perf-report-email.py | |||
| @@ -19,8 +19,6 @@ import socket | |||
| 19 | import subprocess | 19 | import subprocess |
| 20 | import sys | 20 | import sys |
| 21 | import tempfile | 21 | import tempfile |
| 22 | from email.mime.image import MIMEImage | ||
| 23 | from email.mime.multipart import MIMEMultipart | ||
| 24 | from email.mime.text import MIMEText | 22 | from email.mime.text import MIMEText |
| 25 | 23 | ||
| 26 | 24 | ||
| @@ -29,19 +27,6 @@ logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") | |||
| 29 | log = logging.getLogger('oe-build-perf-report') | 27 | log = logging.getLogger('oe-build-perf-report') |
| 30 | 28 | ||
| 31 | 29 | ||
| 32 | # Find js scaper script | ||
| 33 | SCRAPE_JS = os.path.join(os.path.dirname(__file__), '..', 'lib', 'build_perf', | ||
| 34 | 'scrape-html-report.js') | ||
| 35 | if not os.path.isfile(SCRAPE_JS): | ||
| 36 | log.error("Unableto find oe-build-perf-report-scrape.js") | ||
| 37 | sys.exit(1) | ||
| 38 | |||
| 39 | |||
| 40 | class ReportError(Exception): | ||
| 41 | """Local errors""" | ||
| 42 | pass | ||
| 43 | |||
| 44 | |||
| 45 | def check_utils(): | 30 | def check_utils(): |
| 46 | """Check that all needed utils are installed in the system""" | 31 | """Check that all needed utils are installed in the system""" |
| 47 | missing = [] | 32 | missing = [] |
| @@ -77,137 +62,19 @@ def parse_args(argv): | |||
| 77 | "the email parts") | 62 | "the email parts") |
| 78 | parser.add_argument('--text', | 63 | parser.add_argument('--text', |
| 79 | help="Plain text message") | 64 | help="Plain text message") |
| 80 | parser.add_argument('--html', | ||
| 81 | help="HTML peport generated by oe-build-perf-report") | ||
| 82 | parser.add_argument('--phantomjs-args', action='append', | ||
| 83 | help="Extra command line arguments passed to PhantomJS") | ||
| 84 | 65 | ||
| 85 | args = parser.parse_args(argv) | 66 | args = parser.parse_args(argv) |
| 86 | 67 | ||
| 87 | if not args.html and not args.text: | 68 | if not args.text: |
| 88 | parser.error("Please specify --html and/or --text") | 69 | parser.error("Please specify --text") |
| 89 | 70 | ||
| 90 | return args | 71 | return args |
| 91 | 72 | ||
| 92 | 73 | ||
| 93 | def decode_png(infile, outfile): | 74 | def send_email(text_fn, subject, recipients, copy=[], blind_copy=[]): |
| 94 | """Parse/decode/optimize png data from a html element""" | ||
| 95 | with open(infile) as f: | ||
| 96 | raw_data = f.read() | ||
| 97 | |||
| 98 | # Grab raw base64 data | ||
| 99 | b64_data = re.sub('^.*href="data:image/png;base64,', '', raw_data, 1) | ||
| 100 | b64_data = re.sub('">.+$', '', b64_data, 1) | ||
| 101 | |||
| 102 | # Replace file with proper decoded png | ||
| 103 | with open(outfile, 'wb') as f: | ||
| 104 | f.write(base64.b64decode(b64_data)) | ||
| 105 | |||
| 106 | subprocess.check_output(['optipng', outfile], stderr=subprocess.STDOUT) | ||
| 107 | |||
| 108 | |||
| 109 | def mangle_html_report(infile, outfile, pngs): | ||
| 110 | """Mangle html file into a email compatible format""" | ||
| 111 | paste = True | ||
| 112 | png_dir = os.path.dirname(outfile) | ||
| 113 | with open(infile) as f_in: | ||
| 114 | with open(outfile, 'w') as f_out: | ||
| 115 | for line in f_in.readlines(): | ||
| 116 | stripped = line.strip() | ||
| 117 | # Strip out scripts | ||
| 118 | if stripped == '<!--START-OF-SCRIPTS-->': | ||
| 119 | paste = False | ||
| 120 | elif stripped == '<!--END-OF-SCRIPTS-->': | ||
| 121 | paste = True | ||
| 122 | elif paste: | ||
| 123 | if re.match('^.+href="data:image/png;base64', stripped): | ||
| 124 | # Strip out encoded pngs (as they're huge in size) | ||
| 125 | continue | ||
| 126 | elif 'www.gstatic.com' in stripped: | ||
| 127 | # HACK: drop references to external static pages | ||
| 128 | continue | ||
| 129 | |||
| 130 | # Replace charts with <img> elements | ||
| 131 | match = re.match('<div id="(?P<id>\w+)"', stripped) | ||
| 132 | if match and match.group('id') in pngs: | ||
| 133 | f_out.write('<img src="cid:{}"\n'.format(match.group('id'))) | ||
| 134 | else: | ||
| 135 | f_out.write(line) | ||
| 136 | |||
| 137 | |||
| 138 | def scrape_html_report(report, outdir, phantomjs_extra_args=None): | ||
| 139 | """Scrape html report into a format sendable by email""" | ||
| 140 | tmpdir = tempfile.mkdtemp(dir='.') | ||
| 141 | log.debug("Using tmpdir %s for phantomjs output", tmpdir) | ||
| 142 | |||
| 143 | if not os.path.isdir(outdir): | ||
| 144 | os.mkdir(outdir) | ||
| 145 | if os.path.splitext(report)[1] not in ('.html', '.htm'): | ||
| 146 | raise ReportError("Invalid file extension for report, needs to be " | ||
| 147 | "'.html' or '.htm'") | ||
| 148 | |||
| 149 | try: | ||
| 150 | log.info("Scraping HTML report with PhangomJS") | ||
| 151 | extra_args = phantomjs_extra_args if phantomjs_extra_args else [] | ||
| 152 | subprocess.check_output(['phantomjs', '--debug=true'] + extra_args + | ||
| 153 | [SCRAPE_JS, report, tmpdir], | ||
| 154 | stderr=subprocess.STDOUT) | ||
| 155 | |||
| 156 | pngs = [] | ||
| 157 | images = [] | ||
| 158 | for fname in os.listdir(tmpdir): | ||
| 159 | base, ext = os.path.splitext(fname) | ||
| 160 | if ext == '.png': | ||
| 161 | log.debug("Decoding %s", fname) | ||
| 162 | decode_png(os.path.join(tmpdir, fname), | ||
| 163 | os.path.join(outdir, fname)) | ||
| 164 | pngs.append(base) | ||
| 165 | images.append(fname) | ||
| 166 | elif ext in ('.html', '.htm'): | ||
| 167 | report_file = fname | ||
| 168 | else: | ||
| 169 | log.warning("Unknown file extension: '%s'", ext) | ||
| 170 | #shutil.move(os.path.join(tmpdir, fname), outdir) | ||
| 171 | |||
| 172 | log.debug("Mangling html report file %s", report_file) | ||
| 173 | mangle_html_report(os.path.join(tmpdir, report_file), | ||
| 174 | os.path.join(outdir, report_file), pngs) | ||
| 175 | return (os.path.join(outdir, report_file), | ||
| 176 | [os.path.join(outdir, i) for i in images]) | ||
| 177 | finally: | ||
| 178 | shutil.rmtree(tmpdir) | ||
| 179 | |||
| 180 | def send_email(text_fn, html_fn, image_fns, subject, recipients, copy=[], | ||
| 181 | blind_copy=[]): | ||
| 182 | """Send email""" | ||
| 183 | # Generate email message | 75 | # Generate email message |
| 184 | text_msg = html_msg = None | 76 | with open(text_fn) as f: |
| 185 | if text_fn: | 77 | msg = MIMEText("Yocto build performance test report.\n" + f.read(), 'plain') |
| 186 | with open(text_fn) as f: | ||
| 187 | text_msg = MIMEText("Yocto build performance test report.\n" + | ||
| 188 | f.read(), 'plain') | ||
| 189 | if html_fn: | ||
| 190 | html_msg = msg = MIMEMultipart('related') | ||
| 191 | with open(html_fn) as f: | ||
| 192 | html_msg.attach(MIMEText(f.read(), 'html')) | ||
| 193 | for img_fn in image_fns: | ||
| 194 | # Expect that content id is same as the filename | ||
| 195 | cid = os.path.splitext(os.path.basename(img_fn))[0] | ||
| 196 | with open(img_fn, 'rb') as f: | ||
| 197 | image_msg = MIMEImage(f.read()) | ||
| 198 | image_msg['Content-ID'] = '<{}>'.format(cid) | ||
| 199 | html_msg.attach(image_msg) | ||
| 200 | |||
| 201 | if text_msg and html_msg: | ||
| 202 | msg = MIMEMultipart('alternative') | ||
| 203 | msg.attach(text_msg) | ||
| 204 | msg.attach(html_msg) | ||
| 205 | elif text_msg: | ||
| 206 | msg = text_msg | ||
| 207 | elif html_msg: | ||
| 208 | msg = html_msg | ||
| 209 | else: | ||
| 210 | raise ReportError("Neither plain text nor html body specified") | ||
| 211 | 78 | ||
| 212 | pw_data = pwd.getpwuid(os.getuid()) | 79 | pw_data = pwd.getpwuid(os.getuid()) |
| 213 | full_name = pw_data.pw_gecos.split(',')[0] | 80 | full_name = pw_data.pw_gecos.split(',')[0] |
| @@ -245,25 +112,16 @@ def main(argv=None): | |||
| 245 | 112 | ||
| 246 | try: | 113 | try: |
| 247 | log.debug("Storing email parts in %s", outdir) | 114 | log.debug("Storing email parts in %s", outdir) |
| 248 | html_report = images = None | ||
| 249 | if args.html: | ||
| 250 | html_report, images = scrape_html_report(args.html, outdir, | ||
| 251 | args.phantomjs_args) | ||
| 252 | |||
| 253 | if args.to: | 115 | if args.to: |
| 254 | log.info("Sending email to %s", ', '.join(args.to)) | 116 | log.info("Sending email to %s", ', '.join(args.to)) |
| 255 | if args.cc: | 117 | if args.cc: |
| 256 | log.info("Copying to %s", ', '.join(args.cc)) | 118 | log.info("Copying to %s", ', '.join(args.cc)) |
| 257 | if args.bcc: | 119 | if args.bcc: |
| 258 | log.info("Blind copying to %s", ', '.join(args.bcc)) | 120 | log.info("Blind copying to %s", ', '.join(args.bcc)) |
| 259 | send_email(args.text, html_report, images, args.subject, | 121 | send_email(args.text, args.subject, args.to, args.cc, args.bcc) |
| 260 | args.to, args.cc, args.bcc) | ||
| 261 | except subprocess.CalledProcessError as err: | 122 | except subprocess.CalledProcessError as err: |
| 262 | log.error("%s, with output:\n%s", str(err), err.output.decode()) | 123 | log.error("%s, with output:\n%s", str(err), err.output.decode()) |
| 263 | return 1 | 124 | return 1 |
| 264 | except ReportError as err: | ||
| 265 | log.error(err) | ||
| 266 | return 1 | ||
| 267 | finally: | 125 | finally: |
| 268 | if not args.outdir: | 126 | if not args.outdir: |
| 269 | log.debug("Wiping %s", outdir) | 127 | log.debug("Wiping %s", outdir) |
