wic: Add mic w/pykickstart

This is the starting point for the implemention described in [YOCTO
3847] which came to the conclusion that it would make sense to use
kickstart syntax to implement image creation in OpenEmbedded.  I
subsequently realized that there was an existing tool that already
implemented image creation using kickstart syntax, the Tizen/Meego mic
tool.  As such, it made sense to use that as a starting point - this
commit essentially just copies the relevant Python code from the MIC
tool to the scripts/lib dir, where it can be accessed by the
previously created wic tool.

Most of this will be removed or renamed by later commits, since we're
initially focusing on partitioning only.  Care should be taken so that
we can easily add back any additional functionality should we decide
later to expand the tool, though (we may also want to contribute our
local changes to the mic tool to the Tizen project if it makes sense,
and therefore should avoid gratuitous changes to the original code if
possible).

Added the /mic subdir from Tizen mic repo as a starting point:

 git clone git://review.tizen.org/tools/mic.git

 For reference, the top commit:

 commit 20164175ddc234a17b8a12c33d04b012347b1530
 Author: Gui Chen <gui.chen@intel.com>
 Date:   Sun Jun 30 22:32:16 2013 -0400

    bump up to 0.19.2

Also added the /plugins subdir, moved to under the /mic subdir (to
match the default plugin_dir location in mic.conf.in, which was
renamed to yocto-image.conf (moved and renamed by later patches) and
put into /scripts.

(From OE-Core rev: 31f0360f1fd4ebc9dfcaed42d1c50d2448b4632e)

Signed-off-by: Tom Zanussi <tom.zanussi@linux.intel.com>
Signed-off-by: Saul Wold <sgw@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 530 insertions, 0 deletions

diff --git a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/progress.py b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/progress.py
new file mode 100644
index 0000000000..02db524e76
--- /dev/null
+++ b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/progress.py
@@ -0,0 +1,530 @@
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the 
+#      Free Software Foundation, Inc., 
+#      59 Temple Place, Suite 330, 
+#      Boston, MA  02111-1307  USA
+
+# This file is part of urlgrabber, a high-level cross-protocol url-grabber
+# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
+
+# $Id: progress.py,v 1.7 2005/08/19 21:59:07 mstenner Exp $
+
+import sys
+import time
+import math
+import thread
+    
+class BaseMeter:
+    def __init__(self):
+        self.update_period = 0.3 # seconds
+
+        self.filename   = None
+        self.url        = None
+        self.basename   = None
+        self.text       = None
+        self.size       = None
+        self.start_time = None
+        self.last_amount_read = 0
+        self.last_update_time = None
+        self.re = RateEstimator()
+        
+    def start(self, filename=None, url=None, basename=None,
+              size=None, now=None, text=None):
+        self.filename = filename
+        self.url      = url
+        self.basename = basename
+        self.text     = text
+
+        #size = None #########  TESTING
+        self.size = size
+        if not size is None: self.fsize = format_number(size) + 'B'
+
+        if now is None: now = time.time()
+        self.start_time = now
+        self.re.start(size, now)
+        self.last_amount_read = 0
+        self.last_update_time = now
+        self._do_start(now)
+        
+    def _do_start(self, now=None):
+        pass
+
+    def update(self, amount_read, now=None):
+        # for a real gui, you probably want to override and put a call
+        # to your mainloop iteration function here
+        if now is None: now = time.time()
+        if (now >= self.last_update_time + self.update_period) or \
+               not self.last_update_time:
+            self.re.update(amount_read, now)
+            self.last_amount_read = amount_read
+            self.last_update_time = now
+            self._do_update(amount_read, now)
+
+    def _do_update(self, amount_read, now=None):
+        pass
+
+    def end(self, amount_read, now=None):
+        if now is None: now = time.time()
+        self.re.update(amount_read, now)
+        self.last_amount_read = amount_read
+        self.last_update_time = now
+        self._do_end(amount_read, now)
+
+    def _do_end(self, amount_read, now=None):
+        pass
+        
+class TextMeter(BaseMeter):
+    def __init__(self, fo=sys.stderr):
+        BaseMeter.__init__(self)
+        self.fo = fo
+
+    def _do_update(self, amount_read, now=None):
+        etime = self.re.elapsed_time()
+        fetime = format_time(etime)
+        fread = format_number(amount_read)
+        #self.size = None
+        if self.text is not None:
+            text = self.text
+        else:
+            text = self.basename
+        if self.size is None:
+            out = '\r%-60.60s    %5sB %s ' % \
+                  (text, fread, fetime)
+        else:
+            rtime = self.re.remaining_time()
+            frtime = format_time(rtime)
+            frac = self.re.fraction_read()
+            bar = '='*int(25 * frac)
+
+            out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \
+                  (text, frac*100, bar, fread, frtime)
+
+        self.fo.write(out)
+        self.fo.flush()
+
+    def _do_end(self, amount_read, now=None):
+        total_time = format_time(self.re.elapsed_time())
+        total_size = format_number(amount_read)
+        if self.text is not None:
+            text = self.text
+        else:
+            text = self.basename
+        if self.size is None:
+            out = '\r%-60.60s    %5sB %s ' % \
+                  (text, total_size, total_time)
+        else:
+            bar = '='*25
+            out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s     ' % \
+                  (text, 100, bar, total_size, total_time)
+        self.fo.write(out + '\n')
+        self.fo.flush()
+
+text_progress_meter = TextMeter
+
+class MultiFileHelper(BaseMeter):
+    def __init__(self, master):
+        BaseMeter.__init__(self)
+        self.master = master
+
+    def _do_start(self, now):
+        self.master.start_meter(self, now)
+
+    def _do_update(self, amount_read, now):
+        # elapsed time since last update
+        self.master.update_meter(self, now)
+
+    def _do_end(self, amount_read, now):
+        self.ftotal_time = format_time(now - self.start_time)
+        self.ftotal_size = format_number(self.last_amount_read)
+        self.master.end_meter(self, now)
+
+    def failure(self, message, now=None):
+        self.master.failure_meter(self, message, now)
+
+    def message(self, message):
+        self.master.message_meter(self, message)
+
+class MultiFileMeter:
+    helperclass = MultiFileHelper
+    def __init__(self):
+        self.meters = []
+        self.in_progress_meters = []
+        self._lock = thread.allocate_lock()
+        self.update_period = 0.3 # seconds
+        
+        self.numfiles         = None
+        self.finished_files   = 0
+        self.failed_files     = 0
+        self.open_files       = 0
+        self.total_size       = None
+        self.failed_size      = 0
+        self.start_time       = None
+        self.finished_file_size = 0
+        self.last_update_time = None
+        self.re = RateEstimator()
+
+    def start(self, numfiles=None, total_size=None, now=None):
+        if now is None: now = time.time()
+        self.numfiles         = numfiles
+        self.finished_files   = 0
+        self.failed_files     = 0
+        self.open_files       = 0
+        self.total_size       = total_size
+        self.failed_size      = 0
+        self.start_time       = now
+        self.finished_file_size = 0
+        self.last_update_time = now
+        self.re.start(total_size, now)
+        self._do_start(now)
+
+    def _do_start(self, now):
+        pass
+
+    def end(self, now=None):
+        if now is None: now = time.time()
+        self._do_end(now)
+        
+    def _do_end(self, now):
+        pass
+
+    def lock(self): self._lock.acquire()
+    def unlock(self): self._lock.release()
+
+    ###########################################################
+    # child meter creation and destruction
+    def newMeter(self):
+        newmeter = self.helperclass(self)
+        self.meters.append(newmeter)
+        return newmeter
+    
+    def removeMeter(self, meter):
+        self.meters.remove(meter)
+        
+    ###########################################################
+    # child functions - these should only be called by helpers
+    def start_meter(self, meter, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        self._lock.acquire()
+        try:
+            if not meter in self.in_progress_meters:
+                self.in_progress_meters.append(meter)
+                self.open_files += 1
+        finally:
+            self._lock.release()
+        self._do_start_meter(meter, now)
+        
+    def _do_start_meter(self, meter, now):
+        pass
+        
+    def update_meter(self, meter, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        if (now >= self.last_update_time + self.update_period) or \
+               not self.last_update_time:
+            self.re.update(self._amount_read(), now)
+            self.last_update_time = now
+            self._do_update_meter(meter, now)
+
+    def _do_update_meter(self, meter, now):
+        pass
+
+    def end_meter(self, meter, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        self._lock.acquire()
+        try:
+            try: self.in_progress_meters.remove(meter)
+            except ValueError: pass
+            self.open_files     -= 1
+            self.finished_files += 1
+            self.finished_file_size += meter.last_amount_read
+        finally:
+            self._lock.release()
+        self._do_end_meter(meter, now)
+
+    def _do_end_meter(self, meter, now):
+        pass
+
+    def failure_meter(self, meter, message, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        self._lock.acquire()
+        try:
+            try: self.in_progress_meters.remove(meter)
+            except ValueError: pass
+            self.open_files     -= 1
+            self.failed_files   += 1
+            if meter.size and self.failed_size is not None:
+                self.failed_size += meter.size
+            else:
+                self.failed_size = None
+        finally:
+            self._lock.release()
+        self._do_failure_meter(meter, message, now)
+
+    def _do_failure_meter(self, meter, message, now):
+        pass
+
+    def message_meter(self, meter, message):
+        pass
+
+    ########################################################
+    # internal functions
+    def _amount_read(self):
+        tot = self.finished_file_size
+        for m in self.in_progress_meters:
+            tot += m.last_amount_read
+        return tot
+
+
+class TextMultiFileMeter(MultiFileMeter):
+    def __init__(self, fo=sys.stderr):
+        self.fo = fo
+        MultiFileMeter.__init__(self)
+
+    # files: ###/### ###%  data: ######/###### ###%  time: ##:##:##/##:##:##
+    def _do_update_meter(self, meter, now):
+        self._lock.acquire()
+        try:
+            format = "files: %3i/%-3i %3i%%   data: %6.6s/%-6.6s %3i%%   " \
+                     "time: %8.8s/%8.8s"
+            df = self.finished_files
+            tf = self.numfiles or 1
+            pf = 100 * float(df)/tf + 0.49
+            dd = self.re.last_amount_read
+            td = self.total_size
+            pd = 100 * (self.re.fraction_read() or 0) + 0.49
+            dt = self.re.elapsed_time()
+            rt = self.re.remaining_time()
+            if rt is None: tt = None
+            else: tt = dt + rt
+
+            fdd = format_number(dd) + 'B'
+            ftd = format_number(td) + 'B'
+            fdt = format_time(dt, 1)
+            ftt = format_time(tt, 1)
+            
+            out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt))
+            self.fo.write('\r' + out)
+            self.fo.flush()
+        finally:
+            self._lock.release()
+
+    def _do_end_meter(self, meter, now):
+        self._lock.acquire()
+        try:
+            format = "%-30.30s %6.6s    %8.8s    %9.9s"
+            fn = meter.basename
+            size = meter.last_amount_read
+            fsize = format_number(size) + 'B'
+            et = meter.re.elapsed_time()
+            fet = format_time(et, 1)
+            frate = format_number(size / et) + 'B/s'
+            
+            out = '%-79.79s' % (format % (fn, fsize, fet, frate))
+            self.fo.write('\r' + out + '\n')
+        finally:
+            self._lock.release()
+        self._do_update_meter(meter, now)
+
+    def _do_failure_meter(self, meter, message, now):
+        self._lock.acquire()
+        try:
+            format = "%-30.30s %6.6s %s"
+            fn = meter.basename
+            if type(message) in (type(''), type(u'')):
+                message = message.splitlines()
+            if not message: message = ['']
+            out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
+            self.fo.write('\r' + out + '\n')
+            for m in message[1:]: self.fo.write('  ' + m + '\n')
+            self._lock.release()
+        finally:
+            self._do_update_meter(meter, now)
+
+    def message_meter(self, meter, message):
+        self._lock.acquire()
+        try:
+            pass
+        finally:
+            self._lock.release()
+
+    def _do_end(self, now):
+        self._do_update_meter(None, now)
+        self._lock.acquire()
+        try:
+            self.fo.write('\n')
+            self.fo.flush()
+        finally:
+            self._lock.release()
+        
+######################################################################
+# support classes and functions
+
+class RateEstimator:
+    def __init__(self, timescale=5.0):
+        self.timescale = timescale
+
+    def start(self, total=None, now=None):
+        if now is None: now = time.time()
+        self.total = total
+        self.start_time = now
+        self.last_update_time = now
+        self.last_amount_read = 0
+        self.ave_rate = None
+        
+    def update(self, amount_read, now=None):
+        if now is None: now = time.time()
+        if amount_read == 0:
+            # if we just started this file, all bets are off
+            self.last_update_time = now
+            self.last_amount_read = 0
+            self.ave_rate = None
+            return
+
+        #print 'times', now, self.last_update_time
+        time_diff = now         - self.last_update_time
+        read_diff = amount_read - self.last_amount_read
+        self.last_update_time = now
+        self.last_amount_read = amount_read
+        self.ave_rate = self._temporal_rolling_ave(\
+            time_diff, read_diff, self.ave_rate, self.timescale)
+        #print 'results', time_diff, read_diff, self.ave_rate
+        
+    #####################################################################
+    # result methods
+    def average_rate(self):
+        "get the average transfer rate (in bytes/second)"
+        return self.ave_rate
+
+    def elapsed_time(self):
+        "the time between the start of the transfer and the most recent update"
+        return self.last_update_time - self.start_time
+
+    def remaining_time(self):
+        "estimated time remaining"
+        if not self.ave_rate or not self.total: return None
+        return (self.total - self.last_amount_read) / self.ave_rate
+
+    def fraction_read(self):
+        """the fraction of the data that has been read
+        (can be None for unknown transfer size)"""
+        if self.total is None: return None
+        elif self.total == 0: return 1.0
+        else: return float(self.last_amount_read)/self.total
+
+    #########################################################################
+    # support methods
+    def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale):
+        """a temporal rolling average performs smooth averaging even when
+        updates come at irregular intervals.  This is performed by scaling
+        the "epsilon" according to the time since the last update.
+        Specifically, epsilon = time_diff / timescale
+
+        As a general rule, the average will take on a completely new value
+        after 'timescale' seconds."""
+        epsilon = time_diff / timescale
+        if epsilon > 1: epsilon = 1.0
+        return self._rolling_ave(time_diff, read_diff, last_ave, epsilon)
+    
+    def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon):
+        """perform a "rolling average" iteration
+        a rolling average "folds" new data into an existing average with
+        some weight, epsilon.  epsilon must be between 0.0 and 1.0 (inclusive)
+        a value of 0.0 means only the old value (initial value) counts,
+        and a value of 1.0 means only the newest value is considered."""
+        
+        try:
+            recent_rate = read_diff / time_diff
+        except ZeroDivisionError:
+            recent_rate = None
+        if last_ave is None: return recent_rate
+        elif recent_rate is None: return last_ave
+
+        # at this point, both last_ave and recent_rate are numbers
+        return epsilon * recent_rate  +  (1 - epsilon) * last_ave
+
+    def _round_remaining_time(self, rt, start_time=15.0):
+        """round the remaining time, depending on its size
+        If rt is between n*start_time and (n+1)*start_time round downward
+        to the nearest multiple of n (for any counting number n).
+        If rt < start_time, round down to the nearest 1.
+        For example (for start_time = 15.0):
+         2.7  -> 2.0
+         25.2 -> 25.0
+         26.4 -> 26.0
+         35.3 -> 34.0
+         63.6 -> 60.0
+        """
+
+        if rt < 0: return 0.0
+        shift = int(math.log(rt/start_time)/math.log(2))
+        rt = int(rt)
+        if shift <= 0: return rt
+        return float(int(rt) >> shift << shift)
+        
+
+def format_time(seconds, use_hours=0):
+    if seconds is None or seconds < 0:
+        if use_hours: return '--:--:--'
+        else:         return '--:--'
+    else:
+        seconds = int(seconds)
+        minutes = seconds / 60
+        seconds = seconds % 60
+        if use_hours:
+            hours = minutes / 60
+            minutes = minutes % 60
+            return '%02i:%02i:%02i' % (hours, minutes, seconds)
+        else:
+            return '%02i:%02i' % (minutes, seconds)
+            
+def format_number(number, SI=0, space=' '):
+    """Turn numbers into human-readable metric-like numbers"""
+    symbols = ['',  # (none)
+               'k', # kilo
+               'M', # mega
+               'G', # giga
+               'T', # tera
+               'P', # peta
+               'E', # exa
+               'Z', # zetta
+               'Y'] # yotta
+    
+    if SI: step = 1000.0
+    else: step = 1024.0
+
+    thresh = 999
+    depth = 0
+    max_depth = len(symbols) - 1
+    
+    # we want numbers between 0 and thresh, but don't exceed the length
+    # of our list.  In that event, the formatting will be screwed up,
+    # but it'll still show the right number.
+    while number > thresh and depth < max_depth:
+        depth  = depth + 1
+        number = number / step
+
+    if type(number) == type(1) or type(number) == type(1L):
+        # it's an int or a long, which means it didn't get divided,
+        # which means it's already short enough
+        format = '%i%s%s'
+    elif number < 9.95:
+        # must use 9.95 for proper sizing.  For example, 9.99 will be
+        # rounded to 10.0 with the .1f format string (which is too long)
+        format = '%.1f%s%s'
+    else:
+        format = '%.0f%s%s'
+        
+    return(format % (float(number or 0), space, symbols[depth]))