Cleaning up the downloader module and adding progress bar support to it for http downloads and using this for the image downloading parts - which should now show a better download progress status...

devstack/component.py

@@ -161,14 +161,16 @@ class PkgInstallComponent(ComponentBase):
                 msg = "No uri entry found at config location [%s]" % \
                     (cfg_helpers.make_id(cfg_section, cfg_key))
                 raise excp.ConfigException(msg)
+            # Activate da download!
             self.tracewriter.download_happened(target_loc, uri)
-            dirs_made = down.download(target_loc, uri, branch)
+            dirs_made = down.download(uri, target_loc, branch=branch)
             # Here we ensure this is always added so that
             # if a keep old happens then this of course
             # won't be recreated, but if u uninstall without keeping old
             # then this won't be deleted this time around
             # adding it in is harmless and will make sure its removed.
-            dirs_made.append(target_loc)
+            if target_loc not in dirs_made:
+                dirs_made.append(target_loc)
             self.tracewriter.dirs_made(*dirs_made)
         return len(locations)
 

devstack/downloader.py

@@ -15,8 +15,11 @@
 #    under the License.
 
 
-from urlparse import urlparse
 import re
+import urllib
+import urlparse
+
+import progressbar
 
 from devstack import log as logging
 from devstack import shell as sh
@@ -30,30 +33,92 @@ CHECKOUT_CMD = ['git', 'checkout']
 PULL_CMD = ['git', 'pull']
 
 
-def _gitdownload(storewhere, uri, branch=None):
+class Downloader(object):
-    dirsmade = list()
+
-    if sh.isdir(storewhere):
+    def __init__(self, uri, store_where):
-        LOG.info("Updating code located at [%s]" % (storewhere))
+        self.uri = uri
-        cmd = CHECKOUT_CMD + [GIT_MASTER_BRANCH]
+        self.store_where = store_where
-        sh.execute(*cmd, cwd=storewhere)
+
-        cmd = PULL_CMD
+    def download(self):
-        sh.execute(*cmd, cwd=storewhere)
+        raise NotImplementedError()
-    else:
-        LOG.info("Downloading from [%s] to [%s]" % (uri, storewhere))
-        dirsmade.extend(sh.mkdirslist(storewhere))
-        cmd = CLONE_CMD + [uri, storewhere]
-        sh.execute(*cmd)
-    if branch and branch != GIT_MASTER_BRANCH:
-        LOG.info("Adjusting git branch to [%s]" % (branch))
-        cmd = CHECKOUT_CMD + [branch]
-        sh.execute(*cmd, cwd=storewhere)
-    return dirsmade
 
 
-def download(storewhere, uri, branch=None):
+class GitDownloader(Downloader):
-    up = urlparse(uri)
+
+    def __init__(self, uri, store_where, **kargs):
+        Downloader.__init__(self, uri, store_where)
+        self.branch = kargs.get('branch')
+
+    def download(self):
+        dirsmade = list()
+        if sh.isdir(self.store_where):
+            LOG.info("Updating using git: located at %r" % (self.store_where))
+            cmd = CHECKOUT_CMD + [GIT_MASTER_BRANCH]
+            sh.execute(*cmd, cwd=self.store_where)
+            cmd = PULL_CMD
+            sh.execute(*cmd, cwd=self.store_where)
+        else:
+            LOG.info("Downloading using git: %r to %r" % (self.uri, self.store_where))
+            dirsmade.extend(sh.mkdirslist(self.store_where))
+            cmd = CLONE_CMD + [self.uri, self.store_where]
+            sh.execute(*cmd)
+        if self.branch and self.branch != GIT_MASTER_BRANCH:
+            LOG.info("Adjusting branch using git: %r" % (self.branch))
+            cmd = CHECKOUT_CMD + [self.branch]
+            sh.execute(*cmd, cwd=self.store_where)
+        return dirsmade
+
+
+class HttpDownloader(Downloader):
+
+    def __init__(self, uri, store_where, **kargs):
+        Downloader.__init__(self, uri, store_where)
+        self.quiet = kargs.get('quiet', False)
+        self.p_bar = None
+
+    def _make_bar(self, size):
+        widgets = [
+            'Fetching: ', progressbar.Percentage(),
+            ' ', progressbar.Bar(),
+            ' ', progressbar.ETA(),
+            ' ', progressbar.FileTransferSpeed(),
+        ]
+        return progressbar.ProgressBar(widgets=widgets, maxval=size)
+
+    def _report(self, blocks, block_size, total_size):
+        if self.quiet:
+            return
+        byte_down = blocks * block_size
+        if not self.p_bar:
+            self.p_bar = self._make_bar(total_size)
+            self.p_bar.start()
+        if byte_down > self.p_bar.maxval:
+            # This seems to happen, huh???
+            pass
+        else:
+            self.p_bar.update(byte_down)
+
+    def download(self):
+        LOG.info('Downloading using http: %r to %r', self.uri, self.store_where)
+        dirsmade = sh.mkdirslist(sh.dirname(self.store_where))
+        try:
+            urllib.urlretrieve(self.uri, self.store_where, self._report)
+        finally:
+            if self.p_bar:
+                self.p_bar.finish()
+                self.p_bar = None
+        return dirsmade
+
+
+def download(uri, storewhere, **kargs):
+    downloader_cls = None
+    up = urlparse.urlparse(uri)
     if up and up.scheme.lower() == "git" or GIT_EXT_REG.match(up.path):
-        return _gitdownload(storewhere, uri, branch)
+        downloader_cls = GitDownloader
-    else:
+    elif up and up.scheme.lower() == "http":
-        msg = "Currently we do not know how to download from uri [%s]" % (uri)
+        downloader_cls = HttpDownloader
+    if not downloader_cls:
+        msg = "Currently we do not know how to download from uri: %r" % (uri)
         raise NotImplementedError(msg)
+    downloader = downloader_cls(uri, storewhere, **kargs)
+    return downloader.download()

devstack/image/creator.py

@@ -19,12 +19,13 @@ import json
 import os
 import tarfile
 import tempfile
-import urllib
 import urllib2
 
+from devstack import downloader as down
 from devstack import log
 from devstack import shell
 from devstack import utils
+
 from devstack.components import keystone
 
 
@@ -61,27 +62,9 @@ class Image(object):
         self.initrd_id = ''
         self.tmp_folder = None
         self.registry = ImageRegistry(token)
-        self.last_report = 0
-
-    def _format_progress(self, curr_size, total_size):
-        if curr_size > total_size:
-            curr_size = total_size
-        progress = ("%d" % (curr_size)) + "b"
-        progress += "/"
-        progress += ("%d" % (total_size)) + "b"
-        perc_done = "%.02f" % (((curr_size) / (float(total_size)) * 100.0)) + "%"
-        return "[%s](%s)" % (progress, perc_done)
-
-    def _report(self, blocks, block_size, size):
-        downloaded = blocks * block_size
-        if (downloaded - self.last_report) > Image.REPORTSIZE:
-            progress = self._format_progress((blocks * block_size), size)
-            LOG.info('Download progress: %s', progress)
-            self.last_report = downloaded
 
     def _download(self):
-        LOG.info('Downloading %s to %s', self.url, self.download_file_name)
+        return down.download(self.url, self.download_file_name)
-        urllib.urlretrieve(self.url, self.download_file_name, self._report)
 
     def _unpack(self):
         parts = self.download_name.split('.')

devstack/utils.py

@@ -17,7 +17,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
 import os
 import random
 import re