From 1bc9559a02e271e35dadd388b498f290d7940671 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 17 Mar 2012 13:02:39 -0700 Subject: [PATCH] Cleaning up the downloader module and adding progress bar support to it for http downloads and using this for the image downloading parts - which should now show a better download progress status... --- devstack/component.py | 6 +- devstack/downloader.py | 113 ++++++++++++++++++++++++++++++-------- devstack/image/creator.py | 23 +------- devstack/utils.py | 1 - 4 files changed, 96 insertions(+), 47 deletions(-) diff --git a/devstack/component.py b/devstack/component.py index 9c259014..d9e5253d 100644 --- a/devstack/component.py +++ b/devstack/component.py @@ -161,14 +161,16 @@ class PkgInstallComponent(ComponentBase): msg = "No uri entry found at config location [%s]" % \ (cfg_helpers.make_id(cfg_section, cfg_key)) raise excp.ConfigException(msg) + # Activate da download! self.tracewriter.download_happened(target_loc, uri) - dirs_made = down.download(target_loc, uri, branch) + dirs_made = down.download(uri, target_loc, branch=branch) # Here we ensure this is always added so that # if a keep old happens then this of course # won't be recreated, but if u uninstall without keeping old # then this won't be deleted this time around # adding it in is harmless and will make sure its removed. - dirs_made.append(target_loc) + if target_loc not in dirs_made: + dirs_made.append(target_loc) self.tracewriter.dirs_made(*dirs_made) return len(locations) diff --git a/devstack/downloader.py b/devstack/downloader.py index b48b60cb..5c5213a8 100644 --- a/devstack/downloader.py +++ b/devstack/downloader.py @@ -15,8 +15,11 @@ # under the License. -from urlparse import urlparse import re +import urllib +import urlparse + +import progressbar from devstack import log as logging from devstack import shell as sh @@ -30,30 +33,92 @@ CHECKOUT_CMD = ['git', 'checkout'] PULL_CMD = ['git', 'pull'] -def _gitdownload(storewhere, uri, branch=None): - dirsmade = list() - if sh.isdir(storewhere): - LOG.info("Updating code located at [%s]" % (storewhere)) - cmd = CHECKOUT_CMD + [GIT_MASTER_BRANCH] - sh.execute(*cmd, cwd=storewhere) - cmd = PULL_CMD - sh.execute(*cmd, cwd=storewhere) - else: - LOG.info("Downloading from [%s] to [%s]" % (uri, storewhere)) - dirsmade.extend(sh.mkdirslist(storewhere)) - cmd = CLONE_CMD + [uri, storewhere] - sh.execute(*cmd) - if branch and branch != GIT_MASTER_BRANCH: - LOG.info("Adjusting git branch to [%s]" % (branch)) - cmd = CHECKOUT_CMD + [branch] - sh.execute(*cmd, cwd=storewhere) - return dirsmade +class Downloader(object): + + def __init__(self, uri, store_where): + self.uri = uri + self.store_where = store_where + + def download(self): + raise NotImplementedError() -def download(storewhere, uri, branch=None): - up = urlparse(uri) +class GitDownloader(Downloader): + + def __init__(self, uri, store_where, **kargs): + Downloader.__init__(self, uri, store_where) + self.branch = kargs.get('branch') + + def download(self): + dirsmade = list() + if sh.isdir(self.store_where): + LOG.info("Updating using git: located at %r" % (self.store_where)) + cmd = CHECKOUT_CMD + [GIT_MASTER_BRANCH] + sh.execute(*cmd, cwd=self.store_where) + cmd = PULL_CMD + sh.execute(*cmd, cwd=self.store_where) + else: + LOG.info("Downloading using git: %r to %r" % (self.uri, self.store_where)) + dirsmade.extend(sh.mkdirslist(self.store_where)) + cmd = CLONE_CMD + [self.uri, self.store_where] + sh.execute(*cmd) + if self.branch and self.branch != GIT_MASTER_BRANCH: + LOG.info("Adjusting branch using git: %r" % (self.branch)) + cmd = CHECKOUT_CMD + [self.branch] + sh.execute(*cmd, cwd=self.store_where) + return dirsmade + + +class HttpDownloader(Downloader): + + def __init__(self, uri, store_where, **kargs): + Downloader.__init__(self, uri, store_where) + self.quiet = kargs.get('quiet', False) + self.p_bar = None + + def _make_bar(self, size): + widgets = [ + 'Fetching: ', progressbar.Percentage(), + ' ', progressbar.Bar(), + ' ', progressbar.ETA(), + ' ', progressbar.FileTransferSpeed(), + ] + return progressbar.ProgressBar(widgets=widgets, maxval=size) + + def _report(self, blocks, block_size, total_size): + if self.quiet: + return + byte_down = blocks * block_size + if not self.p_bar: + self.p_bar = self._make_bar(total_size) + self.p_bar.start() + if byte_down > self.p_bar.maxval: + # This seems to happen, huh??? + pass + else: + self.p_bar.update(byte_down) + + def download(self): + LOG.info('Downloading using http: %r to %r', self.uri, self.store_where) + dirsmade = sh.mkdirslist(sh.dirname(self.store_where)) + try: + urllib.urlretrieve(self.uri, self.store_where, self._report) + finally: + if self.p_bar: + self.p_bar.finish() + self.p_bar = None + return dirsmade + + +def download(uri, storewhere, **kargs): + downloader_cls = None + up = urlparse.urlparse(uri) if up and up.scheme.lower() == "git" or GIT_EXT_REG.match(up.path): - return _gitdownload(storewhere, uri, branch) - else: - msg = "Currently we do not know how to download from uri [%s]" % (uri) + downloader_cls = GitDownloader + elif up and up.scheme.lower() == "http": + downloader_cls = HttpDownloader + if not downloader_cls: + msg = "Currently we do not know how to download from uri: %r" % (uri) raise NotImplementedError(msg) + downloader = downloader_cls(uri, storewhere, **kargs) + return downloader.download() diff --git a/devstack/image/creator.py b/devstack/image/creator.py index 4c0aeae7..bd8fd00f 100644 --- a/devstack/image/creator.py +++ b/devstack/image/creator.py @@ -19,12 +19,13 @@ import json import os import tarfile import tempfile -import urllib import urllib2 +from devstack import downloader as down from devstack import log from devstack import shell from devstack import utils + from devstack.components import keystone @@ -61,27 +62,9 @@ class Image(object): self.initrd_id = '' self.tmp_folder = None self.registry = ImageRegistry(token) - self.last_report = 0 - - def _format_progress(self, curr_size, total_size): - if curr_size > total_size: - curr_size = total_size - progress = ("%d" % (curr_size)) + "b" - progress += "/" - progress += ("%d" % (total_size)) + "b" - perc_done = "%.02f" % (((curr_size) / (float(total_size)) * 100.0)) + "%" - return "[%s](%s)" % (progress, perc_done) - - def _report(self, blocks, block_size, size): - downloaded = blocks * block_size - if (downloaded - self.last_report) > Image.REPORTSIZE: - progress = self._format_progress((blocks * block_size), size) - LOG.info('Download progress: %s', progress) - self.last_report = downloaded def _download(self): - LOG.info('Downloading %s to %s', self.url, self.download_file_name) - urllib.urlretrieve(self.url, self.download_file_name, self._report) + return down.download(self.url, self.download_file_name) def _unpack(self): parts = self.download_name.split('.') diff --git a/devstack/utils.py b/devstack/utils.py index e472ad77..c981c8fa 100644 --- a/devstack/utils.py +++ b/devstack/utils.py @@ -17,7 +17,6 @@ # License for the specific language governing permissions and limitations # under the License. -import json import os import random import re