x/anvil
Code Issues Proposed changes

Cleaning up the downloader module and adding progress bar support to it for http downloads and using this for the image downloading parts - which should now show a better download progress status...

Browse Source
This commit is contained in:
Joshua Harlow 2012-03-17 13:02:39 -07:00
parent 419686f229
commit 1bc9559a02
4 changed files with 96 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
devstack/component.py
View File

@ -161,14 +161,16 @@ class PkgInstallComponent(ComponentBase):
msg = "No uri entry found at config location [%s]" % \
(cfg_helpers.make_id(cfg_section, cfg_key))
raise excp.ConfigException(msg)
# Activate da download!
self.tracewriter.download_happened(target_loc, uri)
dirs_made = down.download(target_loc, uri, branch)
dirs_made = down.download(uri, target_loc, branch=branch)
# Here we ensure this is always added so that
# if a keep old happens then this of course
# won't be recreated, but if u uninstall without keeping old
# then this won't be deleted this time around
# adding it in is harmless and will make sure its removed.
dirs_made.append(target_loc)
if target_loc not in dirs_made:
dirs_made.append(target_loc)
self.tracewriter.dirs_made(*dirs_made)
return len(locations)

113
devstack/downloader.py
View File

@ -15,8 +15,11 @@
# under the License.
from urlparse import urlparse
import re
import urllib
import urlparse
import progressbar
from devstack import log as logging
from devstack import shell as sh
@ -30,30 +33,92 @@ CHECKOUT_CMD = ['git', 'checkout']
PULL_CMD = ['git', 'pull']
def _gitdownload(storewhere, uri, branch=None):
dirsmade = list()
if sh.isdir(storewhere):
LOG.info("Updating code located at [%s]" % (storewhere))
cmd = CHECKOUT_CMD + [GIT_MASTER_BRANCH]
sh.execute(*cmd, cwd=storewhere)
cmd = PULL_CMD
sh.execute(*cmd, cwd=storewhere)
else:
LOG.info("Downloading from [%s] to [%s]" % (uri, storewhere))
dirsmade.extend(sh.mkdirslist(storewhere))
cmd = CLONE_CMD + [uri, storewhere]
sh.execute(*cmd)
if branch and branch != GIT_MASTER_BRANCH:
LOG.info("Adjusting git branch to [%s]" % (branch))
cmd = CHECKOUT_CMD + [branch]
sh.execute(*cmd, cwd=storewhere)
return dirsmade
class Downloader(object):
def __init__(self, uri, store_where):
self.uri = uri
self.store_where = store_where
def download(self):
raise NotImplementedError()
def download(storewhere, uri, branch=None):
up = urlparse(uri)
class GitDownloader(Downloader):
def __init__(self, uri, store_where, **kargs):
Downloader.__init__(self, uri, store_where)
self.branch = kargs.get('branch')
def download(self):
dirsmade = list()
if sh.isdir(self.store_where):
LOG.info("Updating using git: located at %r" % (self.store_where))
cmd = CHECKOUT_CMD + [GIT_MASTER_BRANCH]
sh.execute(*cmd, cwd=self.store_where)
cmd = PULL_CMD
sh.execute(*cmd, cwd=self.store_where)
else:
LOG.info("Downloading using git: %r to %r" % (self.uri, self.store_where))
dirsmade.extend(sh.mkdirslist(self.store_where))
cmd = CLONE_CMD + [self.uri, self.store_where]
sh.execute(*cmd)
if self.branch and self.branch != GIT_MASTER_BRANCH:
LOG.info("Adjusting branch using git: %r" % (self.branch))
cmd = CHECKOUT_CMD + [self.branch]
sh.execute(*cmd, cwd=self.store_where)
return dirsmade
class HttpDownloader(Downloader):
def __init__(self, uri, store_where, **kargs):
Downloader.__init__(self, uri, store_where)
self.quiet = kargs.get('quiet', False)
self.p_bar = None
def _make_bar(self, size):
widgets = [
'Fetching: ', progressbar.Percentage(),
' ', progressbar.Bar(),
' ', progressbar.ETA(),
' ', progressbar.FileTransferSpeed(),
]
return progressbar.ProgressBar(widgets=widgets, maxval=size)
def _report(self, blocks, block_size, total_size):
if self.quiet:
return
byte_down = blocks * block_size
if not self.p_bar:
self.p_bar = self._make_bar(total_size)
self.p_bar.start()
if byte_down > self.p_bar.maxval:
# This seems to happen, huh???
pass
else:
self.p_bar.update(byte_down)
def download(self):
LOG.info('Downloading using http: %r to %r', self.uri, self.store_where)
dirsmade = sh.mkdirslist(sh.dirname(self.store_where))
try:
urllib.urlretrieve(self.uri, self.store_where, self._report)
finally:
if self.p_bar:
self.p_bar.finish()
self.p_bar = None
return dirsmade
def download(uri, storewhere, **kargs):
downloader_cls = None
up = urlparse.urlparse(uri)
if up and up.scheme.lower() == "git" or GIT_EXT_REG.match(up.path):
return _gitdownload(storewhere, uri, branch)
else:
msg = "Currently we do not know how to download from uri [%s]" % (uri)
downloader_cls = GitDownloader
elif up and up.scheme.lower() == "http":
downloader_cls = HttpDownloader
if not downloader_cls:
msg = "Currently we do not know how to download from uri: %r" % (uri)
raise NotImplementedError(msg)
downloader = downloader_cls(uri, storewhere, **kargs)
return downloader.download()

23
devstack/image/creator.py
View File

@ -19,12 +19,13 @@ import json
import os
import tarfile
import tempfile
import urllib
import urllib2
from devstack import downloader as down
from devstack import log
from devstack import shell
from devstack import utils
from devstack.components import keystone
@ -61,27 +62,9 @@ class Image(object):
self.initrd_id = ''
self.tmp_folder = None
self.registry = ImageRegistry(token)
self.last_report = 0
def _format_progress(self, curr_size, total_size):
if curr_size > total_size:
curr_size = total_size
progress = ("%d" % (curr_size)) + "b"
progress += "/"
progress += ("%d" % (total_size)) + "b"
perc_done = "%.02f" % (((curr_size) / (float(total_size)) * 100.0)) + "%"
return "[%s](%s)" % (progress, perc_done)
def _report(self, blocks, block_size, size):
downloaded = blocks * block_size
if (downloaded - self.last_report) > Image.REPORTSIZE:
progress = self._format_progress((blocks * block_size), size)
LOG.info('Download progress: %s', progress)
self.last_report = downloaded
def _download(self):
LOG.info('Downloading %s to %s', self.url, self.download_file_name)
urllib.urlretrieve(self.url, self.download_file_name, self._report)
return down.download(self.url, self.download_file_name)
def _unpack(self):
parts = self.download_name.split('.')

1
devstack/utils.py
View File

@ -17,7 +17,6 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import os
import random
import re