142 lines
4.5 KiB
Python
142 lines
4.5 KiB
Python
# Copyright ....
|
|
|
|
import git
|
|
import logging
|
|
import os
|
|
import select
|
|
import subprocess
|
|
import time
|
|
|
|
|
|
class GitRepository(object):
|
|
|
|
""" Manage a git repository for our uses """
|
|
log = logging.getLogger("rcbau-ci.GitRepository")
|
|
|
|
def __init__(self, remote_url, local_path):
|
|
self.remote_url = remote_url
|
|
self.local_path = local_path
|
|
self._ensure_cloned()
|
|
|
|
self.repo = git.Repo(self.local_path)
|
|
|
|
def fetch(self, ref):
|
|
# The git.remote.fetch method may read in git progress info and
|
|
# interpret it improperly causing an AssertionError. Because the
|
|
# data was fetched properly subsequent fetches don't seem to fail.
|
|
# So try again if an AssertionError is caught.
|
|
origin = self.repo.remotes.origin
|
|
self.log.debug("Fetching %s from %s" % (ref, origin))
|
|
|
|
try:
|
|
origin.fetch(ref)
|
|
except AssertionError:
|
|
origin.fetch(ref)
|
|
|
|
def checkout(self, ref):
|
|
self.log.debug("Checking out %s" % ref)
|
|
return self.repo.git.checkout(ref)
|
|
|
|
def _ensure_cloned(self):
|
|
if not os.path.exists(self.local_path):
|
|
self.log.debug("Cloning from %s to %s" % (self.remote_url,
|
|
self.local_path))
|
|
git.Repo.clone_from(self.remote_url, self.local_path)
|
|
|
|
""" ####UNUSED>....
|
|
def _make_ssh_wrappesubprocessr(self, key):
|
|
name = os.path.join(self.config['git_working_dir'], '.ssh_wrapper')
|
|
fd = open(name, 'w')
|
|
fd.write('#!/bin/bash\n')
|
|
fd.write('ssh -i %s $@\n' % key)
|
|
fd.close()
|
|
os.chmod(name, 0755)
|
|
os.environ['GIT_SSH'] = name
|
|
|
|
def construct_git_url(self, project_name):
|
|
url = 'ssh://%s@%s:%s/%s' % (
|
|
self.config['gerrit_server']['user'],
|
|
self.config['gerrit_server']['host'],
|
|
self.config['gerrit_server']['port'],
|
|
project_name
|
|
)
|
|
url = 'https://%s/%s' % (
|
|
self.config['gerrit_server']['host'],
|
|
project_name
|
|
)
|
|
return url
|
|
"""
|
|
|
|
|
|
def execute_to_log(cmd, logfile, timeout=-1,
|
|
watch_logs=[
|
|
('[syslog]', '/var/log/syslog'),
|
|
('[sqlslo]', '/var/log/mysql/slow-queries.log'),
|
|
('[sqlerr]', '/var/log/mysql/error.log')
|
|
],
|
|
heartbeat=True
|
|
):
|
|
""" Executes a command and logs the STDOUT/STDERR and output of any
|
|
supplied watch_logs from logs into a new logfile
|
|
|
|
watch_logs is a list of tuples with (name,file) """
|
|
|
|
if not os.path.isdir(os.path.dirname(logfile)):
|
|
os.makedirs(os.path.dirname(logfile))
|
|
|
|
logger = logging.getLogger('execute_to_log')
|
|
log_hanlder = logging.FileHandler(logfile)
|
|
log_formatter = logging.Formatter('%(asctime)s %(message)s')
|
|
log_hanlder.setFormatter(log_formatter)
|
|
logger.addHandler(log_hanlder)
|
|
|
|
descriptors = {}
|
|
|
|
for watch_file in watch_logs:
|
|
fd = os.open(watch_file[1], os.O_RDONLY)
|
|
os.lseek(fd, 0, os.SEEK_END)
|
|
descriptors[fd] = dict(
|
|
name=watch_file[0],
|
|
poll=select.POLLIN
|
|
)
|
|
|
|
cmd += ' 2>&1'
|
|
start_time = time.time()
|
|
p = subprocess.Popen(
|
|
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
|
|
descriptors[p.stdout.fileno()] = dict(
|
|
name='[stdout]',
|
|
poll=(select.POLLIN | select.POLLHUP)
|
|
)
|
|
descriptors[p.stderr.fileno()] = dict(
|
|
name='[stderr]',
|
|
poll=(select.POLLIN | select.POLLHUP)
|
|
)
|
|
|
|
poll_obj = select.poll()
|
|
for fd, descriptor in descriptors.items():
|
|
poll_obj.register(fd, descriptor['poll'])
|
|
|
|
last_heartbeat = time.time()
|
|
|
|
while p.poll() is None:
|
|
if timeout > 0 and time.time() - start_time > timeout:
|
|
# Append to logfile
|
|
logger.info("[timeout]")
|
|
os.kill(p.pid, 9)
|
|
|
|
for fd, flag in poll_obj.poll(0):
|
|
lines = os.read(fd, 1024 * 1024)
|
|
for l in lines.split('\n'):
|
|
if len(l) > 0:
|
|
l = '%s %s' % (descriptors[fd]['name'], l)
|
|
logger.info(l)
|
|
last_heartbeat = time.time()
|
|
|
|
if time.time() - last_heartbeat > 30:
|
|
# Append to logfile
|
|
logger.info("[heartbeat]")
|
|
last_heartbeat = time.time()
|
|
|
|
logger.info('[script exit code = %d]' % p.returncode) |