From 1ab465f923ff805183be98049682db31ee6673aa Mon Sep 17 00:00:00 2001 From: Joshua Hesketh Date: Fri, 26 Jul 2013 13:57:28 +1000 Subject: [PATCH] restructure project slightly --- testing.log | 30 ---------------- README.md => turbo_hipster/README.md | 0 {lib => turbo_hipster/lib}/__init__.py | 0 {lib => turbo_hipster/lib}/utils.py | 35 ++++++++++++------- .../task_plugins}/__init__.py | 0 .../gate_real_db_upgrade/__init__.py | 0 .../gate_real_db_upgrade/task.py | 0 .../test_sqlalchemy_migrations.sh | 0 .../worker_manager.py | 0 .../worker_server.py | 0 10 files changed, 23 insertions(+), 42 deletions(-) delete mode 100644 testing.log rename README.md => turbo_hipster/README.md (100%) rename {lib => turbo_hipster/lib}/__init__.py (100%) rename {lib => turbo_hipster/lib}/utils.py (86%) rename {task_plugins => turbo_hipster/task_plugins}/__init__.py (100%) rename {task_plugins => turbo_hipster/task_plugins}/gate_real_db_upgrade/__init__.py (100%) rename {task_plugins => turbo_hipster/task_plugins}/gate_real_db_upgrade/task.py (100%) rename {task_plugins => turbo_hipster/task_plugins}/gate_real_db_upgrade/test_sqlalchemy_migrations.sh (100%) rename worker_manager.py => turbo_hipster/worker_manager.py (100%) rename worker_server.py => turbo_hipster/worker_server.py (100%) diff --git a/testing.log b/testing.log deleted file mode 100644 index e009ae6..0000000 --- a/testing.log +++ /dev/null @@ -1,30 +0,0 @@ -2013-07-25 16:21:35,001 [stdout] PING localhost (127.0.0.1) 56(84) bytes of data. -2013-07-25 16:21:35,001 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=1 ttl=64 time=0.017 ms -2013-07-25 16:21:36,000 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=2 ttl=64 time=0.018 ms -2013-07-25 16:21:37,000 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=3 ttl=64 time=0.019 ms -2013-07-25 16:21:38,000 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=4 ttl=64 time=0.022 ms -2013-07-25 16:21:38,000 [stdout] --- localhost ping statistics --- -2013-07-25 16:21:38,000 [stdout] 4 packets transmitted, 4 received, 0% packet loss, time 2998ms -2013-07-25 16:21:38,000 [stdout] rtt min/avg/max/mdev = 0.017/0.019/0.022/0.002 ms -2013-07-25 16:22:41,579 [stdout] PING localhost (127.0.0.1) 56(84) bytes of data. -2013-07-25 16:22:41,579 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=1 ttl=64 time=0.018 ms -2013-07-25 16:22:42,578 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=2 ttl=64 time=0.015 ms -2013-07-25 16:22:43,577 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=3 ttl=64 time=0.026 ms -2013-07-25 16:22:44,577 [stdout] 64 bytes from localhost (127.0.0.1): icmp_req=4 ttl=64 time=0.026 ms -2013-07-25 16:22:44,577 [stdout] --- localhost ping statistics --- -2013-07-25 16:22:44,577 [stdout] 4 packets transmitted, 4 received, 0% packet loss, time 2998ms -2013-07-25 16:22:44,577 [stdout] rtt min/avg/max/mdev = 0.015/0.021/0.026/0.005 ms -2013-07-25 16:23:14,577 [heartbeat] -2013-07-25 16:25:04,155 [stderr] Usage: ping [-LRUbdfnqrvVaAD] [-c count] [-i interval] [-w deadline] -2013-07-25 16:25:04,155 [stderr] [-p pattern] [-s packetsize] [-t ttl] [-I interface] -2013-07-25 16:25:04,155 [stderr] [-M pmtudisc-hint] [-m mark] [-S sndbuf] -2013-07-25 16:25:04,155 [stderr] [-T tstamp-options] [-Q tos] [hop1 ...] destination -2013-07-25 16:38:37,987 [stderr] Usage: ping [-LRUbdfnqrvVaAD] [-c count] [-i interval] [-w deadline] -2013-07-25 16:38:37,987 [stderr] [-p pattern] [-s packetsize] [-t ttl] [-I interface] -2013-07-25 16:38:37,987 [stderr] [-M pmtudisc-hint] [-m mark] [-S sndbuf] -2013-07-25 16:38:37,987 [stderr] [-T tstamp-options] [-Q tos] [hop1 ...] destination -2013-07-25 16:43:17,558 [stderr] Usage: ping [-LRUbdfnqrvVaAD] [-c count] [-i interval] [-w deadline] -2013-07-25 16:43:17,558 [stderr] [-p pattern] [-s packetsize] [-t ttl] [-I interface] -2013-07-25 16:43:17,558 [stderr] [-M pmtudisc-hint] [-m mark] [-S sndbuf] -2013-07-25 16:43:17,558 [stderr] [-T tstamp-options] [-Q tos] [hop1 ...] destination -2013-07-25 16:43:17,558 [script exit code = 2] diff --git a/README.md b/turbo_hipster/README.md similarity index 100% rename from README.md rename to turbo_hipster/README.md diff --git a/lib/__init__.py b/turbo_hipster/lib/__init__.py similarity index 100% rename from lib/__init__.py rename to turbo_hipster/lib/__init__.py diff --git a/lib/utils.py b/turbo_hipster/lib/utils.py similarity index 86% rename from lib/utils.py rename to turbo_hipster/lib/utils.py index 99379b3..e2762ae 100644 --- a/lib/utils.py +++ b/turbo_hipster/lib/utils.py @@ -97,7 +97,8 @@ def execute_to_log(cmd, logfile, timeout=-1, os.lseek(fd, 0, os.SEEK_END) descriptors[fd] = dict( name=watch_file[0], - poll=select.POLLIN + poll=select.POLLIN, + lines='' ) cmd += ' 2>&1' @@ -106,11 +107,7 @@ def execute_to_log(cmd, logfile, timeout=-1, cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) descriptors[p.stdout.fileno()] = dict( - name='[stdout]', - poll=(select.POLLIN | select.POLLHUP) - ) - descriptors[p.stderr.fileno()] = dict( - name='[stderr]', + name='[output]', poll=(select.POLLIN | select.POLLHUP) ) @@ -120,6 +117,21 @@ def execute_to_log(cmd, logfile, timeout=-1, last_heartbeat = time.time() + def process(fd): + """ Write the fd to log """ + descriptors[fd]['lines'] += os.read(fd, 1024 * 1024) + # Avoid partial lines by only processing input with breaks + if lines[fd].find('\n') != -1: + elems = descriptors[fd]['lines'].split('\n') + # Take all but the partial line + for l in elems[:-1]: + if len(l) > 0: + l = '%s %s' % (descriptors[fd]['name'], l) + logger.info(l) + last_heartbeat = time.time() + # Place the partial line back into lines to be processed + descriptors[fd]['lines'] = elems[-1] + while p.poll() is None: if timeout > 0 and time.time() - start_time > timeout: # Append to logfile @@ -127,16 +139,15 @@ def execute_to_log(cmd, logfile, timeout=-1, os.kill(p.pid, 9) for fd, flag in poll_obj.poll(0): - lines = os.read(fd, 1024 * 1024) - for l in lines.split('\n'): - if len(l) > 0: - l = '%s %s' % (descriptors[fd]['name'], l) - logger.info(l) - last_heartbeat = time.time() + process(fd) if time.time() - last_heartbeat > 30: # Append to logfile logger.info("[heartbeat]") last_heartbeat = time.time() + # Do one last write to get the remaining lines + for fd, flag in poll_obj.poll(0): + process(fd) + logger.info('[script exit code = %d]' % p.returncode) diff --git a/task_plugins/__init__.py b/turbo_hipster/task_plugins/__init__.py similarity index 100% rename from task_plugins/__init__.py rename to turbo_hipster/task_plugins/__init__.py diff --git a/task_plugins/gate_real_db_upgrade/__init__.py b/turbo_hipster/task_plugins/gate_real_db_upgrade/__init__.py similarity index 100% rename from task_plugins/gate_real_db_upgrade/__init__.py rename to turbo_hipster/task_plugins/gate_real_db_upgrade/__init__.py diff --git a/task_plugins/gate_real_db_upgrade/task.py b/turbo_hipster/task_plugins/gate_real_db_upgrade/task.py similarity index 100% rename from task_plugins/gate_real_db_upgrade/task.py rename to turbo_hipster/task_plugins/gate_real_db_upgrade/task.py diff --git a/task_plugins/gate_real_db_upgrade/test_sqlalchemy_migrations.sh b/turbo_hipster/task_plugins/gate_real_db_upgrade/test_sqlalchemy_migrations.sh similarity index 100% rename from task_plugins/gate_real_db_upgrade/test_sqlalchemy_migrations.sh rename to turbo_hipster/task_plugins/gate_real_db_upgrade/test_sqlalchemy_migrations.sh diff --git a/worker_manager.py b/turbo_hipster/worker_manager.py similarity index 100% rename from worker_manager.py rename to turbo_hipster/worker_manager.py diff --git a/worker_server.py b/turbo_hipster/worker_server.py similarity index 100% rename from worker_server.py rename to turbo_hipster/worker_server.py