From ec5801989bf1553ba91898995b3cc5b3dcedae23 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Thu, 17 Dec 2009 14:54:10 -0800 Subject: [PATCH] daemon: Automatically compress our log files After server startup compress any stale logs which aren't yet compressed, and then once every 24 hours compress any other logs that haven't yet been compressed. This saves disk space for the archived logs, and its useful for us to do it automatically because most Linux distributions don't know where our log files are, so they can't automatically compress them along with other system logs. WorkQueue needed to be modified to not remove periodic tasks from the running task list, so the compressor can be seen in the dump from gerrit show-queue. Change-Id: I523bfdc0005d8c8112591f3460b799fde9abe123 Signed-off-by: Shawn O. Pearce --- .../java/com/google/gerrit/pgm/Daemon.java | 2 + .../google/gerrit/pgm/util/ErrorLogFile.java | 6 +- .../gerrit/pgm/util/LogFileCompressor.java | 139 ++++++++++++++++++ .../google/gerrit/server/git/WorkQueue.java | 5 +- 4 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java index d05b50c99a..add0a93de2 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java @@ -21,6 +21,7 @@ import com.google.gerrit.httpd.WebModule; import com.google.gerrit.lifecycle.LifecycleManager; import com.google.gerrit.pgm.http.jetty.JettyEnv; import com.google.gerrit.pgm.http.jetty.JettyModule; +import com.google.gerrit.pgm.util.LogFileCompressor; import com.google.gerrit.pgm.util.ErrorLogFile; import com.google.gerrit.pgm.util.RuntimeShutdown; import com.google.gerrit.pgm.util.SiteProgram; @@ -140,6 +141,7 @@ public class Daemon extends SiteProgram { private Injector createSysInjector() { final List modules = new ArrayList(); + modules.add(new LogFileCompressor.Module()); modules.add(cfgInjector.getInstance(GerritGlobalModule.class)); if (httpd) { modules.add(new CanonicalWebUrlModule() { diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/ErrorLogFile.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/ErrorLogFile.java index ade4a64776..7a7f144e36 100644 --- a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/ErrorLogFile.java +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/ErrorLogFile.java @@ -30,6 +30,8 @@ import org.apache.log4j.spi.LoggingEvent; import java.io.File; public class ErrorLogFile { + static final String LOG_NAME = "error_log"; + public static void errorOnlyConsole() { LogManager.resetConfiguration(); @@ -56,10 +58,10 @@ public class ErrorLogFile { layout.setConversionPattern("[%d] %-5p %c %x: %m%n"); final DailyRollingFileAppender dst = new DailyRollingFileAppender(); - dst.setName("error_log"); + dst.setName(LOG_NAME); dst.setLayout(layout); dst.setEncoding("UTF-8"); - dst.setFile(new File(logdir, "error_log").getAbsolutePath()); + dst.setFile(new File(logdir, LOG_NAME).getAbsolutePath()); dst.setImmediateFlush(true); dst.setAppend(true); dst.setThreshold(Level.INFO); diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java new file mode 100644 index 0000000000..c579e67f4d --- /dev/null +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java @@ -0,0 +1,139 @@ +// Copyright (C) 2009 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.pgm.util; + +import static java.util.concurrent.TimeUnit.HOURS; + +import com.google.gerrit.lifecycle.LifecycleListener; +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.gerrit.server.config.SitePaths; +import com.google.gerrit.server.git.WorkQueue; +import com.google.inject.Inject; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; + +/** Compresses the old error logs. */ +public class LogFileCompressor implements Runnable { + private static final Logger log = + LoggerFactory.getLogger(LogFileCompressor.class); + + public static class Module extends LifecycleModule { + @Override + protected void configure() { + listener().to(Lifecycle.class); + } + } + + static class Lifecycle implements LifecycleListener { + private final WorkQueue queue; + private final LogFileCompressor compresser; + + @Inject + Lifecycle(final WorkQueue queue, final LogFileCompressor compressor) { + this.queue = queue; + this.compresser = compressor; + } + + @Override + public void start() { + queue.getDefaultQueue().scheduleWithFixedDelay(compresser, 1, 24, HOURS); + } + + @Override + public void stop() { + } + } + + private final File logs_dir; + + @Inject + LogFileCompressor(final SitePaths site) { + logs_dir = site.logs_dir; + } + + @Override + public void run() { + final File[] list = logs_dir.listFiles(); + if (list == null) { + return; + } + + for (final File entry : list) { + if (!isLive(entry) && !isCompressed(entry) && isLogFile(entry)) { + compress(entry); + } + } + } + + private boolean isLive(final File entry) { + final String name = entry.getName(); + return ErrorLogFile.LOG_NAME.equals(name); + } + + private boolean isCompressed(final File entry) { + final String name = entry.getName(); + return name.endsWith(".gz") // + || name.endsWith(".zip") // + || name.endsWith(".bz2"); + } + + private boolean isLogFile(final File entry) { + return entry.isFile(); + } + + private void compress(final File src) { + final File dir = src.getParentFile(); + final File dst = new File(dir, src.getName() + ".gz"); + final File tmp = new File(dir, ".tmp." + src.getName()); + try { + final InputStream in = new FileInputStream(src); + try { + OutputStream out = new GZIPOutputStream(new FileOutputStream(tmp)); + try { + final byte[] buf = new byte[2048]; + int n; + while (0 < (n = in.read(buf))) { + out.write(buf, 0, n); + } + } finally { + out.close(); + } + } finally { + in.close(); + } + if (!tmp.renameTo(dst)) { + throw new IOException("Cannot rename " + tmp + " to " + dst); + } + src.delete(); + } catch (IOException e) { + log.error("Cannot compress " + src, e); + tmp.delete(); + } + } + + @Override + public String toString() { + return "Log File Compressor"; + } +} diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/WorkQueue.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/WorkQueue.java index 5d804b6179..0ba38b1621 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/WorkQueue.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/WorkQueue.java @@ -169,7 +169,10 @@ public class WorkQueue { @Override protected void afterExecute(Runnable r, Throwable t) { - remove((Task) r); + final Task task = (Task) r; + if (!task.isPeriodic()) { + remove(task); + } super.afterExecute(r, t); }