daemon: Automatically compress our log files

After server startup compress any stale logs which aren't yet
compressed, and then once every 24 hours compress any other logs
that haven't yet been compressed.

This saves disk space for the archived logs, and its useful for us
to do it automatically because most Linux distributions don't know
where our log files are, so they can't automatically compress them
along with other system logs.

WorkQueue needed to be modified to not remove periodic tasks from
the running task list, so the compressor can be seen in the dump
from gerrit show-queue.

Change-Id: I523bfdc0005d8c8112591f3460b799fde9abe123
Signed-off-by: Shawn O. Pearce <sop@google.com>

gerrit-pgm/src/main/java/com/google/gerrit/pgm/Daemon.java

@@ -21,6 +21,7 @@ import com.google.gerrit.httpd.WebModule;
 import com.google.gerrit.lifecycle.LifecycleManager;
 import com.google.gerrit.pgm.http.jetty.JettyEnv;
 import com.google.gerrit.pgm.http.jetty.JettyModule;
+import com.google.gerrit.pgm.util.LogFileCompressor;
 import com.google.gerrit.pgm.util.ErrorLogFile;
 import com.google.gerrit.pgm.util.RuntimeShutdown;
 import com.google.gerrit.pgm.util.SiteProgram;
@@ -140,6 +141,7 @@ public class Daemon extends SiteProgram {
 
   private Injector createSysInjector() {
     final List<Module> modules = new ArrayList<Module>();
+    modules.add(new LogFileCompressor.Module());
     modules.add(cfgInjector.getInstance(GerritGlobalModule.class));
     if (httpd) {
       modules.add(new CanonicalWebUrlModule() {

gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/ErrorLogFile.java

@@ -30,6 +30,8 @@ import org.apache.log4j.spi.LoggingEvent;
 import java.io.File;
 
 public class ErrorLogFile {
+  static final String LOG_NAME = "error_log";
+
   public static void errorOnlyConsole() {
     LogManager.resetConfiguration();
 
@@ -56,10 +58,10 @@ public class ErrorLogFile {
     layout.setConversionPattern("[%d] %-5p %c %x: %m%n");
 
     final DailyRollingFileAppender dst = new DailyRollingFileAppender();
-    dst.setName("error_log");
+    dst.setName(LOG_NAME);
     dst.setLayout(layout);
     dst.setEncoding("UTF-8");
-    dst.setFile(new File(logdir, "error_log").getAbsolutePath());
+    dst.setFile(new File(logdir, LOG_NAME).getAbsolutePath());
     dst.setImmediateFlush(true);
     dst.setAppend(true);
     dst.setThreshold(Level.INFO);

gerrit-pgm/src/main/java/com/google/gerrit/pgm/util/LogFileCompressor.java

@@ -0,0 +1,139 @@
+// Copyright (C) 2009 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.pgm.util;
+
+import static java.util.concurrent.TimeUnit.HOURS;
+
+import com.google.gerrit.lifecycle.LifecycleListener;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.git.WorkQueue;
+import com.google.inject.Inject;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+
+/** Compresses the old error logs. */
+public class LogFileCompressor implements Runnable {
+  private static final Logger log =
+      LoggerFactory.getLogger(LogFileCompressor.class);
+
+  public static class Module extends LifecycleModule {
+    @Override
+    protected void configure() {
+      listener().to(Lifecycle.class);
+    }
+  }
+
+  static class Lifecycle implements LifecycleListener {
+    private final WorkQueue queue;
+    private final LogFileCompressor compresser;
+
+    @Inject
+    Lifecycle(final WorkQueue queue, final LogFileCompressor compressor) {
+      this.queue = queue;
+      this.compresser = compressor;
+    }
+
+    @Override
+    public void start() {
+      queue.getDefaultQueue().scheduleWithFixedDelay(compresser, 1, 24, HOURS);
+    }
+
+    @Override
+    public void stop() {
+    }
+  }
+
+  private final File logs_dir;
+
+  @Inject
+  LogFileCompressor(final SitePaths site) {
+    logs_dir = site.logs_dir;
+  }
+
+  @Override
+  public void run() {
+    final File[] list = logs_dir.listFiles();
+    if (list == null) {
+      return;
+    }
+
+    for (final File entry : list) {
+      if (!isLive(entry) && !isCompressed(entry) && isLogFile(entry)) {
+        compress(entry);
+      }
+    }
+  }
+
+  private boolean isLive(final File entry) {
+    final String name = entry.getName();
+    return ErrorLogFile.LOG_NAME.equals(name);
+  }
+
+  private boolean isCompressed(final File entry) {
+    final String name = entry.getName();
+    return name.endsWith(".gz") //
+        || name.endsWith(".zip") //
+        || name.endsWith(".bz2");
+  }
+
+  private boolean isLogFile(final File entry) {
+    return entry.isFile();
+  }
+
+  private void compress(final File src) {
+    final File dir = src.getParentFile();
+    final File dst = new File(dir, src.getName() + ".gz");
+    final File tmp = new File(dir, ".tmp." + src.getName());
+    try {
+      final InputStream in = new FileInputStream(src);
+      try {
+        OutputStream out = new GZIPOutputStream(new FileOutputStream(tmp));
+        try {
+          final byte[] buf = new byte[2048];
+          int n;
+          while (0 < (n = in.read(buf))) {
+            out.write(buf, 0, n);
+          }
+        } finally {
+          out.close();
+        }
+      } finally {
+        in.close();
+      }
+      if (!tmp.renameTo(dst)) {
+        throw new IOException("Cannot rename " + tmp + " to " + dst);
+      }
+      src.delete();
+    } catch (IOException e) {
+      log.error("Cannot compress " + src, e);
+      tmp.delete();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "Log File Compressor";
+  }
+}

gerrit-server/src/main/java/com/google/gerrit/server/git/WorkQueue.java

@@ -169,7 +169,10 @@ public class WorkQueue {
 
     @Override
     protected void afterExecute(Runnable r, Throwable t) {
-      remove((Task<?>) r);
+      final Task<?> task = (Task<?>) r;
+      if (!task.isPeriodic()) {
+        remove(task);
+      }
       super.afterExecute(r, t);
     }