ScanTrackingIds: Utility to update tracking ids in the database

This simple command looks at every change's most recent patch set
and reindexes the associated tracking information, based on the
current trackingid blocks in gerrit.config.  Administrators can
run this command to reindex existing records after making changes.

Bug: issue 124
Change-Id: I152ce430f84b70b6f84510f1abefc8da6037e746
Signed-off-by: Shawn O. Pearce <sop@google.com>

Documentation/config-gerrit.txt

@@ -1560,9 +1560,11 @@ By default a shade of yellow, `FFFFCC`.
 [[trackingid]] Section trackingid
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Tagged footer lines containing references to external tracking
-systems, parsed out of the commit message and saved in Gerrit's
-database.
+Tagged footer lines containing references to external
+tracking systems, parsed out of the commit message and
+saved in Gerrit's database.  After making changes to
+this section, existing changes must be reindexed with the
+link:pgm-ScanTrackingIds.html[ScanTrackingIds] program.
 
 The tracking ids are serachable using tr:<tracking id> or
 bug:<tracking id>.

Documentation/pgm-ScanTrackingIds.txt

@@ -0,0 +1,51 @@
+ScanTrackingIds
+===============
+
+NAME
+----
+ScanTrackingIds - Rescan changes to index trackingids
+
+SYNOPSIS
+--------
+[verse]
+'java' -jar gerrit.war 'ScanTrackingIds' -d <SITE_PATH>
+
+DESCRIPTION
+-----------
+Scans every known change and updates the indexed tracking
+ids associated with the change, after editing the trackingid
+configuration in gerrit.config.
+
+This task can take quite some time, but can run in the background
+concurrently to the server if the database is MySQL or PostgreSQL.
+If the database is H2, this task must be run by itself.
+
+OPTIONS
+-------
+
+-d::
+\--site-path::
+	Location of the gerrit.config file, and all other per-site
+	configuration data, supporting libaries and log files.
+
+\--threads::
+	Number of threads to perform the scan work with.  Defaults to
+	twice the number of CPUs available.
+
+CONTEXT
+-------
+This command can only be run on a server which has direct
+connectivity to the metadata database, and local access to the
+managed Git repositories.
+
+EXAMPLES
+--------
+To rescan all known trackingids:
+
+====
+	$ java -jar gerrit.war ScanTrackingIds -d site_path --threads 16
+====
+
+GERRIT
+------
+Part of link:index.html[Gerrit Code Review]

Documentation/pgm-index.txt

@@ -18,6 +18,9 @@ link:pgm-daemon.html[daemon]::
 link:pgm-gsql.html[gsql]::
 	Administrative interface to idle database.
 
+link:pgm-ScanTrackingIds.html[ScanTrackingIds]::
+	Rescan all changes after configuring trackingids.
+
 version::
   Display the release version of Gerrit Code Review.
 

ReleaseNotes/ReleaseNotes-2.1.3.txt

@@ -35,7 +35,8 @@ id numbers.  Site administrators can configure trackingid sections
 in gerrit.config to parse and extract issue tracking links from
 a commit message's footer, and have them indexed by Gerrit.
 Users can search for relevant changes using the search operator
-`tr:`, for example `tr:432181`.
+`tr:`, for example `tr:432181`.  Administrators can index existing
+change records using the ScanTrackingIds program.
 
 * List branches/tags containing a merged change
 +

gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java

@@ -0,0 +1,182 @@
+// Copyright (C) 2010 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.pgm;
+
+import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER;
+
+import com.google.gerrit.lifecycle.LifecycleManager;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.pgm.util.SiteProgram;
+import com.google.gerrit.reviewdb.Change;
+import com.google.gerrit.reviewdb.PatchSet;
+import com.google.gerrit.reviewdb.Project;
+import com.google.gerrit.reviewdb.ReviewDb;
+import com.google.gerrit.server.ChangeUtil;
+import com.google.gerrit.server.config.TrackingFooters;
+import com.google.gerrit.server.git.GitRepositoryManager;
+import com.google.gerrit.server.git.LocalDiskRepositoryManager;
+import com.google.gwtorm.client.OrmException;
+import com.google.gwtorm.client.SchemaFactory;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+
+import org.eclipse.jgit.errors.IncorrectObjectTypeException;
+import org.eclipse.jgit.errors.MissingObjectException;
+import org.eclipse.jgit.errors.RepositoryNotFoundException;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.lib.TextProgressMonitor;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.revwalk.RevWalk;
+import org.kohsuke.args4j.Option;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Scan changes and update the trackingid information for them. */
+public class ScanTrackingIds extends SiteProgram {
+  @Option(name = "--threads", usage = "Number of concurrent threads to run")
+  private int threads = 2 * Runtime.getRuntime().availableProcessors();
+
+  private final LifecycleManager manager = new LifecycleManager();
+  private final TextProgressMonitor monitor = new TextProgressMonitor();
+  private List<Change> todo;
+
+  private Injector dbInjector;
+  private Injector gitInjector;
+
+  @Inject
+  private TrackingFooters footers;
+
+  @Inject
+  private GitRepositoryManager gitManager;
+
+  @Inject
+  private SchemaFactory<ReviewDb> database;
+
+  @Override
+  public int run() throws Exception {
+    if (threads <= 0) {
+      threads = 1;
+    }
+
+    dbInjector = createDbInjector(MULTI_USER);
+    gitInjector = dbInjector.createChildInjector(new LifecycleModule() {
+      @Override
+      protected void configure() {
+        bind(GitRepositoryManager.class).to(LocalDiskRepositoryManager.class);
+        listener().to(LocalDiskRepositoryManager.Lifecycle.class);
+      }
+    });
+
+    manager.add(dbInjector, gitInjector);
+    manager.start();
+    gitInjector.injectMembers(this);
+
+    final ReviewDb db = database.open();
+    try {
+      todo = db.changes().all().toList();
+      synchronized (monitor) {
+        monitor.beginTask("Scanning changes", todo.size());
+      }
+    } finally {
+      db.close();
+    }
+
+    final List<Worker> workers = new ArrayList<Worker>(threads);
+    for (int tid = 0; tid < threads; tid++) {
+      Worker t = new Worker();
+      t.start();
+      workers.add(t);
+    }
+    for (Worker t : workers) {
+      t.join();
+    }
+    synchronized (monitor) {
+      monitor.endTask();
+    }
+    manager.stop();
+    return 0;
+  }
+
+  private void scan(ReviewDb db, Change change) {
+    final Project.NameKey project = change.getDest().getParentKey();
+    final Repository git;
+    try {
+      git = gitManager.openRepository(project.get());
+    } catch (RepositoryNotFoundException e) {
+      return;
+    }
+    try {
+      PatchSet ps = db.patchSets().get(change.currentPatchSetId());
+      if (ps == null || ps.getRevision() == null
+          || ps.getRevision().get() == null) {
+        return;
+      }
+      ChangeUtil.updateTrackingIds(db, change, footers, parse(git, ps)
+          .getFooterLines());
+    } catch (OrmException error) {
+      System.err.println("ERR " + error.getMessage());
+    } catch (IOException error) {
+      System.err.println("ERR Cannot scan " + change.getId() + ": "
+          + error.getMessage());
+    } finally {
+      git.close();
+    }
+  }
+
+  private RevCommit parse(final Repository git, PatchSet ps)
+      throws MissingObjectException, IncorrectObjectTypeException, IOException {
+    return new RevWalk(git).parseCommit(ObjectId.fromString(ps.getRevision()
+        .get()));
+  }
+
+  private Change next() {
+    synchronized (todo) {
+      if (todo.isEmpty()) {
+        return null;
+      }
+      return todo.remove(todo.size() - 1);
+    }
+  }
+
+  private class Worker extends Thread {
+    @Override
+    public void run() {
+      ReviewDb db;
+      try {
+        db = database.open();
+      } catch (OrmException e) {
+        e.printStackTrace();
+        return;
+      }
+      try {
+        for (;;) {
+          Change change = next();
+          if (change == null) {
+            break;
+          }
+          scan(db, change);
+          synchronized (monitor) {
+            monitor.update(1);
+          }
+        }
+      } finally {
+        db.close();
+      }
+    }
+  }
+}

gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java

@@ -90,4 +90,7 @@ public interface ChangeAccess extends Access<Change, Change.Id> {
   @Query("WHERE open = false AND status = ? AND sortKey < ? ORDER BY sortKey DESC LIMIT ?")
   ResultSet<Change> allClosedNext(char status, String sortKey, int limit)
       throws OrmException;
+
+  @Query
+  ResultSet<Change> all() throws OrmException;
 }

gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java

@@ -20,17 +20,24 @@ import com.google.gerrit.reviewdb.Change;
 import com.google.gerrit.reviewdb.PatchSet;
 import com.google.gerrit.reviewdb.PatchSetApproval;
 import com.google.gerrit.reviewdb.ReviewDb;
+import com.google.gerrit.reviewdb.TrackingId;
+import com.google.gerrit.server.config.TrackingFooter;
+import com.google.gerrit.server.config.TrackingFooters;
 import com.google.gerrit.server.git.MergeQueue;
 import com.google.gwtorm.client.AtomicUpdate;
 import com.google.gwtorm.client.OrmConcurrencyException;
 import com.google.gwtorm.client.OrmException;
 
+import org.eclipse.jgit.revwalk.FooterLine;
 import org.eclipse.jgit.util.Base64;
 import org.eclipse.jgit.util.NB;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
+import java.util.regex.Matcher;
 
 public class ChangeUtil {
   private static int uuidPrefix;
@@ -75,6 +82,59 @@ public class ChangeUtil {
     computeSortKey(c);
   }
 
+  public static void updateTrackingIds(ReviewDb db, Change change,
+      TrackingFooters trackingFooters, List<FooterLine> footerLines)
+      throws OrmException {
+    if (trackingFooters.getTrackingFooters().isEmpty() || footerLines.isEmpty()) {
+      return;
+    }
+
+    final Set<TrackingId> want = new HashSet<TrackingId>();
+    final Set<TrackingId> have = new HashSet<TrackingId>( //
+        db.trackingIds().byChange(change.getId()).toList());
+
+    for (final TrackingFooter footer : trackingFooters.getTrackingFooters()) {
+      for (final FooterLine footerLine : footerLines) {
+        if (footerLine.matches(footer.footerKey())) {
+          // supporting multiple tracking-ids on a single line
+          final Matcher m = footer.match().matcher(footerLine.getValue());
+          while (m.find()) {
+            if (m.group().isEmpty()) {
+              continue;
+            }
+
+            String idstr;
+            if (m.groupCount() > 0) {
+              idstr = m.group(1);
+            } else {
+              idstr = m.group();
+            }
+
+            if (idstr.isEmpty()) {
+              continue;
+            }
+            if (idstr.length() > TrackingId.TRACKING_ID_MAX_CHAR) {
+              continue;
+            }
+
+            want.add(new TrackingId(change.getId(), idstr, footer.system()));
+          }
+        }
+      }
+    }
+
+    // Only insert the rows we don't have, and delete rows we don't match.
+    //
+    final Set<TrackingId> toInsert = new HashSet<TrackingId>(want);
+    final Set<TrackingId> toDelete = new HashSet<TrackingId>(have);
+
+    toInsert.removeAll(have);
+    toDelete.removeAll(want);
+
+    db.trackingIds().insert(toInsert);
+    db.trackingIds().delete(toDelete);
+  }
+
   public static void submit(PatchSet.Id patchSetId, IdentifiedUser user, ReviewDb db, MergeQueue merger)
       throws OrmException {
     final Change.Id changeId = patchSetId.getParentKey();

gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java

@@ -894,7 +894,7 @@ public class ReceiveCommits implements PreReceiveHook, PostReceiveHook {
       log.error("Cannot send email for new change " + change.getId(), e);
     }
 
-    addTrackingIds(change, footerLines);
+    ChangeUtil.updateTrackingIds(db, change, trackingFooters, footerLines);
     hooks.doPatchsetCreatedHook(change, ps);
   }
 
@@ -1194,63 +1194,11 @@ public class ReceiveCommits implements PreReceiveHook, PostReceiveHook {
       log.error("Cannot send email for new patch set " + ps.getId(), e);
     }
 
-    addTrackingIds(change, footerLines);
+    ChangeUtil.updateTrackingIds(db, change, trackingFooters, footerLines);
     sendMergedEmail(result);
     return result != null ? result.info.getKey() : null;
   }
 
-  private void addTrackingIds(final Change change,
-      final List<FooterLine> footerLines) throws OrmException {
-    if (trackingFooters.getTrackingFooters().isEmpty() || footerLines.isEmpty()) {
-      return;
-    }
-
-    final Set<TrackingId> want = new HashSet<TrackingId>();
-    final Set<TrackingId> have = new HashSet<TrackingId>( //
-        db.trackingIds().byChange(change.getId()).toList());
-
-    for (final TrackingFooter footer : trackingFooters.getTrackingFooters()) {
-      for (final FooterLine footerLine : footerLines) {
-        if (footerLine.matches(footer.footerKey())) {
-          // supporting multiple tracking-ids on a single line
-          final Matcher m = footer.match().matcher(footerLine.getValue());
-          while (m.find()) {
-            if (m.group().isEmpty()) {
-              continue;
-            }
-
-            String idstr;
-            if (m.groupCount() > 0) {
-              idstr = m.group(1);
-            } else {
-              idstr = m.group();
-            }
-
-            if (idstr.isEmpty()) {
-              continue;
-            }
-            if (idstr.length() > TrackingId.TRACKING_ID_MAX_CHAR) {
-              continue;
-            }
-
-            want.add(new TrackingId(change.getId(), idstr, footer.system()));
-          }
-        }
-      }
-    }
-
-    // Only insert the rows we don't have, and delete rows we don't match.
-    //
-    final Set<TrackingId> toInsert = new HashSet<TrackingId>(want);
-    final Set<TrackingId> toDelete = new HashSet<TrackingId>(have);
-
-    toInsert.removeAll(have);
-    toDelete.removeAll(want);
-
-    db.trackingIds().insert(toInsert);
-    db.trackingIds().delete(toDelete);
-  }
-
   static boolean parentsEqual(RevCommit a, RevCommit b) {
     if (a.getParentCount() != b.getParentCount()) {
       return false;