From e800b1e0f3452e5be1537a67f1fa3e44a58c6dda Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Wed, 16 Jun 2010 17:33:43 -0700 Subject: [PATCH] ScanTrackingIds: Utility to update tracking ids in the database This simple command looks at every change's most recent patch set and reindexes the associated tracking information, based on the current trackingid blocks in gerrit.config. Administrators can run this command to reindex existing records after making changes. Bug: issue 124 Change-Id: I152ce430f84b70b6f84510f1abefc8da6037e746 Signed-off-by: Shawn O. Pearce --- Documentation/config-gerrit.txt | 8 +- Documentation/pgm-ScanTrackingIds.txt | 51 +++++ Documentation/pgm-index.txt | 3 + ReleaseNotes/ReleaseNotes-2.1.3.txt | 3 +- .../google/gerrit/pgm/ScanTrackingIds.java | 182 ++++++++++++++++++ .../google/gerrit/reviewdb/ChangeAccess.java | 3 + .../com/google/gerrit/server/ChangeUtil.java | 60 ++++++ .../gerrit/server/git/ReceiveCommits.java | 56 +----- 8 files changed, 308 insertions(+), 58 deletions(-) create mode 100644 Documentation/pgm-ScanTrackingIds.txt create mode 100644 gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java diff --git a/Documentation/config-gerrit.txt b/Documentation/config-gerrit.txt index ccaabc706a..814d32ec20 100644 --- a/Documentation/config-gerrit.txt +++ b/Documentation/config-gerrit.txt @@ -1560,9 +1560,11 @@ By default a shade of yellow, `FFFFCC`. [[trackingid]] Section trackingid ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Tagged footer lines containing references to external tracking -systems, parsed out of the commit message and saved in Gerrit's -database. +Tagged footer lines containing references to external +tracking systems, parsed out of the commit message and +saved in Gerrit's database. After making changes to +this section, existing changes must be reindexed with the +link:pgm-ScanTrackingIds.html[ScanTrackingIds] program. The tracking ids are serachable using tr: or bug:. diff --git a/Documentation/pgm-ScanTrackingIds.txt b/Documentation/pgm-ScanTrackingIds.txt new file mode 100644 index 0000000000..4ab4a02070 --- /dev/null +++ b/Documentation/pgm-ScanTrackingIds.txt @@ -0,0 +1,51 @@ +ScanTrackingIds +=============== + +NAME +---- +ScanTrackingIds - Rescan changes to index trackingids + +SYNOPSIS +-------- +[verse] +'java' -jar gerrit.war 'ScanTrackingIds' -d + +DESCRIPTION +----------- +Scans every known change and updates the indexed tracking +ids associated with the change, after editing the trackingid +configuration in gerrit.config. + +This task can take quite some time, but can run in the background +concurrently to the server if the database is MySQL or PostgreSQL. +If the database is H2, this task must be run by itself. + +OPTIONS +------- + +-d:: +\--site-path:: + Location of the gerrit.config file, and all other per-site + configuration data, supporting libaries and log files. + +\--threads:: + Number of threads to perform the scan work with. Defaults to + twice the number of CPUs available. + +CONTEXT +------- +This command can only be run on a server which has direct +connectivity to the metadata database, and local access to the +managed Git repositories. + +EXAMPLES +-------- +To rescan all known trackingids: + +==== + $ java -jar gerrit.war ScanTrackingIds -d site_path --threads 16 +==== + +GERRIT +------ +Part of link:index.html[Gerrit Code Review] diff --git a/Documentation/pgm-index.txt b/Documentation/pgm-index.txt index ad54e09826..c6430adc35 100644 --- a/Documentation/pgm-index.txt +++ b/Documentation/pgm-index.txt @@ -18,6 +18,9 @@ link:pgm-daemon.html[daemon]:: link:pgm-gsql.html[gsql]:: Administrative interface to idle database. +link:pgm-ScanTrackingIds.html[ScanTrackingIds]:: + Rescan all changes after configuring trackingids. + version:: Display the release version of Gerrit Code Review. diff --git a/ReleaseNotes/ReleaseNotes-2.1.3.txt b/ReleaseNotes/ReleaseNotes-2.1.3.txt index bd4718b446..a89a2853a9 100644 --- a/ReleaseNotes/ReleaseNotes-2.1.3.txt +++ b/ReleaseNotes/ReleaseNotes-2.1.3.txt @@ -35,7 +35,8 @@ id numbers. Site administrators can configure trackingid sections in gerrit.config to parse and extract issue tracking links from a commit message's footer, and have them indexed by Gerrit. Users can search for relevant changes using the search operator -`tr:`, for example `tr:432181`. +`tr:`, for example `tr:432181`. Administrators can index existing +change records using the ScanTrackingIds program. * List branches/tags containing a merged change + diff --git a/gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java new file mode 100644 index 0000000000..e62fe40050 --- /dev/null +++ b/gerrit-pgm/src/main/java/com/google/gerrit/pgm/ScanTrackingIds.java @@ -0,0 +1,182 @@ +// Copyright (C) 2010 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.pgm; + +import static com.google.gerrit.server.schema.DataSourceProvider.Context.MULTI_USER; + +import com.google.gerrit.lifecycle.LifecycleManager; +import com.google.gerrit.lifecycle.LifecycleModule; +import com.google.gerrit.pgm.util.SiteProgram; +import com.google.gerrit.reviewdb.Change; +import com.google.gerrit.reviewdb.PatchSet; +import com.google.gerrit.reviewdb.Project; +import com.google.gerrit.reviewdb.ReviewDb; +import com.google.gerrit.server.ChangeUtil; +import com.google.gerrit.server.config.TrackingFooters; +import com.google.gerrit.server.git.GitRepositoryManager; +import com.google.gerrit.server.git.LocalDiskRepositoryManager; +import com.google.gwtorm.client.OrmException; +import com.google.gwtorm.client.SchemaFactory; +import com.google.inject.Inject; +import com.google.inject.Injector; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.kohsuke.args4j.Option; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** Scan changes and update the trackingid information for them. */ +public class ScanTrackingIds extends SiteProgram { + @Option(name = "--threads", usage = "Number of concurrent threads to run") + private int threads = 2 * Runtime.getRuntime().availableProcessors(); + + private final LifecycleManager manager = new LifecycleManager(); + private final TextProgressMonitor monitor = new TextProgressMonitor(); + private List todo; + + private Injector dbInjector; + private Injector gitInjector; + + @Inject + private TrackingFooters footers; + + @Inject + private GitRepositoryManager gitManager; + + @Inject + private SchemaFactory database; + + @Override + public int run() throws Exception { + if (threads <= 0) { + threads = 1; + } + + dbInjector = createDbInjector(MULTI_USER); + gitInjector = dbInjector.createChildInjector(new LifecycleModule() { + @Override + protected void configure() { + bind(GitRepositoryManager.class).to(LocalDiskRepositoryManager.class); + listener().to(LocalDiskRepositoryManager.Lifecycle.class); + } + }); + + manager.add(dbInjector, gitInjector); + manager.start(); + gitInjector.injectMembers(this); + + final ReviewDb db = database.open(); + try { + todo = db.changes().all().toList(); + synchronized (monitor) { + monitor.beginTask("Scanning changes", todo.size()); + } + } finally { + db.close(); + } + + final List workers = new ArrayList(threads); + for (int tid = 0; tid < threads; tid++) { + Worker t = new Worker(); + t.start(); + workers.add(t); + } + for (Worker t : workers) { + t.join(); + } + synchronized (monitor) { + monitor.endTask(); + } + manager.stop(); + return 0; + } + + private void scan(ReviewDb db, Change change) { + final Project.NameKey project = change.getDest().getParentKey(); + final Repository git; + try { + git = gitManager.openRepository(project.get()); + } catch (RepositoryNotFoundException e) { + return; + } + try { + PatchSet ps = db.patchSets().get(change.currentPatchSetId()); + if (ps == null || ps.getRevision() == null + || ps.getRevision().get() == null) { + return; + } + ChangeUtil.updateTrackingIds(db, change, footers, parse(git, ps) + .getFooterLines()); + } catch (OrmException error) { + System.err.println("ERR " + error.getMessage()); + } catch (IOException error) { + System.err.println("ERR Cannot scan " + change.getId() + ": " + + error.getMessage()); + } finally { + git.close(); + } + } + + private RevCommit parse(final Repository git, PatchSet ps) + throws MissingObjectException, IncorrectObjectTypeException, IOException { + return new RevWalk(git).parseCommit(ObjectId.fromString(ps.getRevision() + .get())); + } + + private Change next() { + synchronized (todo) { + if (todo.isEmpty()) { + return null; + } + return todo.remove(todo.size() - 1); + } + } + + private class Worker extends Thread { + @Override + public void run() { + ReviewDb db; + try { + db = database.open(); + } catch (OrmException e) { + e.printStackTrace(); + return; + } + try { + for (;;) { + Change change = next(); + if (change == null) { + break; + } + scan(db, change); + synchronized (monitor) { + monitor.update(1); + } + } + } finally { + db.close(); + } + } + } +} diff --git a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java index b589877f8f..dba2a5824b 100644 --- a/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java +++ b/gerrit-reviewdb/src/main/java/com/google/gerrit/reviewdb/ChangeAccess.java @@ -90,4 +90,7 @@ public interface ChangeAccess extends Access { @Query("WHERE open = false AND status = ? AND sortKey < ? ORDER BY sortKey DESC LIMIT ?") ResultSet allClosedNext(char status, String sortKey, int limit) throws OrmException; + + @Query + ResultSet all() throws OrmException; } diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java b/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java index 2f28f984ed..740ce9c527 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/ChangeUtil.java @@ -20,17 +20,24 @@ import com.google.gerrit.reviewdb.Change; import com.google.gerrit.reviewdb.PatchSet; import com.google.gerrit.reviewdb.PatchSetApproval; import com.google.gerrit.reviewdb.ReviewDb; +import com.google.gerrit.reviewdb.TrackingId; +import com.google.gerrit.server.config.TrackingFooter; +import com.google.gerrit.server.config.TrackingFooters; import com.google.gerrit.server.git.MergeQueue; import com.google.gwtorm.client.AtomicUpdate; import com.google.gwtorm.client.OrmConcurrencyException; import com.google.gwtorm.client.OrmException; +import org.eclipse.jgit.revwalk.FooterLine; import org.eclipse.jgit.util.Base64; import org.eclipse.jgit.util.NB; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; public class ChangeUtil { private static int uuidPrefix; @@ -75,6 +82,59 @@ public class ChangeUtil { computeSortKey(c); } + public static void updateTrackingIds(ReviewDb db, Change change, + TrackingFooters trackingFooters, List footerLines) + throws OrmException { + if (trackingFooters.getTrackingFooters().isEmpty() || footerLines.isEmpty()) { + return; + } + + final Set want = new HashSet(); + final Set have = new HashSet( // + db.trackingIds().byChange(change.getId()).toList()); + + for (final TrackingFooter footer : trackingFooters.getTrackingFooters()) { + for (final FooterLine footerLine : footerLines) { + if (footerLine.matches(footer.footerKey())) { + // supporting multiple tracking-ids on a single line + final Matcher m = footer.match().matcher(footerLine.getValue()); + while (m.find()) { + if (m.group().isEmpty()) { + continue; + } + + String idstr; + if (m.groupCount() > 0) { + idstr = m.group(1); + } else { + idstr = m.group(); + } + + if (idstr.isEmpty()) { + continue; + } + if (idstr.length() > TrackingId.TRACKING_ID_MAX_CHAR) { + continue; + } + + want.add(new TrackingId(change.getId(), idstr, footer.system())); + } + } + } + } + + // Only insert the rows we don't have, and delete rows we don't match. + // + final Set toInsert = new HashSet(want); + final Set toDelete = new HashSet(have); + + toInsert.removeAll(have); + toDelete.removeAll(want); + + db.trackingIds().insert(toInsert); + db.trackingIds().delete(toDelete); + } + public static void submit(PatchSet.Id patchSetId, IdentifiedUser user, ReviewDb db, MergeQueue merger) throws OrmException { final Change.Id changeId = patchSetId.getParentKey(); diff --git a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java index 899f377a20..231d31d789 100644 --- a/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java +++ b/gerrit-server/src/main/java/com/google/gerrit/server/git/ReceiveCommits.java @@ -894,7 +894,7 @@ public class ReceiveCommits implements PreReceiveHook, PostReceiveHook { log.error("Cannot send email for new change " + change.getId(), e); } - addTrackingIds(change, footerLines); + ChangeUtil.updateTrackingIds(db, change, trackingFooters, footerLines); hooks.doPatchsetCreatedHook(change, ps); } @@ -1194,63 +1194,11 @@ public class ReceiveCommits implements PreReceiveHook, PostReceiveHook { log.error("Cannot send email for new patch set " + ps.getId(), e); } - addTrackingIds(change, footerLines); + ChangeUtil.updateTrackingIds(db, change, trackingFooters, footerLines); sendMergedEmail(result); return result != null ? result.info.getKey() : null; } - private void addTrackingIds(final Change change, - final List footerLines) throws OrmException { - if (trackingFooters.getTrackingFooters().isEmpty() || footerLines.isEmpty()) { - return; - } - - final Set want = new HashSet(); - final Set have = new HashSet( // - db.trackingIds().byChange(change.getId()).toList()); - - for (final TrackingFooter footer : trackingFooters.getTrackingFooters()) { - for (final FooterLine footerLine : footerLines) { - if (footerLine.matches(footer.footerKey())) { - // supporting multiple tracking-ids on a single line - final Matcher m = footer.match().matcher(footerLine.getValue()); - while (m.find()) { - if (m.group().isEmpty()) { - continue; - } - - String idstr; - if (m.groupCount() > 0) { - idstr = m.group(1); - } else { - idstr = m.group(); - } - - if (idstr.isEmpty()) { - continue; - } - if (idstr.length() > TrackingId.TRACKING_ID_MAX_CHAR) { - continue; - } - - want.add(new TrackingId(change.getId(), idstr, footer.system())); - } - } - } - } - - // Only insert the rows we don't have, and delete rows we don't match. - // - final Set toInsert = new HashSet(want); - final Set toDelete = new HashSet(have); - - toInsert.removeAll(have); - toDelete.removeAll(want); - - db.trackingIds().insert(toInsert); - db.trackingIds().delete(toDelete); - } - static boolean parentsEqual(RevCommit a, RevCommit b) { if (a.getParentCount() != b.getParentCount()) { return false;