Add support "rebase" subcommand

The rebase commands merges Debian packages from one branch
to another. The way that this works is the following:

1. Determine the current deployment of a running system.
2. Checkout the current deployment from an ostree repo.
3. Determine the packages installed in the current deployment.
4. Checkout the target branch to deploy.
5. Determine the packages installed in the target branch.
6. Determine the delta of the old packages that is not installed
   in the target branch.
7. Install the packakge delta in the target branch.
8. Commit the changes.
9. Deploy the new branch.

Story: 2010867
Task: 48556

Change-Id: Ibd302712c3131c64978df50a32d3b1d075eac0e3
Signed-off-by: Charles Short <charles.short@windriver.com>

apt_ostree/apt.py

@@ -6,6 +6,7 @@ SPDX-License-Identifier: Apache-2.0
 """
 
 import logging
+import os
 import subprocess
 import sys
 
@@ -33,30 +34,43 @@ class Apt:
 
     def apt_update(self, rootfs):
         """Run apt-get update."""
+        self.logging.info("Running apt-update")
         r = run_sandbox_command(
             ["apt-get", "update", "-y"],
-            rootfs)
+            rootfs,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE)
         if r.returncode != 0:
             self.logging.error("Failed to run apt-get update.")
         return r
 
-    def apt_install(self, packages, rootfs):
+    def apt_install(self, cache, packages, rootfs):
         """Run apt-get install."""
-        cmd = ["apt-get", "install"]
-        if packages:
-            cmd += packages
-        r = run_sandbox_command(cmd, rootfs)
-        if r.returncode != 0:
-            self.logging.error("Failed to run apt-get install.")
+        env = os.environ.copy()
+        env["DEBIAN_FRONTEND"] = "noninteractive"
+        for package in packages:
+            version = self.get_version(cache, package)
+            pkg = self.apt_package(cache, package)
+            if not pkg.is_installed:
+                self.logging.info(f"Installing {package} ({version}).")
+            else:
+                self.logging.info(
+                    f"Skipping {package} ({version}), already installed.")
+            cmd = ["apt-get", "-y", "install", package]
+            r = run_sandbox_command(cmd, rootfs, env=env,
+                                    stdout=subprocess.PIPE,
+                                    stderr=subprocess.PIPE)
+            if r.returncode != 0:
+                self.logging.error("Failed to run apt-get install")
         return r
 
     def apt_list(self, rootfs, action):
         """Show package versions."""
         return run_sandbox_command(
-                ["apt", "list", action],
-                rootfs,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.DEVNULL)
+            ["apt", "list", action],
+            rootfs,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL)
 
     def apt_upgrade(self, rootfs):
         """Run apt-get upgrade."""
@@ -90,6 +104,14 @@ class Apt:
     def get_version(self, cache, package):
         return self.apt_package(cache, package).candidate.version
 
+    def get_installed_packages(self, cache):
+        """Get a list of installed packages."""
+        pkgs = set()
+        for pkg in cache:
+            if pkg.is_installed:
+                pkgs.add(pkg.name)
+        return pkgs
+
     def get_dependencies(self, cache, packages, deps, predeps, all_deps):
         """Get installed versions."""
         for pkg in packages:

apt_ostree/cmd/rebase.py

@@ -0,0 +1,40 @@
+"""
+Copyright (c) 2023 Wind River Systems, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+
+"""
+
+import errno
+import sys
+
+import click
+
+from apt_ostree.cmd.options import branch_argument
+from apt_ostree.cmd.options import reboot_option
+from apt_ostree.cmd import pass_state_context
+from apt_ostree.rebase import Rebase
+
+
+@click.command(
+    help="Switch to a new tree.")
+@pass_state_context
+@reboot_option
+@click.option(
+    "--update",
+    help="Pull objects from ostree repository",
+    is_flag=True,
+    default=False
+)
+@branch_argument
+def rebase(state, reboot, update, branch):
+    try:
+        Rebase(state).rebase(update)
+    except KeyboardInterrupt:
+        click.secho("\n" + ("Exiting at your request."))
+        sys.exit(130)
+    except BrokenPipeError:
+        sys.exit()
+    except OSError as error:
+        if error.errno == errno.ENOSPC:
+            sys.exit("error - No space left on device.")

apt_ostree/cmd/shell.py

@@ -16,6 +16,7 @@ from apt_ostree.cmd.install import install
 from apt_ostree.cmd.options import debug_option
 from apt_ostree.cmd.options import workspace_option
 from apt_ostree.cmd import pass_state_context
+from apt_ostree.cmd.rebase import rebase
 from apt_ostree.cmd.remotes import remote
 from apt_ostree.cmd.repo import repo
 from apt_ostree.cmd.status import status
@@ -46,10 +47,12 @@ def main():
     cli(prog_name="apt-ostree")
 
 
+# apt-ostree subcommands
 cli.add_command(apt)
 cli.add_command(compose)
 cli.add_command(deploy)
 cli.add_command(install)
+cli.add_command(rebase)
 cli.add_command(remote)
 cli.add_command(repo)
 cli.add_command(status)

apt_ostree/deploy.py

@@ -89,9 +89,10 @@ class Deploy:
         with self.console.status("Cleaning up."):
             shutil.rmtree(rootfs)
 
-    def get_sysroot(self):
+    def get_sysroot(self, branch=None):
         """Checkout the commit to the specified directory."""
-        branch = self.ostree.get_branch()
+        if branch is None:
+            branch = self.ostree.get_branch()
         rev = self.ostree.ostree_ref(branch)
         with self.console.status(f"Checking out {rev[:10]}..."):
             self.workdir = self.workdir.joinpath(branch)

apt_ostree/ostree.py

@@ -95,6 +95,42 @@ class Ostree:
                 sys.exit(1)
         return repo
 
+    def fetch(self, remote, branch, progress=None):
+        """Fetch an object from a remote repository."""
+        cancellable = None
+        progress = OSTree.AsyncProgress.new()
+        progress.connect('changed', self._progress_cb)
+
+        repo = self.open_ostree()
+
+        # Pull Options
+        pull_options = {
+            'depth': GLib.Variant('i', -1),
+            'refs': GLib.Variant('as', (branch,)),
+        }
+        try:
+            repo.pull_with_options(remote,
+                                   GLib.Variant('a{sv}', pull_options),
+                                   progress, cancellable)
+        except GLib.GError as e:
+            self.logging.error(f"Fetch failed: {e.message}")
+            sys.exit(1)
+
+    def _progress_cb(self, async_progress):
+        """Show whats happening."""
+        status = async_progress.get_status()
+        outstanding_fetches = async_progress.get_uint('outstanding-fetches')
+        if status:
+            print(f'OUTPUT:Status: {status}')
+        elif outstanding_fetches > 0:
+            fetched = async_progress.get_uint('fetched')
+            requested = async_progress.get_uint('requested')
+            if requested == 0:
+                percent = 0.0
+            else:
+                percent = fetched / requested
+            print(f'PROGRESS:{percent}')
+
     def ostree_checkout(self, branch, rootfs):
         """Checkout a branch from an ostree repository."""
         repo = self.open_ostree()
@@ -111,9 +147,6 @@ class Ostree:
         """Find the commit id for a given reference."""
         repo = self.open_ostree()
         ret, rev = repo.resolve_rev(branch, True)
-        if not rev:
-            self.logging.error(f"{branch} not found in {self.state.repo}")
-            sys.exit(1)
         return rev
 
     def get_branch(self):

apt_ostree/packages.py

@@ -66,7 +66,7 @@ class Packages:
                 commit += f"- {dep} ({version})\n"
 
         # Step 4 - Install the valid packages.
-        self.apt.apt_install(packages, rootfs)
+        self.apt.apt_install(cache, packages, rootfs)
 
         # Step 5 - Run post staging steps.
         self.deploy.poststaging(rootfs)

apt_ostree/rebase.py

@@ -0,0 +1,158 @@
+"""
+Copyright (c) 2023 Wind River Systems, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+"""
+
+import logging
+import sys
+
+from rich.console import Console
+
+from apt_ostree.apt import Apt
+from apt_ostree.deploy import Deploy
+from apt_ostree.ostree import Ostree
+
+
+class Rebase:
+    def __init__(self, state):
+        self.state = state
+        self.console = Console()
+        self.ostree = Ostree(self.state)
+        self.deploy = Deploy(self.state)
+        self.apt = Apt(self.state)
+        self.logging = logging.getLogger(__name__)
+
+    def rebase(self, update):
+        """Switch to another branch."""
+        # Verify the branch is formatted correctly.
+        try:
+            (remote, branch) = self.state.branch.split(":")
+        except KeyError:
+            self.logging.error(
+                "Branch must be in the format of <remote>:<branch>"
+            )
+            sys.exit(1)
+
+        # Make sure that we have remotes configured.
+        if len(self._get_remotes()) == 0:
+            self.logging.error("No remotes configured.")
+            sys.exit(1)
+
+        if update:
+            self.logging.info(f"Pulling {branch} from {remote}.")
+            with self.console.status(f"Fetching {branch} from {remote}"):
+                self._fetch(remote, branch)
+            sys.exit(1)
+        else:
+            # Get the current deployment, check for
+            # a deployed sysroot.
+            sysroot = self.ostree.get_sysroot()
+            if sysroot is None:
+                self.logging.error("Not running ostree.")
+                sys.exit(1)
+
+            csum = sysroot.get_booted_deployment().get_csum()
+            origin = sysroot.get_booted_deployment().get_origin()
+            refspec = origin.get_string('origin', 'refspec')
+            if refspec is None:
+                self.logging.error("Unable to determine branch.")
+                sys.exit(1)
+
+            self.logging.info(f"Rebasing {refspec} on to {self.state.branch}.")
+            self.logging.info(f"Local deployment: {refspec} ({csum[:10]}).")
+
+            # Get a list of manually installed packages.
+            # These include packages that were installed after an
+            # initial boostrap as well.
+            current_packages = self.get_current_packages(refspec, csum)
+
+            # If we didnt fetch the latest repository do it now.
+            self.logging.info(f"Pulling {branch} from {remote}.")
+            self._fetch(remote, branch)
+
+            # Prepare the new branch for deployment.
+            ref = self.ostree.ostree_ref(self.state.branch)
+            if ref is None:
+                self.logging.error(f"Failed to fetch {self.state.branch}")
+                sys.exit(1)
+
+            self.logging.debug(f"Fetched {self.state.branch} ({ref[:10]})")
+
+            # Rebase onto the pristine branch.
+            rootfs = self.deploy.get_sysroot(self.state.branch)
+            if rootfs is None:
+                self.logging.error(f"Unable to checkout {self.state.branch}")
+                sys.exit(1)
+
+            # Prestaging
+            self.deploy.prestaging(rootfs)
+            self.apt.apt_update(rootfs)
+            cache = self.apt.cache(rootfs)
+            new_packages = self.apt.get_installed_packages(cache)
+
+            # Determine packages that are missing and install them.
+            self.logging.debug(
+                f"Deteriming package delta between {refspec} "
+                f"and {self.state.branch}."
+            )
+            packages = list(current_packages - new_packages)
+            if len(packages) != 0:
+                commit = f"Resynchronize {self.state.branch} ({ref[:10]})"\
+                         f" with {refspec} ({csum[:10]}).\n\n"
+                subject = "Resynchronize package list"
+                for pkg in packages:
+                    version = self.apt.get_version(cache, pkg)
+                    commit += f" - {pkg} {version}"
+                    self.logging.info(f"Installing {pkg} ({version}).")
+                self.apt.apt_install(cache, packages, rootfs)
+            else:
+                subject = f"Updated {origin}"
+                commit = "Previous deployment: {csum[:10]}"
+
+            self.deploy.poststaging(rootfs)
+
+            self.logging.info(f"Commiting to {self.state.branch}.")
+            self.ostree.ostree_commit(
+                root=str(rootfs),
+                branch=self.state.branch,
+                repo=self.state.repo,
+                subject=subject,
+                msg=commit
+            )
+
+            self.deploy.cleanup(rootfs)
+
+    def _get_remotes(self):
+        """List of remotes configured."""
+        return [
+            refs for refs in self.ostree.remotes_list()
+            if refs != "origin"]
+
+    def _fetch(self, remote, branch):
+        """Wrapper around ostree fertch."""
+        self.ostree.fetch(remote, branch)
+
+    def get_current_packages(self, branch, ref):
+        """Steps to prepare the systeem before a rebase."""
+        self.logging.debug(
+            f"Deploying current deployment {branch} ({ref[:10]})")
+        rootfs = self.deploy.get_sysroot(branch)
+        if not rootfs.exists():
+            self.logging.error("Unable to determine rootfs: {rootfs}")
+            sys.exit(1)
+
+        # Deploy the current booted deployment and run apt-update
+        # to configure apt.
+        self.deploy.prestaging(rootfs)
+        self.apt.apt_update(rootfs)
+
+        # Check for packages that are installed.
+        self.logging.debug("Querying installed packages.")
+        cache = self.apt.cache(rootfs)
+        pkgs = self.apt.get_installed_packages(cache)
+
+        # Just remove the directory since we are done with it
+        self.deploy.cleanup(rootfs)
+
+        return pkgs