From c0fd3e0894c214ddacc158baa7ebe6980989cd81 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Thu, 7 May 2020 08:52:48 -0700 Subject: [PATCH] Pull and prune docker images together We noticed that our zuul scheduler was running out of disk and one of the causes of this is we are pulling all of the wonderful new zuul images and not pruning them. This happens because we were only pruning when (re)starting services and we don't do that automatically with Zuul. Address this by always pruning after pulling even if we don't restart services. This should be safe because prune will leave the latest tagged images as well as the running images. This should keep our disk consumption down. Change-Id: Ibdd22ac42d86781f1e87c3d11e05fd8f99677167 --- playbooks/roles/nodepool-builder/tasks/main.yaml | 4 ++++ playbooks/roles/nodepool-launcher/tasks/main.yaml | 6 ++++++ playbooks/roles/nodepool-launcher/tasks/start.yaml | 4 ---- playbooks/roles/zuul-merger/tasks/main.yaml | 6 ++++++ playbooks/roles/zuul-merger/tasks/start.yaml | 4 ---- playbooks/roles/zuul-scheduler/tasks/main.yaml | 6 ++++++ playbooks/roles/zuul-scheduler/tasks/start.yaml | 5 ----- playbooks/roles/zuul-web/tasks/main.yaml | 6 ++++++ playbooks/roles/zuul-web/tasks/start.yaml | 4 ---- 9 files changed, 28 insertions(+), 17 deletions(-) diff --git a/playbooks/roles/nodepool-builder/tasks/main.yaml b/playbooks/roles/nodepool-builder/tasks/main.yaml index 3d51d3e98d..7c33fffc86 100644 --- a/playbooks/roles/nodepool-builder/tasks/main.yaml +++ b/playbooks/roles/nodepool-builder/tasks/main.yaml @@ -41,6 +41,10 @@ cmd: docker-compose up -d chdir: /etc/nodepool-builder-compose/ +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + - name: Disable old service service: name: nodepool-builder diff --git a/playbooks/roles/nodepool-launcher/tasks/main.yaml b/playbooks/roles/nodepool-launcher/tasks/main.yaml index 85799ea964..f77c189ecf 100644 --- a/playbooks/roles/nodepool-launcher/tasks/main.yaml +++ b/playbooks/roles/nodepool-launcher/tasks/main.yaml @@ -54,6 +54,12 @@ include_tasks: start.yaml when: nodepool_launcher_start | bool +# We can prune here as it should leave the "latest" tagged images +# as well as the currently running images. +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + - name: Disable old service service: name: nodepool-launcher diff --git a/playbooks/roles/nodepool-launcher/tasks/start.yaml b/playbooks/roles/nodepool-launcher/tasks/start.yaml index 02b8bd3905..bdfefc51bf 100644 --- a/playbooks/roles/nodepool-launcher/tasks/start.yaml +++ b/playbooks/roles/nodepool-launcher/tasks/start.yaml @@ -2,7 +2,3 @@ shell: cmd: docker-compose up -d chdir: /etc/nodepool-docker/ - -- name: Run docker prune to cleanup unneeded images - shell: - cmd: docker image prune -f diff --git a/playbooks/roles/zuul-merger/tasks/main.yaml b/playbooks/roles/zuul-merger/tasks/main.yaml index 0b407ed020..c550c0695b 100644 --- a/playbooks/roles/zuul-merger/tasks/main.yaml +++ b/playbooks/roles/zuul-merger/tasks/main.yaml @@ -51,6 +51,12 @@ include_tasks: start.yaml when: zuul_merger_start | bool +# We can prune here as it should leave the "latest" tagged images +# as well as the currently running images. +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + - name: Disable old service service: name: zuul-merger diff --git a/playbooks/roles/zuul-merger/tasks/start.yaml b/playbooks/roles/zuul-merger/tasks/start.yaml index 68d603a4ee..c488671545 100644 --- a/playbooks/roles/zuul-merger/tasks/start.yaml +++ b/playbooks/roles/zuul-merger/tasks/start.yaml @@ -2,7 +2,3 @@ shell: cmd: docker-compose up -d chdir: /etc/zuul-merger - -- name: Run docker prune to cleanup unneeded images - shell: - cmd: docker image prune -f diff --git a/playbooks/roles/zuul-scheduler/tasks/main.yaml b/playbooks/roles/zuul-scheduler/tasks/main.yaml index 012a43d350..77d501cfc0 100644 --- a/playbooks/roles/zuul-scheduler/tasks/main.yaml +++ b/playbooks/roles/zuul-scheduler/tasks/main.yaml @@ -72,6 +72,12 @@ include_tasks: start.yaml when: zuul_scheduler_start is defined and zuul_scheduler_start | bool +# We can prune here as it should leave the "latest" tagged images +# as well as the currently running images. +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + - name: Disable old service service: name: zuul-scheduler diff --git a/playbooks/roles/zuul-scheduler/tasks/start.yaml b/playbooks/roles/zuul-scheduler/tasks/start.yaml index 05902c7158..bee386e55e 100644 --- a/playbooks/roles/zuul-scheduler/tasks/start.yaml +++ b/playbooks/roles/zuul-scheduler/tasks/start.yaml @@ -2,8 +2,3 @@ shell: cmd: docker-compose up -d chdir: /etc/zuul-scheduler - -# TODO: put this in a handler -- name: Run docker prune to cleanup unneeded images - shell: - cmd: docker image prune -f diff --git a/playbooks/roles/zuul-web/tasks/main.yaml b/playbooks/roles/zuul-web/tasks/main.yaml index 1f1d2a8e06..5757b62218 100644 --- a/playbooks/roles/zuul-web/tasks/main.yaml +++ b/playbooks/roles/zuul-web/tasks/main.yaml @@ -100,6 +100,12 @@ include_tasks: start.yaml when: zuul_web_start | bool +# We can prune here as it should leave the "latest" tagged images +# as well as the currently running images. +- name: Run docker prune to cleanup unneeded images + shell: + cmd: docker image prune -f + - name: Disable old service service: name: 'zuul-{{ item }}' diff --git a/playbooks/roles/zuul-web/tasks/start.yaml b/playbooks/roles/zuul-web/tasks/start.yaml index 1aa4b16cd6..1498e6ee8f 100644 --- a/playbooks/roles/zuul-web/tasks/start.yaml +++ b/playbooks/roles/zuul-web/tasks/start.yaml @@ -2,7 +2,3 @@ shell: cmd: docker-compose up -d chdir: /etc/zuul-web - -- name: Run docker prune to cleanup unneeded images - shell: - cmd: docker image prune -f