Publish specs jobs for *-specs

The gate jobs already publish the specs as drafts and it
would be very beneficial to actually see those online at a permanent
place - rather than going to github.

This patch:
* Sets up a new site specs.openstack.org
* Creates new job-template '{name}-publish-specs' for publishing specs
* Creates new specs-job job-group for gating and publishing specs
* Converts all *-specs repositories to use the new job-group
* Creates Zuul specs-job macro to use these jobs
* Replaces those *-specs repositories that use already docs and python27
  gates to use the new job-group and thus get published.

The patch does not take care of publishing of repositories that do not
use both the docs and python27 gates, this will be done separately.

implements bp publish-specs

Co-Authored-By: Andreas Jaeger <aj@suse.de>
Change-Id: Icc1c780cea8cd24c7a108429f84b2450056c6eed

doc/source/static.rst

@@ -16,6 +16,7 @@ At a Glance
   * http://docs-draft.openstack.org
   * http://status.openstack.org
   * http://pypi.openstack.org
+  * http://specs.openstack.org
 :Puppet:
   * :file:`modules/openstack_project/manifests/static.pp`
 :Projects:

modules/openstack_project/files/jenkins_job_builder/config/projects.yaml

@@ -298,7 +298,7 @@
     node: 'bare-precise || bare-trusty'
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
 
 - project:
     name: sahara
@@ -319,11 +319,9 @@
     name: sahara-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
-      - gate-{name}-python27
 
 - project:
     name: python-saharaclient
@@ -661,10 +659,9 @@
     name: glance-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
 
 - project:
     name: glance.store
@@ -696,10 +693,9 @@
     name: heat-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
 
 - project:
     name: python-heatclient
@@ -778,11 +774,9 @@
     name: keystone-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
 
 - project:
     name: image-api
@@ -894,8 +888,7 @@
     tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
 
 - project:
     name: nova-docker
@@ -922,10 +915,9 @@
     name: swift-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
 
 
 - project:
@@ -1072,12 +1064,9 @@
     name: oslo-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
-
 
 - project:
     name: python-ceilometerclient
@@ -1249,12 +1238,9 @@
     name: neutron-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
-
 
 - project:
     name: trove
@@ -1336,11 +1322,9 @@
     name: marconi-specs
     github-org: openstack
     node: bare-precise
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
 
 - project:
     name: python-marconiclient
@@ -1369,11 +1353,9 @@
     name: designate-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
 
 - project:
     name: billingstack
@@ -1668,11 +1650,9 @@
     name: qa-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
-
 
 - project:
     name: zuul
@@ -1761,11 +1741,9 @@
     name: ceilometer-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
 
 - project:
     name: tripleo-heat-templates
@@ -1802,11 +1780,9 @@
     name: tripleo-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
 
 - project:
     name: puppet-glance
@@ -2276,11 +2252,9 @@
     name: ironic-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-python27
+      - specs-jobs
-      - gate-{name}-docs
 
 - project:
     name: ironic
@@ -2353,10 +2327,9 @@
     name: barbican-specs
     github-org: openstack
     node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
 
 - project:
     name: python-barbicanclient
@@ -2612,12 +2585,10 @@
 - project:
     name: docs-specs
     github-org: openstack
-    node: bare-precise
+    node: 'bare-precise || bare-trusty'
-    tarball-site: tarballs.openstack.org
 
     jobs:
-      - gate-{name}-docs
+      - specs-jobs
-      - gate-{name}-python27
 
 - project:
     name: operations-guide

modules/openstack_project/files/jenkins_job_builder/config/specs-jobs.yaml

@@ -0,0 +1,28 @@
+# Jobs for the various *-specs repositories
+
+- job-template:
+    name: '{name}-publish-specs'
+    node: 'bare-precise || bare-trusty'
+
+    builders:
+      - revoke-sudo
+      - gerrit-git-prep
+      - docs:
+          github-org: '{github-org}'
+          project: '{name}'
+
+    publishers:
+      - ftp:
+          site: 'specs.openstack.org'
+          source: 'doc/build/html/**'
+          target: '{github-org}/{name}'
+          remove-prefix: 'doc/build/html'
+          excludes: ''
+      - console-log
+
+- job-group:
+    name: specs-jobs
+    jobs:
+      - gate-{name}-docs
+      - gate-{name}-python27
+      - '{name}-publish-specs'

modules/openstack_project/files/zuul/layout.yaml

@@ -379,6 +379,16 @@ project-templates:
       - 'gate-{name}-chef-lint'
       - 'gate-{name}-chef-unit'
 
+  - name: specs-jobs
+    check:
+      - 'gate-{name}-docs'
+      - 'gate-{name}-python27'
+    gate:
+      - 'gate-{name}-docs'
+      - 'gate-{name}-python27'
+    post:
+      - '{name}-publish-specs'
+
 jobs:
   - name: ^.*$
     parameter-function: set_node_options_default_precise
@@ -765,12 +775,8 @@ projects:
       - ceilometer-merge-release-tags
 
   - name: openstack/ceilometer-specs
-    check:
+    template:
-      - gate-ceilometer-specs-python27
+      - name: specs-jobs
-      - gate-ceilometer-specs-docs
-    gate:
-      - gate-ceilometer-specs-python27
-      - gate-ceilometer-specs-docs
 
   - name: openstack/cinder
     template:
@@ -859,12 +865,8 @@ projects:
       - gate-designate-devstack-dsvm
 
   - name: openstack/designate-specs
-    check:
+    template:
-      - gate-designate-specs-python27
+      - name: specs-jobs
-      - gate-designate-specs-docs
-    gate:
-      - gate-designate-specs-python27
-      - gate-designate-specs-docs
 
   - name: openstack/glance
     template:
@@ -1009,12 +1011,8 @@ projects:
       - ironic-merge-release-tags
 
   - name: openstack/ironic-specs
-    check:
+    template:
-      - gate-ironic-specs-python27
+      - name: specs-jobs
-      - gate-ironic-specs-docs
-    gate:
-      - gate-ironic-specs-python27
-      - gate-ironic-specs-docs
 
   - name: openstack/keystone
     template:
@@ -1085,12 +1083,8 @@ projects:
       - marconi-docs
 
   - name: openstack/marconi-specs
-    check:
+    template:
-      - gate-marconi-specs-python27
+      - name: specs-jobs
-      - gate-marconi-specs-docs
-    gate:
-      - gate-marconi-specs-python27
-      - gate-marconi-specs-docs
 
   - name: openstack/neutron
     template:
@@ -1124,12 +1118,8 @@ projects:
       - neutron-merge-release-tags
 
   - name: openstack/neutron-specs
-    check:
+    template:
-      - gate-neutron-specs-python27
+      - name: specs-jobs
-      - gate-neutron-specs-docs
-    gate:
-      - gate-neutron-specs-python27
-      - gate-neutron-specs-docs
 
   - name: openstack/nova
     template:
@@ -1168,12 +1158,8 @@ projects:
       - nova-merge-release-tags
 
   - name: openstack/nova-specs
-    check:
+    template:
-      - gate-nova-specs-python27
+      - name: specs-jobs
-      - gate-nova-specs-docs
-    gate:
-      - gate-nova-specs-python27
-      - gate-nova-specs-docs
 
   - name: stackforge/nova-docker
     template:
@@ -1205,12 +1191,8 @@ projects:
       - sahara-merge-release-tags
 
   - name: openstack/sahara-specs
-    check:
+    template:
-      - gate-sahara-specs-docs
+      - name: specs-jobs
-      - gate-sahara-specs-python27
-    gate:
-      - gate-sahara-specs-docs
-      - gate-sahara-specs-python27
 
   - name: openstack/stevedore
     template:
@@ -1928,12 +1910,8 @@ projects:
       - gate-oslo-incubator-python33
 
   - name: openstack/oslo-specs
-    check:
+    template:
-      - gate-oslo-specs-python27
+      - name: specs-jobs
-      - gate-oslo-specs-docs
-    gate:
-      - gate-oslo-specs-python27
-      - gate-oslo-specs-docs
 
   - name: openstack/oslo.config
     template:
@@ -2352,12 +2330,8 @@ projects:
       - tripleo-incubator-docs
 
   - name: openstack/tripleo-specs
-    check:
+    template:
-      - gate-tripleo-specs-python27
+      - name: specs-jobs
-      - gate-tripleo-specs-docs
-    gate:
-      - gate-tripleo-specs-python27
-      - gate-tripleo-specs-docs
 
   - name: openstack/trove-integration
     check:
@@ -2405,12 +2379,8 @@ projects:
       - identity-api-tox-doc-publishdocs
 
   - name: openstack/keystone-specs
-    check:
+    template:
-      - gate-keystone-specs-python27
+      - name: specs-jobs
-      - gate-keystone-specs-docs
-    gate:
-      - gate-keystone-specs-python27
-      - gate-keystone-specs-docs
 
   - name: openstack/image-api
     template:
@@ -2454,12 +2424,8 @@ projects:
       - openstack-api-ref
 
   - name: openstack/docs-specs
-    check:
+    template:
-      - gate-docs-specs-docs
+      - name: specs-jobs
-      - gate-docs-specs-python27
-    gate:
-      - gate-docs-specs-docs
-      - gate-docs-specs-python27
 
   - name: openstack/openstack-manuals
     template:
@@ -3372,11 +3338,11 @@ projects:
 
   - name: openstack/congress-specs
     check:
-      - gate-congress-specs-docs
       - gate-congress-specs-python27
+      - gate-congress-specs-docs
     gate:
-      - gate-congress-specs-docs
       - gate-congress-specs-python27
+      - gate-congress-specs-docs
 
   - name: stackforge/cookbook-openstack-block-storage
     template:

modules/openstack_project/manifests/static.pp

@@ -280,4 +280,30 @@ class openstack_project::static (
     source  => 'puppet:///modules/openstack_project/run_bandersnatch.py',
   }
 
+  ###########################################################
+  # Specs
+
+  apache::vhost { 'specs.openstack.org':
+    port     => 80,
+    priority => '50',
+    docroot  => '/srv/static/specs',
+    require  => File['/srv/static/specs'],
+  }
+
+  file { '/srv/static/specs':
+    ensure  => directory,
+    owner   => 'jenkins',
+    group   => 'jenkins',
+    require => User['jenkins'],
+  }
+
+  file { '/srv/static/specs/robots.txt':
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0444',
+    source  => 'puppet:///modules/openstack_project/disallow_robots.txt',
+    require => File['/srv/static/specs'],
+  }
+
 }