From 1875a89e4bb29090eae521eb4cdb48003bbce1a6 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 17 Dec 2012 13:14:12 -0800 Subject: [PATCH] Add robots.txt for logs and docs-draft. Indexing build logs isn't particularly useful though it's a lot of data. And indexing draft documentation is actually counter productive (since it might be wrong). So adopt a robots.txt for both vhosts. Change-Id: I8a61a0e7dacb96b26217d31a89a6ae94d9c5f62e Reviewed-on: https://review.openstack.org/18265 Reviewed-by: Clark Boylan Approved: Jeremy Stanley Reviewed-by: Jeremy Stanley Tested-by: Jenkins --- .../files/disallow_robots.txt | 2 ++ modules/openstack_project/manifests/static.pp | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 modules/openstack_project/files/disallow_robots.txt diff --git a/modules/openstack_project/files/disallow_robots.txt b/modules/openstack_project/files/disallow_robots.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/modules/openstack_project/files/disallow_robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/modules/openstack_project/manifests/static.pp b/modules/openstack_project/manifests/static.pp index 56fc6d24b1..697094d838 100644 --- a/modules/openstack_project/manifests/static.pp +++ b/modules/openstack_project/manifests/static.pp @@ -61,10 +61,28 @@ class openstack_project::static ( ensure => directory, } + file { '/srv/static/logs/robots.txt': + ensure => present, + owner => 'root', + group => 'root', + mode => '0444', + source => 'puppet:///modules/openstack_project/disallow_robots.txt', + require => File['/srv/static/logs'], + } + file { '/srv/static/docs-draft': ensure => directory, } + file { '/srv/static/docs-draft/robots.txt': + ensure => present, + owner => 'root', + group => 'root', + mode => '0444', + source => 'puppet:///modules/openstack_project/disallow_robots.txt', + require => File['/srv/static/docs-draft'], + } + cron { 'gziplogs': user => 'root', hour => '*/6',