Merge "Removed pypi.<region> mirror manifests"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
75abc5bc4b
@ -556,15 +556,6 @@ node /^mirror\..*\.openstack\.org$/ {
|
||||
}
|
||||
}
|
||||
|
||||
# Legacy machines in each region to run pypi package mirrors.
|
||||
# Node-OS: precise
|
||||
node /^pypi\..*\.openstack\.org$/ {
|
||||
$group = "pypi"
|
||||
class { 'openstack_project::pypi':
|
||||
sysadmins => hiera('sysadmins', []),
|
||||
}
|
||||
}
|
||||
|
||||
# A machine to run ODSREG in preparation for summits.
|
||||
# Node-OS: trusty
|
||||
node 'design-summit-prep.openstack.org' {
|
||||
|
||||
@ -1,51 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import requests
|
||||
import subprocess
|
||||
|
||||
|
||||
def setup_logging(logger):
|
||||
ch = logging.StreamHandler()
|
||||
formatter = logging.Formatter(
|
||||
'%(asctime)s %(levelname)s: %(message)s')
|
||||
ch.setFormatter(formatter)
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.addHandler(ch)
|
||||
|
||||
|
||||
def main():
|
||||
logger = logging.getLogger('bandersnatch')
|
||||
setup_logging(logger)
|
||||
|
||||
stale = dict()
|
||||
output = subprocess.check_output(
|
||||
['bandersnatch', 'mirror'], stderr=subprocess.STDOUT)
|
||||
for line in output.split('\n'):
|
||||
print(line)
|
||||
if 'Expected PyPI serial' in line:
|
||||
url = line.split("for request ")[1].split()[0]
|
||||
stale[url] = True
|
||||
for url in stale.keys():
|
||||
logger.info('Purging %s' % url)
|
||||
response = requests.request('PURGE', url)
|
||||
if not response.ok:
|
||||
logger.error('Failed to purge %s: %s' % (url, response.text))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1,58 +0,0 @@
|
||||
# == Class: openstack_project::pypi
|
||||
#
|
||||
class openstack_project::pypi (
|
||||
$vhost_name = $::fqdn,
|
||||
$sysadmins = [],
|
||||
) {
|
||||
|
||||
class { 'openstack_project::server':
|
||||
iptables_public_tcp_ports => [22, 80],
|
||||
sysadmins => $sysadmins,
|
||||
}
|
||||
|
||||
$mirror_root = '/srv/static'
|
||||
$pypi_root = "${mirror_root}/mirror"
|
||||
$www_root = "${pypi_root}/web"
|
||||
$wheel_root = "${www_root}/wheel"
|
||||
|
||||
if ! defined(File[$mirror_root]) {
|
||||
file { $mirror_root:
|
||||
ensure => directory,
|
||||
}
|
||||
}
|
||||
|
||||
class { 'openstack_project::pypi_mirror':
|
||||
data_directory => "${pypi_root}",
|
||||
require => File[$mirror_root]
|
||||
}
|
||||
|
||||
class { 'openstack_project::wheel_mirror':
|
||||
data_directory => "${wheel_root}",
|
||||
require => Class['Openstack_project::Pypi_mirror'],
|
||||
}
|
||||
|
||||
include ::httpd
|
||||
|
||||
if ! defined(Httpd::Mod['rewrite']) {
|
||||
httpd::mod { 'rewrite':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
::httpd::vhost { $vhost_name:
|
||||
port => 80,
|
||||
priority => '50',
|
||||
docroot => $www_root,
|
||||
require => Class['Openstack_project::Pypi_mirror'],
|
||||
template => 'openstack_project/pypi.vhost.erb',
|
||||
}
|
||||
|
||||
file { "${www_root}/robots.txt":
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0444',
|
||||
source => 'puppet:///modules/openstack_project/disallow_robots.txt',
|
||||
require => Class['Openstack_project::Pypi_mirror'],
|
||||
}
|
||||
}
|
||||
@ -1,65 +0,0 @@
|
||||
# == Class: openstack_project::pypi_mirror
|
||||
#
|
||||
class openstack_project::pypi_mirror (
|
||||
$data_directory = '/srv/static/mirror',
|
||||
$cron_frequency = '*/5',
|
||||
) {
|
||||
|
||||
file { "${data_directory}":
|
||||
ensure => directory,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
}
|
||||
|
||||
file { "${data_directory}/web":
|
||||
ensure => directory,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
require => File["${data_directory}"],
|
||||
}
|
||||
|
||||
package { 'bandersnatch':
|
||||
ensure => 'latest',
|
||||
provider => 'pip',
|
||||
}
|
||||
|
||||
file { '/etc/bandersnatch.conf':
|
||||
ensure => present,
|
||||
content => template('openstack_project/bandersnatch.conf.erb'),
|
||||
}
|
||||
|
||||
file { '/var/log/bandersnatch':
|
||||
ensure => directory,
|
||||
}
|
||||
|
||||
file { '/var/run/bandersnatch':
|
||||
ensure => directory,
|
||||
}
|
||||
|
||||
cron { 'bandersnatch':
|
||||
minute => $cron_frequency,
|
||||
command => 'flock -n /var/run/bandersnatch/mirror.lock timeout -k 2m 30m run-bandersnatch >>/var/log/bandersnatch/mirror.log 2>&1',
|
||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
}
|
||||
|
||||
include logrotate
|
||||
logrotate::file { 'bandersnatch':
|
||||
log => '/var/log/bandersnatch/mirror.log',
|
||||
options => [
|
||||
'compress',
|
||||
'copytruncate',
|
||||
'missingok',
|
||||
'rotate 7',
|
||||
'daily',
|
||||
'notifempty',
|
||||
],
|
||||
}
|
||||
|
||||
file { '/usr/local/bin/run-bandersnatch':
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => 'puppet:///modules/openstack_project/run_bandersnatch.py',
|
||||
}
|
||||
}
|
||||
@ -1,39 +0,0 @@
|
||||
[mirror]
|
||||
; The directory where the mirror data will be stored.
|
||||
directory = <%= @data_directory %>
|
||||
|
||||
; The PyPI server which will be mirrored.
|
||||
; master = https://testpypi.python.org
|
||||
; scheme for PyPI server MUST be https
|
||||
master = https://pypi.python.org
|
||||
|
||||
; The network socket timeout to use for all connections. This is set to a
|
||||
; somewhat aggressively low value: rather fail quickly temporarily and re-run
|
||||
; the client soon instead of having a process hang infinitely and have TCP not
|
||||
; catching up for ages.
|
||||
timeout = 10
|
||||
|
||||
; Number of worker threads to use for parallel downloads.
|
||||
; Recommendations for worker thread setting:
|
||||
; - leave the default of 3 to avoid overloading the pypi master
|
||||
; - official servers located in data centers could run 20 workers
|
||||
; - anything beyond 50 is probably unreasonable and avoided by bandersnatch
|
||||
workers = 3
|
||||
|
||||
; Whether to stop a sync quickly after an error is found or whether to continue
|
||||
; syncing but not marking the sync as successful. Value should be "true" or
|
||||
; "false".
|
||||
stop-on-error = false
|
||||
|
||||
; Whether or not files that have been deleted on the master should be deleted
|
||||
; on the mirror, too.
|
||||
; IMPORTANT: if you are running an official mirror than you *need* to leave
|
||||
; this on.
|
||||
delete-packages = true
|
||||
|
||||
[statistics]
|
||||
; A glob pattern matching all access log files that should be processed to
|
||||
; generate daily access statistics that will be aggregated on the master PyPI.
|
||||
access-log-pattern = /var/log/apache2/pypi.*openstack.org_access.*
|
||||
|
||||
; vim: set ft=cfg:
|
||||
Loading…
Reference in New Issue
Block a user