Merge "Removed pypi.<region> mirror manifests"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
75abc5bc4b
@ -556,15 +556,6 @@ node /^mirror\..*\.openstack\.org$/ {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# Legacy machines in each region to run pypi package mirrors.
|
|
||||||
# Node-OS: precise
|
|
||||||
node /^pypi\..*\.openstack\.org$/ {
|
|
||||||
$group = "pypi"
|
|
||||||
class { 'openstack_project::pypi':
|
|
||||||
sysadmins => hiera('sysadmins', []),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# A machine to run ODSREG in preparation for summits.
|
# A machine to run ODSREG in preparation for summits.
|
||||||
# Node-OS: trusty
|
# Node-OS: trusty
|
||||||
node 'design-summit-prep.openstack.org' {
|
node 'design-summit-prep.openstack.org' {
|
||||||
|
|||||||
@ -1,51 +0,0 @@
|
|||||||
#!/usr/bin/python
|
|
||||||
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
# you may not use this file except in compliance with the License.
|
|
||||||
# You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
||||||
# implied.
|
|
||||||
# See the License for the specific language governing permissions and
|
|
||||||
# limitations under the License.
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import requests
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
|
|
||||||
def setup_logging(logger):
|
|
||||||
ch = logging.StreamHandler()
|
|
||||||
formatter = logging.Formatter(
|
|
||||||
'%(asctime)s %(levelname)s: %(message)s')
|
|
||||||
ch.setFormatter(formatter)
|
|
||||||
logger.setLevel(logging.INFO)
|
|
||||||
logger.addHandler(ch)
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
logger = logging.getLogger('bandersnatch')
|
|
||||||
setup_logging(logger)
|
|
||||||
|
|
||||||
stale = dict()
|
|
||||||
output = subprocess.check_output(
|
|
||||||
['bandersnatch', 'mirror'], stderr=subprocess.STDOUT)
|
|
||||||
for line in output.split('\n'):
|
|
||||||
print(line)
|
|
||||||
if 'Expected PyPI serial' in line:
|
|
||||||
url = line.split("for request ")[1].split()[0]
|
|
||||||
stale[url] = True
|
|
||||||
for url in stale.keys():
|
|
||||||
logger.info('Purging %s' % url)
|
|
||||||
response = requests.request('PURGE', url)
|
|
||||||
if not response.ok:
|
|
||||||
logger.error('Failed to purge %s: %s' % (url, response.text))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@ -1,58 +0,0 @@
|
|||||||
# == Class: openstack_project::pypi
|
|
||||||
#
|
|
||||||
class openstack_project::pypi (
|
|
||||||
$vhost_name = $::fqdn,
|
|
||||||
$sysadmins = [],
|
|
||||||
) {
|
|
||||||
|
|
||||||
class { 'openstack_project::server':
|
|
||||||
iptables_public_tcp_ports => [22, 80],
|
|
||||||
sysadmins => $sysadmins,
|
|
||||||
}
|
|
||||||
|
|
||||||
$mirror_root = '/srv/static'
|
|
||||||
$pypi_root = "${mirror_root}/mirror"
|
|
||||||
$www_root = "${pypi_root}/web"
|
|
||||||
$wheel_root = "${www_root}/wheel"
|
|
||||||
|
|
||||||
if ! defined(File[$mirror_root]) {
|
|
||||||
file { $mirror_root:
|
|
||||||
ensure => directory,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
class { 'openstack_project::pypi_mirror':
|
|
||||||
data_directory => "${pypi_root}",
|
|
||||||
require => File[$mirror_root]
|
|
||||||
}
|
|
||||||
|
|
||||||
class { 'openstack_project::wheel_mirror':
|
|
||||||
data_directory => "${wheel_root}",
|
|
||||||
require => Class['Openstack_project::Pypi_mirror'],
|
|
||||||
}
|
|
||||||
|
|
||||||
include ::httpd
|
|
||||||
|
|
||||||
if ! defined(Httpd::Mod['rewrite']) {
|
|
||||||
httpd::mod { 'rewrite':
|
|
||||||
ensure => present,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
::httpd::vhost { $vhost_name:
|
|
||||||
port => 80,
|
|
||||||
priority => '50',
|
|
||||||
docroot => $www_root,
|
|
||||||
require => Class['Openstack_project::Pypi_mirror'],
|
|
||||||
template => 'openstack_project/pypi.vhost.erb',
|
|
||||||
}
|
|
||||||
|
|
||||||
file { "${www_root}/robots.txt":
|
|
||||||
ensure => present,
|
|
||||||
owner => 'root',
|
|
||||||
group => 'root',
|
|
||||||
mode => '0444',
|
|
||||||
source => 'puppet:///modules/openstack_project/disallow_robots.txt',
|
|
||||||
require => Class['Openstack_project::Pypi_mirror'],
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,65 +0,0 @@
|
|||||||
# == Class: openstack_project::pypi_mirror
|
|
||||||
#
|
|
||||||
class openstack_project::pypi_mirror (
|
|
||||||
$data_directory = '/srv/static/mirror',
|
|
||||||
$cron_frequency = '*/5',
|
|
||||||
) {
|
|
||||||
|
|
||||||
file { "${data_directory}":
|
|
||||||
ensure => directory,
|
|
||||||
owner => 'root',
|
|
||||||
group => 'root',
|
|
||||||
}
|
|
||||||
|
|
||||||
file { "${data_directory}/web":
|
|
||||||
ensure => directory,
|
|
||||||
owner => 'root',
|
|
||||||
group => 'root',
|
|
||||||
require => File["${data_directory}"],
|
|
||||||
}
|
|
||||||
|
|
||||||
package { 'bandersnatch':
|
|
||||||
ensure => 'latest',
|
|
||||||
provider => 'pip',
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/etc/bandersnatch.conf':
|
|
||||||
ensure => present,
|
|
||||||
content => template('openstack_project/bandersnatch.conf.erb'),
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/var/log/bandersnatch':
|
|
||||||
ensure => directory,
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/var/run/bandersnatch':
|
|
||||||
ensure => directory,
|
|
||||||
}
|
|
||||||
|
|
||||||
cron { 'bandersnatch':
|
|
||||||
minute => $cron_frequency,
|
|
||||||
command => 'flock -n /var/run/bandersnatch/mirror.lock timeout -k 2m 30m run-bandersnatch >>/var/log/bandersnatch/mirror.log 2>&1',
|
|
||||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
|
||||||
}
|
|
||||||
|
|
||||||
include logrotate
|
|
||||||
logrotate::file { 'bandersnatch':
|
|
||||||
log => '/var/log/bandersnatch/mirror.log',
|
|
||||||
options => [
|
|
||||||
'compress',
|
|
||||||
'copytruncate',
|
|
||||||
'missingok',
|
|
||||||
'rotate 7',
|
|
||||||
'daily',
|
|
||||||
'notifempty',
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
file { '/usr/local/bin/run-bandersnatch':
|
|
||||||
ensure => present,
|
|
||||||
owner => 'root',
|
|
||||||
group => 'root',
|
|
||||||
mode => '0755',
|
|
||||||
source => 'puppet:///modules/openstack_project/run_bandersnatch.py',
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,39 +0,0 @@
|
|||||||
[mirror]
|
|
||||||
; The directory where the mirror data will be stored.
|
|
||||||
directory = <%= @data_directory %>
|
|
||||||
|
|
||||||
; The PyPI server which will be mirrored.
|
|
||||||
; master = https://testpypi.python.org
|
|
||||||
; scheme for PyPI server MUST be https
|
|
||||||
master = https://pypi.python.org
|
|
||||||
|
|
||||||
; The network socket timeout to use for all connections. This is set to a
|
|
||||||
; somewhat aggressively low value: rather fail quickly temporarily and re-run
|
|
||||||
; the client soon instead of having a process hang infinitely and have TCP not
|
|
||||||
; catching up for ages.
|
|
||||||
timeout = 10
|
|
||||||
|
|
||||||
; Number of worker threads to use for parallel downloads.
|
|
||||||
; Recommendations for worker thread setting:
|
|
||||||
; - leave the default of 3 to avoid overloading the pypi master
|
|
||||||
; - official servers located in data centers could run 20 workers
|
|
||||||
; - anything beyond 50 is probably unreasonable and avoided by bandersnatch
|
|
||||||
workers = 3
|
|
||||||
|
|
||||||
; Whether to stop a sync quickly after an error is found or whether to continue
|
|
||||||
; syncing but not marking the sync as successful. Value should be "true" or
|
|
||||||
; "false".
|
|
||||||
stop-on-error = false
|
|
||||||
|
|
||||||
; Whether or not files that have been deleted on the master should be deleted
|
|
||||||
; on the mirror, too.
|
|
||||||
; IMPORTANT: if you are running an official mirror than you *need* to leave
|
|
||||||
; this on.
|
|
||||||
delete-packages = true
|
|
||||||
|
|
||||||
[statistics]
|
|
||||||
; A glob pattern matching all access log files that should be processed to
|
|
||||||
; generate daily access statistics that will be aggregated on the master PyPI.
|
|
||||||
access-log-pattern = /var/log/apache2/pypi.*openstack.org_access.*
|
|
||||||
|
|
||||||
; vim: set ft=cfg:
|
|
||||||
Loading…
Reference in New Issue
Block a user