From 841e7e2f5736657e68eba9408d029b0032904dea Mon Sep 17 00:00:00 2001 From: Michael Krotscheck Date: Fri, 8 Apr 2016 15:39:06 -0700 Subject: [PATCH] Added NPM Mirror to mirror_update This patch adds a cron job to our mirror_update servers which will begin mirroring the npm registry into our own AFS mirror directories, under the /npm path. It uses two major components: First, the registry-static project, available here: https://github.com/davglass/registry-static Secondly, a custom implementation of the fs-blob-store named afs-blob-store, which creates a virtual filesystem that automatically translates paths to accomodate AFS' folder size limit. This project is currently still hosted on github, however will be added to OpenStack soon: https://github.com/krotscheck/js-afs-blob-store This will require creating ACL's as well as the AFS principal before this can be landed. Change-Id: Ib78852fc5cde4fb04fe6439e01a591ca2781ce21 --- manifests/site.pp | 1 + .../manifests/mirror_update.pp | 34 ++++++++++++ .../openstack_project/manifests/npm_mirror.pp | 52 +++++++++++++++++++ .../templates/npm-mirror-update.sh | 28 ++++++++++ 4 files changed, 115 insertions(+) create mode 100644 modules/openstack_project/manifests/npm_mirror.pp create mode 100644 modules/openstack_project/templates/npm-mirror-update.sh diff --git a/manifests/site.pp b/manifests/site.pp index 85c0b17edd..943080e4d1 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -543,6 +543,7 @@ node 'mirror-update.openstack.org' { bandersnatch_keytab => hiera('bandersnatch_keytab'), admin_keytab => hiera('afsadmin_keytab'), reprepro_keytab => hiera('reprepro_keytab'), + npm_keytab => hiera('npm_keytab'), sysadmins => hiera('sysadmins', []), } } diff --git a/modules/openstack_project/manifests/mirror_update.pp b/modules/openstack_project/manifests/mirror_update.pp index 81932e9021..a1429011f0 100644 --- a/modules/openstack_project/manifests/mirror_update.pp +++ b/modules/openstack_project/manifests/mirror_update.pp @@ -5,6 +5,7 @@ class openstack_project::mirror_update ( $bandersnatch_keytab = '', $reprepro_keytab = '', $admin_keytab = '', + $npm_keytab = '', ) { class { 'openstack_project::server': @@ -12,6 +13,11 @@ class openstack_project::mirror_update ( afs => true, } + class { 'openstack_project::npm_mirror': + data_directory => '/afs/.openstack.org/mirror/npm', + uri_rewrite => 'localhost', + } + class { 'bandersnatch': bandersnatch_source => 'https://bitbucket.org/jeblair/bandersnatch', } @@ -30,6 +36,13 @@ class openstack_project::mirror_update ( content => $bandersnatch_keytab, } + file { '/etc/npm.keytab': + owner => 'root', + group => 'root', + mode => '0400', + content => $npm_keytab, + } + file { '/etc/afsadmin.keytab': owner => 'root', group => 'root', @@ -45,6 +58,14 @@ class openstack_project::mirror_update ( source => 'puppet:///modules/openstack_project/bandersnatch-mirror-update.sh', } + file { '/usr/local/bin/npm-mirror-update': + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + content => template('openstack_project/npm-mirror-update.sh'), + } + cron { 'bandersnatch': user => $user, minute => '*/5', @@ -58,6 +79,19 @@ class openstack_project::mirror_update ( ] } + cron { 'npm-mirror-update': + user => $user, + minute => '*/5', + command => 'flock -n /var/run/npm-mirror-update/mirror.lock npm-mirror-update >>/var/log/npm-mirror-update/mirror.log 2>&1', + environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', + require => [ + File['/usr/local/bin/npm-mirror-update'], + File['/etc/afsadmin.keytab'], + File['/etc/npm.keytab'], + Class['openstack_project::npm_mirror'], + ] + } + # TODO(clarkb) this setup needs to go in a class of its own. It is not # in the define because it is common to all reprepro mirrors. package { 'reprepro': diff --git a/modules/openstack_project/manifests/npm_mirror.pp b/modules/openstack_project/manifests/npm_mirror.pp new file mode 100644 index 0000000000..3eefe147bc --- /dev/null +++ b/modules/openstack_project/manifests/npm_mirror.pp @@ -0,0 +1,52 @@ +# == Class: openstack_project::npm_mirror +# +class openstack_project::npm_mirror ( + $uri_rewrite, + $data_directory, +) { + + file { $data_directory: + ensure => directory, + owner => 'root', + group => 'root', + } + + class { '::nodejs': + repo_url_suffix => 'node_4.x', + } + + # See: https://github.com/davglass/registry-static/pull/45 + package { 'patch-package-json': + ensure => '0.0.4', + provider => 'npm', + require => Class['nodejs'], + } + + package { 'follow-registry': + ensure => '2.0.0', + provider => 'npm', + require => [ + Class['nodejs'], + ] + } + + # The registry mirroring script. + package { 'registry-static': + ensure => '2.2.0', + provider => 'npm', + require => [ + Class['nodejs'], + Package['follow-registry'], + Package['patch-package-json'], + ] + } + + # The afs-blob-store file structure rewriter. + package { 'afs-blob-store': + ensure => '1.0.1', + provider => 'npm', + require => [ + Class['nodejs'], + ] + } +} diff --git a/modules/openstack_project/templates/npm-mirror-update.sh b/modules/openstack_project/templates/npm-mirror-update.sh new file mode 100644 index 0000000000..61416303b1 --- /dev/null +++ b/modules/openstack_project/templates/npm-mirror-update.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2016 Hewlett Packard Enterprise Development Corporation, LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -e + +CMD="/usr/bin/registry-static" +CMD_ARGS="-d <%= @uri_rewrite %> -o <%= @data_directory %> --blobstore afs-blob-store" + +echo "Obtaining npm tokens and running registry-static." +k5start -t -f /etc/npm.keytab service/npm -- timeout -k 2m 30m $CMD $CMD_ARGS + +echo "registry-static completed successfully, running vos release." +k5start -t -f /etc/afsadmin.keytab service/afsadmin -- vos release -v mirror.npm + +echo "Done."