diff --git a/porthos/README.md b/porthos/README.md index 1d35bfb..bc89043 100644 --- a/porthos/README.md +++ b/porthos/README.md @@ -122,8 +122,7 @@ For instance to create a key on athos The `tier_id` value should be a number. If the value is negative, the tier number is calculated by an hash function, based on the system identifier. -If `tier_id` is not a number, both `auth.php` and `subscription.php` reply with -403 - forbidden. +If `tier_id` is not a number, `stable/` is served instead. ### `icat` field @@ -150,19 +149,16 @@ from `/etc/porthos/repos.conf`. Upstream YUM rsync URLs are defined there. The following commands are executed automatically, as defined in `porthos.cron`: - `repo-bulk-pull` creates a snapshot date-dir (e.g. `d20180301`) under - `dest_dir` with differences from upstream repositories. It sets `t0` to point at - it. -- `repo-bulk-shift [N]` updates `t1` ... `tN` links by shifting tiers of one position - the optional `N` parameter creates missing links up to N - 1. + `dest_dir` with differences from upstream repositories. - `repo-bulk-cleanup` erases stale snapshots directories The following commands are designed for Porthos initialization, to recover from errors, or implement low-level actions: - `repo-bulk-hinit` runs initial synchronization from upstream repositories (-f disables the check for already existing directories) -- `repo-head-init` initial/override synchronization of head from a specific upstream repo -- `repo-head-rollback` roll back head to a previous snapshot for a specific repo -- `repo-tier-pull` create a new upstream snapshot for a specific repo -- `repo-tier-delete` delete repomd.xml from a given tier or snapshot +- `repo-head-init` synchronization of head from a specific upstream repo +- `repo-head-rollback` roll back a repository head state to a previous snapshot state +- `repo-snapshot-create` create a new repository snapshot +- `repo-snapshot-delete` delete repomd.xml from a given repository snapshot - `repo-rpm-lookup` seek the given RPM in every snapshot for a given repository - `xrsync` run rsync safely, trying to repeat the operation if it fails @@ -179,8 +175,9 @@ it fails as soon as no snapshot is found, or if an invalid repository identifier is issued. Some times it is desirable to re-sync the head repository, without generating a -new snapshot, like `repo-tier-pull` does. That happens if an upstream repo was -fixed before being shifted. In that case run `repo-head-init` as follow: +new snapshot, like `repo-snapshot-create` does. That happens if an upstream repo +was fixed quickly and the bogus RPM never entered any snapshot. In that case run +`repo-head-init` as follow: repo-head-init -n -f 7.6.1810/nethserver-updates/x86_64 @@ -190,9 +187,9 @@ command to run even if the repository was already initialized. If one or more snapshots contain a bogus RPM it is possible to delete the whole repository metadata (repomd.xml) file with the following command: - repo-tier-delete d20190702/7.6.1810/nethserver-updates/x86_64 d20190630/7.6.1810/nethserver-updates/x86_64 + repo-snapshot-delete d20190702/7.6.1810/nethserver-updates/x86_64 d20190630/7.6.1810/nethserver-updates/x86_64 -The correct snapshot (or tier) name can be found starting from the RPM name with: +The correct snapshot name can be found starting from the RPM name with: repo-rpm-lookup bogus-rpm-1.2.3-1.ns7.noarch.rpm d20190702/7.6.1810/nethserver-updates/x86_64 @@ -200,9 +197,9 @@ The correct snapshot (or tier) name can be found starting from the RPM name with The two commands can be combined together with `xargs`: - repo-rpm-lookup bogus-rpm-1.2.3-1.ns7.noarch.rpm | xargs -- repo-tier-delete + repo-rpm-lookup bogus-rpm-1.2.3-1.ns7.noarch.rpm | xargs -- repo-snapshot-delete -If the RPM is found under `head/`, `repo-tier-delete` safely ignores it. +If the RPM is found under `head/`, `repo-snapshot-delete` safely ignores it. ## Automated schedule diff --git a/porthos/root/etc/cron.d/porthos.cron b/porthos/root/etc/cron.d/porthos.cron index e9986de..3a4f9eb 100644 --- a/porthos/root/etc/cron.d/porthos.cron +++ b/porthos/root/etc/cron.d/porthos.cron @@ -5,9 +5,8 @@ RANDOM=20 CRON_TZ=UTC -0 21 * * 1 root /usr/local/bin/repo-bulk-pull -0 21 * * 2-4 root /usr/local/bin/repo-bulk-shift -0 22 * * 1-4 root /usr/local/bin/repo-bulk-cleanup +0 21 * * 0-4 root /usr/local/bin/repo-bulk-pull +0 22 * * 0-4 root /usr/local/bin/repo-bulk-cleanup >/dev/null # end diff --git a/porthos/root/etc/nginx/conf.d/porthos.conf b/porthos/root/etc/nginx/conf.d/porthos.conf index b3bb8bb..c97acc3 100644 --- a/porthos/root/etc/nginx/conf.d/porthos.conf +++ b/porthos/root/etc/nginx/conf.d/porthos.conf @@ -38,29 +38,9 @@ server { return 200 "pong\n"; } - location ~ "^/T0/(.*)" { - internal; - try_files /t0/$1 /head/$1; - } - - location ~ "^/T1/(.*)" { - internal; - try_files /t1/$1 /t0/$1 /head/$1; - } - - location ~ "^/T2/(.*)" { - internal; - try_files /t2/$1 /t1/$1 /t0/$1 /head/$1; - } - - location ~ "^/T3/(.*)" { - internal; - try_files /t3/$1 /t2/$1 /t1/$1 /t0/$1 /head/$1; - } - # add another location for tier "Tx" here... - location ~ "^/(t[0-9]|d[0-9]{8}|head)(/.*)" { + location ~ "^/(d[0-9]{8}|head)(/.*)" { internal; } diff --git a/porthos/root/srv/porthos/script/auth.php b/porthos/root/srv/porthos/script/auth.php index feb41d1..a59bdea 100644 --- a/porthos/root/srv/porthos/script/auth.php +++ b/porthos/root/srv/porthos/script/auth.php @@ -22,6 +22,7 @@ require_once("lib.php"); require_once("config-" . $_SERVER['PORTHOS_SITE'] . ".php"); +ini_set('date.timezone', $config['timezone']); $uri = parse_uri($_SERVER['DOCUMENT_URI']); @@ -54,7 +55,6 @@ if($config['legacy_auth']) { $valid_credentials = $valid_credentials || $_SERVER['PHP_AUTH_USER'] === $_SERVER['PHP_AUTH_PW']; } -$has_access_disabled = ! is_numeric($access['tier_id']); if($access['tier_id'] < 0) { $hash = 0; @@ -62,22 +62,29 @@ $hash += ord($c); } $hash = $hash % 256; - if($hash < 13) { // 5% + if($hash < 26) { // 10% $tier_id = 0; - } elseif($hash < 51) { // +15% = 20% + } elseif($hash < 77) { // +20% = 30% $tier_id = 1; - } elseif($hash < 128) { // +30% = 50% + } else { // +70% = 100% $tier_id = 2; - } else { // +50% = 100% - $tier_id = 3; } - $tier_id += $config['tier_id_base']; } else { - $tier_id = intval($access['tier_id']); + $tier_id = $access['tier_id']; +} + +$is_tier_request = is_numeric($tier_id) && $uri['prefix'] == 'autoupdate'; +if($is_tier_request && $valid_credentials) { + // Seeking a snapshot is a time-consuming op. Ensure we have valid + // credentials before running it! + $snapshot = lookup_snapshot($uri['full_path'], $tier_id, $config['week_size']); +} else { + $snapshot = 'head'; } if(basename($uri['rest']) == 'repomd.xml') { - header('Cache-Control: private'); + // repomd.xml is the entry point of repository (meta)data: let's keep track + // of every client access to repositories: application_log(json_encode(array( 'porthos_site' => $_SERVER['PORTHOS_SITE'], 'connection' => $_SERVER['CONNECTION'] ?: '', @@ -86,19 +93,18 @@ 'repo' => $uri['repo'], 'version' => $uri['version'], 'arch' => $uri['arch'], - 'tier_id' => $uri['prefix'] == 'autoupdate' ? $tier_id : NULL, + 'tier_id' => $is_tier_request ? $tier_id : FALSE, 'tier_auto' => isset($hash), 'tls' => isset($_SERVER['HTTPS']), - 'auth_response' => ! $valid_credentials ? 'bad_credentials' : $has_access_disabled ? 'bad_access' : 'pass', + 'auth_response' => ! $valid_credentials ? 'bad_credentials' : 'pass', + 'snapshot' => $snapshot, ))); } -if ($has_access_disabled || ! $valid_credentials) { +if (! $valid_credentials) { + // Exit here, after sending the application_log record for repomd.xml requests. exit_http(403); } -if($uri['prefix'] == 'autoupdate') { - return_file('/T' . $tier_id . $uri['full_path']); -} else { - return_file('/head' . $uri['full_path']); -} +header('Cache-Control: private'); +return_file('/' . $snapshot . $uri['full_path']); diff --git a/porthos/root/srv/porthos/script/config-porthos.php b/porthos/root/srv/porthos/script/config-porthos.php index abc6a17..3521940 100644 --- a/porthos/root/srv/porthos/script/config-porthos.php +++ b/porthos/root/srv/porthos/script/config-porthos.php @@ -47,8 +47,11 @@ // if FALSE, check the user name and the password separately $config['legacy_auth'] = FALSE; -// tier_id_base (int) -// this is the base/minimum tier_id value, when automatic tier_id (-1) is set. -// - be sure that (tier_id_base + 3) < "number of tiers" -// - run repo-bulk-shift N (with N = "number of tiers") to add more tier links -$config['tier_id_base'] = 0; +// week_size (int) +// number of days/snapshots per week. Ensure this value is consistent with +// your crontab settings +$config['week_size'] = 5; + +// timezone (string) +// the PHP timezone for this application +$config['timezone'] = 'UTC'; diff --git a/porthos/root/srv/porthos/script/lib.php b/porthos/root/srv/porthos/script/lib.php index c82d4a4..e94317c 100644 --- a/porthos/root/srv/porthos/script/lib.php +++ b/porthos/root/srv/porthos/script/lib.php @@ -70,3 +70,27 @@ function parse_uri($uri) { $parts = array_merge($parts, $matches); return $parts; } + +function get_snapshot_timestamp($snapshot_name) { + $parts = array(); + if(!$snapshot_name || !preg_match('/^d(?\d\d\d\d)(?\d\d)(?\d\d)$/', $snapshot_name, $parts)) { + return time(); + } + return mktime(0, 0, 0, $parts['month'], $parts['day'], $parts['year']); +} + +function lookup_snapshot($path, $tier_id = 0, $week_size = 5) { + $root_path = "/srv/porthos/webroot/"; + $snapshots = array_reverse(array_map('basename', glob($root_path . "d20*"))); + $last_snapshot_day_id = date('w', get_snapshot_timestamp($snapshots[0])); + // $monday_offset formula: + // ($last_snapshot_day_id-1): rebase on Mondays + // ($last_snapshot_day_id > $tier_id ? 0 : $week_size): select current week Monday or previous one + $monday_offset = ($last_snapshot_day_id-1) + ($last_snapshot_day_id > $tier_id ? 0 : $week_size); + for($i = min($monday_offset, count($snapshots) - 1); $i >= 0; $i--) { + if(is_file($root_path . $snapshots[$i] . '/' . $path)) { + break; + } + } + return $i < 0 ? 'head' : $snapshots[$i]; +} \ No newline at end of file diff --git a/porthos/root/srv/porthos/script/subscription.php b/porthos/root/srv/porthos/script/subscription.php index 4897352..958478c 100644 --- a/porthos/root/srv/porthos/script/subscription.php +++ b/porthos/root/srv/porthos/script/subscription.php @@ -47,8 +47,7 @@ if($config['legacy_auth']) { $valid_credentials = $valid_credentials || $_SERVER['PHP_AUTH_USER'] === $_SERVER['PHP_AUTH_PW']; } -$has_access_disabled = ! is_numeric($access['tier_id']) || $access['icat'] === FALSE; -if ($has_access_disabled || ! $valid_credentials) { +if (! $valid_credentials) { exit_http(403); } diff --git a/porthos/root/usr/local/bin/repo-bulk-cleanup b/porthos/root/usr/local/bin/repo-bulk-cleanup index abc5b0d..5f25076 100755 --- a/porthos/root/usr/local/bin/repo-bulk-cleanup +++ b/porthos/root/usr/local/bin/repo-bulk-cleanup @@ -26,8 +26,8 @@ shopt -s nullglob extglob . /etc/porthos/repos.conf cd ${dest_dir} -# 4 weeks before last repo-tier-pull: -stale_ts=$[ $(stat -c %Y head) - (86400*28) ] +# 2 weeks before last repo-bulk-pull: +stale_ts=$[ $(stat -c %Y head) - (86400*14) ] declare -A targets @@ -38,11 +38,6 @@ for D in d+([0-9]); do fi done -# Always preserve a symlinked tier -for T in t[0-9]; do - unset "targets[$(readlink ${T})]" -done - if [[ -n ${!targets[@]} ]]; then echo "[NOTICE] removing stale tier(s) ${!targets[@]}" rm -rf ${!targets[@]} diff --git a/porthos/root/usr/local/bin/repo-bulk-pull b/porthos/root/usr/local/bin/repo-bulk-pull index 3137c03..af96ca9 100755 --- a/porthos/root/usr/local/bin/repo-bulk-pull +++ b/porthos/root/usr/local/bin/repo-bulk-pull @@ -27,10 +27,7 @@ date_dir=d$(date +%Y%m%d) trap 'kill $(jobs -p)' INT HUP TERM for repo in ${!repos[@]}; do - /usr/local/bin/repo-tier-pull $* ${repo} ${date_dir} & + /usr/local/bin/repo-snapshot-create $* ${repo} ${date_dir} & done wait - -cd ${dest_dir} -ln -sfT ${date_dir} t0 diff --git a/porthos/root/usr/local/bin/repo-bulk-shift b/porthos/root/usr/local/bin/repo-bulk-shift deleted file mode 100755 index 53b3880..0000000 --- a/porthos/root/usr/local/bin/repo-bulk-shift +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/bash - -# -# Copyright (C) 2018 Nethesis S.r.l. -# http://www.nethesis.it - nethserver@nethesis.it -# -# This script is part of NethServer. -# -# NethServer is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, -# or any later version. -# -# NethServer is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with NethServer. If not, see COPYING. -# - -set -e -shopt -s nullglob - -min_tiers=${1:-1} - -. /etc/porthos/repos.conf -cd ${dest_dir} -tiers=( t[0-9] ) - -# Shift up tier states -for ((ti=${#tiers[@]}-1; ti>0; ti--)); do - date_dir=$(readlink ${tiers[$((ti - 1))]}) - if [[ $? != 0 || ! -d ${date_dir} ]]; then - echo "[ERROR] Bad tier link ${tiers[$((ti - 1))]}" - continue - fi - ln -sfT ${date_dir} t${ti} -done - -# Fill missing tiers up to $min_tiers -if (( $min_tiers > 1 )); then - for ((ti=0; ti < $min_tiers - 1; ti++)); do - date_dir=$(readlink t${ti}) - if [[ $? != 0 || ! -d ${date_dir} ]]; then - echo "[ERROR] Bad tier link t${ti}" - continue - fi - - next_tier=t$((ti + 1)) - if [[ ! -e ${next_tier} ]]; then - echo "[NOTICE] initializing ${next_tier}" - ln -sfT ${date_dir} ${next_tier} - fi - done -fi \ No newline at end of file diff --git a/porthos/root/usr/local/bin/repo-head-init b/porthos/root/usr/local/bin/repo-head-init index 8744f03..b774e79 100755 --- a/porthos/root/usr/local/bin/repo-head-init +++ b/porthos/root/usr/local/bin/repo-head-init @@ -61,7 +61,7 @@ fi head_dir=${dest_dir}/head/${repo_id} if [[ -z ${force} && -d ${head_dir} ]]; then - echo "[ERROR] directory exists, did you mean repo-tier-pull? (${head_dir})" + echo "[ERROR] directory exists: ${head_dir}" exit 1 fi diff --git a/porthos/root/usr/local/bin/repo-tier-pull b/porthos/root/usr/local/bin/repo-snapshot-create similarity index 89% rename from porthos/root/usr/local/bin/repo-tier-pull rename to porthos/root/usr/local/bin/repo-snapshot-create index d7384f6..cfed7ae 100755 --- a/porthos/root/usr/local/bin/repo-tier-pull +++ b/porthos/root/usr/local/bin/repo-snapshot-create @@ -62,10 +62,10 @@ fi if [[ -d ${backup_dir} ]]; then if [[ -n ${force} ]]; then - echo "[WARNING] the already existing repository backup directory ${date_dir}/${repo_id} has been removed" + echo "[WARNING] the already existing repository snapshot directory ${date_dir}/${repo_id} has been removed" rm -rf ${backup_dir} else - echo "[WARNING] the repository backup directory already exists. Quit now ${backup_dir}" + echo "[WARNING] the repository snapshot directory already exists. Quit now ${backup_dir}" exit 0 fi fi diff --git a/porthos/root/usr/local/bin/repo-tier-delete b/porthos/root/usr/local/bin/repo-snapshot-delete similarity index 100% rename from porthos/root/usr/local/bin/repo-tier-delete rename to porthos/root/usr/local/bin/repo-snapshot-delete