...
 
Commits (11)
---
- name: debian bootstrap fact gathering
hosts: all
user: root
gather_facts: False
# Install the basics required to gather facts.
# This shouldn't be run normally, however, can't find a way to
# conditionally run it so far.
tasks:
- name: update apt repository
action: raw apt-get -q -y update
- name: install python
action: raw apt-get -q -y install python
# the command succeeds (returns code 0) if python needs simplejson
- name: check if python is old enough to need simplejson
action: raw python -c 'import sys; sys.stdout.write("%s" % (sys.version_info<(2,6)))'
register: need_simplejson
- name: ensure other prereqs installed
action: raw apt-get -qy install python-simplejson
when: need_simplejson.stdout
- name: ensure other prereqs installed
action: raw apt-get -qy install python-paramiko python-yaml python-jinja2 python-apt python-docker
- name: update packages
tags:
- update
hosts: all
user: root
roles:
- role: apt-upgrade
\ No newline at end of file
---
- name: bootstrap fact gathering
hosts: all
user: root
gather_facts: False
# Probe the system for package management type
tasks:
- name: check package management
action: raw apt-get
ignore_errors: yes
register: has_apt
# For now we don't support other package management systems!
- name: fail if no apt package management
fail:
msg: We currently only support Linux with apt
when: not has_apt
- import_playbook: bootstrap-debian.yml
when: has_apt
# Maybe add these somewhere later.
# # Needs to be included before sshd, since root needs to have a key installed
# # before sshd port changes when bootstrapping
# - role: ssh-key
# ssh_key_user: root
# ssh_key_pubfile: "{{userdefs.root.pubkey}}"
# - role: sshd
# sshd_port: "{{sshd.port}}"
# # ... moves port
......@@ -4,18 +4,13 @@ use warnings;
use FindBin qw($Bin);
my $ansible_root = shift
or die "Please supply a path to the ansible root directory\n";
$ENV{PATH} = "$Bin/../ansible-src/bin:$ENV{PATH}";
$ansible_root =~ s{/+$}{};
my $env_setup = "$Bin/../ansible-src/hacking/env-setup";
my $ansible = "$ansible_root/bin/ansible";
-x $ansible
or die "No ansible executable can be found at $ansible\n";
# Set the environment so that pass will use the referenced
# password-store directory, not ~/.password-store.
$ENV{PATH} = "$ansible_root/bin:$ENV{PATH}";
#$ENV{ANSIBLE_HOSTS} = "$Bin/ansible_hosts";
my $env_setup = "$ansible_root/hacking/env-setup";
$ENV{PASSWORD_STORE_DIR} = "$Bin/../pass";
exec '/bin/bash', '--rcfile', $env_setup;
# apt-update
This role simply runs an `apt update`
---
- name: update apt
apt: update_cache=yes
# apt-upgrade
This role simply runs an `apt update; apt upgrade`
---
dependencies:
- role: apt-update
---
- name: update apt safely
apt: upgrade=safe
async: 600
poll: 5
......@@ -37,5 +37,13 @@ pg_dump_to_s3_encrypt_path: /usr/bin/gpgwrapper
# Options for encrypt
pg_dump_to_s3_encrypt_opts: "-ek {{pg_dump_to_s3_keychain_path}}"
# Path of select-expired-backups executable
pg_dump_to_s3_select_expired_backups_path: /usr/bin/select-expired-backups
# Defines the backup origin date and rotation schedule
# Be careful about quoting! The quotes will stay in the options
# verbatim.
pg_dump_to_s3_select_expired_backups_opts: -c 7/4/3/4/2 -i 2019-01-24 -p dump.pg.%Y-%m-%d-%H%M.gz.gpg
# What to put into the systemd timer section
pg_dump_to_s3_systemd_timer_section: OnCalendar=00:40:00
This diff is collapsed.
---
- name: reload systemd services
systemd:
daemon_reload: yes
......@@ -16,7 +16,7 @@
- gpg # should be v2
- rclone
- name: install pg-dump-to-s3 and gpgwrapper script
- name: install pg-dump-to-s3 and pg-prune-from-s3 scripts
template:
src: "{{item}}.j2"
dest: "/usr/bin/{{item}}"
......@@ -25,7 +25,18 @@
mode: 0755
with_items:
- pg-dump-to-s3
- pg-prune-from-s3
- name: install select-expired-backups and gpgwrapper scripts
copy:
src: "{{item}}"
dest: "/usr/bin/{{item}}"
owner: root
group: root
mode: 0755
with_items:
- gpgwrapper
- select-expired-backups
- name: create paths
file:
......@@ -66,9 +77,11 @@
with_items:
- pg-dump-to-s3.service
- pg-dump-to-s3.timer
notify: reload systemd services
- name: enable pg-dump-to-s3 timed service
systemd:
name: pg-dump-to-s3.timer
state: started
enabled: yes
daemon_reload: yes
......@@ -3,4 +3,5 @@ Description=pg-dump-to-s3 backup
[Service]
Type=oneshot
ExecStart=/usr/bin/pg-prune-from-s3 execute
ExecStart=/usr/bin/pg-dump-to-s3
#!/bin/bash
# Prunes backup data stored on an S3 service
# {{ansible_managed}}
# Utilises a script {{ pg_dump_to_s3_select_expired_backups_path|basename }}
#
# This implements the backup rotation schedule, indicating which files
# have expired and can be deleted. See the inline documentation of
# that script for more information about this. For the purposes of
# this script, all you need to know is that it accepts a list of
# filenames on the input which include dates which can be parsed, one
# file per line, and it outputs the subset of these which can be
# deleted (also one per line). Options to the script define a nominal
# origin date and rotation period for the backup schedule. Backups are
# assumed to be made daily.
#
# If run with no arguments, it merely prints the commands executed to
# query the backups, and shows the commands which would be used to
# prune them, without actually deleting them. To delete, the first
# parameter must be the word 'execute'.
set -o errexit
set -o pipefail
SELECT_EXPIRED_BACKUPS="{{ pg_dump_to_s3_select_expired_backups_path }}"
SELECT_OPTS="{{ pg_dump_to_s3_select_expired_backups_opts }}"
RCLONE="{{ pg_dump_to_s3_rclone_path }}"
DUMP_STEM="{{ pg_dump_to_s3_archive_path_stem | basename }}"
RCLONE_CONFIG="{{ pg_dump_to_s3_rclone_conf_path}}"
DESTURL="{{ pg_dump_to_s3_desturl }}"
echo "selecting prunable backups ..."
echo " $RCLONE --max-depth 1 ls $DESTURL | grep -e '$DUMP_STEM' | sed 's/^ *[0-9]* //' | $SELECT_EXPIRED_BACKUPS $SELECT_OPTS"
export RCLONE_CONFIG
expired=$(
"$RCLONE" --max-depth 1 ls "$DESTURL" \
| /bin/grep -e "$DUMP_STEM" \
| /bin/sed 's/^ *[0-9]* //' \
| "$SELECT_EXPIRED_BACKUPS" $SELECT_OPTS
) || {
echo "selecting expired backups failed."
exit 1
}
echo "pruning backups ..."
if [[ $1 == 'execute' ]]; then
rc=0
for file in $expired; do
path="${DESTURL%/}/$file"
printf " $RCLONE delete ${DESTURL%/}/%s\n" $file
"$RCLONE" delete "$path" || {
echo "failed to delete $path"
rc=1
}
done
exit $rc
else
[[ -n "$expired" ]] && printf " $RCLONE delete ${DESTURL%/}/%s\n" $expired
echo "no deletions performed - add the parameter 'execute' to delete"
fi
These are test scripts for select-expired-backups.
Run them (from this directory), and the output should indicate if they
pass or not.
If they don't pass, something is wrong.
package schedules;
# This datastructure is a map of schedule names to schedule tables.
# Tables show how the schedule is calculated, and are used to test
# the scheduling code.
#
# Use this command to regenerate the data:
#
# select-expired-backups --cycles 5/4/3 --initial 2019-01/24 --dump
#
# Columns as follows:
# +------------------ Index number of the backup (zero based)
# v +------------- Cycle counts as 4 irregular base encoded integers
# v +------- Level of backup
# v +--- Lifetime in days
# v +- Retention: . means expired, else bash64 encoded age
# v
{
543 => [split '\n', <<HERE],
0 0,0,0,0 4 60 0
1 0,0,0,1 0 5 10
2 0,0,0,2 0 5 210
3 0,0,0,3 0 5 3210
4 0,0,0,4 0 5 43210
5 0,0,1,0 1 20 543210
6 0,0,1,1 0 5 6.43210
7 0,0,1,2 0 5 7..43210
8 0,0,1,3 0 5 8...43210
9 0,0,1,4 0 5 9....43210
10 0,0,2,0 1 20 a....543210
11 0,0,2,1 0 5 b....6.43210
12 0,0,2,2 0 5 c....7..43210
13 0,0,2,3 0 5 d....8...43210
14 0,0,2,4 0 5 e....9....43210
15 0,0,3,0 1 20 f....a....543210
16 0,0,3,1 0 5 g....b....6.43210
17 0,0,3,2 0 5 h....c....7..43210
18 0,0,3,3 0 5 i....d....8...43210
19 0,0,3,4 0 5 j....e....9....43210
20 0,1,0,0 2 60 k....f....a....543210
21 0,1,0,1 0 5 l....g....b....6.43210
22 0,1,0,2 0 5 m....h....c....7..43210
23 0,1,0,3 0 5 n....i....d....8...43210
24 0,1,0,4 0 5 o....j....e....9....43210
25 0,1,1,0 1 20 p.........f....a....543210
26 0,1,1,1 0 5 q.........g....b....6.43210
27 0,1,1,2 0 5 r.........h....c....7..43210
28 0,1,1,3 0 5 s.........i....d....8...43210
29 0,1,1,4 0 5 t.........j....e....9....43210
30 0,1,2,0 1 20 u..............f....a....543210
31 0,1,2,1 0 5 v..............g....b....6.43210
32 0,1,2,2 0 5 w..............h....c....7..43210
33 0,1,2,3 0 5 x..............i....d....8...43210
34 0,1,2,4 0 5 y..............j....e....9....43210
35 0,1,3,0 1 20 z...................f....a....543210
36 0,1,3,1 0 5 A...................g....b....6.43210
37 0,1,3,2 0 5 B...................h....c....7..43210
38 0,1,3,3 0 5 C...................i....d....8...43210
39 0,1,3,4 0 5 D...................j....e....9....43210
40 0,2,0,0 2 60 E...................k....f....a....543210
41 0,2,0,1 0 5 F...................l....g....b....6.43210
42 0,2,0,2 0 5 G...................m....h....c....7..43210
43 0,2,0,3 0 5 H...................n....i....d....8...43210
44 0,2,0,4 0 5 I...................o....j....e....9....43210
45 0,2,1,0 1 20 J...................p.........f....a....543210
46 0,2,1,1 0 5 K...................q.........g....b....6.43210
47 0,2,1,2 0 5 L...................r.........h....c....7..43210
48 0,2,1,3 0 5 M...................s.........i....d....8...43210
49 0,2,1,4 0 5 N...................t.........j....e....9....43210
50 0,2,2,0 1 20 O...................u..............f....a....543210
51 0,2,2,1 0 5 P...................v..............g....b....6.43210
52 0,2,2,2 0 5 Q...................w..............h....c....7..43210
53 0,2,2,3 0 5 R...................x..............i....d....8...43210
54 0,2,2,4 0 5 S...................y..............j....e....9....43210
55 0,2,3,0 1 20 T...................z...................f....a....543210
56 0,2,3,1 0 5 U...................A...................g....b....6.43210
57 0,2,3,2 0 5 V...................B...................h....c....7..43210
58 0,2,3,3 0 5 W...................C...................i....d....8...43210
59 0,2,3,4 0 5 X...................D...................j....e....9....43210
60 1,0,0,0 3 60 ....................E...................k....f....a....543210
61 1,0,0,1 0 5 ....................F...................l....g....b....6.43210
HERE
};
#!/usr/bin/perl
package Test;
use strict;
use warnings;
use Test::More;
# This is a test script for pg-prune-from-s3.j2
require_ok '../files/select-expired-backups';
my $schedules = require './schedules.pm';
my $scheduler = new Schedule(cycles => [5, 4, 3], initial => '2019-01-01');
isa_ok $scheduler, 'Schedule';
is_deeply [$scheduler->day_cycles], [5, 20, 60], 'day_cycles are correct';
my @debug_out = map { $scheduler->debug_index($_) } 0..61;
#print "$_\n" for @debug_out; # DEBUG
# This tests that the computation does what we want, for a specific case.
is_deeply \@debug_out, $schedules->{543}, 'dump_indexes match the expected values';
# Test some random counter bases (from the documentation)
my @etc = (initial => '2019-01-01');
is_deeply [Schedule->new(cycles => [10,10,10], @etc)->decompose(356)], [0,3,5,6],
"356 in basis 10,10,10 is 0,3,5,6";
is_deeply [Schedule->new(cycles => [2,2,2,2], @etc)->decompose(0xFA)], [15,1,0,1,0],
"0xFA in basis 2,2,2,2 is 15,1,0,1,0";
is_deeply [Schedule->new(cycles => [5,4,3], @etc)->decompose(119)], [1,2,3,4],
"119 in basis 5,4,3 is 1,2,3,4";
done_testing;
#!/usr/bin/perl
use strict;
use warnings;
use Test::More;
use Time::Piece;
use Time::Seconds;
# This is a test script for select-expired-backups
# Redirects the output of anything run in the $block and returns the
# captured standard input/output.
sub capture(&) {
my ($block) = @_;
my $dest = '';
do {
local (*STDOUT, *STDERR);
open STDOUT, '>>', \$dest or die $!;
# open STDERR, '>>', \$dest or die $!;
eval { $block->() };
};
return $dest;
}
require_ok '../files/select-expired-backups';
# Alias fully qualified names into our namespace, for convenience
{
no strict 'refs';
*{$_} = \&{$SelectExpiredBackups::{$_}}
for qw(parse_args trim);
}
######################################################################
# Test ancillary functions
for my $str ('trimmed ham', 'trimmed ham ', 'trimmed ham', ' trimmed ham ') {
is trim($str), 'trimmed ham', "trimming '$str' correctly";
}
for my $str (qw(x2019-02-01y x2019-2-1y)) {
my $time = Schedule::_parse_time('something', $str, 'x%Y-%m-%dy', 'querty');
isa_ok $time, 'Time::Piece';
is $time->datetime, '2019-02-01T00:00:00', "parsed $str correctly";
}
for my $str (qw(1 2019 2019-02 19-02-01 2019-02-01-01 2019_02_01 2019-o1-o2 2019-02-01x x2019-02-01)) {
eval {
Schedule::_parse_time('something', $str, '%Y-%m-%d', 'querty');
};
ok !@$, "parsing $str fails correctly";
}
######################################################################
# Test feeding params to parse_args
# This is the backup schedule start date. Just an arbitrary date.
my $initial = Time::Piece->strptime('2019-2-1', '%Y-%m-%d');
#use Data::Dumper; print Dumper \@dates;
# This describes the schedule in a visual way. Expiries are marked
# with an underscore. It describes the first 62 days of a 5/4/3 nested
# rotation schedule.
my $schedules = require './schedules.pm';
my @expected_schedule_543 = @{ $schedules->{543} };
ok @expected_schedule_543 > 1, "more than one test schedule item exists";
# Strip all data but the final schedule part
s/^.* // for @expected_schedule_543;
# Compute a list of date strings, one for every day of the schedule
# period.
my @dates = map { ($initial + ONE_DAY*$_)->ymd } 0..$#expected_schedule_543;
# Check that every day of the backup matches our expected schedule.
for my $ix (0..$#expected_schedule_543) {
diag "schedule $expected_schedule_543[$ix] for day $ix";
my %expiry = map {
$dates[$_] => substr $expected_schedule_543[$ix], $_, 1
} 0..length($expected_schedule_543[$ix])-1;
# Calculate the dates we expect to be marked expired
# from the expiry schedule string
my @expected_expired = grep { $expiry{$_} && $expiry{$_} eq '.' } sort @dates;
my @expected_unexpired = grep { !$expiry{$_} || $expiry{$_} ne '.' } sort @dates;
# Get the output of parse_args for the full list of @dates (passed as params)
my $out = capture {
parse_args(
'--initial', $initial->ymd,
'--now', ($initial + $ix*ONE_DAY)->ymd,
qw(--cycles 5/4/3), @dates
);
};
is $out, join("\n", @expected_expired, ''), @expected_expired." expiries for day $ix";
# Get the output of parse_args for the full list of @dates (passed as params)
# inverted output
$out = capture {
parse_args(
'--initial', $initial->ymd,
'--now', ($initial + $ix*ONE_DAY)->ymd,
'--invert',
qw(--cycles 5/4/3), @dates
);
};
is $out, join("\n", @expected_unexpired, ''), @expected_unexpired." preserved for day $ix";
# Get the output of parse_args for the full list of @dates (passed on STDIN)
$out = capture {
local *STDIN;
my $stream = join "\n", @dates;
open STDIN, "<", \$stream
or die $!;
parse_args(
'--initial', $initial->ymd,
'--now', ($initial + $ix*ONE_DAY)->ymd,
qw(--cycles 5/4/3)
);
};
is $out, join("\n", @expected_expired, ''), @expected_expired." expiries for day $ix";
# Get the output of parse_args for the full list of @dates (passed
# as params) however with prefixes and suffixes. NOTE:
# Time::Piece::strptime seems to happily ignore missing suffixes
# if one is specified (but warns if they are present when not
# specified). Seems to be a potential bug, but for our purposes
# this is probably ok to ignore.
$out = capture {
parse_args(
'--initial', $initial->ymd,
'--now', ($initial + $ix*ONE_DAY)->ymd,
qw(--cycles 5/4/3 --pattern foo.%Y-%m-%d.bak),
map "foo.$_.bak", @dates
);
};
is $out, join("\n", map("foo.$_.bak", @expected_expired), ''), @expected_expired." expiries for day $ix";
}
done_testing;