1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
#!/usr/bin/python3
# coding: utf-8
''''''
# Copyright (C) 2016 Antoine Beaupré <anarcat@debian.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import division, absolute_import
from __future__ import print_function, unicode_literals
import argparse
import logging
import time
import sys
try:
from fabric import Config
except ImportError:
sys.stderr.write('cannot find fabric, install with `apt install python3-fabric`') # noqa: E501
raise
# get the safe_open hack
import fabric_tpa # noqa: F401
from fabric_tpa import host
from fabric_tpa.reboot import (DEFAULT_DELAY_DOWN,
DEFAULT_DELAY_UP,
DEFAULT_DELAY_HOSTS,
DEFAULT_DELAY_SHUTDOWN,
shutdown_and_wait)
# TODO: don't use argparse: use Fabric's "Fab" program wrapper. We
# don't do this yet because we haven't figured out how to do the
# "sleep between hosts" policy. This probably requires overriding the
# Executor class? See also https://github.com/fabric/fabric/issues/2069
def parse_args(args=sys.argv[1:]):
parser = argparse.ArgumentParser(description=__doc__,
epilog='''''')
parser.add_argument('--verbose', '-v', dest='log_level',
action='store_const', const='info', default='warning')
parser.add_argument('--debug', '-d', dest='log_level',
action='store_const', const='debug', default='warning')
parser.add_argument('--force', '-f', action='store_true',
help='force reboot even if not needed')
parser.add_argument('--skip-ganeti-checks', action='store_true',
help='forcibly reboot Ganeti hosts without checking')
parser.add_argument('--skip-ganeti-empty', action='store_true',
help='reboot instances on the node as well, avoiding migrations')
# TODO: autodetect from LDAP. current documented behavior is
# according to rebootPolicy:
#
# rotation: delay_shutdown=30 minutes
# justdoit: delay_shutdown=10 minute
# manual: ignore! let the operator call reboots the normal way
# (e.g. with hosts listed by hand here)
#
# "manual" seems to be the default policy
parser.add_argument('--hosts', '-H', nargs='+',
help="host(s) to reboot, can be comma-separated")
parser.add_argument('--dryrun', '-n', action='store_true',
help='do not reboot servers (but do migrate)')
parser.add_argument('--delay-down', default=DEFAULT_DELAY_DOWN, type=int,
help='how long to wait for host to shutdown (default: %(default)s seconds)') # noqa: E501
parser.add_argument('--delay-up', default=DEFAULT_DELAY_UP, type=int,
help='how long to wait for host to come back up (default: %(default)s seconds)') # noqa: E501
parser.add_argument('--delay-hosts', default=DEFAULT_DELAY_HOSTS, type=int,
help='how long to wait between hosts (default: %(default)s seconds)') # noqa: E501
parser.add_argument('--delay-shutdown', default=DEFAULT_DELAY_SHUTDOWN,
type=int, help='delay, in minutes, passed to the shutdown command (default: %(default)s minutes)') # noqa: E501
parser.add_argument('--reason', default='rebooting for security upgrades',
help='reason to give users (default: %(default)s)')
return parser.parse_args(args=args)
def main(args):
config = Config({
'run': {
'dry': args.dryrun,
}
})
first = True
# split each hostname on comma, like `fab -H` does
for hostname in [x for h in args.hosts for x in h.split(',')]:
con = host.find_context(hostname, config=config)
# TODO: check if reboot required with needrestart instead of
# this?
#
# needrestart -p is a little dumb though: it marks everything
# as CRITICAL, so there's no distinction between requiring a
# reboot (microcode, kernel, systemd/dbus/qemu) and a restart
# (user-level services)
#
# see also:
# https://github.com/xneelo/hetzner-needrestart/issues/23
logging.info('checking if host %s requires a reboot', con.host)
kernel_and_libs = con.run(
"/usr/lib/nagios/plugins/dsa-check-running-kernel "
" && ! (/usr/lib/nagios/plugins/dsa-check-libs "
" | egrep --color 'systemd|dbus-daemon|qemu-system-x86') ",
warn=True,
)
microcode = con.run("/usr/lib/nagios/plugins/dsa-check-ucode-intel ", warn=True)
if kernel_and_libs.ok and (
microcode.stdout.startswith('OK') or
microcode.stdout.startswith('UNKNOWN')):
if args.force:
logging.warning('rebooting anyways because of --force')
else:
logging.info('host %s does not require a reboot, skipping', hostname)
continue
if first:
first = False
else:
logging.info('sleeping %d seconds before rebooting %s',
args.delay_hosts, hostname)
time.sleep(args.delay_hosts)
delay_shutdown = args.delay_shutdown
logging.info('rebooting host %s', hostname)
if not shutdown_and_wait(con,
reason=args.reason,
delay_down=args.delay_down,
delay_up=args.delay_up,
delay_shutdown=delay_shutdown,
ganeti_checks=not args.skip_ganeti_checks,
ganeti_empty=not args.skip_ganeti_empty):
logging.error('rebooting host %s failed, aborting', hostname)
break
logging.info('done with host %s', hostname)
# TODO: rebalance ganeti cluster if nodes were migrated
if __name__ == '__main__':
args = parse_args()
logging.basicConfig(format='%(message)s', level=args.log_level.upper())
# override default logging policies in submodules
#
# without this, we get debugging info from paramiko with --verbose
for mod in 'fabric', 'paramiko', 'invoke':
logging.getLogger(mod).setLevel('WARNING')
try:
main(args)
except Exception as e:
logging.error('unexpected exception during reboot: [%r] %s', e, e)
if args.log_level.upper() == 'DEBUG':
import traceback
import pdb
import sys
traceback.print_exc()
pdb.post_mortem()
sys.exit(1)
|
