Commit d7d635c5 authored by Seamus Lee's avatar Seamus Lee
Browse files

Merge branch 'master' of lab.civicrm.org:infra/ops

parents 64b082be 159086ab
---
# IPv4 allocation for 192.95.2.128/29 (.129 to .134) and 2607:5300:60:71d0::/64
# - padthai.civicrm.org = 167.114.158.208 = 2607:5300:60:71d0::/64
# - test-1.civicrm.org = 192.95.2.130 = 2607:5300:60:71d0:100::/72
# - test-2.civicrm.org = 192.95.2.131 = 2607:5300:60:71d0:200::/72
# - botdylan.civicrm.org = 192.95.2.132 = 2607:5300:60:71d0:300::/72
# - www-test.civicrm.org = 192.95.2.134 = 2607:5300:60:71d0:500::/72
# - www-demo.civicrm.org = 192.95.2.129 = 2607:5300:60:71d0:600::/72
# - 192.95.2.133 : free (was training.c.o)
# ZFS partition setup
# - created new extended partition on all 3 disks with cfdisk
# - created new regular linux partition on all 3 disks (/dev/sdX5) with cfdisk
# - ran "partprobe"
# - created new pool: zpool create zpadthai raidz /dev/sda5 /dev/sdb5 /dev/sdc5
# - created new partitions:
# - zfs create -V 150G zpadthai/test-1
# - zfs create -V 50G zpadthai/test-2 # not created yet - 2018-09-13
# - zfs create -V 30G zpadthai/www-test
# - zfs create -V 30G zpadthai/botdylan
# - zfs create -V 30G zpadthai/www-demo
# New VM creation:
# head -15 /etc/preseeds/botdylan.civicrm.org/preseed.cfg
# .. then copy-paste the example command (I use ctrl+click to select without comments)
# virsh autostart test-1
# virsh autostart test-2
# virsh autostart botdylan
# virsh autostart www-test
# virsh autostart www-demo
# There are probably better ways to automate this, but for now, check:
# ethtool -P eno1
# or the initial file setup by OVH: /etc/systemd/network/50-default.network
kvm_main_mac_address: 0c:c4:7a:47:d9:7c
kvm_main_ipv4_address: 167.114.158.208
kvm_main_ipv4_netmask: 24
kvm_main_ipv4_network: 167.114.158.0
kvm_main_ipv4_broadcast: 167.114.158.255
kvm_main_ipv4_gateway: 167.114.158.254
kvm_main_ipv4_dns: 213.186.33.99
kvm_main_ipv6_address: "2607:5300:60:71d0::"
kvm_main_ipv6_netmask: 64
kvm_main_ipv6_dns: "2001:41d0:3:163::1"
kvm_main_ipv6_gateway: "2607:5300:60:71ff:ff:ff:ff:ff"
kvm_zfs_pool: zpadthai
kvm_hosts:
- botdylan.civicrm.org
- test-1.civicrm.org
- www-test.civicrm.org
- www-demo.civicrm.org
# IPv4 allocation for 192.95.2.128/29 (.129 to .134) and 2607:5300:203:6713::/64
# | 167.114.158.208 | 2607:5300:203:6713::/64 | padthai.civicrm.org |
# | 192.95.2.130 | 2607:5300:203:6713:100::/72 | test-1.civicrm.org |
# | 192.95.2.131 | 2607:5300:203:6713:200::/72 | (free ) |
# | 192.95.2.132 | 2607:5300:203:6713:300::/72 | botdylan |
# | 192.95.2.134 | 2607:5300:203:6713:500::/72 | www-test |
# | 192.95.2.129 | 2607:5300:203:6713:600::/72 | www-demo |
# | 192.95.2.133 | 2607:5300:203:6713:700::/72 | free (was training.c.o) |
kvm_main_ipv4_address: 51.161.13.19
kvm_main_ipv4_netmask: 255.255.255.0
kvm_main_ipv4_network: 51.161.13.127
kvm_main_ipv4_broadcast: 51.161.13.255
kvm_main_ipv4_gateway: 51.161.13.254
kvm_main_ipv6_address: "2607:5300:0203:6713::"
kvm_main_ipv6_netmask: "64"
kvm_main_ipv6_gateway: "2607:5300:0203:67ff:ff:ff:ff:ff"
kvm_main_ipv6_dns: "2001:41d0:3:163::1"
kvm_main_mac_address: "d0:50:99:d5:0a:55"
kvm_main_ipv4_dns: 8.8.8.8
kvm_zfs_pool: zpaella
kvm_hosts:
# - eu1.civi-go.net
# - eu2.civi-go.net
# - scc.symbiotic.coop
# - makoa1.symbiotic.coop
......@@ -6,7 +6,7 @@
# admins to classify servers so that we have a better idea of what they do.
#
barbecue.civicrm.org
padthai.civicrm.org
paella.civicrm.org
newsushi.civicrm.osuosl.org ansible_ssh_host=140.211.166.28
backups-1.civicrm.org
botdylan.civicrm.org
......@@ -86,8 +86,8 @@ test.civicrm.org
www-cxn-2.civicrm.osuosl.org
[kvm-servers]
padthai.civicrm.org
barbecue.civicrm.org
paella.civicrm.org
[servers:children]
apache-servers
......
---
# Adjust this to your needs
# If your infrastructure is dual-stack ipv6/ipv4, make sure to include all IPs.
ufw_munin_allow_src:
- 127.0.0.1
---
- name: restart ufw
service: name=ufw state=restarted
- name: restart rsyslog
service: name=rsyslog state=restarted
- name: refresh sysctl
shell: sysctl -p
......@@ -23,10 +23,8 @@
- less
- libpam-yubico
- ncdu
- ntpdate
- tmux
- ufw
- unattended-upgrades
- tree
- unzip
- vim
- vnstat
......@@ -37,6 +35,42 @@
- packages
- common-all
# KVM servers run openntpd; usually not recommended to install both.
- apt: name={{ item }} state=present install_recommends=no
with_items:
- ntpdate
when: "'kvm-servers' not in group_names"
tags:
- packages
- common-all
# Not required since Debian10
# - name: rsyslog | Deploy conf file to avoid annoying warnings
# template: src=etc/rsyslog.conf dest=/etc/rsyslog.conf owner=root group=root mode=0644
# notify: restart rsyslog
# tags:
# - common-all
# - common-all-rsyslog
- name: sysctl | Deploy custom sysctl settings
template: src=etc/sysctl.d/95-symbiotic.conf dest=/etc/sysctl.d/95-symbiotic.conf owner=root group=root mode=0644
notify: refresh sysctl
tags:
- common-all
- common-all-sysctl
- name: common | Deploy keyboard default conf
template: src=etc/default/keyboard dest=/etc/default/keyboard owner=root group=root mode=0644
tags:
- common-all
- name: network | Deploy IPv4 configuration
template: src=etc/network/interfaces dest=/etc/network/interfaces owner=root group=root mode=0644
when: preseed_ipv4_address is defined
tags:
- common-all
- common-all-ipv4
- name: network | Deploy IPv6 configuration script
template: src=etc/network/if-up.d/civicrm-ipv6 dest=/etc/network/if-up.d/civicrm-ipv6 owner=root group=root mode=0755
when: preseed_ipv6_address is defined
......
......@@ -2,5 +2,5 @@
deb http://ftp.ca.debian.org/debian/ {{ ansible_distribution_release }} main contrib
deb http://security.debian.org/ {{ ansible_distribution_release }}/updates main contrib
# stretch-updates, previously known as 'volatile'
# previously known as 'volatile'
deb http://ftp.ca.debian.org/debian/ {{ ansible_distribution_release }}-updates main
# {{ ansible_managed }}
#
# This is mostly required for kvm servers.
XKBMODEL="pc105"
XKBLAYOUT="us"
XKBVARIANT=""
XKBOPTIONS=""
BACKSPACE="guess"
# {{ ansible_managed }}
#
# For more information, see interfaces(5).
source /etc/network/interfaces.d/*
auto lo
iface lo inet loopback
{% if preseed_ipv4_bridged_gateway is defined %}
allow-hotplug {{ preseed_network_interface }}
iface {{ preseed_network_interface }} inet static
address {{ preseed_ipv4_address }}/32
broadcast {{ preseed_ipv4_address }}
post-up /sbin/ip route add {{ preseed_ipv4_bridged_gateway }} dev {{ preseed_network_interface }}
post-up /sbin/ip route add default via {{ preseed_ipv4_bridged_gateway }} dev {{ preseed_network_interface }}
dns-nameservers 8.8.8.8
dns-search {{ preseed_domain }}
{% else %}
allow-hotplug {{ preseed_network_interface }}
iface {{ preseed_network_interface }} inet static
address {{ preseed_ipv4_address }}
netmask {{ preseed_ipv4_netmask }}
network {{ preseed_ipv4_network }}
broadcast {{ preseed_ipv4_broadcast }}
post-up /sbin/ip route add {{ preseed_ipv4_gateway }} dev {{ preseed_network_interface }}
post-up /sbin/ip route add default via {{ preseed_ipv4_gateway }} dev {{ preseed_network_interface }}
dns-nameservers 8.8.8.8
dns-search {{ preseed_domain }}
{% endif %}
# {{ ansible_managed }}
#
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
#################
#### MODULES ####
#################
$ModLoad imuxsock # provides support for local system logging
$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability
# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514
# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514
###########################
#### GLOBAL DIRECTIVES ####
###########################
#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog
#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf
###############
#### RULES ####
###############
#
# First some standard log files. Log by facility.
#
auth,authpriv.* /var/log/auth.log
*.*;auth,authpriv.none -/var/log/syslog
#cron.* /var/log/cron.log
daemon.* -/var/log/daemon.log
kern.* -/var/log/kern.log
lpr.* -/var/log/lpr.log
mail.* -/var/log/mail.log
user.* -/var/log/user.log
#
# Logging for the mail system. Split it up so that
# it is easy to write scripts to parse these files.
#
mail.info -/var/log/mail.info
mail.warn -/var/log/mail.warn
mail.err /var/log/mail.err
#
# Logging for INN news system.
#
news.crit /var/log/news/news.crit
news.err /var/log/news/news.err
news.notice -/var/log/news/news.notice
#
# Some "catch-all" log files.
#
*.=debug;\
auth,authpriv.none;\
news.none;mail.none -/var/log/debug
*.=info;*.=notice;*.=warn;\
auth,authpriv.none;\
cron,daemon.none;\
mail,news.none -/var/log/messages
#
# Emergencies are sent to everybody logged in.
#
*.emerg :omusrmsg:*
#
# I like to have messages displayed on the console, but only on a virtual
# console I usually leave idle.
#
#daemon,mail.*;\
# news.=crit;news.=err;news.=notice;\
# *.=debug;*.=info;\
# *.=notice;*.=warn /dev/tty8
# The named pipe /dev/xconsole is for the `xconsole' utility. To use it,
# you must invoke `xconsole' with the `-file' option:
#
# $ xconsole -file /dev/xconsole [...]
#
# NOTE: adjust the list below, or you'll go crazy if you have a reasonably
# busy site..
#
# [ML] SYMBIOTIC
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=783687
#
# daemon.*;mail.*;\
# news.err;\
# *.=debug;*.=info;\
# *.=notice;*.=warn |/dev/xconsole
# {{ ansible_managed }}
# https://askubuntu.com/questions/118273/what-are-icmp-redirects-and-should-they-be-blocked
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.default.accept_redirects = 0
net.ipv4.conf.all.secure_redirects = 0
net.ipv4.conf.default.secure_redirects = 0
net.ipv6.conf.all.accept_redirects = 0
net.ipv6.conf.default.accept_redirects = 0
net.ipv6.conf.all.secure_redirects = 0
net.ipv6.conf.default.secure_redirects = 0
# http://www.forensicswiki.org/wiki/TCP_timestamps
net.ipv4.tcp_timestamps=0
---
# The VM user password is changed once the VM is booted, so this doesn't need to be saved
kvm_preseed_password: "{{ lookup('password', '/dev/null chars=ascii_letters length=15') }}"
kvm_preseed_language: en
kvm_preseed_country: US
kvm_preseed_locale: en_US
......@@ -2,51 +2,70 @@
# NB: netcat-openbsd is required if using virt-manager GUI (requires -U option).
- apt: name={{ item }} state=present install_recommends=no default_release=jessie-backports
with_items:
- linux-image-amd64
- linux-headers-amd64
when: ansible_distribution_release == "jessie"
- apt: name={{ item }} state=present install_recommends=no
with_items:
- linux-image-amd64
- linux-headers-amd64
when: ansible_distribution_release != "jessie"
tags:
- packages
# FIXME: on Stretch, requires "contrib"
- apt: name={{ item }} state=present install_recommends=no
with_items:
- openntpd
- kvm
- qemu-kvm
- virtinst
- bridge-utils
- netcat-openbsd
- nvme-cli
- parted
- zfs-dkms
- zfsutils-linux
- zfs-zed
tags:
- packages
- apt: name={{ item }} state=present install_recommends=no
with_items:
- libvirt-bin
- libvirt-daemon
- libvirt-daemon-system
when: ansible_distribution_release == "jessie"
- name: Modprobe zfs
shell: modprobe zfs
tags:
- kvm-server-zfs
- name: zfs | make sure that the module is loaded at boot
copy:
content: "zfs"
dest: "/etc/modules-load.d/zfs.conf"
owner: "root"
group: "root"
mode: '0644'
tags:
- kvm-server-zfs
- name: zfs | modprobe arc memory limit
copy:
content: "options zfs zfs_arc_max=1073741824"
dest: "/etc/modprobe.d/zfs.conf"
owner: "root"
group: "root"
mode: '0644'
tags:
- kvm-server-zfs
- apt: name={{ item }} state=present install_recommends=no
with_items:
- zfsutils-linux
- zfs-zed
- libvirt-clients
- libvirt-daemon
- libvirt-daemon-system
when: ansible_distribution_release == "stretch"
- virt-top
- service: name=openntpd state=started enabled=yes
# TODO:
# - had to "rm /boot/bzImage-3.14.32-xxxx-grs-ipv6-64" otherwise it would
# boot automatically on this kernel, and this causes issues with dkms for ZFS.
# - network interface configuration
- name: Ensure that OVH defaults are absent
file: path="/etc/systemd/network/{{ item }}" state=absent
with_items:
- 50-default.network
- 50-public-interface.link
- pub.network
tags:
- kvm-server-networkd
- name: Deploy the network interfaces configuration
template:
......@@ -57,22 +76,11 @@
mode: 0644
with_items:
- 50-br0.netdev
- 50-default.network
- 50-br0.network
- 50-network-interface.network
tags:
- kvm-server-networkd
# FIXME: not sure if these are the correct file names, some are the same as above.
# - name: Ensure that OVH defaults are absent
# file: path="/etc/systemd/network/{{ item }}" state=absent
# with_items:
# - 50-br0.netdev
# - 50-default.network
# - 50-public-interface.link
# TODO: reduce network timeout delay
- name: Create networking.service.d directory
file: path="/etc/systemd/system/networking.service.d/" state=directory mode=0755 owner=root group=root
......@@ -80,9 +88,30 @@
# [Service]
# TimeoutStartSec=15
# TODO: reload systemd (systemctl daemon-reload)
# Enable IP forwarding in /etc/sysctl.d/99-sysctl.conf by uncommenting:
# - net.ipv4.ip_forward=1
# - net.ipv6.conf.all.forwarding=1
- name: network | Deploy the network interfaces configuration
template:
src: "etc/network/interfaces"
dest: "/etc/network/interfaces"
owner: "root"
group: "root"
mode: 0644
tags:
- kvm-server-networkd
- name: network | Disable Debian old networking
systemd:
name: networking
enabled: no
tags:
- kvm-server-networkd
- name: network | Enable systemd-networkd
systemd:
name: systemd-networkd
enabled: yes
tags:
- kvm-server-networkd
- name: kvm preseeds | Create preseed directory
file: path="/etc/preseeds" state=directory mode=0750 owner=root group=root
......@@ -100,3 +129,9 @@
with_items: "{{ kvm_hosts }}"
tags:
- kvm-server-preseeds
- name: Generate preseed start script for VMs on this host
template: src=etc/preseeds/host/start.sh dest=/etc/preseeds/{{ item }}/start.sh owner=root group=root mode=0755
with_items: "{{ kvm_hosts }}"
tags:
- kvm-server-preseeds
......@@ -29,28 +29,3 @@ iface br0 inet6 static
post-up /sbin/ip -6 route add default via {{ kvm_main_ipv6_gateway }}
pre-down /sbin/ip -6 route del default via {{ kvm_main_ipv6_gateway }}
pre-down /sbin/ip -6 route del {{ kvm_main_ipv6_gateway }} dev br0
# auto br0
# iface br0 inet static
# address 167.114.158.208
# netmask 255.255.255.0
# network 167.114.158.0
# broadcast 167.114.158.255
# gateway 167.114.158.254
# bridge_ports eth0
# bridge_waitport 0
# bridge_stp off
# bridge_fd 0
# up echo 0 > /sys/devices/virtual/net/$IFACE/bridge/multicast_snooping
#
# iface br0:pub inet static
# address 192.95.2.129
# netmask 255.255.255.248
#
# iface br0 inet6 static
# address 2607:5300:60:71d0::
# netmask 64
# post-up /sbin/ip -6 route add 2607:5300:60:71ff:ff:ff:ff:ff dev br0
# post-up /sbin/ip -6 route add default via 2607:5300:60:71ff:ff:ff:ff:ff
# pre-down /sbin/ip -6 route del default via 2607:5300:60:71ff:ff:ff:ff:ff
# pre-down /sbin/ip -6 route del 2607:5300:60:71ff:ff:ff:ff:ff dev br0
#
# virt-install --name {{ hostvars[item]['preseed_hostname'] }} --ram {{ hostvars[item]['preseed_ram_mb'] }} --disk path=/dev/zvol/{{ kvm_zfs_pool }}/{{ hostvars[item]['preseed_hostname'] }} \
# --vcpus {{ hostvars[item]['preseed_vcpus'] }} --os-type linux --os-variant virtio26 --network bridge=br0,mac={{ hostvars[item]['preseed_macaddr'] }} --graphics vnc,listen=127.0.0.1 \
# --noautoconsole --location 'http://ftp.ca.debian.org/debian/dists/stretch/main/installer-amd64/' \
# --initrd-inject=/etc/preseeds/{{ hostvars[item]['preseed_hostname'] }}.{{ hostvars[item]['preseed_domain'] }}/preseed.cfg
#
# {{ ansible_managed }}
#
# To start the installation, run:
# /etc/preseeds/{{ hostvars[item]['preseed_hostname'] }}.{{ hostvars[item]['preseed_domain'] }}/start.sh
#### Contents of the preconfiguration file (for stretch)
#### Contents of the preconfiguration file (for buster)
### Localization
# Preseeding only locale sets language, country and locale.
# d-i debian-installer/locale string en_US
# The values can also be preseeded individually for greater flexibility.
d-i debian-installer/language string en
d-i debian-installer/country string US
d-i debian-installer/locale string en_US.UTF-8
d-i debian-installer/language string {{ hostvars[item]['preseed_language'] }}
d-i debian-installer/country string {{ hostvars[item]['preseed_country'] }}
d-i debian-installer/locale string {{ hostvars[item]['preseed_locale'] }}
# Keyboard selection.
d-i keyboard-configuration/xkb-keymap select us
......@@ -27,7 +24,7 @@ d-i netcfg/enable boolean true
# netcfg will choose an interface that has link if possible. This makes it
# skip displaying a list if there is more than one interface.
d-i netcfg/choose_interface select auto
d-i netcfg/choose_interface select {{ hostvars[item]['preseed_network_interface'] }}
# If you prefer to configure the network manually, uncomment this line and
# the static network configuration below.
......@@ -35,15 +32,14 @@ d-i netcfg/disable_autoconfig boolean true
# Static network configuration.
# IPv4
# FIXME: [ML] network config seems to fail?
# comment out these settings (ipaddress, netmask, gateway) and do it manually.
d-i netcfg/get_ipaddress string {{ hostvars[item]['preseed_ipv4_address'] }}
d-i netcfg/get_netmask string {{ hostvars[item]['preseed_ipv4_netmask'] }}
d-i netcfg/get_gateway string {{ hostvars[item]['preseed_ipv4_gateway'] }}
# d-i netcfg/get_gateway string {{ hostvars[item]['preseed_ipv4_gateway'] }}
d-i netcfg/get_gateway string none
d-i netcfg/get_nameservers string {{ hostvars[item]['preseed_ipv4_nameservers'] }}
d-i netcfg/confirm_static boolean true
# IPv6
# IPv6 - not working
#d-i netcfg/get_ipaddress string fc00::2
#d-i netcfg/get_netmask string ffff:ffff:ffff:ffff::
#d-i netcfg/get_gateway string fc00::1
......@@ -275,5 +271,28 @@ d-i finish-install/reboot_in_progress note
# still a usable /target directory. You can chroot to /target and use it