#!/bin/sh
#
# Written by Alex Landau <alex.landau@xorcom.com>
# Modified by Leonid Fainshtein <Leonid@xorcom.com>
# Modified by Oron Peled <oron.peled@xorcom.com>
# Copyright (C) 2009-2024, Xorcom Ltd.
# All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#

set -e

# Don't let astribank udev hook re-arm the twinstar watchdog
NO_AUTOJUMP_FILE="/var/tmp/twinstar_no_autojump"
OS_ID=$( . "/etc/os-release" && echo "${ID}"-"${VERSION_ID}")

CFG_FOLDER=/usr/share/twinstar/
. ${CFG_FOLDER}twinstar_config.conf
#set -x

die() {
	echo >&2 "$@"
	exit 1
}

info() {
	echo >&2 "I: $@"
}

show_step() {
	echo >&2 "---------------------------------------------------------------------- $1"
}

valid_ip() {
	echo "$1" | grep -qP '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}[[:space:]]*$'
}

valid_ip_netmask() {
	echo "$1" | grep -qP '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/[0-9]{1,2}[[:space:]]*$'
}

check_lic() {
	rc=0
	licState=$(xlic list | sed -n '/twinstar-plus/{n;n;p;}' | awk -F' : ' '{print $2}')
	if [ "$licState" != "Active" ] && [ "$licState" != "Grace" ]; then
		echo "E: The Twinstar Plus (twinstar-plus) license is not found on this server."
		rc=1
	fi
	licState=$($SSH xlic list | sed -n '/twinstar-plus/{n;n;p;}' | awk -F' : ' '{print $2}')
	if [ "$licState" != "Active" ] && [ "$licState" != "Grace" ]; then
		echo "E: The Twinstar Plus (twinstar-plus) license is not found on the second server."
		rc=1
	fi
	return $rc
}

handle_twinstar_config() {
	# Defaults
	if [ -z "$DRBD_PART" ]; then
		if [ -b /dev/md2 ]; then
			DRBD_PART="/dev/md2"
		else
			DRBD_PART="/dev/sda2"
		fi
	fi
	[ -z "$DRBD_PART_A" ] && DRBD_PART_A="$DRBD_PART"
	[ -z "$DRBD_PART_B" ] && DRBD_PART_B="$DRBD_PART"
	if [ -z "$CLUSTER_INTERFACE" ]; then
		CLUSTER_INTERFACE='nic0'
		interfaces="nic0 eth0"
		for i in $interfaces; do
			if [ -e "/sys/class/net/$i" ]; then
				CLUSTER_INTERFACE="$i"
				break
			fi
		done
	fi

	# Sanity checks
	[ -z "$CLUSTER_IP" ] && die "Must define CLUSTER_IP"
	[ -z "$DRBD_PART_A" ] && die "Must define DRBD_PART_A"
	[ -z "$DRBD_PART_B" ] && die "Must define DRBD_PART_B"
	[ -z "$SRV_A_IP" ] && die "Must define SRV_A_IP"
	[ -z "$SRV_B_IP" ] && die "Must define SRV_B_IP"

	[ -z "$COROSYNC_RINGNUMBER" ] && die "Must define COROSYNC_RINGNUMBER"
	[ -z "$COROSYNC_BINDNETADDR" ] && die "Must define COROSYNC_MCBINDNETADDR"
	[ -z "$COROSYNC_MCASTADDR" ] && die "Must define COROSYNC_MCASTADDR"
	[ -z "$COROSYNC_MCASTPORT" ] && die "Must define COROSYNC_MCASTPORT"

	DHCPD_ENABLE=${DHCPD_ENABLE:-no}
	DNSMASQ_ENABLE=${DNSMASQ_ENABLE:-no}
	TFTPD_ENABLE=${TFTPD_ENABLE:-no}
	FAIL2BAN_ENABLE=${FAIL2BAN_ENABLE:-no}
	COMPLETE_CONCIERGE=${COMPLETE_CONCIERGE:-no}
	ELASTIX_CALL_CENTER_ENABLE=${ELASTIX_CALL_CENTER_ENABLE:-no}
	FOP2_ENABLE=${FOP2_ENABLE:-no}
	SUPERVISION_ENABLE=${SUPERVISION_ENABLE:-no}
	CCSTATS_ENABLE=${CCSTATS_ENABLE:-no}
	STATEXPLORER_ENABLE=${STATEXPLORER_ENABLE:-no}
	EMAIL_NOTIFY=${EMAIL_NOTIFY:-no}
	EPMPNP_ENABLE=${EPMPNP_ENABLE:-no}
	NIC_CARRIER_GUARD_MONITOR=${NIC_CARRIER_GUARD_MONITOR:-no}
	COROSYNC_CFG_TEMPL=${CFG_FOLDER}${COROSYNC_CFG_TEMPL:-corosync-mcast.tpl}
	
	# The values of Yes/No parameters will be converted to 1/0
	PARAMS_YES_NO="DHCPD_ENABLE DNSMASQ_ENABLE TFTPD_ENABLE FAIL2BAN_ENABLE COMPLETE_CONCIERGE ELASTIX_CALL_CENTER_ENABLE FOP2_ENABLE SUPERVISION_ENABLE CCSTATS_ENABLE STATEXPLORER_ENABLE EMAIL_NOTIFY EPMPNP_ENABLE NIC_CARRIER_GUARD_MONITOR"

	if ! valid_ip_netmask "$CLUSTER_IP"; then
		echo >&2 "E: The CLUSTER_IP parameter is wrong. Please check that it is defined in the CDIR notation. For example: 1.2.3.4/24"
		exit 1
	fi
	CLUSTER_NETMASK=`echo "$CLUSTER_IP" | sed 's,^.*/,,'`
	CLUSTER_IP_ONLY=`echo "$CLUSTER_IP" | sed 's,/.*$,,'`
	[ -z "$CLUSTER_NETMASK" ] && die "The CLUSTER_IP should have '/netmask' suffix"
	[ -z "$CLUSTER_IP_ONLY" ] && die "Bad CLUSTER_IP='$CLUSTER_IP' (should be IP/NETMASK)"

	if ! valid_ip "$SRV_B_IP"; then
		echo >&2 "E: The SRV_B_IP parameter is wrong ($SRV_B_IP)."
		exit 1
	fi

	if ! valid_ip "$COROSYNC_BINDNETADDR"; then
		echo >&2 "E: The COROSYNC_BINDNETADDR parameter is wrong ($COROSYNC_BINDNETADDR)."
		exit 1
	fi

	if ! valid_ip "$COROSYNC_MCASTADDR"; then
		echo >&2 "E: The COROSYNC_MCASTADDR parameter is wrong ($COROSYNC_MCASTADDR)."
		exit 1
	fi

	for i in $PARAMS_YES_NO; do
		eval "val=\$${i}"
		case "$val" in
		[Yy][Ee][Ss])
			eval ${i}=1
			;;
		*)
			eval ${i}=0
			;;
		esac
		#eval echo "${i}=\$${i}"
	done
	SSH="ssh -i $HOME/.ssh/twinstar root@$SRV_B_IP"
	SCP="scp -i $HOME/.ssh/twinstar"
}

run_both()
{
	eval "$@"
	$SSH "$@"
}

subst_corosync_conf() {
	sed \
		-e "s/[@]COROSYNC_RINGNUMBER[@]/$COROSYNC_RINGNUMBER/g"	\
		-e "s/[@]COROSYNC_BINDNETADDR[@]/$COROSYNC_BINDNETADDR/g"	\
		-e "s/[@]COROSYNC_MCASTADDR[@]/$COROSYNC_MCASTADDR/g"	\
		-e "s/[@]COROSYNC_MCASTPORT[@]/$COROSYNC_MCASTPORT/g"	\
		-e "s/[@]NODEID[@]/$nodeid/g" \
		-e "s/[@]COROSYNC_IP_A[@]/$SRV_A_IP/g" \
		-e "s/[@]COROSYNC_IP_B[@]/$SRV_B_IP/g" 
}

echo_corosync_conf() {
	[ -f "$COROSYNC_CFG_TEMPL" ] || die "Corosync configuration templet file ($COROSYNC_CFG_TEMPL) doesn't exist"
	nodeid="$1"
	[ "$nodeid" = '' ] && die "Missing nodeid parameter"
	
	cat "$COROSYNC_CFG_TEMPL" | subst_corosync_conf
}

echo_drbd_conf()
{
	cat <<'EOF'
resource "r0" {
	protocol A;
	disk {
		on-io-error pass_on;
	}
	startup {
		wfc-timeout 5;
		outdated-wfc-timeout 3;
		degr-wfc-timeout 3;
		outdated-wfc-timeout 2;
	}
	syncer {
		rate 10M;
		verify-alg md5;
	}
	net {
		after-sb-0pri discard-older-primary;
		after-sb-1pri discard-secondary;
		after-sb-2pri call-pri-lost-after-sb;
	}
	handlers {
		pri-lost-after-sb "/sbin/reboot";
	}
	on AHN {
		device /dev/drbd0;
		disk PART_A;
		address AIP:7789;
		meta-disk internal;
	}
	on BHN {
		device /dev/drbd0;
		disk PART_B;
		address BIP:7789;
		meta-disk internal;
	}
}
EOF
}

echo_dirs()
{

	cat <<'EOF'
/etc/asterisk
/etc/ombutel
/var/lib/asterisk
/var/lib/mysql
/var/lib/ombutel
/var/log/asterisk
/var/log/cpbx
/var/spool/asterisk
EOF
if [ "$OS_ID" = "centos-7" ]; then
  cat << 'EOF'
/etc/httpd
EOF
else 
  cat << 'EOF'
/etc/apache2
EOF
fi
if [ $DHCPD_ENABLE = 1 ]; then
  cat << 'EOF'
/var/lib/dhcpd
EOF
fi
if [ $DNSMASQ_ENABLE = 1 ]; then
	if [ "$OS_ID" = "centos-7" ]; then
cat << 'EOF'
/etc/dnsmasq.d
/var/lib/dnsmasq
EOF
	else
cat << 'EOF'
/etc/dnsmasq.d
/var/lib/misc
EOF
	fi
fi
if [ $FOP2_ENABLE = 1 ]; then
  cat << 'EOF'
/etc/fop2
/var/lib/fop2
/usr/share/fop2/plugins/auth
EOF
fi
if [ $TFTPD_ENABLE = 1 ]; then
  cat << 'EOF'
/tftpboot
EOF
fi
if [ $FAIL2BAN_ENABLE = 1 ]; then
  cat << 'EOF'
/etc/fail2ban
EOF
fi
}

update_hosts() {
	#$1-IP; $2-hostname
	addr=`awk -v h1=$2 '$0 ~ h1 {print $1}' /etc/hosts`
	if [ "$addr" = "" ]; then
		echo "$1 $2" >> /etc/hosts
	elif [ "$1" != "$addr" ]; then
		sed -i "s/[[:space:]]*\([^[:space:]]*\)[[:space:]]*\($2.*\)/$1 \2/" /etc/hosts
	fi

}

create_drbd_conf()
{
	cfg_name="/etc/drbd.d/r0.res"
	test -b "$DRBD_PART_A" || die "$0: $DRBD_PART_A: Not a block device. Aborting."
	$SSH "test -b '$DRBD_PART_B'" || die "$0: $DRBD_PART_B: Not a block device. Aborting."
	CMD_GETHOSTNAME="uname -n"
	MASTER_DRBD_IP="$SRV_A_IP"
	MASTER_HOSTNAME=`eval "$CMD_GETHOSTNAME"`
	SLAVE_DRBD_IP="$SRV_B_IP"
	SLAVE_HOSTNAME=`$SSH "$CMD_GETHOSTNAME"`
	echo_drbd_conf | sed \
		-e "s/AIP/$MASTER_DRBD_IP/g"	\
		-e "s/AHN/$MASTER_HOSTNAME/g"	\
		-e "s/BIP/$SLAVE_DRBD_IP/g"	\
		-e "s/BHN/$SLAVE_HOSTNAME/g"	\
		-e "s|PART_A|$DRBD_PART_A|g"	\
		-e "s|PART_B|$DRBD_PART_B|g"	\
		> "$cfg_name"
	cat "$cfg_name" | $SSH "cat > '$cfg_name'"
	info "Updating /etc/hosts"
	update_hosts "$MASTER_DRBD_IP" "$MASTER_HOSTNAME"
	update_hosts "$SLAVE_DRBD_IP" "$SLAVE_HOSTNAME"
	cat "/etc/hosts" | $SSH "cat > '/etc/hosts'"
}

create_nic_carrier_conf()
{
	if [ "$NIC_CARRIER_GUARD_MONITOR" = 1 ]; then
		#Check the parameters
		NIC_CARRIER_GUARD_MONITORED_NICS=${NIC_CARRIER_GUARD_MONITORED_NICS:-"nic0"}
		NIC_CARRIER_GUARD_POLL_TO=${NIC_CARRIER_GUARD_POLL_TO:-5}
		NIC_CARRIER_GUARD_THRESHOLD=${NIC_CARRIER_GUARD_THRESHOLD:-2}
		#Create configuration on both servers
		echo "NIC_CARRIER_GUARD_MONITORED_NICS=\"$NIC_CARRIER_GUARD_MONITORED_NICS\"" > /etc/default/nic-carrier-guard
		echo "NIC_CARRIER_GUARD_POLL_TO=$NIC_CARRIER_GUARD_POLL_TO" >> /etc/default/nic-carrier-guard
		echo "NIC_CARRIER_GUARD_THRESHOLD=$NIC_CARRIER_GUARD_THRESHOLD" >> /etc/default/nic-carrier-guard
		echo "NIC_CARRIER_GUARD_DEBUG=0" >> /etc/default/nic-carrier-guard
		cat "/etc/default/nic-carrier-guard" | $SSH "cat > '/etc/default/nic-carrier-guard'"
		#Enable and start the service on both servers
		run_both systemctl restart nic-carrier-guard.service 2>/dev/null || :
		run_both systemctl enable nic-carrier-guard.service 2>/dev/null || :
	else
		run_both systemctl stop nic-carrier-guard.service 2>/dev/null || :
		run_both systemctl disable nic-carrier-guard.service 2>/dev/null || :
	fi
}

create_notify_conf()
{
	if [ "$EMAIL_NOTIFY" = 1 ]; then
		cfg_file='/etc/twinstar/runconf.d/notify.conf'
		sed -i -e "s/\(ADDR=\).*$/\1\"$EMAIL_ADDR\"/" \
			   -e "s/\(ADDR_FROM=\).*$/\1\"$EMAIL_ADDR_FROM\"/" \
				$cfg_file
		cat $cfg_file | $SSH "cat > $cfg_file"
	fi
}

set_systemd_services() {
	run_both systemctl disable asterisk.service  2>/dev/null || :
	run_both systemctl disable mariadb.service   2>/dev/null || :
	run_both systemctl disable amiserv5.service   2>/dev/null || :
	run_both systemctl disable fop2.service   2>/dev/null || :
	if [ "$OS_ID" = "centos-7" ]; then
		run_both systemctl disable httpd.service  2>/dev/null || :
	else
		run_both systemctl disable apache2.service  2>/dev/null || :
	fi
	if [ "$DNSMASQ_ENABLE" = 1 ]; then 
		run_both systemctl disable dnsmasq.service  2>/dev/null || :
	fi
	run_both systemctl disable epmpnp.service  2>/dev/null || :
}

all_services() {
	echo "
		ipaddr		1
		srcaddr		1
		drbd		1
		filesystem	1
		mysql		1
		asterisk	1
		astribanks	1
		apache		1
		ccstatspro	$CCSTATS_ENABLE
		statexplorer $STATEXPLORER_ENABLE
		cconcierge	$COMPLETE_CONCIERGE
		fop2		$FOP2_ENABLE
		superv		$SUPERVISION_ENABLE
		epmpnp		$EPMPNP_ENABLE
		cpbx		1
		dnsmasq		$DNSMASQ_ENABLE
		firewall	1
		fail2ban	$FAIL2BAN_ENABLE
		notify 		$EMAIL_NOTIFY
		" | sed -e 's/^[ \t]*//' -e 's/[ \t]*$//'
}

is_wanted_service() {
	service_name="$1"
	all_services | awk -vname="$service_name" '$1 == name { print $2 }'
}

enable_wanted_services() {
	info "Enabling services:"
	for i in `all_services | awk '{ print $1 }'`; do
		case "`is_wanted_service "$i"`" in
		1)
			echo "	Adding:	$i"
			run_both /usr/sbin/twinstar-resources add "$i"
			;;
		*)
			echo "	Removing:	$i"
			run_both /usr/sbin/twinstar-resources remove "$i"
			;;
		esac
	done
}

create_corosync_conf() {
	tws_cfgdir='/etc/twinstar/runconf.d'
	cfg_file='/etc/corosync/corosync.conf'
	authkey='/etc/corosync/authkey'
	if [ ! -f "$authkey" ]; then
		die "Missing '$authkey' -- first run corosync-keygen"
	fi
	set -e
	run_both systemctl stop twinstar-manager.service 2>/dev/null || :
	run_both systemctl stop twinstar.service 2>/dev/null || :
	run_both systemctl stop corosync.service 2>/dev/null || :
	echo_corosync_conf 1 > "$cfg_file.tmp"
	mv -f "$cfg_file.tmp" "$cfg_file"
	echo_corosync_conf 2 | $SSH "cat > $cfg_file.tmp"
	$SSH mv -f "$cfg_file.tmp" "$cfg_file"

	rm -f ${tws_cfgdir}/ifcfg-{eth0,nic0}.conf
	$SSH rm -f ${tws_cfgdir}/ifcfg-{eth0,nic0}.conf
	cat <<EOF > "$tws_cfgdir/ifcfg-${CLUSTER_INTERFACE}.conf"
export OCF_RESKEY_nic=$CLUSTER_INTERFACE
export OCF_RESKEY_ip=$CLUSTER_IP_ONLY
export OCF_RESKEY_cidr_netmask=$CLUSTER_NETMASK
EOF
	cat <<EOF > "$tws_cfgdir/srcaddr.conf"
export OCF_RESKEY_ipaddress=$CLUSTER_IP_ONLY
export OCF_RESKEY_cidr_netmask=$CLUSTER_NETMASK
EOF
	for f in "$tws_cfgdir/"*.conf; do
		$SCP "$f" "root@$SRV_B_IP:$f"
	done

	echo 1 > /etc/twinstar/preferred_master

	enable_wanted_services
}

copy_dahdi_config() {
	# Copy /etc/dahdi to slave
	tar czf - -C / etc/dahdi | $SSH "rm -rf /etc/dahdi && tar xzf - -C /"
}

prep_drbd() {
	if [ -h /etc/asterisk -a ! -r "$CONFIG_TAR" ]; then
		echo >&2 "E: /etc/asterisk is a symbolic link and no config.tar.gz specified. Aborting."
		exit 1
	fi

	set +e
	run_both service drbd start
	set -e

	# Unmount /replica if it exists (shouldn't exist on a clean installation)
	if [ "`awk '$2 == "/replica"' /proc/mounts`" != "" ]; then
		fuser -k -m /replica || :
		umount -f /replica || :
	fi

	# Create /etc/drbd.conf
	create_drbd_conf

	# Initialize DRBD on $DRBD_PART
	run_both drbdadm down r0
	dd if=/dev/zero of=$DRBD_PART_A bs=1M count=1
	$SSH "dd if=/dev/zero 'of=$DRBD_PART_B' bs=1M count=1"
	run_both echo yes \| drbdadm create-md r0

	#They need to do the initial handshake, so they know their sizes.
	run_both drbdadm up r0
	drbdadm wait-connect r0

	#They are now Connected Secondary/Secondary UpToDate/UpToDate. On the system to be Primary:
	drbdadm primary --force r0

	# Make filesystem
	mkfs.ext4 /dev/drbd0
	tune2fs -c 0 -i 0 /dev/drbd0
	mkdir -p /replica
	mount /dev/drbd0 /replica

	# On master, move directories to /replica
	TAR_FILE=/usr/share/twinstar/config-`date '+%Y%m%d%H%M%S'`.tar.gz
	tar czf $TAR_FILE --acls -C / `echo_dirs`
	if [ "$CONFIG_TAR" ]; then
		tar xzf "$CONFIG_TAR" --acls  -C /replica
	else
		tar xzf "$TAR_FILE" --acls -C /replica
	fi
	echo_dirs | while read dir; do
		rm -rf $dir
		ln -s /replica$dir $dir
	done
	
	if [ "$OS_ID" = "centos-7" ]; then	
		# On Master, fix symbolic links inside of /replica/etc/httpd
		pushd /replica/etc/httpd
		rm logs modules
		ln -s ../../../var/log/httpd logs
		ln -s ../../../usr/lib64/httpd/modules modules
		popd
	fi

	# On slave, make symlinks
	$SSH tar czf $TAR_FILE -C / `echo_dirs`
	$SSH "mkdir -p /replica"
	echo_dirs | while read dir; do
		$SSH "rm -rf $dir && ln -s /replica$dir $dir" < /dev/null
	done
	/usr/sbin/twinstar disable-wd || :
	info "A reboot is required on BOTH servers"
}

create_configurations() {
	create_corosync_conf
	create_drbd_conf
	create_nic_carrier_conf
	set_systemd_services
	create_notify_conf
}

reconfigure_asterisk() {
	# Make sure we are master
	show_step "Make us MASTER and wait for run-steps to complete"
	twinstar-ctl take-over
	twinstar-ctl freeze
	twinstar-ctl wait

	show_step "Reconfiguring Asterisk"
	touch $NO_AUTOJUMP_FILE && trap "rm -f $NO_AUTOJUMP_FILE" EXIT
	run_both "rm -f /etc/dahdi/xpp_order"
	/usr/share/ombutel/scripts/ddetect -r
	rm -f $NO_AUTOJUMP_FILE && trap - EXIT
	twinstar-ctl unfreeze
	ab_num=`lsusb |grep e4e4:1162 |wc -l`
	if [ "$ab_num" -gt "0" ]; then
		/usr/share/dahdi/twinstar-check-astribanks || die "Problems with Astribanks!!"
	fi
	copy_dahdi_config
	# Copy the /usr/share/twinstar/twinstar_config.conf to slave
	cat /usr/share/twinstar/twinstar_config.conf | $SSH "cat > /usr/share/twinstar/twinstar_config.conf"
}

reconfigure_corosync() {
	show_step "Reconfiguring Network"
	# Deactivate the current cluster IP
	/usr/share/twinstar/run.d/09-srcaddr.run stop
	/usr/share/twinstar/run.d/08-ipaddr.run stop

	create_configurations

	# Activate the current cluster IP
	/usr/share/twinstar/run.d/08-ipaddr.run start
	/usr/share/twinstar/run.d/09-srcaddr.run start

	run_both systemctl stop twinstar-manager.service > /dev/null 2>&1 || :
	run_both systemctl stop twinstar.service > /dev/null 2>&1 || :
	run_both systemctl stop corosync.service > /dev/null 2>&1 || :
	systemctl daemon-reload
	run_both systemctl start corosync.service
	run_both systemctl start twinstar.service
	run_both systemctl start twinstar-manager.service
	node_count=`twinstar-ctl status | sed -n 's/^NodeCount:[ \t]*//p'`
	[ "$node_count" = 2 ] || die "E: Cluster communication problem (twinstar-status reports NodeCount != 2)"

	info "Ensure that this server is Master"
	twinstar-ctl take-over
	info "Wait for services activation."
	twinstar-ctl wait

	# Copy the /usr/share/twinstar/twinstar_config.conf to slave
	cat /usr/share/twinstar/twinstar_config.conf | $SSH "cat > /usr/share/twinstar/twinstar_config.conf"
}

setup_keys() {
	[ "$#" -eq 2 ] || die "E: setup_keys() -- missing 2 <IP> parameters"
	this="$1"
	other="$2"
	sshkey="$HOME/.ssh/twinstar"
	authkeys="$HOME/.ssh/authorized_keys"

	info "Generate and copy SSH key"
	mkdir -p ~/.ssh -m 600
	if ! ssh-keygen -l -f "$sshkey" >/dev/null 2>&1; then
		ssh-keygen -t rsa -N '' -f "$sshkey"
	fi
	ssh-copy-id -i "$sshkey" root@$other
	if [ `ssh -i "$sshkey" root@$other "echo ok"` != "ok" ]; then
		die "E: Can't login to $other after setting up keys. Aborting."
	fi
	scp -q -i "$sshkey" "$sshkey" "root@$other:$sshkey"
	key_contents=`cat "$sshkey.pub"`
	if ! grep -q "$key_contents" "$authkeys" 2>/dev/null; then
		echo "$key_contents" >>"$authkeys"
	fi
	# Make sure the SSH host key is OK:
	dual_ssh_cmd="ssh -qt -i $sshkey root@$other ssh -qt -i $sshkey root@$this"
	if ! $dual_ssh_cmd true; then
		die "E: Can't login from $other to $this after setting up keys. Aborting."
	fi

	# Generate and copy COROSYNC shared-secret key
	info "Generate COROSYNC key"
	(
		set -e
		umask 277
		dd if=/dev/urandom bs=128 count=1 2>/dev/null > /etc/corosync/authkey
	)
	scp -i "$sshkey" /etc/corosync/authkey "$other:/etc/corosync/authkey"

	info "Keys have been set up successfully."
}

prep_system() {
	show_step "Setting up SSH + COROSYNC keys"
	setup_keys "$SRV_A_IP" "$SRV_B_IP"

	# Set default master
	is_default_master='/etc/twinstar/preferred_master'
	touch "$is_default_master"
	$SSH "rm -f '$is_default_master'"

	# Set hostname on both servers
	hostnamectl --static set-hostname srv-a
	$SSH 'hostnamectl --static set-hostname srv-b'

	# Stop COROSYNC
	show_step "Stop TWINSTAR + COROSYNC + DRBD"
	run_both systemctl stop twinstar-manager.service > /dev/null 2>&1 || :
	run_both systemctl stop twinstar.service > /dev/null 2>&1 || :
	run_both systemctl stop corosync.service > /dev/null 2>&1 || :

	# Order is important
	$SSH /usr/sbin/twinstar-resources stop || :
	$SSH 'service drbd stop' || :
	service drbd stop

	if [ "$OS_ID" = "centos-7" ]; then
		show_step "Restart networking"
		networksrvice="network.service"
		systemctl restart network.service
		sleep 1
		$SSH "systemctl restart network.service"
	fi

	# Generate configuration only (DRBD and COROSYNC)
	show_step "Generate configuration files (DRBD + COROSYNC)"
	create_configurations

	# Start DRBD service
	show_step "Start drbd"
	run_both service drbd start
	drbdadm wait-connect r0
	drbdadm primary --force r0
	mount /dev/drbd0 /replica || :

	# Remove /etc/dahdi/xpp_order -- it's legacy.
	run_both rm -f /etc/dahdi/xpp_order

	# Start all services on srv-a. It will mount /replica.
	reconfigure_corosync

	# Configure both servers
	$SSH twinstar jump  || :
	reconfigure_asterisk

	# Try to wait for Astribanks as well
	if [ -f /etc/dahdi ]; then
		show_step "Wait for Astribanks if any"
		/usr/share/dahdi/waitfor_xpds
		dahdi_waitfor_span_assignments assigned
	fi

	show_step "Enabling Ombutel user interface"
	dir=`dirname $0`
	$dir/setup_ombutel
	
	set_systemd_services
}

update()
{
	show_step "Updating the current server"
	drbdadm disconnect r0
	if [ "$OS_ID" = "centos-7" ]; then
		yum clean all
		yum -y update
		systemctl restart fop2 || :
	else
		apt update
		apt -y full-upgrade
	fi
	show_step "Updating the second server"
	$SSH '/usr/share/twinstar/run.d/10-drbd.run start'
	$SSH '/usr/share/twinstar/run.d/15-filesystem.run start'
	$SSH '/usr/share/twinstar/run.d/20-mysql.run start'
	$SSH '/usr/share/twinstar/run.d/25-asterisk.run start'
	$SSH '/usr/share/twinstar/run.d/30-apache.run start'
	if [ "$OS_ID" = "centos-7" ]; then
		$SSH "yum clean all"
		$SSH "yum -y update"
	else
		$SSH "apt update"
		$SSH "apt -y full-upgrade"
	fi
	$SSH '/usr/share/twinstar/run.d/30-apache.run stop'
	$SSH '/usr/share/twinstar/run.d/25-asterisk.run stop'
	$SSH '/usr/share/twinstar/run.d/20-mysql.run stop'
	$SSH '/usr/share/twinstar/run.d/15-filesystem.run stop'
	$SSH '/usr/share/twinstar/run.d/10-drbd.run stop'
	drbdadm connect r0
	show_step "Twinstar servers upgrade completed!"
}

usage()
{
	echo >&2 "Usage: $0 <action> [opts]"
	echo >&2 "       Actions:"
	echo >&2 "          keys                     -- Generate SSH and Corosync keys."
	echo >&2 "          reconfigure [-i] [-a]    -- Change IP and/or Asterisk/DAHDI configuration."
	echo >&2 "          system-prep              -- Prepare a new system after Rapid Recovery images"
	echo >&2 "                                      were installed on both servers."
	exit 1
}

TEMP=`getopt -o "c:ai" --long conftar:,asterisk,ip -n 'twinstar_config' -- "$@"`
if [ $? != 0 ]; then
	usage
fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
while true ; do
	case "$1" in
		-c|--conftar)
			CONFIG_TAR="$2"
			shift 2
			;;
                -a|--asterisk)
			DO_ASTERISK_CFG=1
			shift
			;;
                -i|--ip)
			DO_IP_CFG=1
			shift
			;;
                --) shift
			break
			;;
                *)
			die "E: Internal getopt error"
			;;
	esac
done
if [ "$#" -lt 1 ]; then
	echo >&2 "E: Missing action"
	usage
fi
action="$1"
shift

handle_twinstar_config

case "$action" in
keys)
	setup_keys "$SRV_A_IP" "$SRV_B_IP"
	;;
drbd-prep)
	if [ "$DO_IP_CFG" != '' -o "$DO_ASTERISK_CFG" != '' ]; then
		echo >&2 "E: Should not specify '-a' and/or '-i' for '$action'"
		usage
	fi
	prep_drbd
	;;
system-prep)
	prep_system
	show_step "Check license."
	check_lic || :
	show_step "FINISHED"
	;;
reconfigure)
	if [ "$DO_IP_CFG" = '' -a "$DO_ASTERISK_CFG" = '' ]; then
		echo >&2 "E: Reconfigure what? (should specify '-a' and/or '-i')"
		usage
	fi
	[ "$DO_IP_CFG" = 1 ] && reconfigure_corosync
	[ "$DO_ASTERISK_CFG" = 1 ] && reconfigure_asterisk
	show_step "Check license."
	check_lic || :
	show_step "FINISHED"
	;;
update)
	update
	show_step "FINISHED"
	;;
*)
	echo >&2 "E: Bad action '$action'"
	usage
	;;
esac

exit 0
