#!/bin/sh

set -e
#set -x

usage () {
	echo "Usage: $0 <CLUSTER_NAME> <FROM_OS_RELEASE> <TO_OS_RELEASE>"
	exit 1
}

if [ $# != 3 ] ; then
	usage
fi

ALL_SERVICES_CTRL="aodh-api aodh-evaluator aodh-expirer aodh-listener aodh-notifier barbican-api barbican-keystone-listener barbican-worker ceilometer-agent-central ceilometer-agent-notification ceilometer-polling cinder-api cinder-backup cinder-scheduler cinder-volume cloudkitty-api cloudkitty-processor glance-api gnocchi-api gnocchi-metricd gnocchi-statsd heat-api-cfn heat-api heat-engine keystone magnum-api magnum-conductor neutron-api neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent neutron-rpc-server nova-api nova-api-metadata nova-compute nova-conductor nova-novncproxy nova-scheduler octavia-api octavia-health-manager octavia-housekeeping octavia-worker panko-api placement-api neutron-api neutron-rpc-server"
ALL_SERVICES_VOL="cinder-backup cinder-volume"
ALL_SERVICES_NET="neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent"
ALL_SERVICES_COMP="ceilometer-agent-compute cinder-backup cinder-volume neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent nova-compute"

CLUSTER_NAME=${1}
FROM_OS_RELEASE=${2}
TO_OS_RELEASE=${3}
FROM_DEB_RELEASE=buster
TO_DEB_RELEASE=buster

DOMAIN_NAME=$(ocicli -csv cluster-show ${CLUSTER_NAME} | grep Domain: | cut -d, -f2)

# Make a survey of all the hosts we have in the cluster
ALL_CTRL=$(ocicli -csv machine-list -a | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ONE_CTRL=$(ocicli -csv machine-list -a | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname LIMIT 1" | tr '\n' ' ')
ONE_CTRL_IP=$(echo ${ONE_CTRL} | cut -d, -f1)
ALL_COMP=$(ocicli -csv machine-list -a | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='compute' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_VOL=$(ocicli -csv machine-list -a | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='volume' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_NET=$(ocicli -csv machine-list -a | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='network' AND cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
ALL_NODES=$(ocicli -csv machine-list -a | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE cluster='${CLUSTER_NAME}' AND status='installed' ORDER BY hostname" | tr '\n' ' ')

# Get to know if we have specific nodes for:
# - volume
# - network
# - cephosd
if [ $(ocicli -csv machine-list -a | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='cephosd'") = "0" ] ; then
	HAS_OSD="no"
else
	HAS_OSD="yes"
fi

if [ $(ocicli -csv machine-list -a | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='volume'") = "0" ] ; then
	HAS_VOL="no"
else
	HAS_VOL="yes"
fi

if [ $(ocicli -csv machine-list -a | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='network'") = "0" ] ; then
	HAS_NET="no"
else
	HAS_NET="yes"
fi

if [ $(ocicli cluster-show ${CLUSTER_NAME} | grep "Self signed API cert:" | awk '{print $5}')"" = "yes" ] ; then
	SELF_SIGNED_API_CERT=yes
else
	SELF_SIGNED_API_CERT=no
fi

#########################
### GENERIC FUNCTIONS ###
#########################

RED="\033[1;31m"
NO_COL="\033[0m"
GREEN="\033[1;32m"
green_echo () {
	echo ${GREEN}${1}${NO_COL}
}
red_echo () {
	echo ${RED}${1}${NO_COL}
}

sshi () {
	ssh -i /etc/openstack-cluster-installer/id_rsa $@
}

scpi () {
	scp -i /etc/openstack-cluster-installer/id_rsa $@
}

wait_for_ssh () {
	local COUNT CYCLES OTCI_CAN_SSH SSH_HOST
	SYSUSERNAME=root
	# This is 15 minutes
	COUNT=900
	CYCLES=0
	OTCI_CAN_SSH=no
	SSH_HOST=${1}
	echo -n "${GREEN}---> Waiting for ssh to be up for ${SSH_HOST}:${NO_COL}"
	ssh-keygen -f ~/.ssh/known_hosts -R ${SSH_HOST} 1>/dev/null 2>/dev/null || true
	while [ "${OTCI_CAN_SSH}" != "yes" ] && [ ${COUNT} != 0 ] ; do
		if ssh -i /etc/openstack-cluster-installer/id_rsa -o "ConnectTimeout 2" ${SYSUSERNAME}@${SSH_HOST} 'echo -n ""' 2>/dev/null ; then
			OTCI_CAN_SSH=yes
			echo "${GREEN}ok.${NO_COL}"
		else
			COUNT=$(( ${COUNT} - 1 ))
			CYCLES=$(( ${CYCLES} + 1 ))
			sleep 2
			echo -n "."
		fi
	done
	if [ ${OTCI_CAN_SSH} = "yes" ] ; then
		sleep 1
	else
		echo "${RED}timeout.${NO_COL}"
		exit 1
	fi
}

disable_puppet () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop puppet"
	sshi root@${HOST_IP} "systemctl disable puppet"
	sshi root@${HOST_IP} "systemctl mask puppet"
	sshi root@${HOST_IP} "rm -f /usr/sbin/policy-rc.d"
}

enable_puppet () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask puppet"
	sshi root@${HOST_IP} "systemctl enable puppet"
	sshi root@${HOST_IP} "systemctl start puppet"
}

switch_to_release () {
	local HOST HOST_IP HOST_NAME FROM_DEB_RELEASE FROM_OS_RELEASE TO_DEB_RELEASE TO_OS_RELEASE
	HOST=${1}
	FROM_DEB_RELEASE=${2}
	FROM_OS_RELEASE=${3}
	TO_DEB_RELEASE=${4}
	TO_OS_RELEASE=${5}

	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> Switching ${HOST_NAME} / ${HOST_IP} to ${TO_DEB_RELEASE}-${TO_OS_RELEASE}"
	sshi root@${HOST_IP} "if [ -e /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list ] ; then mv /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list ; fi"
	sshi root@${HOST_IP} "sed -i \"s/${FROM_OS_RELEASE}/${TO_OS_RELEASE}/g\" /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list"
	sshi root@${HOST_IP} "curl http://${TO_DEB_RELEASE}-${TO_OS_RELEASE}.debian.net/debian/dists/pubkey.gpg >pubkey.txt ; apt-key add pubkey.txt ; rm pubkey.txt"
	sshi root@${HOST_IP} "apt-get update"
}

# Example call:
# iterate_on_hosts "192.168.101.2,z-controller-1 192.168.101.3,z-controller-2" switch_to_release buster rocky buster stein
# Example transformation for the first host:
# switch_to_release 192.168.101.2,z-controller-1 buster rocky buster stein
iterate_on_hosts () {
	local HOST_LIST
	HOST_LIST=${1}
	FUNCTION=${2}
	shift
	shift
	for i in ${HOST_LIST} ; do
		${FUNCTION} ${i} $@
	done
}

#####################
### OCI FUNCTIONS ###
#####################

switch_oci_to_release () {
	local FROM_DEB_RELEASE FROM_OS_RELEASE TO_DEB_RELEASE TO_OS_RELEASE
	FROM_DEB_RELEASE=${1}
	FROM_OS_RELEASE=${2}
	TO_DEB_RELEASE=${3}
	TO_OS_RELEASE=${4}

	green_echo "===> Switch OCI to use ${TO_OS_RELEASE}"
	if [ -e /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list ] ; then
		mv /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list
	fi
	# That one should be the standard on all OCI setup
	if [ -e /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list ] ; then
		sed -i "s/${FROM_OS_RELEASE}/${TO_OS_RELEASE}/g" /etc/apt/sources.list.d/${TO_DEB_RELEASE}-${TO_OS_RELEASE}.list
	fi
	# That one is used in the PoC
	if [ -e /etc/apt/sources.list.d/openstack.list ] ; then
		sed -i "s/${FROM_OS_RELEASE}/${TO_OS_RELEASE}/g" /etc/apt/sources.list.d/openstack.list
	fi
	wget http://${TO_DEB_RELEASE}-${TO_OS_RELEASE}.debian.net/debian/dists/pubkey.gpg
	apt-key add pubkey.gpg
	rm pubkey.gpg
	apt-get update
	DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold
	DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold puppet-module-placement
	/etc/init.d/puppet-master restart
	# Looks like what's bellow doesn't work, and we have to use sys-v :(
	#systemctl puppet-master restart
	sed -i s/openstack_release=${FROM_OS_RELEASE}/openstack_release=${TO_OS_RELEASE}/ /etc/openstack-cluster-installer/openstack-cluster-installer.conf
}

########################
### KEYSTONE UPGRADE ###
########################
stop_keystone_services () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping keystone on ${HOST_NAME}"
	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		# If we're with Train, Keystone is still running on Apache, so we disable it there and restart apache.
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:5000//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:35357//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "a2dissite 10-keystone_wsgi_admin.conf ; a2dissite 10-keystone_wsgi_main.conf"
		sshi root@${HOST_IP} "systemctl restart apache2"
		sshi root@${HOST_IP} "HOSTNAME=\$(hostname --fqdn) ; if [ -e /etc/keystone/ssl/private/\${HOSTNAME}.key ] ; then mv /etc/keystone/ssl/private/\${HOSTNAME}.key /etc/keystone/ssl/private/\${HOSTNAME}.pem ; fi"
	fi
	sshi root@${HOST_IP} "systemctl stop keystone || true ; systemctl disable keystone || true"
}

upgrade_keystone () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading keystone on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold keystone"
}

start_keystone_services () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting keystone on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask keystone ; systemctl enable keystone ; systemctl start keystone"
}

cluster_upgrade_keystone () {
	green_echo "===> Stopping Keystone on all controllers"
	iterate_on_hosts "${ALL_CTRL}" stop_keystone_services

	green_echo "===> Upgrading Keystone on all controllers"
	iterate_on_hosts "${ALL_CTRL}" upgrade_keystone

	green_echo "===> Running keystone-manage db_sync"
	sshi root@${ONE_CTRL_IP} "keystone-manage db_sync"

	green_echo "===> Starting keystone on all controllers"
	iterate_on_hosts "${ALL_CTRL}" start_keystone_services
}

#######################
### NEUTRON UPGRADE ###
#######################
upgrade_ovs () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading OVS on ${HOST_NAME}"
	# Because we're rebooting just after this, let's install latest kernel at the same time...

	if sshi root@${HOST_IP} "dpkg-query -W openvswitch-common" ; then
		CUR_OVS_VERSION=$(sshi root@${HOST_IP} "dpkg-query -W openvswitch-common | awk '{print \$2}'" 2>/dev/null)
		NEXT_OVS_VERSION=$(sshi root@${HOST_IP} "apt-cache policy openvswitch-common | grep Candidate: | awk '{print \$2}'" 2>/dev/null)

		if dpkg --compare-versions ${NEXT_OVS_VERSION} gt ${CUR_OVS_VERSION} ; then
			sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold openvswitch-common openvswitch-switch python3-openvswitch linux-image-amd64"

			green_echo "-> Rebooting ${HOST_NAME} after OVS upgrade"
			sshi root@${HOST_IP} "reboot" || true
			sleep 5
			wait_for_ssh ${HOST_IP}

			green_echo "-> Waiting 10 seconds for network to settle on ${HOST_NAME} after OVS upgrade"
			sleep 10
			green_echo "-> Restarting neutron-openvswitch-agent on ${HOST_NAME}"
			sshi root@${HOST_IP} "if [ -e /etc/init.d/neutron-openvswitch-agent ] ; then /etc/init.d/neutron-openvswitch-agent stop ; sleep 1 ; /etc/init.d/neutron-openvswitch-agent start ; fi"

			green_echo "-> Waiting 20 seconds for OVS agent to settle on ${HOST_NAME}"
			sleep 10
			green_echo "-> Restarting neutron-l3-agent on ${HOST_NAME}"
			sshi root@${HOST_IP} "if [ -e /etc/init.d/neutron-l3-agent ] ; then /etc/init.d/neutron-l3-agent stop ; sleep 1 ; /etc/init.d/neutron-l3-agent start ; fi"
		fi
	fi
}

stop_neutron_server () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Neutron on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop neutron-api ; systemctl stop neutron-rpc-server ; systemctl disable neutron-api ; systemctl disable neutron-rpc-server ; systemctl mask neutron-api ; systemctl mask neutron-rpc-server"
}

start_neutron_server () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Neutron on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask neutron-api ; systemctl unmask neutron-rpc-server ; systemctl enable neutron-api ; systemctl enable neutron-rpc-server ; systemctl start neutron-api ; systemctl start neutron-rpc-server"
}

upgrade_neutron_server () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrade Neutron on ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-api"
	sshi root@${HOST_IP} "if dpkg-query -W python3-neutron-fwaas 2>&1 ; then DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron-fwaas ; fi"
}

upgrade_network_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Dist-upgrading ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold"
}

upgrade_compute_neutron_agents () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Neutron on ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent"
}

upgrade_network_agents_on_controller () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Neutron on ${HOST_NAME}"
	# Fix [nova]username is defined twice in neutron.conf
	sshi root@${HOST_IP} "sed -i 's/^username = nova//' /etc/neutron/neutron.conf"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold neutron-common python3-neutron neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent"
}

cluster_upgrade_neutron () {
	green_echo "===> Upgrading openvswitch-switch on all hosts (if needed)"
	if [ "${HAS_NET}" = "yes" ] ; then
		iterate_on_hosts "${ALL_NET}" upgrade_ovs
	else
		iterate_on_hosts "${ALL_CTRL}" upgrade_ovs
	fi
	iterate_on_hosts "${ALL_COMP}" upgrade_ovs

	green_echo "===> Upgrading neutron API and RPC servers on all controllers"
	iterate_on_hosts "${ALL_CTRL}" stop_neutron_server

	green_echo "===> Upgrading neutron API and RPC servers on controllers"
	iterate_on_hosts "${ALL_CTRL}" upgrade_neutron_server

	green_echo "===> Upgrading neutron db on controllers"
	sshi root@${ONE_CTRL_IP} "neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/ml2_conf.ini upgrade head"

	green_echo "===> Restarting Neutron API and RPC servers"
	iterate_on_hosts "${ALL_CTRL}" start_neutron_server

	if [ "${HAS_NET}" = "yes" ] ; then
		green_echo "===> Upgrading Neutron agents on network nodes"
		iterate_on_hosts "${ALL_NET}" upgrade_network_node
	else
		green_echo "===> Upgrading Neutron agents on network nodes"
		iterate_on_hosts "${ALL_CTRL}" upgrade_network_agents_on_controller
	fi

	green_echo "===> Upgrading Neutron agents on compute nodes"
	iterate_on_hosts "${ALL_COMP}" upgrade_compute_neutron_agents
}

####################
### NOVA UPGRADE ###
####################
stop_nova_servers () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Nova on ${HOST_NAME}"
	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		# If we're with Train, Nova is still running on Apache, so we disable it there and restart apache.
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:8774//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "a2dissite 10-nova_api_wsgi.conf"
		sshi root@${HOST_IP} "systemctl restart apache2"
	fi
	if [ "${FROM_OS_RELEASE}" = "stein" ] ; then
		sshi root@${HOST_IP} "apt-get purge -y nova-consoleauth nova-placement-api"
	fi
	for i in nova-api nova-conductor nova-consoleauth nova-novncproxy nova-placement-api nova-scheduler nova-serialproxy nova-spicehtml5proxy nova-xenvncproxy ; do
		sshi root@${HOST_IP} "if [ -e /etc/init.d/$i ] ; then systemctl stop $i ; systemctl disable $i ; systemctl mask $i ; fi"
	done
}

upgrade_nova_servers () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Nova on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold nova-api nova-common nova-conductor nova-consoleproxy nova-scheduler python3-nova"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get remove -y nova-placement-api"
}

start_nova_servers () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Nova on ${HOST_NAME}"
	for i in nova-api nova-conductor nova-consoleauth nova-novncproxy nova-scheduler nova-xenvncproxy nova-api nova-api-metadata ; do
		sshi root@${HOST_IP} "if [ -e /etc/init.d/$i ] ; then systemctl unmask $i ; systemctl enable $i ; systemctl start $i ; fi"
	done
}

upgrade_compute (){
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold"
	sshi root@${HOST_IP} "oci-puppet || true"
}

install_placement (){
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Installing placement on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold placement-api python3-osc-placement python3-placement placement-common"
	# Get the placement connection from nova and set it for placement
	sshi root@${HOST_IP} "if [ -d /etc/placement ] && ! [ -r /etc/placement/placement.conf ] && [ -r /usr/share/placement-common/placement.conf ] ; then cp /usr/share/placement-common/placement.conf /etc/placement ; chown placement:placement /etc/placement/placement.conf ; chmod 640 /etc/placement/placement.conf ; fi"
	sshi root@${HOST_IP} ". /usr/share/openstack-pkg-tools/pkgos_func ; TMPFILE=\$(mktemp); cat /etc/nova/nova.conf | grep -v '^#' >\${TMPFILE} ; pkgos_inifile get \${TMPFILE} placement_database connection ; rm -f \${TMPFILE} ; pkgos_inifile set /etc/placement/placement.conf placement_database connection \${RET}"
}

cluster_upgrade_nova () {
	green_echo "===> Upgrading nova on controllers"
	iterate_on_hosts "${ALL_CTRL}" stop_nova_servers
	iterate_on_hosts "${ALL_CTRL}" upgrade_nova_servers
	green_echo "-> Running nova-manage api_db sync"
	sshi root@${ONE_CTRL_IP} "nova-manage api_db sync"
	green_echo "-> Running nova-manage db sync"
	sshi root@${ONE_CTRL_IP} "nova-manage db sync"
	iterate_on_hosts "${ALL_CTRL}" start_nova_servers

	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		iterate_on_hosts "${ALL_CTRL}" install_placement
		sshi root@${ONE_CTRL_IP} "placement-manage db stamp b4ed3a175331"
		sshi root@${ONE_CTRL_IP} "placement-manage db sync"
	fi

	green_echo "===> dist-upgrade of all computes"
	iterate_on_hosts "${ALL_COMP}" upgrade_compute
	green_echo "===> Running nova-manage db online_data_migrations"
	sshi root@${ONE_CTRL_IP} "nova-manage db online_data_migrations || true"
}

######################
### CINDER UPGRADE ###
######################
stop_cinder_volume_and_backup () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Cinder backup and volume on ${HOST_NAME}"
	for i in cinder-volume cinder-backup ; do
		sshi root@${HOST_IP} "systemctl stop $i ; systemctl disable $i ; systemctl mask $i"
	done
}

start_cinder_volume_and_backup () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Cinder backup and volume on ${HOST_NAME}"
	for i in cinder-volume cinder-backup ; do
		sshi root@${HOST_IP} "systemctl unmask $i ; systemctl enable $i ; systemctl start $i"
	done
}

stop_cinder_servers () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Cinder API and Scheduler on ${HOST_NAME}"
	if [ "${FROM_OS_RELEASE}" = "rocky" ] ; then
		# If we're with Train, Cinder is still running on Apache, so we disable it there and restart apache.
		sshi root@${HOST_IP} "sed -i \"s/Listen ${HOST_IP}:8776//\" /etc/apache2/ports.conf"
		sshi root@${HOST_IP} "a2dissite 10-cinder_wsgi.conf"
		sshi root@${HOST_IP} "systemctl restart apache2"
	fi
	for i in cinder-api cinder-scheduler ; do
		sshi root@${HOST_IP} "systemctl stop $i ; systemctl disable $i ; systemctl mask $i"
	done
}

upgrade_cinder_controller () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Cinder on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold python3-cinder cinder-scheduler cinder-common cinder-api"
}

start_cinder_servers () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Cinder API and Scheduler on ${HOST_NAME}"
	for i in cinder-api cinder-scheduler ; do
		sshi root@${HOST_IP} "systemctl unmask $i ; systemctl enable $i ; systemctl start $i"
	done
}

start_cinder_volume_and_backup () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Cinder Volume and Backup on ${HOST_NAME}"
	for i in cinder-volume cinder-backup ; do
		sshi root@${HOST_IP} "if dpkg-query -W $i ; then systemctl unmask $i ; systemctl enable $i ; systemctl start $i ; fi"
	done
}

dist_upgrade_volume_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> dist-upgrade on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold"
	sshi root@${HOST_IP} "oci-puppet || true"
}

cluster_upgrade_cinder () {
	if [ "${HAS_VOL}" = "yes" ] ; then
		green_echo "===> Stopping all cinder-volume services on volume nodes"
		iterate_on_hosts "${ALL_VOL}" stop_cinder_volume_and_backup
	fi
	if [ "${HAS_OSD}" = "yes" ] ; then
		green_echo "===> Stopping all cinder-volume services on compute nodes"
		iterate_on_hosts "${ALL_COMP}" stop_cinder_volume_and_backup
	fi

	green_echo "===> Stopping cinder on controllers"
	iterate_on_hosts "${ALL_CTRL}" stop_cinder_servers
	green_echo "===> Upgrading cinder on all controllers"
	iterate_on_hosts "${ALL_CTRL}" upgrade_cinder_controller
	sshi root@${ONE_CTRL_IP} "cinder-manage db sync"
	green_echo "===> Starting cinder on all controllers"
	iterate_on_hosts "${ALL_CTRL}" start_cinder_servers

	if [ "${HAS_VOL}" = "yes" ] ; then
		green_echo "===> Dist-upgrading cinder-volume nodes"
		iterate_on_hosts "${ALL_VOL}" dist_upgrade_volume_node
		green_echo "===> Starting cinder-volume services on volume nodes"
		iterate_on_hosts "${ALL_VOL}" start_cinder_volume_and_backup
	fi
	if [ "${HAS_OSD}" = "yes" ] ; then
		green_echo "===> Starting cinder-volume services on compute nodes"
		iterate_on_hosts "${ALL_VOL}" start_cinder_volume_and_backup
	fi
}

######################
### GLANCE UPGRADE ###
######################
stop_glance_api () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping Glance API on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop glance-api ; systemctl disable glance-api ; systemctl mask glance-api"
}

upgrade_glance () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading Glance API on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold glance glance-common glance-api python3-glance"
}

start_glance_api () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting Glance API on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask glance-api ; systemctl enable glance-api ; systemctl start glance-api"
}

cluster_upgrade_glance () {
	green_echo "===> Stopping glance-api on controllers"
	iterate_on_hosts "${ALL_CTRL}" stop_glance_api

	green_echo "===> Upgrading glance on controllers"
	iterate_on_hosts "${ALL_CTRL}" upgrade_glance
	sshi root@${ONE_CTRL_IP} "glance-manage db sync"

	green_echo "===> Starting glance-api on controllers"
	iterate_on_hosts "${ALL_CTRL}" start_glance_api
}

#######################
### HORIZON REMOVAL ###
#######################
# It's actually easier to just remove horizon and wait until
# puppet reinstalls it, rather than attempting an upgrade.

remove_horizon () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Removing horizon on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get purge -y -o Dpkg::Options::=--force-confold python3-django-horizon"
}

cluster_remove_horizon () {
	green_echo "===> Removing horizon"
	iterate_on_hosts "${ALL_CTRL}" remove_horizon
}


################
### FINALIZE ###
################
distupgrade_everyone () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Dist-upgrading ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold"
}

run_puppet_on_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Running puppet on ${HOST_NAME}"
	sshi root@${HOST_IP} "oci-puppet || true"
}

restart_all_services_on_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	shift
	for i in $@ ; do
		green_echo "-> Restarting ${i} on ${HOST_NAME}"
		sshi root@${HOST_IP} "if [ -e /etc/init.d/${i} ] ; then /etc/init.d/${i} stop ; sleep 1 ; /etc/init.d/${i} start ; fi"
	done
}

cluster_upgrade_finilize () {
	green_echo "===> Distupgrading everyone"
	iterate_on_hosts "${ALL_NODES}" distupgrade_everyone

	green_echo "===> Activating initial setup"
	ocicli cluster-set ${CLUSTER_NAME} --initial-cluster-setup yes

	green_echo "===> Run puppet on all nodes"
	iterate_on_hosts "${ALL_NODES}" run_puppet_on_node

	green_echo "===> Removing initial setup"
	ocicli cluster-set ${CLUSTER_NAME} --initial-cluster-setup no

	green_echo "===> Restarting all services everywhere"
	iterate_on_hosts "${ALL_CTRL}" restart_all_services_on_node ${ALL_SERVICES_CTRL}
	iterate_on_hosts "${ALL_COMP}" restart_all_services_on_node ${ALL_SERVICES_COMP}
	iterate_on_hosts "${ALL_VOL}" restart_all_services_on_node ${ALL_SERVICES_VOL}
	iterate_on_hosts "${ALL_NET}" restart_all_services_on_node ${ALL_SERVICES_NET}

	green_echo "===> Enabling puppet everywhere"
	iterate_on_hosts "${ALL_NODES}" enable_puppet
}

#######################
### START OF SCRIPT ###
#######################

green_echo "===> Disabling puppet everywhere"
iterate_on_hosts "${ALL_NODES}" disable_puppet

switch_oci_to_release ${FROM_DEB_RELEASE} ${FROM_OS_RELEASE} ${TO_DEB_RELEASE} ${TO_OS_RELEASE}

green_echo "===> Switch the sources.list everywhere to ${TO_DEB_RELEASE}-${TO_OS_RELEASE}"
iterate_on_hosts "${ALL_NODES}" switch_to_release ${FROM_DEB_RELEASE} ${FROM_OS_RELEASE} ${TO_DEB_RELEASE} ${TO_OS_RELEASE}

cluster_upgrade_keystone
cluster_upgrade_neutron
cluster_upgrade_nova
if [ "${HAS_VOL}" = "yes" ] || [ "${HAS_OSD}" = "yes" ] ; then
	cluster_upgrade_cinder
fi
cluster_upgrade_glance
cluster_remove_horizon
cluster_upgrade_finilize
