get-all-logs/get-all-logs.sh

#!/bin/bash
version="23/02/23"

PRMT="[$(whoami)@$(hostname -s) ]"
for ONE in vm dci;
do 
	test -d /opt/ispsystem/${ONE} && PLATFORM=${ONE};
done
	LicenseToken=$(grep -oP '\"LicenseToken\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"LicenseToken\"://g' -e 's/\"//g')
	CurrentVersion=$(grep -oP '\"CurrentVersion\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"CurrentVersion\"://g' -e 's/\"//g')
	Stage=$(grep -oP '\"Stage\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"Stage\"://g' -e 's/\"//g')
	DomainName=$(grep -oP '\"DomainName\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"DomainName\"://g' -e 's/\"//g')


about_platform(){
# информация о платформе
#

if [[ -z $logtype ]];then
	logtype=$1;
fi;	


if [[ -z "${PLATFORM}" ]]; then
	echo "No ISPsystem Software found" && exit;
else
	LOGDIR="${HOME}/${PLATFORM}_${logtype}_$(date +"%Y_%m_%d_%H_%M_%S")" && mkdir -p ${LOGDIR}
	ARCNAME=$(date +"${HOME}/${PLATFORM}_${logtype}_%Y_%m_%d_%H_%M_%S.tar.gz")
	cp /opt/ispsystem/${PLATFORM}/config.json ${LOGDIR}
	cp /opt/ispsystem/${PLATFORM}/install.log ${LOGDIR}
	cp /opt/ispsystem/${PLATFORM}/docker-compose.yaml ${LOGDIR}

fi;

}

newlog() {
#делаем файл лога	
 LOGINFO=${LOGDIR}/$1 && date > ${LOGINFO}
}

cmdlog() {
# выводим в stdout и выполняем команду
 test -z "${LOGINFO}" && return
 local CMD=$*;(echo "${PRMT} ${CMD}";echo;eval ${CMD};echo) >> ${LOGINFO}
}

about_server() {
#информация о сервере	
newlog hostinfo_$(hostname).log
echo "log $LOGINFO"
cmdlog hostnamectl
cmdlog cat /etc/*release
cmdlog lscpu
cmdlog timedatectl
cmdlog free -m 
cmdlog df -h --exclude tmpfs --exclude devtmpfs --exclude squashfs --exclude overlay
cmdlog ip -br a
}

about_docker(){
#информация о docker	
newlog docker_$(hostname).log
cmdlog docker version
cmdlog docker ps
cmdlog "docker inspect --format ' {{.Name}} - {{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq)"
}

plaform_logs(){
# логи из docker контейнеров	
	DOCKER_CONTAINER_NAMES=`docker ps --format '{{.Names}}'`

	SERVICES=($DOCKER_CONTAINER_NAMES)
	cd ${LOGDIR}
	for service in ${SERVICES[@]}
		do
		 echo -e "----\033[0;31mCopying logs from $service\033[0m----\n"
	 	mkdir -p $service
	 	docker cp $service:/var/log/. $service/.
	 	docker logs $service > $service/${service}_stdout.log 2>&1
	done
}

about_license(){
# информация о лицензии	
	curl -m 180 -vvv -k "https://license6.ispsystem.com/lic6/check" -H "X-auth-token:$LicenseToken" > ${LOGDIR}/license_${PLATFORM}.log 2>&1
	echo -e "\n${PLATFORM} Version:$CurrentVersion\n" >>${LOGDIR}/license_${PLATFORM}.log;
	echo "${PLATFORM} LicenseToken:$LicenseToken" >>${LOGDIR}/license_${PLATFORM}.log;
	echo "${PLATFORM} Stage:$Stage" >>${LOGDIR}/license_${PLATFORM}.log; 
	echo "${PLATFORM} DomainName:$DomainName" >>${LOGDIR}/license_${PLATFORM}.log;

	auth_back_dir="${LOGDIR}/${PLATFORM}_auth_back_1" && mkdir -p ${auth_back_dir}
	input_dir="${LOGDIR}/${PLATFORM}_input_1" && mkdir -p ${input_dir}
	docker cp ${PLATFORM}_auth_back_1:/var/log/. ${auth_back_dir} 
	docker cp ${PLATFORM}_input_1:/var/log/nginx/. ${input_dir}
}

finish() {
# пакуем логи и выводим общую информацию о платформе в конце сбора	
	tar -cvzf ${ARCNAME} -C ${LOGDIR} .
	echo -e "\nISPsystem Software found:\n";
	echo "${PLATFORM} DomainName:$DomainName";	
	echo "${PLATFORM} LicenseToken:$LicenseToken"
	dist_type=$(curl -s -m 120 -k "https://license6.ispsystem.com/lic6/check" -H "X-auth-token:5a99a084a0334c808143708cfbc24ec7:iWuiY_A5q02jwZlKcFV0Yw" | grep -Po '"dist_type":.*?[^\\]"' | sed -e 's/\"dist_type\"://g' -e 's/\"//g')
	echo "${PLATFORM} CurrentVersion:$CurrentVersion $dist_type"
	echo "${PLATFORM} Stage:$Stage"
	echo -e "\ndirectory with logs $LOGDIR\n"
	echo -e "Please send this logs archive to ISP system support\n"
	du -h ${ARCNAME}	
}

mysql_collect(){
# выполнение запросов к БД	
	logfile=$1;
	query=$2;
	fullcmd="docker exec -it mysql bash -c \"mysql isp -p\\\$MYSQL_ROOT_PASSWORD -e "$query"\" >> ${LOGDIR}/$logfile";
	
    echo $fullcmd;
    echo -e "\n exec $query \n">> ${LOGDIR}/$logfile;    
    eval $fullcmd;
}


node_collect() {
# выполнение команды на узле	
    ip_addr=$1;
    ssh_port=$2;
    cmd=$4;
    logfile=$3;
	docker exec --tty --interactive vm_box ssh -o BatchMode=yes -o ConnectTimeout=5 -o "StrictHostKeyChecking no" -q -i /opt/ispsystem/vm/etc/.ssh/vmmgr.1 $ip_addr -p $ssh_port 'exit 0';
	_RCODE=$?
	if [ $_RCODE -ne 0 ]
	then
#    	echo "unable to ssh, host is not accessible";
#    	connected=0;
    	echo "node $ip_addr on $node_port not connected!";
      	echo "node $ip_addr on $node_port not connected!">> ${LOGDIR}/$logfile;
      	echo -e "\n not executed $cmd \n">> ${LOGDIR}/$logfile;    
	else
#    	echo "connected $ip_addr $ssh_port";
#    	connected=1;
      	fullcmd="docker exec --tty --interactive vm_box ssh -o \"StrictHostKeyChecking no\" -i /opt/ispsystem/vm/etc/.ssh/vmmgr.1 $ip_addr -p $ssh_port $cmd >> ${LOGDIR}/$logfile";
      	echo $fullcmd;
      	echo -e "\n $cmd \n">> ${LOGDIR}/$logfile;    
      	eval $fullcmd;
	fi
}

_help(){
# справка о скрипте

echo -e "git.isptech.ru:ISPsystem/get-all-logs.git \n Log's script ver.: $version Usage : $0 [arguments]";
echo -e "\n  arguments:\n";
echo -e "    -logs or 1";
echo -e "    -short-logs or 2";
echo -e "    -lic or 3";
echo -e "    -node or 4 <IP-address> <SSH-port>\n";

PS3='Please enter your choice(enter number): '
# интерактивное меню
options=("Full logs" "Short logs" "License's logs" "Node's logs" "Quit")
select opt in "${options[@]}"
	do
	    case $opt in
	        "Full logs")
				logtype="logs";
				about_platform;
				_logs;
				break;
	            ;;
	        "Short logs")
				logtype="short-logs";
				about_platform;
				_short_logs;
				break;
	            ;;
	        "License's logs")
				logtype="lic";
				about_platform;
	           	_lic;
	           	break;
	            ;;
	        "Node's logs")
				logtype="node";
	        	read -p "Enter IP address: " node_ip
	        	read -p "Enter port: " node_port
	        	about_platform;
				_node $node_ip $node_port;	        	
				break;
	            ;;
	        "Quit")
	            break
	            ;;
	        *) echo "invalid option $REPLY";;
 	   esac
	done

}

_gen_key() {
	
}

_logs() {
# общая функция сбора полных логов	
	about_server;
	about_docker;
	about_license;
	plaform_logs;
	finish;
}

_short_logs() {
# общая функция сбора сокращенных логов	
	about_server;
	about_docker;
	about_license;
	plaform_logs;
	find ${LOGDIR} -type f -mtime +7 -delete;
	finish;
}

_lic() {
# общая функция сбора информации о лицензии	
	about_server;
	about_docker;
	about_license;
	finish;
}

_vm_host() {
# планируется сбор логов о виртуальной машине	
# так можно сделать проверку переданного параметра, либо id либо имя ВМ,если начинается с символа решетки значит id
# echo '#12444' | grep -Pq '^#[0-9]+$' && echo by-id || echo by-name
	echo "it's coming soon!";
	vm_name=$1;
	vm_id=$2;
}

_node() {
# общая функция сбор логов об узле	
	node_ip=$1;
	node_port=$2;
	mysql_collect "${node_ip}_db.log" "'select id,name,comment,cluster,ram_mib,max_vcpu_number,cpu_number,ip_addr,ssh_port,host_max_num,state,connection_error,libvirt_error,gray_ips,disk_params,property,host_creation_blocked,mon_installed,mon_install_date,metrics_timestamp,os_version,kernel_version,uptime,libvirt_version,qemu_version,overselling,hostname,vnc_addr,selinux_enforcing,problems,hdd_overselling,ip_addr_v6,bird_error,lxd_error,lxd_version,host_limit,socket_number,ha_state,ha_disable_failed,ha_error,update_dc_networks_failed,network_settings_lock,network_error,frr_error,host_filter,vm5_data,additional_data,vcpu_limit,spice_error from vm_node where ip_addr=\\\"$node_ip\\\"\\G;'"
declare -A cmd
cmd["0 hostnamectl"]="hostnamectl";
cmd["1 release"]="\"cat /etc/*release\"";
cmd["2 dmesg"]="\"dmesg | grep -i -E 'error|failed|critical|bug|panic'\"";
cmd["3 services"]="\"journalctl | grep -i -E 'error|failed|critical|bug|panic'\"";
cmd["4 services"]="\"systemctl status libvirtd -l\"";
cmd["5 services"]="\"systemctl status gomon -l\"";
cmd["6 services"]="\"systemctl show libvirtd\"";
cmd["7 sshd_config"]="\"cat /etc/ssh/sshd_config\"";
cmd["8 hwinfo"]="lscpu";
cmd["9 hwinfo"]="lscpu | grep -i -E 'socket|core|thread'";
cmd["10 hwinfo"]="dmidecode";
cmd["11 time"]="timedatectl";
cmd["12 time"]="uptime";
cmd["13 time"]="\"last reboot | head -10\"";
cmd["14 virsh"]="\"virsh list --all\"";
cmd["15 virsh"]="\"virsh pool-list --all\"";
cmd["16 virsh"]="\"virt-host-validate\"";
cmd["17 mem"]="\"free -m\"";
cmd["18 network"]="\"ip -br a\"";
cmd["19 network"]="\"brctl show\"";
cmd["20 network"]="\"curl -v download.ispsystem.com\"";
cmd["21 firewall"]="\"systemctl status nftables\"";
cmd["22 firewall"]="\"systemctl status firewalld\"";
cmd["23 firewall"]="\"firewall-cmd --list-ports\"";
cmd["24 firewall"]="\"nft list ruleset\"";
cmd["25 firewall"]="\"ss -tulpn | grep 16514\"";
cmd["26 firewall"]="\"ss -tulpn '( sport >= 49152 and sport <= 49215 )'\"";
cmd["27 firewall"]="\"ss -tulpn '( sport >= 5900 and sport <= 6900 )'\"";
cmd["28 firewall"]="\"ss -tulpn '( sport = 179 or sport = 4789 )'\"";
cmd["29 firewall"]="\"ss -tulpn | grep 16514\"";
cmd["30 firewall"]="\"ss -tulpn | grep -E ':179|:4789'\"";
cmd["31 disks"]="\"df -h --exclude tmpfs --exclude devtmpfs --exclude squashfs --exclude overlay\"";
cmd["32 disks"]="\"df -i\"";
cmd["33 disks"]="lsblk";
cmd["34 virsh"]="\"lvdisplay\"";
cmd["35 virsh"]="\"pvdisplay\"";
cmd["36 virsh"]="\"vgdisplay\"";
cmd["37 packages"]="\"yum repolist\"";
cmd["38 packages"]="\"yum history list\"";

for k in "${!cmd[@]}";do
typelog=$(echo $k | cut -d ' ' -f2);
	#echo $typelog;
	#echo -e "$k=${cmd[$k]}";
	node_collect $node_ip $node_port "${node_ip}_${typelog}.log" "${cmd[$k]}";
done

	echo "collected logs about node $node_ip $node_port";
	finish;
}

# обработка аргументов
if [[ "$1" == "-logs" || "$1" == "1" ]]; then
	logtype="logs";
	about_platform;
	_logs;
elif [[ "$1" == "-short-logs" || "$1" == "2"  ]]; then
	logtype="short-logs";
	about_platform;
	_short_logs;
elif [[ "$1" == "-lic" || "$1" == "3"  ]]; then
	logtype="lic";
	about_platform;
	_lic;
elif [[ "$1" == "-node" || "$1" == "4"  ]]; then
	logtype="node";
	if [[ -z $2 || -z $3 ]]; then
		echo "please set IP address and port of node!";
       	read -p "Enter IP address: " node_ip
       	read -p "Enter port: " node_port
       	about_platform;
		_node $node_ip $node_port;	        	
		exit;
	else
		about_platform;
		_node $2 $3;
	fi	
#elif [[ "$1" == "-vm-host" || "$1" == "5"  ]]; then
#	_vm_host;
else
	_help;
fi