408 lines
13 KiB
Bash
408 lines
13 KiB
Bash
#!/bin/bash
|
|
|
|
# about script
|
|
version="17/06/2023"
|
|
|
|
PRMT="[$(whoami)@$(hostname -s) ]"
|
|
|
|
function askYesNo {
|
|
QUESTION=$1
|
|
DEFAULT=$2
|
|
if [ "$DEFAULT" = true ]; then
|
|
OPTIONS="[Y/n]"
|
|
DEFAULT="y"
|
|
else
|
|
OPTIONS="[y/N]"
|
|
DEFAULT="n"
|
|
fi
|
|
read -p "$QUESTION $OPTIONS " -n 1 -s -r INPUT
|
|
INPUT=${INPUT:-${DEFAULT}}
|
|
echo ${INPUT}
|
|
if [[ "$INPUT" =~ ^[yY]$ ]]; then
|
|
ANSWER=true
|
|
else
|
|
ANSWER=false
|
|
fi
|
|
}
|
|
|
|
_logs3days(){
|
|
files3d=($(find . -maxdepth 1 ! -iname "*log*.sh" -mtime -3 -iname "*logs*" -o -mtime -3 -iname "*vm_lic*" -o -mtime -3 -iname "*vm_node*" -o -mtime -3 -iname "*short-logs*"));
|
|
if [[ ${#files3d[@]} >0 ]];then
|
|
echo -e "\n*********************** platform's logs for last 3 days *************************\n";
|
|
for item3d in ${files3d[*]}
|
|
do
|
|
# printf " %s\n" $item
|
|
du -sh $item3d
|
|
done
|
|
echo -e "\n*********************************************************************************\n";
|
|
elif [[ ${#files3d[@]} == 0 ]];then
|
|
echo -e "\n************************** Logs are not founded:( You Can Do It! ****************\n";
|
|
fi;
|
|
}
|
|
|
|
_remove_logs (){
|
|
files=($(find . -maxdepth 1 -mtime +3 -iname "*logs*" -o -mtime +3 -iname "*vm_lic*" -o -mtime +3 -iname "*vm_node*" -o -mtime +3 -iname "*short-logs*" ! -iname "get*log*.sh"));
|
|
|
|
if [[ ${#files[@]} >0 ]];then
|
|
echo -e "\n************* Founded platform logs older than last 3 days **********************\n";
|
|
for item in ${files[*]}
|
|
do
|
|
# printf " %s\n" $item
|
|
du -sh $item
|
|
done
|
|
|
|
askYesNo "Do you want to delete these logs?" false;
|
|
DOIT=$ANSWER;
|
|
|
|
if [ "$DOIT" = true ]; then
|
|
for item in ${files[*]}
|
|
do
|
|
printf "\n>>>>> Deleting %s \n" $item;
|
|
rm -rfv $item;
|
|
done
|
|
echo "Platform logs older than 3 days have been deleted!";
|
|
fi;
|
|
fi;
|
|
}
|
|
|
|
about_platform(){
|
|
# common info about platform
|
|
#
|
|
|
|
if [[ -z $logtype ]];then
|
|
logtype=$1;
|
|
fi;
|
|
|
|
for ONE in vm dci;
|
|
do
|
|
test -d /opt/ispsystem/${ONE} && PLATFORM=${ONE};
|
|
done
|
|
|
|
if [[ -z "${PLATFORM}" ]]; then
|
|
echo "No ISPsystem Software found" && exit;
|
|
else
|
|
LOGDIR="${HOME}/${PLATFORM}_${logtype}_$(date +"%Y_%m_%d_%H_%M_%S")" && mkdir -p ${LOGDIR}
|
|
ARCNAME=$(date +"${HOME}/${PLATFORM}_${logtype}_%Y_%m_%d_%H_%M_%S.tar.gz")
|
|
LicenseToken=$(grep -oP '\"LicenseToken\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"LicenseToken\"://g' -e 's/\"//g')
|
|
CurrentVersion=$(grep -oP '\"CurrentVersion\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"CurrentVersion\"://g' -e 's/\"//g')
|
|
Stage=$(grep -oP '\"Stage\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"Stage\"://g' -e 's/\"//g')
|
|
DomainName=$(grep -oP '\"DomainName\":"[^"]*\"' /opt/ispsystem/${PLATFORM}/config.json | sed -e 's/\"DomainName\"://g' -e 's/\"//g')
|
|
cp /opt/ispsystem/${PLATFORM}/config.json ${LOGDIR}
|
|
cp /opt/ispsystem/${PLATFORM}/install.log ${LOGDIR}
|
|
cp /opt/ispsystem/${PLATFORM}/docker-compose.yaml ${LOGDIR}
|
|
|
|
fi;
|
|
|
|
}
|
|
|
|
newlog() {
|
|
# make new logfile
|
|
LOGINFO=${LOGDIR}/$1 && date > ${LOGINFO}
|
|
}
|
|
|
|
cmdlog() {
|
|
# output to stdout and running the command
|
|
test -z "${LOGINFO}" && return
|
|
local CMD=$*;(echo "${PRMT} ${CMD}";echo;eval ${CMD};echo) >> ${LOGINFO}
|
|
}
|
|
|
|
about_server() {
|
|
#common info about server
|
|
newlog hostinfo_$(hostname).log
|
|
echo "log $LOGINFO"
|
|
cmdlog hostnamectl
|
|
cmdlog cat /etc/*release
|
|
cmdlog lscpu
|
|
cmdlog timedatectl
|
|
cmdlog free -m
|
|
cmdlog df -h --exclude tmpfs --exclude devtmpfs --exclude squashfs --exclude overlay
|
|
cmdlog ip -br a
|
|
}
|
|
|
|
about_docker(){
|
|
# collecting info about docker configuration
|
|
newlog docker_$(hostname).log
|
|
cmdlog docker version
|
|
cmdlog docker ps
|
|
cmdlog "docker inspect --format ' {{.Name}} - {{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq)"
|
|
}
|
|
|
|
plaform_logs(){
|
|
# logs from docker
|
|
DOCKER_CONTAINER_NAMES=`docker ps --format '{{.Names}}'`
|
|
|
|
SERVICES=($DOCKER_CONTAINER_NAMES)
|
|
cd ${LOGDIR}
|
|
for service in ${SERVICES[@]}
|
|
do
|
|
echo -e "----\033[0;31mCopying logs from $service\033[0m----\n"
|
|
mkdir -p $service
|
|
docker cp $service:/var/log/. $service/.
|
|
docker logs $service > $service/${service}_stdout.log 2>&1
|
|
done
|
|
}
|
|
|
|
about_license(){
|
|
# all logs about license
|
|
curl -m 180 -vvv -k "https://license6.ispsystem.com/lic6/check" -H "X-auth-token:$LicenseToken" > ${LOGDIR}/license_${PLATFORM}.log 2>&1
|
|
echo -e "\n${PLATFORM} Version:$CurrentVersion\n" >>${LOGDIR}/license_${PLATFORM}.log;
|
|
echo "${PLATFORM} LicenseToken:$LicenseToken" >>${LOGDIR}/license_${PLATFORM}.log;
|
|
echo "${PLATFORM} Stage:$Stage" >>${LOGDIR}/license_${PLATFORM}.log;
|
|
echo "${PLATFORM} DomainName:$DomainName" >>${LOGDIR}/license_${PLATFORM}.log;
|
|
|
|
auth_back_dir="${LOGDIR}/${PLATFORM}_auth_back_1" && mkdir -p ${auth_back_dir}
|
|
input_dir="${LOGDIR}/${PLATFORM}_input_1" && mkdir -p ${input_dir}
|
|
docker cp ${PLATFORM}_auth_back_1:/var/log/. ${auth_back_dir}
|
|
docker cp ${PLATFORM}_input_1:/var/log/nginx/. ${input_dir}
|
|
}
|
|
|
|
finish() {
|
|
# packing logs and printing common info at the end.
|
|
tar -cvzf ${ARCNAME} -C ${LOGDIR} .
|
|
echo -e "\nISPsystem Software found:\n";
|
|
echo "${PLATFORM} DomainName: https://$DomainName";
|
|
echo "${PLATFORM} LicenseToken: $LicenseToken"
|
|
dist_type=$(curl -s -m 120 -k "https://license6.ispsystem.com/lic6/check" -H "X-auth-token:5a99a084a0334c808143708cfbc24ec7:iWuiY_A5q02jwZlKcFV0Yw" | grep -Po '"dist_type":.*?[^\\]"' | sed -e 's/\"dist_type\"://g' -e 's/\"//g')
|
|
echo "${PLATFORM} CurrentVersion: $CurrentVersion $dist_type"
|
|
echo "${PLATFORM} Stage: $Stage"
|
|
|
|
echo -e "\ndirectory with logs $LOGDIR\n"
|
|
echo -e "Please send this logs archive to ISP system support\n"
|
|
du -h ${ARCNAME}
|
|
_logs3days;
|
|
_remove_logs;
|
|
}
|
|
|
|
mysql_collect(){
|
|
# running query to DB
|
|
logfile=$1;
|
|
query=$2;
|
|
fullcmd="docker exec -it mysql bash -c \"mysql isp -p\\\$MYSQL_ROOT_PASSWORD -e "$query"\" >> ${LOGDIR}/$logfile";
|
|
|
|
echo $fullcmd;
|
|
echo -e "\n exec $query \n">> ${LOGDIR}/$logfile;
|
|
eval $fullcmd;
|
|
}
|
|
|
|
|
|
node_collect() {
|
|
# running commands on the node
|
|
ip_addr=$1;
|
|
ssh_port=$2;
|
|
cmd=$4;
|
|
logfile=$3;
|
|
docker exec --tty --interactive vm_box ssh -o BatchMode=yes -o ConnectTimeout=5 -o "StrictHostKeyChecking no" -q -i /opt/ispsystem/vm/etc/.ssh/vmmgr.1 $ip_addr -p $ssh_port 'exit 0';
|
|
_RCODE=$?
|
|
if [ $_RCODE -ne 0 ]
|
|
then
|
|
# echo "unable to ssh, host is not accessible";
|
|
echo "node $ip_addr on $node_port not connected!";
|
|
echo "node $ip_addr on $node_port not connected!">> ${LOGDIR}/$logfile;
|
|
echo -e "\n not executed $cmd \n">> ${LOGDIR}/$logfile;
|
|
else
|
|
# echo "connected $ip_addr $ssh_port";
|
|
fullcmd="docker exec --tty --interactive vm_box ssh -o \"StrictHostKeyChecking no\" -i /opt/ispsystem/vm/etc/.ssh/vmmgr.1 $ip_addr -p $ssh_port $cmd >> ${LOGDIR}/$logfile";
|
|
echo $fullcmd;
|
|
echo -e "\n $cmd \n">> ${LOGDIR}/$logfile;
|
|
eval $fullcmd;
|
|
fi
|
|
}
|
|
|
|
_help(){
|
|
echo -e "*********************************************************************************";
|
|
echo -e "*\t\t\tISPsystem technical support\t\t\t\t*";
|
|
echo -e "*\t\t\tLog's script ver.: $version\t\t\t\t*";
|
|
echo -e "*\t\t\t$0 [arguments]\t\t\t*";
|
|
echo -e "*********************************************************************************";
|
|
echo -e "\n\targuments:\n";
|
|
echo -e "\t-logs or 1\t\t\t\t all logs from platform";
|
|
echo -e "\t-short-logs or 2\t\t\t logs for only last 7 days";
|
|
echo -e "\t-lic or 3\t\t\t\t logs about license";
|
|
echo -e "\t-node or 4 <IP-address> <SSH-port>\t node's logs";
|
|
echo -e "\t-clear or 5\t\t\t\t deleting logs older 3 days";
|
|
|
|
_logs3days;
|
|
_remove_logs;
|
|
|
|
echo -e "\t\t\t\t *** MENU ***\t\t\t\t\n";
|
|
PS3='Please enter your choice(enter number): '
|
|
# interactive menu
|
|
options=("Full logs" "Short logs" "License's logs" "Node's logs" "Clear logs" "Quit")
|
|
select opt in "${options[@]}"
|
|
do
|
|
case $opt in
|
|
"Full logs")
|
|
logtype="logs";
|
|
about_platform;
|
|
_logs;
|
|
break;
|
|
;;
|
|
"Short logs")
|
|
logtype="short-logs";
|
|
about_platform;
|
|
_short_logs;
|
|
break;
|
|
;;
|
|
"License's logs")
|
|
logtype="lic";
|
|
about_platform;
|
|
_lic;
|
|
break;
|
|
;;
|
|
"Node's logs")
|
|
logtype="node";
|
|
read -p "Enter IP address: " node_ip
|
|
read -p "Enter port: " node_port
|
|
about_platform;
|
|
_node $node_ip $node_port;
|
|
break;
|
|
;;
|
|
"Clear logs")
|
|
_remove_logs;
|
|
break
|
|
;;
|
|
"Quit")
|
|
break;
|
|
;;
|
|
|
|
*) echo "invalid option $REPLY";;
|
|
esac
|
|
done
|
|
|
|
}
|
|
|
|
_logs() {
|
|
# main function collect all logs
|
|
about_server;
|
|
about_docker;
|
|
about_license;
|
|
plaform_logs;
|
|
finish;
|
|
}
|
|
|
|
_short_logs() {
|
|
# main function collect logs for 7 days
|
|
about_server;
|
|
about_docker;
|
|
about_license;
|
|
plaform_logs;
|
|
find ${LOGDIR} -type f -mtime +7 -delete;
|
|
finish;
|
|
}
|
|
|
|
_lic() {
|
|
# main function about license
|
|
about_server;
|
|
about_docker;
|
|
about_license;
|
|
finish;
|
|
}
|
|
|
|
_vm_host() {
|
|
# collecting info about vm
|
|
# id or VM's name checking params
|
|
# echo '#12444' | grep -Pq '^#[0-9]+$' && echo by-id || echo by-name
|
|
echo "it's coming soon!";
|
|
vm_name=$1;
|
|
vm_id=$2;
|
|
}
|
|
|
|
_node() {
|
|
# main function collecting info about node
|
|
node_ip=$1;
|
|
node_port=$2;
|
|
mysql_collect "${node_ip}_db.log" "'select id,name,comment,cluster,ram_mib,max_vcpu_number,cpu_number,ip_addr,ssh_port,host_max_num,state,connection_error,libvirt_error,gray_ips,disk_params,property,host_creation_blocked,mon_installed,mon_install_date,metrics_timestamp,os_version,kernel_version,uptime,libvirt_version,qemu_version,overselling,hostname,vnc_addr,selinux_enforcing,problems,hdd_overselling,ip_addr_v6,bird_error,lxd_error,lxd_version,host_limit,socket_number,ha_state,ha_disable_failed,ha_error,update_dc_networks_failed,network_settings_lock,network_error,frr_error,host_filter,vm5_data,additional_data,vcpu_limit,spice_error from vm_node where ip_addr=\\\"$node_ip\\\"\\G;'"
|
|
declare -A cmd
|
|
cmd["0 hostnamectl"]="hostnamectl";
|
|
cmd["1 release"]="\"cat /etc/*release\"";
|
|
cmd["2 dmesg"]="\"dmesg | grep -i -E 'error|failed|critical|bug|panic'\"";
|
|
cmd["3 services"]="\"journalctl | grep -i -E 'error|failed|critical|bug|panic'\"";
|
|
cmd["4 services"]="\"systemctl status libvirtd -l\"";
|
|
cmd["5 services"]="\"systemctl status gomon -l\"";
|
|
cmd["6 services"]="\"systemctl show libvirtd\"";
|
|
cmd["7 sshd_config"]="\"cat /etc/ssh/sshd_config\"";
|
|
cmd["8 hwinfo"]="lscpu";
|
|
cmd["9 hwinfo"]="lscpu | grep -i -E 'socket|core|thread'";
|
|
cmd["10 hwinfo"]="dmidecode";
|
|
cmd["11 time"]="timedatectl";
|
|
cmd["12 time"]="uptime";
|
|
cmd["13 time"]="\"last reboot | head -10\"";
|
|
cmd["14 virsh"]="\"virsh list --all\"";
|
|
cmd["15 virsh"]="\"virsh pool-list --all\"";
|
|
cmd["16 virsh"]="\"virt-host-validate\"";
|
|
cmd["17 mem"]="\"free -m\"";
|
|
cmd["18 network"]="\"ip -br a\"";
|
|
cmd["19 network"]="\"brctl show\"";
|
|
cmd["20 network"]="\"curl -v download.ispsystem.com\"";
|
|
cmd["21 firewall"]="\"systemctl status nftables\"";
|
|
cmd["22 firewall"]="\"systemctl status firewalld\"";
|
|
cmd["23 firewall"]="\"firewall-cmd --list-ports\"";
|
|
cmd["24 firewall"]="\"nft list ruleset\"";
|
|
cmd["25 firewall"]="\"ss -tulpn | grep 16514\"";
|
|
cmd["26 firewall"]="\"ss -tulpn '( sport >= 49152 and sport <= 49215 )'\"";
|
|
cmd["27 firewall"]="\"ss -tulpn '( sport >= 5900 and sport <= 6900 )'\"";
|
|
cmd["28 firewall"]="\"ss -tulpn '( sport = 179 or sport = 4789 )'\"";
|
|
cmd["29 firewall"]="\"ss -tulpn | grep 16514\"";
|
|
cmd["30 firewall"]="\"ss -tulpn | grep -E ':179|:4789'\"";
|
|
cmd["31 disks"]="\"df -h --exclude tmpfs --exclude devtmpfs --exclude squashfs --exclude overlay\"";
|
|
cmd["32 disks"]="\"df -i\"";
|
|
cmd["33 disks"]="lsblk";
|
|
cmd["34 virsh"]="\"lvdisplay\"";
|
|
cmd["35 virsh"]="\"pvdisplay\"";
|
|
cmd["36 virsh"]="\"vgdisplay\"";
|
|
cmd["37 packages"]="\"yum repolist\"";
|
|
cmd["38 packages"]="\"yum history list\"";
|
|
cmd["39 mem"]="\"ps -eo pmem,pcpu,pid,ppid,user,stat,args | sort -k 1 -r | head -6\"";
|
|
cmd["40 hwinfo"]="\"ps -eo pcpu,pmem,pid,ppid,user,stat,args | sort -k 1 -r | head -6\"";
|
|
cmd["41 services"]="\"systemctl status libvirtd-ro.socket -l\"";
|
|
cmd["42 services"]="\"systemctl status libvirtd-admin.socket -l\"";
|
|
cmd["43 services"]="\"systemctl status libvirtd.socket -l\"";
|
|
cmd["44 services"]="\"systemctl status libvirtd.service -l\"";
|
|
cmd["44 services"]="\"systemctl list-units\"";
|
|
cmd["45 hwinfo"]="\"sestatus\"";
|
|
|
|
|
|
for k in "${!cmd[@]}";do
|
|
typelog=$(echo $k | cut -d ' ' -f2);
|
|
#echo $typelog;
|
|
#echo -e "$k=${cmd[$k]}";
|
|
node_collect $node_ip $node_port "${node_ip}_${typelog}.log" "${cmd[$k]}";
|
|
done
|
|
|
|
echo "collected logs about node $node_ip $node_port";
|
|
finish;
|
|
}
|
|
|
|
# parsing arguments
|
|
if [[ "$1" == "-logs" || "$1" == "1" ]]; then
|
|
logtype="logs";
|
|
about_platform;
|
|
_logs;
|
|
elif [[ "$1" == "-short-logs" || "$1" == "2" ]]; then
|
|
logtype="short-logs";
|
|
about_platform;
|
|
_short_logs;
|
|
elif [[ "$1" == "-lic" || "$1" == "3" ]]; then
|
|
logtype="lic";
|
|
about_platform;
|
|
_lic;
|
|
elif [[ "$1" == "-node" || "$1" == "4" ]]; then
|
|
logtype="node";
|
|
if [[ -z $2 || -z $3 ]]; then
|
|
echo "please set IP address and port of node!";
|
|
read -p "Enter IP address: " node_ip
|
|
read -p "Enter port: " node_port
|
|
about_platform;
|
|
_node $node_ip $node_port;
|
|
exit;
|
|
else
|
|
about_platform;
|
|
_node $2 $3;
|
|
fi
|
|
elif [[ "$1" == "-clear" || "$1" == "5" ]]; then
|
|
_remove_logs;
|
|
#elif [[ "$1" == "-vm-host" || "$1" == "5" ]]; then
|
|
# _vm_host;
|
|
else
|
|
_help;
|
|
fi |