diff --git a/get-all-logs.sh b/get-all-logs.sh index 051d286..0288ac9 100755 --- a/get-all-logs.sh +++ b/get-all-logs.sh @@ -4,6 +4,9 @@ PRMT="[$(whoami)@$(hostname -s) ]" about_platform(){ +# информация о платформе +# + if [[ -z $logtype ]];then logtype=$1; fi; @@ -33,15 +36,18 @@ fi; } newlog() { +#делаем файл лога LOGINFO=${LOGDIR}/$1 && date > ${LOGINFO} } cmdlog() { +# выводим в stdout и выполняем команду test -z "${LOGINFO}" && return local CMD=$*;(echo "${PRMT} ${CMD}";echo;eval ${CMD};echo) >> ${LOGINFO} } about_server() { +#информация о сервере newlog hostinfo_$(hostname).log echo "log $LOGINFO" cmdlog hostnamectl @@ -54,6 +60,7 @@ cmdlog ip -br a } about_docker(){ +#информация о docker newlog docker_$(hostname).log cmdlog docker version cmdlog docker ps @@ -61,6 +68,7 @@ cmdlog "docker inspect --format ' {{.Name}} - {{range.NetworkSettings.Networks}} } plaform_logs(){ +# логи из docker контейнеров DOCKER_CONTAINER_NAMES=`docker ps --format '{{.Names}}'` SERVICES=($DOCKER_CONTAINER_NAMES) @@ -75,6 +83,7 @@ plaform_logs(){ } about_license(){ +# информация о лицензии curl -vvv -k "https://license6.ispsystem.com/lic6/check" -H "X-auth-token:$LicenseToken" > ${LOGDIR}/license_${PLATFORM}.log 2>&1 echo -e "\n${PLATFORM} Version:$CurrentVersion\n" >>${LOGDIR}/license_${PLATFORM}.log; echo "${PLATFORM} LicenseToken:$LicenseToken" >>${LOGDIR}/license_${PLATFORM}.log; @@ -88,19 +97,21 @@ about_license(){ } finish() { +# пакуем логи и выводим общую информацию о платформе в конце сбора tar -cvzf ${ARCNAME} -C ${LOGDIR} . echo -e "\nISPsystem Software found:\n"; echo "${PLATFORM} DomainName:$DomainName"; - echo "${PLATFORM} LicenseToken=$LicenseToken" - echo "${PLATFORM} CurrentVersion=$CurrentVersion" - echo "${PLATFORM} Stage=$Stage" - + echo "${PLATFORM} LicenseToken:$LicenseToken" + dist_type=$(curl -s -m 120 -k "https://license6.ispsystem.com/lic6/check" -H "X-auth-token:5a99a084a0334c808143708cfbc24ec7:iWuiY_A5q02jwZlKcFV0Yw" | grep -Po '"dist_type":.*?[^\\]"' | sed -e 's/\"dist_type\"://g' -e 's/\"//g') + echo "${PLATFORM} CurrentVersion:$CurrentVersion $dist_type" + echo "${PLATFORM} Stage:$Stage" echo -e "\ndirectory with logs $LOGDIR\n" echo -e "Please send this logs archive to ISP system support\n" du -h ${ARCNAME} } -mysql_connect(){ +mysql_collect(){ +# выполнение запросов к БД logfile=$1; query=$2; fullcmd="docker exec -it mysql bash -c \"mysql isp -p\\\$MYSQL_ROOT_PASSWORD -e "$query"\" >> ${LOGDIR}/$logfile"; @@ -111,27 +122,33 @@ mysql_connect(){ } -node_connect() { +node_collect() { +# выполнение команды на узле ip_addr=$1; ssh_port=$2; cmd=$4; logfile=$3; - $(docker exec --tty --interactive vm_box ssh -o BatchMode=yes -o ConnectTimeout=5 -o "StrictHostKeyChecking no" -q -i /opt/ispsystem/vm/etc/.ssh/vmmgr.1 $ip_addr -p $ssh_port exit) && connected=1 || connected=0; - if [[ ${connected} == 1 ]];then -# echo "node $ip_addr on $node_port connected!"; + docker exec --tty --interactive vm_box ssh -o BatchMode=yes -o ConnectTimeout=5 -o "StrictHostKeyChecking no" -q -i /opt/ispsystem/vm/etc/.ssh/vmmgr.1 $ip_addr -p $ssh_port 'exit 0'; + _RCODE=$? + if [ $_RCODE -ne 0 ] + then +# echo "unable to ssh, host is not accessible"; +# connected=0; + echo "node $ip_addr on $node_port not connected!"; + echo "node $ip_addr on $node_port not connected!">> ${LOGDIR}/$logfile; + echo -e "\n not executed $cmd \n">> ${LOGDIR}/$logfile; + else +# echo "connected $ip_addr $ssh_port"; +# connected=1; fullcmd="docker exec --tty --interactive vm_box ssh -o \"StrictHostKeyChecking no\" -i /opt/ispsystem/vm/etc/.ssh/vmmgr.1 $ip_addr -p $ssh_port $cmd >> ${LOGDIR}/$logfile"; echo $fullcmd; echo -e "\n $cmd \n">> ${LOGDIR}/$logfile; eval $fullcmd; - else - echo "node $ip_addr on $node_port not connected!"; - echo "node $ip_addr on $node_port not connected!">> ${LOGDIR}/$logfile; - echo -e "\n not executed $cmd \n">> ${LOGDIR}/$logfile; - fi + fi } _help(){ - +# справка о скрипте version="23/02/23" echo -e "Log's script ver.: $version Usage : $0 [arguments]"; @@ -142,7 +159,7 @@ echo -e " -lic or 3"; echo -e " -node or 4 \n"; PS3='Please enter your choice(enter number): ' - +# интерактивное меню options=("Full logs" "Short logs" "License's logs" "Node's logs" "Quit") select opt in "${options[@]}" do @@ -183,6 +200,7 @@ select opt in "${options[@]}" } _logs() { +# общая функция сбора полных логов about_server; about_docker; about_license; @@ -191,6 +209,7 @@ _logs() { } _short_logs() { +# общая функция сбора сокращенных логов about_server; about_docker; about_license; @@ -200,6 +219,7 @@ _short_logs() { } _lic() { +# общая функция сбора информации о лицензии about_server; about_docker; about_license; @@ -207,55 +227,57 @@ _lic() { } _vm_host() { +# планируется сбор логов о виртуальной машине echo "it's coming soon!"; vm_name=$1; vm_id=$2; } _node() { +# общая функция сбор логов об узле node_ip=$1; node_port=$2; - mysql_connect "${node_ip}_db.log" "'select id,name,comment,cluster,ram_mib,max_vcpu_number,cpu_number,ip_addr,ssh_port,host_max_num,state,connection_error,libvirt_error,gray_ips,disk_params,property,host_creation_blocked,mon_installed,mon_install_date,metrics_timestamp,os_version,kernel_version,uptime,libvirt_version,qemu_version,overselling,hostname,vnc_addr,selinux_enforcing,problems,hdd_overselling,ip_addr_v6,bird_error,lxd_error,lxd_version,host_limit,socket_number,ha_state,ha_disable_failed,ha_error,update_dc_networks_failed,network_settings_lock,network_error,frr_error,host_filter,vm5_data,additional_data,vcpu_limit,spice_error from vm_node where ip_addr=\\\"$node_ip\\\"\\G;'" - node_connect $node_ip $node_port "${node_ip}_hostnamectl.log" "hostnamectl"; - node_connect $node_ip $node_port "${node_ip}_release.log" "\"cat /etc/*release\""; - node_connect $node_ip $node_port "${node_ip}_dmesg.log" "\"dmesg | grep -i -E 'error|failed|critical|bug|panic'\""; - node_connect $node_ip $node_port "${node_ip}_services.log" "\"journalctl | grep -i -E 'error|failed|critical|bug|panic'\""; - node_connect $node_ip $node_port "${node_ip}_services.log" "\"systemctl status libvirtd -l\""; - node_connect $node_ip $node_port "${node_ip}_services.log" "\"systemctl show libvirtd\""; - node_connect $node_ip $node_port "${node_ip}_sshd_config" "\"cat /etc/ssh/sshd_config\""; + mysql_collect "${node_ip}_db.log" "'select id,name,comment,cluster,ram_mib,max_vcpu_number,cpu_number,ip_addr,ssh_port,host_max_num,state,connection_error,libvirt_error,gray_ips,disk_params,property,host_creation_blocked,mon_installed,mon_install_date,metrics_timestamp,os_version,kernel_version,uptime,libvirt_version,qemu_version,overselling,hostname,vnc_addr,selinux_enforcing,problems,hdd_overselling,ip_addr_v6,bird_error,lxd_error,lxd_version,host_limit,socket_number,ha_state,ha_disable_failed,ha_error,update_dc_networks_failed,network_settings_lock,network_error,frr_error,host_filter,vm5_data,additional_data,vcpu_limit,spice_error from vm_node where ip_addr=\\\"$node_ip\\\"\\G;'" + node_collect $node_ip $node_port "${node_ip}_hostnamectl.log" "hostnamectl"; + node_collect $node_ip $node_port "${node_ip}_release.log" "\"cat /etc/*release\""; + node_collect $node_ip $node_port "${node_ip}_dmesg.log" "\"dmesg | grep -i -E 'error|failed|critical|bug|panic'\""; + node_collect $node_ip $node_port "${node_ip}_services.log" "\"journalctl | grep -i -E 'error|failed|critical|bug|panic'\""; + node_collect $node_ip $node_port "${node_ip}_services.log" "\"systemctl status libvirtd -l\""; + node_collect $node_ip $node_port "${node_ip}_services.log" "\"systemctl show libvirtd\""; + node_collect $node_ip $node_port "${node_ip}_sshd_config" "\"cat /etc/ssh/sshd_config\""; - node_connect $node_ip $node_port "${node_ip}_hwinfo.log" "lscpu"; - node_connect $node_ip $node_port "${node_ip}_hwinfo.log" "lscpu | grep -i -E 'socket|core|thread'"; - node_connect $node_ip $node_port "${node_ip}_hwinfo.log" "dmidecode"; - node_connect $node_ip $node_port "${node_ip}_time.log" "timedatectl"; - node_connect $node_ip $node_port "${node_ip}_time.log" "uptime"; - node_connect $node_ip $node_port "${node_ip}_time.log" "\"last reboot | head -10\""; - node_connect $node_ip $node_port "${node_ip}_virsh.log" "\"virsh list --all\""; - node_connect $node_ip $node_port "${node_ip}_virsh.log" "\"virsh pool-list --all\""; - node_connect $node_ip $node_port "${node_ip}_virsh.log" "\"virt-host-validate\""; - node_connect $node_ip $node_port "${node_ip}_mem.log" "\"free -m\""; - node_connect $node_ip $node_port "${node_ip}_network.log" "\"ip -br a\""; - node_connect $node_ip $node_port "${node_ip}_network.log" "\"brctl show\""; - node_connect $node_ip $node_port "${node_ip}_network.log" "\"curl -v download.ispsystem.com\""; + node_collect $node_ip $node_port "${node_ip}_hwinfo.log" "lscpu"; + node_collect $node_ip $node_port "${node_ip}_hwinfo.log" "lscpu | grep -i -E 'socket|core|thread'"; + node_collect $node_ip $node_port "${node_ip}_hwinfo.log" "dmidecode"; + node_collect $node_ip $node_port "${node_ip}_time.log" "timedatectl"; + node_collect $node_ip $node_port "${node_ip}_time.log" "uptime"; + node_collect $node_ip $node_port "${node_ip}_time.log" "\"last reboot | head -10\""; + node_collect $node_ip $node_port "${node_ip}_virsh.log" "\"virsh list --all\""; + node_collect $node_ip $node_port "${node_ip}_virsh.log" "\"virsh pool-list --all\""; + node_collect $node_ip $node_port "${node_ip}_virsh.log" "\"virt-host-validate\""; + node_collect $node_ip $node_port "${node_ip}_mem.log" "\"free -m\""; + node_collect $node_ip $node_port "${node_ip}_network.log" "\"ip -br a\""; + node_collect $node_ip $node_port "${node_ip}_network.log" "\"brctl show\""; + node_collect $node_ip $node_port "${node_ip}_network.log" "\"curl -v download.ispsystem.com\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"systemctl status nftables\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"systemctl status firewalld\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"firewall-cmd --list-ports\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"nft list ruleset\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"systemctl status nftables\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"systemctl status firewalld\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"firewall-cmd --list-ports\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"nft list ruleset\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn | grep 16514\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn '( sport >= 49152 and sport <= 49215 )'\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn '( sport >= 5900 and sport <= 6900 )'\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn '( sport = 179 or sport = 4789 )'\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn | grep 16514\""; - node_connect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn | grep -E ':179|:4789'\""; - node_connect $node_ip $node_port "${node_ip}_disks.log" "\"df -h --exclude tmpfs --exclude devtmpfs --exclude squashfs --exclude overlay\""; - node_connect $node_ip $node_port "${node_ip}_disks.log" "\"df -i\""; - node_connect $node_ip $node_port "${node_ip}_disks.log" "lsblk"; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn | grep 16514\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn '( sport >= 49152 and sport <= 49215 )'\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn '( sport >= 5900 and sport <= 6900 )'\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn '( sport = 179 or sport = 4789 )'\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn | grep 16514\""; + node_collect $node_ip $node_port "${node_ip}_firewall.log" "\"ss -tulpn | grep -E ':179|:4789'\""; + node_collect $node_ip $node_port "${node_ip}_disks.log" "\"df -h --exclude tmpfs --exclude devtmpfs --exclude squashfs --exclude overlay\""; + node_collect $node_ip $node_port "${node_ip}_disks.log" "\"df -i\""; + node_collect $node_ip $node_port "${node_ip}_disks.log" "lsblk"; echo "collected logs about node $node_ip $node_port"; finish; } - +# обработка аргументов if [[ "$1" == "-logs" || "$1" == "1" ]]; then logtype="logs"; about_platform;