#!/usr/bin/perl use MIME::Lite; use Net::SMTP; use Getopt::Std; use Config::Tiny; ######################################### # # Globals # ######################################### $hostname = qx(hostname -s); chomp($hostname); ### Make a directory with the current epoch as the name $epoch=time(); mkdir("/home/syswatch/$epoch",700); $wrk_dir="/home/syswatch/$epoch"; ######################################### # # Grab the passed command line options # ######################################### my $opt_string = 'hiams:M:'; getopts( "$opt_string", \%opt ) or usage(); usage() if $opt{h}; usage() if ! $opt{s}; ######################################### # # Grab the app values from daemontools env # ######################################### @service_vars = Get_Path_to_Webapps(); print "Configured HOST is: ".$service_vars[0]."\n"; print "Configured DOMAIN is: ".$service_vars[1]."\n"; print "Configured CONTEXT is: ".$service_vars[2]."\n"; ######################################### # Read and parse the config file. ######################################### my $Config = Config::Tiny->new(); $Config = Config::Tiny->read( '/home/syswatch/syswatch.cfg' ); my $no_mail_rcpts = $Config->{_}->{mailrcpts}; my $no_attachments = $Config->{_}->{attachments}; ######################################### # # Check the options # ######################################### if ($opt{a}) { $archive = 1; print "Archive files will be kept\n";} else { $archive = 0; } if ($opt{i}) { print "Processing now because -i was specified\n"; Process_Main(); exit(0); } else { print "You have requested daemonized processing.\n"; while (1) { $proc_load = qx(cat /proc/loadavg); my @load_values = split (/ /, $proc_load); if ($load_values[0] > 2 ) { Process_Main(); sleep 1800; } else { sleep 180; } } } ######################################### # # Main process # ######################################### sub Process_Main() { Dump_Top(); $result = Dump_Threads(); if ($result eq "-1") { return; } print "Gathering Attachments\n"; @attachment_status = Gather_Attachments(); print "Sending Email\n"; Send_Email(@attachment_status) unless $opt{m}; print "cleaning up\n"; Cleanup_or_Archive($archive); } ###################################### # # Send_Email() # Sends a message to an email address # ###################################### sub Send_Email() { #who are we sending to if (! $opt{M}) { my $no_mail_rcpts = $Config->{_}->{mailrcpts}; print "Sending a message to: ".$no_mail_rcpts." recipients with: "; } else { print "Sending a message to ".$opt{M}." because -M was specifiedi, with: "; } my $no_attachments = $Config->{_}->{attachments}; print $no_attachments." attachments\n"; my $message_body = "Please forward the attachment to the appropriate contact.\n"; foreach $as (@attachment_status) { $message_body = $message_body.$as; } while ($no_mail_rcpts) { my $rcpt_address = "rcpt_".$no_mail_rcpts; print "Checking config file for ".$rcpt_address."\n"; my $from_address = 'support@contegix.com'; my $to_address = $opt{M} || $Config->{MAIL}->{$rcpt_address}; print "Sending message to: ".$to_address."\n"; my $mail_host = 'mail.contegix.com'; my $subject = "Process Dump from $hostname"; ### Create the multipart container $msg = MIME::Lite->new ( From => $from_address, To => $to_address, Subject => $subject, Type =>'multipart/mixed' ) or die "Error creating multipart container: $!\n"; MIME::Lite->send('smtp', $mail_host, Timeout=>60); ### $msg->attach( Type =>'TEXT', #Data => "Instance: $opt{s}\n\nPlease forward this to the appropriate Jive contacts." Data => $message_body ); $msg->attach( Type =>'BINARY', Path => "$wrk_dir/Process_Dump.tar.gz", Filename => 'Process_Dump.tar.gz' ); $no_mail_rcpts--; } $msg->send; } ###################################### # # Get a list of services that match request # ###################################### sub Enumerate_Services() { my $service_name = $opt{s}; my $service_command = "svstat /service/j2ee_$service_name |grep [[:space:]]up[[:space:]]"; print "Looking for a service for: ".$service_name."\n"; my @running_service = qx($service_command); if (@running_service) { foreach $rs (@running_service) { chomp($rs); print "Seeing if : ".$rs." matches our server request.\n"; my @service_values = split(/ /, $rs); my $proc_pid = @service_values[3]; $proc_pid =~ s/[()]//gi; my $proc_name = @service_values[0]; $proc_name =~ s/[:]//gi; $processes[0] = $proc_name; $processes[1] = $proc_pid; } print "Yes, ".$processes[0]." matched what we were looking for.\n"; return @processes; } else { print "Weird, I couldn't find a service that matched ".$opt{s}."!\n"; return "-1"; } } ###################################### # # Gather_Attachments() # ###################################### sub Gather_Attachments() { ### move the thread dump file my $log_path = "/service/j2ee_$opt{s}/log/main/"; my @files = <${log_path}/*.u>; foreach my $file (@files) { my @filestats = stat($file); if ($filestats[10] > $lastctime) { $logfile_name = $file; } } chomp($logfile_name); print "Processing logfile named: ".$logfile_name." for thread dump\n"; qx(cp $logfile_name $wrk_dir\/$service_vars[0].thread.txt); ### Get the rest of the attachments while ($no_attachments) { ### Build the variables for the config values to retreive $attach_path = "attach_".$no_attachments."_path"; $attach_name = "attach_".$no_attachments."_name"; ### Get the path to the attachment from the config file $attachment_path = $Config->{ATTACHMENTS}->{$attach_path}; $attachment_path =~ s/\[HOME\]/\/home\/syswatch/gi; $attachment_path =~ s/\[HOST\]/$service_vars[0]/gi; $attachment_path =~ s/\[DOMAIN\]/$service_vars[1]/gi; $attachment_path =~ s/\[WORKDIR\]/$wrk_dir/gi; ### Get the name that the file will be attached as from the config file $attachment_name = $Config->{ATTACHMENTS}->{$attach_name}; $attachment_name =~ s/\[HOME\]/\/home\/syswatch/gi; $attachment_name =~ s/\[HOST\]/$service_vars[0]/gi; $attachment_name =~ s/\[DOMAIN\]/$service_vars[1]/gi; ### If it exist then stuff it in the work directory. If not then skip it. print "Checking to see if ".$attachment_path." exist..."; if ( -f $attachment_path ) { print "yes\n"; qx(cp $attachment_path $wrk_dir/$attachment_name) unless ($attachment_path =~ m[.*$wrk_dir.*]i); print "Adding ".$attachment_path." to the list of attachments, renaming to ".$attachment_name."\n"; push(@attachment_status,"Requested attachment: ".$attachment_name." was included in attached tar file.\n"); } else { print "no!\n"; push(@attachment_status,"Requested attachment: ".$attachment_name." could not be located and was skipped!\n"); } $no_attachments--; } ### Now we have a directory full of log files let tar and compress them (gzip) qx(cd $wrk_dir;tar zcf Process_Dump.tar.gz ./*); return @attachment_status; } ###################################### # # Dump_Top() # ###################################### sub Dump_Top() { my $command_line = "top -n 1 -b -c > ".$wrk_dir."/".$service_vars[0].".top.txt"; qx($command_line); } ###################################### # # Message about this program and how to use it # ###################################### sub usage() { print STDERR << "EOF"; This program follows the request method to dump threads for Jive Software and Mails the output to engineers. usage: $0 -s service_name [-iam] -i : Run now and exit, don't check for system load. -a : Archive the output files -m : Don't email output. Only use if you also specify -a. -s : Service name to process: host.domain.com. (Example: for /service/j2ee_discussions.apple.com/service enter discussions.apple.com ) -M : Override the mailto field. Put in the name to send message to i.e. username\@domain.com example: $0 -s discussions.apple.com -i -a -m EOF exit; } ###################################### # Dump_Threads ###################################### sub Dump_Threads() { @processes = Enumerate_Services(); if (@processes[0] eq "-1") { print "There are no services! So how am I expected to dump the Threads!\n"; return "-1"; } $records = @processes -1; for ($count = $records; $count >= 1; $count--) { print "Working on: ".$processes[0]."\n"; $service_log = $processes[0]."/log"; print "Sending HUP to /service/j2ee_".$opt{s}."/log\n"; qx(/usr/local/bin/svc -h $service_log); sleep 2; print "Sending kill -3 to /service/j2ee_".$opt{s}."\n"; unless (fork) { exec ("sudo kill -3 $processes[1]"); } wait; sleep 10; print "Sending kill -3 to /service/j2ee_".$opt{s}." again.\n"; unless (fork) { exec ("sudo kill -3 $processes[1]"); } wait; sleep 10; print "Sending kill -3 to /service/j2ee_".$opt{s}." last time.\n"; unless (fork) { exec ("sudo kill -3 $processes[1]"); } wait; sleep 10; print "Sending HUP to /service/j2ee_".$opt{s}."/log\n"; qx(/usr/local/bin/svc -h $service_log); sleep 2; } } ###################################### # Cleanup_or_Archive() ###################################### sub Cleanup_or_Archive() { my $action = shift || "0"; # Actions are 1=Archive, 0=Delete Files my $archive_base = $opt{s}; if ( $action eq "1" ) { print "Archiving files....\n"; qx(mv $wrk_dir $opt{s}.`date +\%F-\%H-\%M`); } else { print "Removing files. If you want to keep them remember to use -a\n"; qx(rm -rf $wrk_dir); } } ####################################### # # Build_path_to_webapps # ####################################### sub Get_Path_to_Webapps() { my $host; my $domain; my $context; my $service = $opt{s}; #Get the host,domain, and context from $opt{s}/env files. $host = qx(cat /service/j2ee_$service/env/HOST); chomp($host); $domain = qx(cat /service/j2ee_$service/env/DOMAIN); chomp($domain); @retr_val[0] = $host; @retr_val[1] = $domain; @retr_val[2] = $context; return @retr_val; }