#!/bin/bash

export VROOT=/home/virtual
export LOGDIR=var/log/httpd

export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin

ME=$(basename $0)

USAGE="$ME [--on-host host] [--resolve] 
                    [--status code] [--hide-status] \\
                    [--match-user regex] [--user-length] [--all-users] [--show-user] \\
                    [--action name] [--hide-action] \\
                    [--show-referrer] [--date-pat pat] \\
                    [--on-day pat] [--hide-date] \\
                    [--sort-by [host|date|action|match|status|referrer]] \\
                    [--hide-match] [--match-length N] \\
                    [--output-separator] \\
                    [--hide-host] pat1..patn
$ME --help (this screen)

Example 1:  Search all host access logs for cmd.exe and 
            default.ida attempts.  Just show IP addresses
            so we don't spend all resolving IPs! ;)

$ME --on-day today index.html cmd.exe default.ida

Example 2:  How many times have visitors attempted formmail (we
            want to see host names if we can!)

$ME --resolve formmail.pl

Example 3:  How many times has package.tar.gz been downloaded
            from host foo.com?  Again, lets see the hosts if
            we can get em

$ME --resolve --on-host foo.com package.tar.gz

Example 4:  How many 404s for today?

$ME --resolve --on-host foo.com --status 404 --on-day today

Example 5:  Lets see March log entries, sorted by URI, for requests
            that resulted in a 404.

$ME --sort-by match --date-pat Mar --status 404"

RESOLVE=0
SEARCH_DATE=""
STATUS=""
SHOWACTION=1
ACTION=""
SHOWSTATUS=1
SKIP=0
SHOWMATCH=1
SHOWHOST=1
SHOWDATE=1
SHOWTIME=0
SHOWREFERRER=0
USER=""
SHOWUSER=0
USERLEN=30
PATTERNS=""
MATCH_REFERRER=0
MATCHLEN=44
SEP=' '

#  sorts
DATESORT=0
DATESORT=0
HOSTSORT=0
STATUSSORT=0
ACTIONSORT=0
MATCHSORT=0
REFERRERSORT=0
USERSORT=0

CLIENT=""

for arg
do

    if [ "$SKIP" = "1" ]
    then
        SKIP=0
        continue
    fi

    case $arg in
        --on-host|-oh) shift; SITE=$1; SKIP=1;;
        --from-client|-fc) shift; CLIENT=$1; SKIP=1;;
        --status|-s) shift; STATUS="$1"; SKIP=1;;
        --action|-a) shift; ACTION="$1"; SKIP=1;;
        --match-user|-mu) shift; USER="$1"; SKIP=1;;
        --match-length|-ml) shift; MATCHLEN="$1"; SKIP=1;;
        --user-length|-ul) shift; USERLEN="$1"; SKIP=1;;
        --output-separator|-os) shift; SEP="$1"; SKIP=1;;
        --all-users|-au) SHOWUSER=1; USER='[a-z]';;
        --hide-action|-ha) SHOWACTION=0;;
        --hide-match|-hm) SHOWMATCH=0;;
        --hide-host|-hh) SHOWHOST=0;;
        --hide-status|-hs) SHOWSTATUS=0;;
        --hide-date|-hd) SHOWDATE=0;;
        --show-time|-st) SHOWTIME=1;;
        --sort-by|-sb) shift; SORTBY=$(echo "$1" | tr 'A-Z' 'a-z'); SKIP=1;
            case $SORTBY in
                date|d) DATESORT=1; SORTBY=date;;
                host|h) HOSTSORT=1; SORTBY=host;;
                status|s) STATUSSORT=1; SORTBY=status;;
                match|m) MATCHSORT=1; SORTBY=match;;
                action|a) ACTIONSORT=1; SORTBY=action;;
                referrer|r) REFERRERSORT=1; SORTBY=referrer;;
                user|u) USERSORT=1; SORTBY=user;;
                *) echo "Unknown sort $SORTBY!"
                   exit 1;;
            esac
            ;;
        --match-referrer|-mr) MATCH_REFERRER=1;;
        --show-referrer|-sr) SHOWREFERRER=1;;
        --help|-h)    echo "$USAGE"; exit 0;;
        --on-day|-od) shift; SKIP=1; pat=$1;
                  SEARCH_DATE="$(date  -d "$pat" '+%d/%b/%Y')"
                  ;;
        --date-pat|-dp) shift; SKIP=1; SEARCH_DATE="$1";
                   ;;
        --resolve|-r) RESOLVE=1;;
        --show-user|-su) SHOWUSER=1;;
        *) PATTERNS="$(echo "$PATTERNS $arg" | sed -e 's/^ //' -e 's/ $//')";;
    esac

    shift

done

if [ "x$PATTERNS" = "x" ]
then
    if [ "x$STATUS" = "x"  -a "x$SEARCH_DATE" = "x" -a \
         "x$ACTION" = "x" -a \
         "x$USER" = "x" -a \
         "$xCLIENT" = "x" ]
    then
        echo "Missing patterns, HTTP status, action, user, or date to look for!"
        echo "$USAGE"
        exit 1
    fi
fi

debug() {
    echo "PATTERNS: $PATTERNS"
    echo "SITE:     $SITE"
    echo "DATE:     $SEARCH_DATE"
    echo "RESOLVE:  $RESOLVE"
    echo "STATUS:   $STATUS"
    echo "CLIENT:   $CLIENT"
    echo "  USER:   $USER"
    echo "   SEP:   $SEP"
    exit
}

#debug

search_host() {

    local site="$1"
    local pat="$2"
    local status="$3"


    (cat $VROOT/$site/$LOGDIR/access_log;
     cat $VROOT/$site/$LOGDIR/access_log.[0-9];
     cat $VROOT/$site/$LOGDIR/access_log.[0-9]*[0-9];
     zcat $VROOT/$site/$LOGDIR/access_log*gz;
     cat $VROOT/$site/$LOGDIR/access_log-ssl;
     cat $VROOT/$site/$LOGDIR/access_log.[0-9]-ssl;
     cat $VROOT/$site/$LOGDIR/access_log-ssl[0-9];
     zcat $VROOT/$site/$LOGDIR/access_log-ssl*gz;
    ) 2>/dev/null | \
    perl -n -MSocket -MPOSIX -e '
         BEGIN{
             $DATE = "'"$SEARCH_DATE"'";$STATUS = "'"$STATUS"'"; 
             $RESOLVE = '"$RESOLVE"'; $PAT = "'"$pat"'";
             $MATCH_REFERRER = '"$MATCH_REFERRER"';
             $SHOWREFERRER = '"$SHOWREFERRER"';
             $ACTION = "'"$ACTION"'";
             $CLIENT = "'"$CLIENT"'";
             $USER = "'"$USER"'";

             $SHOWMATCH = '"$SHOWMATCH"';
             $MATCHLEN = '"$MATCHLEN"';
             $SHOWHOST = '"$SHOWHOST"';
             $SHOWDATE = "'$SHOWDATE'";
             $SHOWTIME = "'$SHOWTIME'";
             $SHOWSTATUS = "'$SHOWSTATUS'";
             $SHOWACTION = '"$SHOWACTION"';
             $SHOWUSER = '"$SHOWUSER"';
             $USERLEN = '"$USERLEN"';
             $SEP = "'"$SEP"'";

             #   Sorts

             $SORTBY = "'"$SORTBY"'";

             $HOSTSORT = "'$HOSTSORT'";
             if ($HOSTSORT == 1) {
                 die "No sort by host; field not showing!\n"
                     unless $SHOWHOST;
             }

             $DATESORT = "'$DATESORT'";
             if ($DATESORT == 1) {
                 die "No sort by date; field not showing!\n"
                     unless ($SHOWDATE || $SHOWTIME);
             }

             $STATUSSORT = "'$STATUSSORT'";
             if ($STATUSSORT == 1) {
                 die "No sort by status; field not showing!\n"
                     unless $SHOWSTATUS;
             }

             $MATCHSORT = "'$MATCHSORT'";

             if ($MATCHSORT == 1) {
                 die "No sort by match; field not showing!\n"
                     unless $SHOWMATCH;
             }

             $REFERRERSORT = "'$REFERRERSORT'";

             if ($REFERRERSORT == 1) {
                 die "No sort by match; field not showing!\n"
                     unless $SHOWREFERRER;
             }

             $ACTIONSORT = "'$ACTIONSORT'";
             if ($ACTIONSORT == 1) {
                 die "No sort by action; field not showing!\n"
                     unless $SHOWACTION;
             }

             $USERSORT = "'$USERSORT'";
             if ($USERSORT == 1) {
                 die "No sort by user; field not showing!\n"
                     unless $SHOWUSER;
             }

             $SITE = "'"$site"'";

             my %MONTHS = qw(
                 Jan 0 Feb 1 Mar 2 Apr 3 May 4 Jun 5 
                 Jul 6 Aug 7 Sep 8 Oct 9 Nov 10 Dec 11
             );

             my %DATES = ();

             sub format_date {
                 my $date = shift;
                 my ($day, $mon, $year) = split("/", $date);
                 my ($hour, $min, $sec) = (0,0,0);

                 if ($SHOWTIME) {
                     ($hour, $min, $sec) = 
                         ($date =~ /:(\d+):(\d+):(\d+)/)[0,1,2];
                 }

                 $mon = $MONTHS{$mon};
                 $year -= 1900;

                 my $key = "$day$mon$year$hour$min$sec";

                 if (! exists $DATES{$key}) {
                     $DATES{$key} = 
                         POSIX::mktime($sec, $min, $hour, $day, $mon, $year);
                 }

                 return $DATES{$key};
             }

             my $THIS_YEAR = (localtime())[5] + 1900;

             my @LINES;

             sub to_date {
                 my $long = shift;
                 my ($sec,$min,$hour,$mday, $mon, $yr) = 
                     (localtime($long-3600))[0,1,2,3,4,5];
                 $mon++;
                 $yr += 1900;
                 $mday = "0$mday" if $mday < 10;
                 $mon = "0$mon" if $mon < 10;
                 my $day = "";
                 if ($SHOWDATE) {
                     if ($THIS_YEAR == $yr) {
                         $day = "$mon/$mday";
                     } else {
                         $day = "$mon/$mday/$yr";
                     }
                 }
                 if ($SHOWTIME) {
                     $day .= sprintf(" %02d:%02d:%02d", $hour, $min, $sec);
                 }
                 return $day;
             }

             sub add_line {
                 my $add = shift;
                 push(@LINES, $add);
             }

             sub convert_date {
                 my $line = shift;
                 $line =~ m/(\d{10})/;
                 my $long = $1;
                 return $line unless $long;
                 my $day = to_date($long);
                 $line =~ s/$long/$day/;
                 return $line;
             }

             sub get_lines {
                 return @LINES;
             }

             my %positions = ();

             my $index = 1;

             sub save_position {
                 my $label = shift;
                 return if exists $positions{$label};
                 $positions{$label} = $index;
                 $index++;
             }

             sub get_position {
                 my $label = shift;
                 return $positions{$label};
             }

         }
         
         my $matched_date;

         if ($DATE) {
             next unless /\[.*?${DATE}.*?\]/;
             m#\[(\d+/\w+/\d+:\S+) #;
             $matched_date = format_date($1) if ($SHOWDATE || $SHOWTIME);
         } else {
             m#\[(\d+/\w+/\d+:\S+) #;
             $matched_date = format_date($1) if ($SHOWDATE || $SHOWTIME);
         }

         s/\[.*?\]//;

         my $matched = "";

         my $request = (m/(".*?")/)[0];
         $request =~ s/^"(\S+)//;
         my $action = uc($1);
         $request =~ s#\S+$##;
         s/".*?"//;

         if ($ACTION ne "") {
             next unless $action eq uc($ACTION);
         }

         if ($PAT ne "" && $MATCH_REFERRER == 0) {
             $matched = ($request =~ /(\S*$PAT\S*)/)[0];
             next unless $matched;
         } else {
             $matched = $request;
         }

         my @fields = split(/\s+/, $_);

         my $user = $fields[2];

         if ($USER ne "") {
             next unless $user =~ /$USER/i;
         }

         my $referrer = $fields[5];
         s/".*?"//;
         $referrer =~ s/^"//;
         $referrer =~ s/"$//;

         if ($CLIENT != "") {
             next unless $fields[0] =~ /${CLIENT}/;
         }

         if ($MATCH_REFERRER == 1 && (! $matched)) {
             $matched = ($referrer =~ /(\S*$PAT\S*)/)[0];
             next unless $matched;
         } else {
             #  Unless already matched before
             $matched = $referrer unless $matched;
         }

         my $namehost = $fields[0];
         $namehost =~ s/^.*?://;
         next unless $namehost;

         my $status = $fields[3];
         if ($STATUS) {
             next unless ($status =~ /$STATUS/);
         }

         my $name = $namehost;

         if ($RESOLVE == 1) {
             if (exists $HOSTS{$namehost}) {
                 $name = $HOSTS{$namehost};
             } else {
                 my $nsname = gethostbyaddr(inet_aton($namehost), AF_INET);
                 if ($nsname) {
                     $HOSTS{$namehost} = $nsname;
                     $name = $nsname;
                 }
             }
         }

         my $line = "";


         if ($SHOWHOST == 1) {
             if ($RESOLVE == 1) {
                 $line = sprintf("%-40s", $name);
             } else {
                 $line = sprintf("%-15s", $name);
             }
             save_position("host");
         }

         if ($SHOWUSER == 1) {
             $line = sprintf("%s${SEP}%${USERLEN}s", $line, $user);
             save_position("user");
         }

         if ($SHOWACTION == 1) {
             $line = sprintf("%s${SEP}%-4s", $line, $action);
             save_position("action");
         }

         if ($SHOWMATCH == 1) {
             save_position("match");
             $line = sprintf("%s${SEP}%-${MATCHLEN}s", 
                             $line, substr($matched, 0, $MATCHLEN));
         }

         if ($SHOWREFERRER == 1) {
             save_position("referrer");
             $line = sprintf("%s${SEP}%-44s", $line, substr($referrer, 0, 44));
         }

         if ($SHOWSTATUS == 1) {
             $line = join("${SEP}",$line, $status);
             save_position("status");
         }

         if ($SHOWDATE || $SHOWTIME) {
             $line = join("${SEP}",$line, $matched_date);
             save_position("date");
         }

         $line = join("",$line, "\n");

         add_line($line);

     END{

         my $block = sub { $b <=> $a };

         if ($HOSTSORT == 1 && $SHOWHOST == 1) {
             my $index = get_position("host");
             $block = sub { 
                 $ad = (split("${SEP}", $a))[0];
                 $bd = (split("${SEP}", $b))[0];
                 return $bd <=> $ad;
             };
         } elsif ($DATESORT == 1) {
             my $index = get_position("date");
             $block = sub {
                 $ad = (split("${SEP}", $a))[$index];
                 $bd = (split("${SEP}", $b))[$index];
                 return $ad <=> $bd;
             };
         } elsif ($HOSTSORT == 1) {
             my $index = get_position("host");
             $block = sub {
                 $ad = (split("${SEP}", $a))[$index];
                 $bd = (split("${SEP}", $b))[$index];
                 return $ad cmp $bd;
             };
         } elsif ($STATUSSORT == 1) {
             my $index = get_position("status");
             $block = sub {
                 $ad = (split("${SEP}", $a))[$index];
                 $bd = (split("${SEP}", $b))[$index];
                 return $ad <=> $bd;
             };
         } elsif ($ACTIONSORT == 1) {
             my $index = get_position("action");
             $block = sub {
                 $ad = (split("${SEP}", $a))[$index];
                 $bd = (split("${SEP}", $b))[$index];
                 return $ad cmp $bd;
             };
         } elsif ($MATCHSORT == 1) {
             my $index = get_position("match");
             $block = sub {
                 $ad = (split("${SEP}", $a))[$index];
                 $bd = (split("${SEP}", $b))[$index];
                 return $ad cmp $bd;
             };
         } elsif ($REFERRERSORT == 1) {
             my $index = get_position("referrer");
             $block = sub {
                 $ad = (split("${SEP}", $a))[$index];
                 $bd = (split("${SEP}", $b))[$index];
                 return $ad cmp $bd;
             };
         } elsif ($USERSORT == 1) {
             my $index = get_position("user");
             $block = sub {
                 $ad = (split("${SEP}", $a))[$index];
                 $bd = (split("${SEP}", $b))[$index];
                 return $ad cmp $bd;
             };
         }

         my %UNIQ;

         my $hosts;
         my $matches;

         my %KEYS = ();

         for my $line (get_lines()) {
             $matches++;
             $UNIQ{$line}++;
             my $key = "";
             if ($MATCHSORT == 1) {
                 $key = (split("${SEP}", $line))[get_position("match")];
             } elsif ($REFERRERSORT == 1) {
                 $key = (split("${SEP}", $line))[get_position("referrer")];
             } elsif ($USERSORT == 1) {
                 $key = (split("${SEP}", $line))[get_position("user")];
             } else {
                 $key = ($line =~ m/^(\S+)/)[0];
             }
             $KEYS{$key}++;
         }

         my @keys = keys %UNIQ;
         my $unique_keys = scalar @keys;

         my $label;

         if ($MATCHSORT == 1) {
             $label = "matche";
         } else {
             $label = $SORTBY;
             $label = "entrie" unless $SORTBY;
         }

         if (scalar @keys > 0) {

             $PAT="(any URI)" unless $PAT;
             $STATUS = "any status" unless $STATUS;
             $SORTBY = "# unique matches" unless $SORTBY;
             $DATE = "(any date)" unless $DATE;
             $ACTION = "(any action)" unless $ACTION;
             $USER = "(any user)" unless $user;

             print "Query:   Pattern \"${PAT}\" on $SITE, sorted by $SORTBY\n";
             print "Filters: $USER $ACTION ${DATE} ($STATUS)\n\n";

             print map { my $d = convert_date($_); 
                         if ($SEP ne " ") {
                             $d =~ s/\s*${SEP}\s*/$SEP/g;
                             $d =~ s/${SEP}+/$SEP/g;
                         }
                         $d;
                   }
                   sort $block 
                   map { sprintf "%-3d${SEP}%s", $UNIQ{$_}, $_ }
                   @keys;

             print "----------------\n";
             print "$unique_keys unique ${label}s\n";
             print "$matches total matches\n";
             print "\n";
             print "====================================================\n";
             print "\n";
         }

     }'

}

do_search() {

    pattern="$1"

    SITES=""

    if [ "x$SITE" != "x" ]
    then
        SITES="$SITE"
    else 
        #SITES="$(cd $VROOT; echo *\.*)"
        SITES="$(/usr/local/bin/sitelookup -a domain)"
    fi

    for site in $SITES
    do
        search_host "$site" "$pattern" "$STATUS"
    done

}

if [ "x$PATTERNS" = "x" ]
then
    do_search ""
else
    for pattern in $PATTERNS
    do
        do_search "$pattern"
    done
fi