#!/usr/bin/perl -w
use strict;
use utf8;

use LWP::Simple;
use LWP::UserAgent;
use HTTP::Request;
use HTTP::Response;
use HTML::LinkExtor;
use Encode;

# Subroutine prototypes.
sub ParseText($);			# Returns a string containing the normalized document without HTML markup.
sub ParseDescription($);	# Returns a string containing the description of the HTML document.
sub GenerateIndex();		# Generates an index of the pages crawled.
sub GetBase($);				# Gets the base part of a URL path.
sub AddToRobotFilter($);	# Adds a domain's robots.txt file to the exclude list.
sub IsExcluded($);			# Returns a boolean value indicating whether a page should be visited.
sub LoadKeywords($);		# Reads in a keywords file (assumed to be newline delimited).
sub GetPageRanking($);		# Returns a value between 0 and 1 for the ranking of a page.
sub StripHTML($);			# Gets rid of all HTML tags from a document.


# Settings
my $KEYWORDS_FILE = "pokemon_keywords.txt";
my $MIN_THRESHOLD = 0.5;
my $START_URL = "http://mondo123.tripod.com/main.html";
my $MAX_SPREAD = 3;
my $MAX_RESULTS = 30;
my $OUTPUT_FOLDER = "pokemon_output";

# Initialize the directory
my $oDirectory;
if (!mkdir($OUTPUT_FOLDER, 0777) && !opendir($oDirectory, $OUTPUT_FOLDER)) {
	die "Can't open output directory.";
}

# Initialize page getter.
my $browser = LWP::UserAgent->new();
$browser->timeout(15);

# Get list of keywords.
my %keywords;
LoadKeywords($KEYWORDS_FILE);

# Initialize robot filter.
my %robotpages = ();
my %excludelist = ();
AddToRobotFilter($START_URL);

# Initialize page queue and history.
my @pagequeue;
push (@pagequeue, $START_URL);
my %pagehistory;
$pagehistory{$START_URL} = 0;

# Initialize index.
my %PageIndex;

my $numResults = 0;
# While there are pages left to crawl and we have slots left to fill.
while (@pagequeue != 0 && $numResults < $MAX_RESULTS) {
	# Get the page we will be working on.
	my $currentURL = pop (@pagequeue);
	print @pagequeue." $currentURL\n";
	
	# Download the web page and go on to the next if can't.
	my $request = HTTP::Request->new(GET => $currentURL);
	my $response = $browser->request($request);
	if ($response->is_error()) {
		print "\t\t".$response->status_line."\n";
		next;
	}
	my $contents = $response->content();
	
	# Get page ranking.
	my $pageText = ParseText($contents);
	my $ranking = GetPageRanking($pageText);
	
	print "\t\t$ranking\n";
	
	# Save this page if it has a good ranking.
	if ($ranking >= $MIN_THRESHOLD) {
		open (FILEOUT, ">", "$OUTPUT_FOLDER/$numResults.txt");
		print FILEOUT StripHTML($contents);
		close (FILEOUT);
		
		my $formattedRanking = sprintf("%.4f", $ranking);
		
		if (! defined $PageIndex{$formattedRanking}) {
			$PageIndex{$formattedRanking} = "$numResults^pp^".ParseDescription($contents);
		}
		else {
			$PageIndex{$formattedRanking} .= "&_&$numResults^pp^".ParseDescription($contents);
		}		
		$numResults++;
	}
	
	# Push the links on this page if it has a good score or if the spread is still low.	
	if ($ranking >= $MIN_THRESHOLD || $pagehistory{$currentURL} < $MAX_SPREAD) {
		# Ranking which all of the pages that connect to this one will recieve.
		my $linkSpread;
		if ($ranking >= $MIN_THRESHOLD) {
			$linkSpread = 1;
		}
		else {
			$linkSpread = $pagehistory{$currentURL} + 1;
		}
		
		# Extract links and resolve to current page.
		my $page_parser = HTML::LinkExtor->new(undef, $currentURL);
		$page_parser->parse(decode_utf8 $contents)->eof;
		
		# Process all of the links on the page.
		my @links = $page_parser->links;
		foreach my $link (@links) {
			my $currLink = $$link[2];
			# Make sure the link is HTML.
			$currLink =~ s/\?.*//;
			$currLink =~ s/\#.*//;
			if ($currLink =~ m/(\/|\/[^\.\/]+|\.html?|\.shtml?|\.cgi|\.jsp|
							   \.asp|\.aspx|\.php|\.pl|\.cfm)[^\/.]*$/) {
				# Add the page to the queue if it hasn't already been added.
				if (! defined $pagehistory{$currLink} &&
					$pagehistory{$currentURL} < 2 && !IsExcluded($currLink)) {
					if (! defined $robotpages{GetBase($currLink)}) {
						AddToRobotFilter($currentURL);
					}
					$pagehistory{$currLink} = $linkSpread;
					push (@pagequeue, $currLink);
				}
			}
		}
	}
}

GenerateIndex();


#####################################################################################


sub ParseText($) {
	my $text = "";
	if ($_[0] =~ m/\<title\>([^<]*)\<\/title\>/i) {
		$text = $1;
	}
	if ($_[0] =~ m/\<body[^>]*\>(.*)\<\/body\>/is) {
		$text .= $1;
	}
	$text =~ s/\<!--[^>]+?--\>//sg; # Commented HTML tags not removed properly.
	$text =~ s/\<script[^>]*\>.+?\<\/script\>//sg;
	$text =~ s/\<[^>]+\>//g;
	$text =~ s/&[A-Za-z#0-9]+;/ /g;
	$text =~ s/\n[\t ]+/\n/g;
	$text =~ s/\W+/ /g;
	return lc($text);
}

#####################################################################################


sub ParseDescription($) {
	my $description = "";
	$_[0] =~ s/\s+/ /;
	if ($_[0] =~ m/\<title\>([^<]+)\<\/title\>/i) {
		$description = $1;
	}
	elsif ($_[0] =~ m/\<h([1-6])\>([^<]+)\<\/h[1-6]\>/i) {
		$description = $2;
	}
	else {
		my $text = ParseText($_[0]);
		my $subLength = 100;
		if (length($text) < 100) {
			$subLength = length($text);
		}
		$description = substr($text, 0, $subLength)
	}
	return $description;
}

#####################################################################################


sub GenerateIndex() {
	open (FILEOUT, ">", $OUTPUT_FOLDER . "/index.html") or die "Can't open output file.";
	
	print FILEOUT "<html><body>\n<h1>Index of downloaded pages.</h1>\n<table>";
	
	my @sortedRanking = reverse sort(keys %PageIndex);
	
	foreach my $rank (@sortedRanking) {
		my @listOfPages = split(/&_&/, $PageIndex{$rank});
		foreach my $result (@listOfPages) {
			my ($fileNumber, $descr) = split(/\^pp\^/, $result);
			print FILEOUT "<tr><td>$rank</td><td><a href=\"$fileNumber.txt\">";
			print FILEOUT "$descr</a></td></tr>\n";
		}
	}
	
	print FILEOUT "</table>\n</body></html>\n";
	
	close (FILEOUT);
}

#####################################################################################


sub AddToRobotFilter($) {
	my $base = GetBase($_[0]);
	if (! defined $robotpages{$base}) {
		$robotpages{$base} = 1;
		my $request = HTTP::Request->new(GET => $base."robots.txt");
		my $response = $browser->request($request);
		if ($response->is_error()) {
			return;
		}
		my $contents = $response->content();
		
		$contents =~ s/#.*//g;
		if ($contents =~ m/^.*User-agent:\s*\*(.*)$/isg) {
			$contents = $1;
		}
		
		my @lines = split(/\n/, $contents);
		
		foreach (@lines) {
			if ($_ =~ m/Disallow:\s*\/(\S*)/i) {
				$excludelist{$base.$1} = 1;
			}
		}
	}
}

#####################################################################################


sub IsExcluded($) {
	my $link = $_[0];
	
	foreach (keys %excludelist) {
		if ($link =~ m#$_#i) {
			return 1;
		}
	}
	
	return 0;
}

#####################################################################################


sub GetBase($) {
	my $returnval = "";
	if ($_[0] =~ m$(http://[^/]+/)$) {
		$returnval = $1;
	}
	return $returnval;
}

#####################################################################################


sub LoadKeywords($) {
	my $INPUT;
	open (INPUT, $_[0]) or die "Can't open $_[0].\n";
	
	while (<INPUT>) {
		$_ =~ s/\W//g;
		$keywords{$_} = $_;
	}
	
	if (scalar keys(%keywords) == 0) {
		die "No words in $_[0].\n";
	}
}

#####################################################################################


sub GetPageRanking($) {
	my @allWords = split(/ /, $_[0]);
	my %uniqueWords = ();
	foreach (@allWords) {
		$uniqueWords{$_} = 0;
	}
	
	my $tokens_found = 0;
	foreach (keys %keywords) {
		if (defined $uniqueWords{$_}) {
			$tokens_found++;
		}
	}
	
	return ($tokens_found / scalar keys(%keywords));
}

#####################################################################################


sub StripHTML($) {
	my $text = "";
	if ($_[0] =~ m/\<title\>([^<]*)\<\/title\>/i) {
		$text = $1;
	}
	if ($_[0] =~ m/\<body[^>]*\>(.*)\<\/body\>/is) {
		$text .= $1;
	}
	$text =~ s/\<!--[^>]+?--\>//sg; # Commented HTML tags not removed properly.
	$text =~ s/\<script[^>]*\>.+?\<\/script\>//sg;
	$text =~ s/\<[^>]+\>//g;
	$text =~ s/&[A-Za-z#0-9]+;/ /g;
	$text =~ s/\n+[\t ]*/\n/g;
	$text =~ s/\n+/\n/g;
	return $text;
}