#!/usr/bin/perl

use gweb;
use gfio;
use strict;
use warnings;

my $EMAILMODE = 1;
my @SEARCH=("ceres","van","rijn");
my @REQUIRED=("van","rijn");
my @WANTED=("seringenhof");
my $INITSITES=100;
my $CURSITE;

if ($EMAILMODE) {
  @SEARCH=('company');
  @REQUIRED=('mail');
  @WANTED=();
}

my @sites=();
my %history = ();
my %mailhistory = ();
my $total=0;
my $todo=0;

my $filename = join("_",@SEARCH)."-".join("_",@REQUIRED)."-".join("_",@WANTED).".txt";
gfio::create($filename);

# init with yahoo
my $search=join('+',@SEARCH);
my $url="http://search.yahoo.com/search?p=$search&pz=$INITSITES";
my $test=gweb::website($url);

my $dat=$test->{content};
my @bll=split(/data-sb=\"\/beacon\/clk/,$dat);
shift @bll;
foreach my $b (@bll) {
  if ($b !~ /srpcache/) {
    if ($b =~ /href=\"(https?:\/\/[^\"]+)/) {
      addsite($1)
    }
  }
}

print "INIT: $total SITES\n";

my $snr=0;
while ($todo) {
  my $site=shift @sites; $todo--; $snr++;
  print "TRY $snr\/$total: $site\n";
  my $data=gweb::website($site);
  $CURSITE=$site;
  process($data->{content});
}

print "\n*** FINISHED CRAWLING $snr/$total SITES ***\n";

sub media {
  my ($url) = @_;
  if ($url =~ /jpg$/i) { return 1 }
  if ($url =~ /jpeg$/i) { return 1 }
  if ($url =~ /png$/i) { return 1 }
  if ($url =~ /gif$/i) { return 1 }
  if ($url =~ /bmp$/i) { return 1 }
  if ($url =~ /pdf$/i) { return 1 }
  if ($url =~ /avi$/i) { return 1 }
  if ($url =~ /mp4$/i) { return 1 }
  if ($url =~ /mpg$/i) { return 1 }
  return 0
}

sub addsite {
  my ($url) = @_;
  if (!$history{$url}) {
    if (media($url)) { return }
    $history{$url}=1;
    push @sites,$url;
    $total++; $todo++;
    print "ADD: $url\n"
  }
}

sub crawl {
  my ($data) = @_;
  my @links = ($data =~ /https?\:\/\/[^\s\t\n\r\"\'\<]+/ig);
  my $cnt=0;
  foreach my $link (@links) {
    # print "RECLINK: $link\n";
    addsite($link); $cnt++; if ($cnt>=100) { return }
  }
}

sub process {
  my ($data) = @_;
  if (!defined $data) { return }
  $data =~ s/\r//g;
  foreach my $rq (@REQUIRED) {
    if ($data !~ /$rq/i) {
      return
    }
  }
  crawl($data);
  if ($EMAILMODE) {
    checkmail($data); return
  }
  $data =~ s/\<[^>]+>/ /g;
  my @lines=split(/\n/,$data);
  foreach my $subline (@lines) {
    my @sls = split(/\.[\s\t]/,$subline);
    foreach my $sl (@sls) {
#      print "LINE: $sl\n";
      my $fnd=1;
      foreach my $want (@WANTED) {
        if ($sl !~ /$want/i) { $fnd=0; last }
      }
      if ($fnd) {
        foundline($sl)
      }
    }
  }
}

sub checkmail {
  my ($data) = @_;
  my @mails = ($data =~ /[a-z0-9\.\-\_\~]+\@[a-z0-9\-\_]+\.[a-z]+/ig);
  foreach my $mail (@mails) {
    if (media($mail)) { next }
    if ($mailhistory{$mail}) { next }
    $mailhistory{$mail} = 1;
    print "EMAIL: $mail\n";
    gfio::append($filename,"$mail\n")
  }
}

sub foundline {
  my ($line) = @_;
  if (!defined $line) { return }
  $line =~ s/\t/ /g;
  $line =~ s/  / /g;
  $line =~ s/^[\s]+//;
  $line =~ s/[\s]+$//;
  print "FOUND: $line\n";
  gfio::append($filename,"$CURSITE\n$line\n\n")
}