#!/usr/bin/perl

=pod

=head1 NAME

tv_grab_pt - Grab TV listings for Portugal.

=head1 SYNOPSIS

tv_grab_pt --help

tv_grab_pt [--config-file FILE] --configure [--gui OPTION]

tv_grab_pt [--config-file FILE] [--output FILE] [--quiet] [--icons]

tv_grab_pt --list-channels

=head1 DESCRIPTION

Output TV listings for several channels available in Portugal.
It supports the public network and the private NetCabo network.

First run B<tv_grab_pt --configure> to choose, which channels you want
to download. Then running B<tv_grab_pt> with no arguments will output
listings in XML format to standard output.

B<--configure> Prompt for which channels,
and write the configuration file.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk', or left blank for the best available choice.
Additional allowed values of OPTION are 'Term' for normal terminal output
(default) and 'TermNoProgressBar' to disable the use of Term::ProgressBar.

B<--config-file FILE> Set the name of the configuration file, the
default is B<~/.xmltv/tv_grab_pt.conf>.  This is the file written by
B<--configure> and read when grabbing.

B<--output FILE> Write to FILE rather than standard output.

B<--quiet> Suppress the progress messages normally written to standard
error.

B<--icons> Fetches channels icons/logos

B<--version> Show the version of the grabber.

B<--help> Print a help message and exit.

=head1 SEE ALSO

L<xmltv(5)>.

=head1 AUTHOR

Bruno Tavares, gawen@users.sourceforge.net, based on tv_grab_es, from Ramon Roca.

Grabber Site : http://bat.is-a-geek.com/XMLGrabPt

=head1 BUGS

=cut

######################################################################
# initializations

use warnings;
use strict;
use XMLTV::Version '$Id: tv_grab_pt,v 1.45 2008/07/22 16:55:46 rmeden Exp $ ';
use XMLTV::Capabilities qw/baseline manualconfig cache/;
use XMLTV::Description 'Portugal';
use Getopt::Long;
use Date::Manip;
use Data::Dumper;
use HTML::TreeBuilder;
use HTML::Entities; # parse entities
use IO::File;
use File::Path;
use File::Basename;
use LWP::UserAgent;
use Unicode::UTF8simple;

use XMLTV;
use XMLTV::Memoize;
use XMLTV::ProgressBar;
use XMLTV::Ask;
use XMLTV::Config_file;
use XMLTV::DST;
use XMLTV::Get_nice;
use XMLTV::Mode;
# Todo: perhaps we should internationalize messages and docs?
use XMLTV::Usage <<END
$0: get Portuguese television listings in XMLTV format
To configure: $0 --configure [--config-file FILE] [--gui OPTION]
To grab listings: $0 [--config-file FILE] [--output FILE] [--quiet] [--offset OFFSET] [--days DAYS] [--icons]
To list channels: $0 --list-channels
END
  ;

my $DOMAIN = 'tvcabo.pt';

# Attributes of the root element in output.
my $HEAD = { 'source-info-url'     => "http://$DOMAIN",
	     'source-data-url'     => "http://www.$DOMAIN/Televisao/Programacao.aspx",
	     'generator-info-name' => 'XMLTV',
	     'generator-info-url'  => 'http://membled.com/work/apps/xmltv/',
	   };

# default language
my $LANG="pt";

# Global channel_data
our @ch_all;

######################################################################
# get options

# Get options, including undocumented --cache option.
XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');
our ($opt_help, $opt_output,
    $opt_configure, $opt_config_file, $opt_gui, $opt_quiet,
    $opt_list_channels,$opt_offset,$opt_days,$opt_fullinfo,$opt_icons);
$opt_quiet  = 0; # default
GetOptions('help'          => \$opt_help,
	   'configure'     => \$opt_configure,
	   'config-file=s' => \$opt_config_file,
	   'gui:s'         => \$opt_gui,
	   'output=s'      => \$opt_output,
	   'quiet'         => \$opt_quiet,
	   'list-channels' => \$opt_list_channels,
	   'offset=i'      => \$opt_offset,
           'days=i'        => \$opt_days,
           'fullinfo'      => \$opt_fullinfo,
	   'icons' 	   => \$opt_icons,
	  )
  or usage(0);
usage(1) if $opt_help;
XMLTV::Ask::init($opt_gui);

# --offset and --days are ignored (we return more data than was
# requested) but at least check the user didn't ask for something
# impossible.
#
$opt_days |= 0;
if ($opt_days > 0) { $opt_days-- }
our $first_day = ($opt_offset || 0);
our $last_day  = $first_day + $opt_days;
die 'cannot grab more than one week ahead' if $first_day >= 7 || $last_day >= 7;

my $mode = XMLTV::Mode::mode('grab', # default
			     $opt_configure => 'configure',
			     $opt_list_channels => 'list-channels',
			    );

# File that stores which channels to download.
my $config_file
  = XMLTV::Config_file::filename($opt_config_file, 'tv_grab_pt', $opt_quiet);

my @config_lines; # used only in grab mode
if ($mode eq 'configure') {
    XMLTV::Config_file::check_no_overwrite($config_file);
    mkpath(dirname($config_file));
}
elsif ($mode eq 'grab') {
    @config_lines = XMLTV::Config_file::read_lines($config_file);
}
elsif ($mode eq 'list-channels') {
    # Config file not used.
}
else { die }

# Whatever we are doing, we need the channels data.
my $token;
my %channels = get_channels(); # sets @ch_all
my %channelnumbers;
my @channels;

my %icons = ();
%icons = get_icons() if $opt_icons;


######################################################################
# write configuration

if ($mode eq 'configure') {
    open(CONF, ">$config_file") or die "cannot write to $config_file: $!";

    # Ask about each channel.
    my @chs = sort keys %channels;
    my @names = map { $channels{$_} } @chs;
    my @qs = map { "add channel $_?" } @names;
    my @want = ask_many_boolean(1, @qs);
    foreach (@chs) {
    	my $w = shift @want;
    	warn("cannot read input, stopping channel questions"), last
    	  if not defined $w;
    	# No need to print to user - XMLTV::Ask is verbose enough.
    
    	# Print a config line, but comment it out if channel not wanted.
    	print CONF '#' if not $w;
    	my $name = shift @names;
    	print CONF "channel $_.$DOMAIN\n";
    }

    close CONF or warn "cannot close $config_file: $!";
    say("Finished configuration.");

    exit();
}


# Not configuration, we must be writing something, either full
# listings or just channels.
#
die if $mode ne 'grab' and $mode ne 'list-channels';

# Options to be used for XMLTV::Writer.
my %w_args;
if (defined $opt_output) {
    my $fh = new IO::File(">$opt_output");
    die "cannot write to $opt_output: $!" if not defined $fh;
    $w_args{OUTPUT} = $fh;
}
$w_args{encoding} = 'UTF-8';
my $writer;
sub start_writing() { ($writer = new XMLTV::Writer(%w_args))->start($HEAD) }

if ($mode eq 'list-channels') {
    start_writing;
    foreach (@ch_all) {
		    $_{'icon'} = [{'src' => $icons{$_}}] if(defined($icons{$_}));
    }
    $writer->write_channel($_) foreach @ch_all;
    $writer->end();
    exit();
}

######################################################################
# We are producing full listings.
die if $mode ne 'grab';

# Read configuration
my $line_num = 1;
foreach (@config_lines) {
    ++$line_num;
    next if not defined;

    # For now, check that $DOMAIN appears on every line.  This
    # ensures we don't have a config file left over from the old
    # grabber.
    #
    if (/^channel:?\s+(.+)\.tvcabo\.pt\s*$/) {
        my $ch_did = $1;
        die if not defined $ch_did;
        push @channels, $ch_did;
    }
    elsif (/^channel/) {
        die <<END
The configuration file is left over from the old tv_grab_pt.  The new
site uses different channels so you need to reconfigure the grabber.
END
          ;
    }
    else {
        warn "$config_file:$line_num: bad line\n";
    }
}

######################################################################
# begin main program

start_writing;

# Assume the listings source uses CET (see BUGS above).
die "No channels specified, run me with --configure\n"
  if not keys %channels;
my @to_get;

# the order in which we fetch the channels matters
# This progress bar is for both downloading and parsing.  Maybe
# they could be separate.
#

my $bar = new XMLTV::ProgressBar('getting listings',
				                          scalar @channels)
  if not $opt_quiet;
foreach my $ch_did (@channels) {
    die if not defined $ch_did;


    my $ch_name=$channels{$ch_did};
    my $channel = { id => "$ch_did.$DOMAIN",
                    'display-name' => [ [ $ch_name ] ],
		  };
    $channel->{'icon'} = [{'src' => $icons{$ch_did}}] if(defined($icons{$ch_did}));

    $writer->write_channel($channel);
}


my $some=0;
our $today_offset;
foreach my $ch_did (@channels) {
	#skip legacy channels...
	next unless $channels{$ch_did};
	foreach (process_table($ch_did)) {
	    $writer->write_programme($_);
	    $some = 1;
	}
	update $bar if $bar;
}
if (not $some) {
  die "no programmes found\n" unless $some;
}

$writer->end();

######################################################################
# subroutine definitions

# Use Log::TraceMessages if installed.
BEGIN {
    eval { require Log::TraceMessages };
    if ($@) {
	*t = sub {};
	*d = sub { '' };
    }
    else {
	*t = \&Log::TraceMessages::t;
	*d = \&Log::TraceMessages::d;
	Log::TraceMessages::check_argv();
    }
}

# Clean up bad characters in HTML.
sub tidy( $ ) {
    for (my $s = shift) {
	# Character 150 seems to be used for 'versus' in sporting
	# events, but I don't know what that is in Portuguese.
	#
	#s/\s\226\s/ vs /g;
	return $_;
    }
}

sub today_offset {
    return $today_offset if defined($today_offset);

    #we need to find today index
    my $td = get_nice_tree $HEAD->{"source-data-url"}, \&tidy;
    my $days_s = $td->find_by_attribute( 'id', 'SelectedDay' );
    my $td_offset = $days_s->attr('value');
    $today_offset = $td_offset;
    $td->delete;
    return $today_offset;
}

sub process_table {
    my ($ch_xmltv_id) = @_;

    t "Getting channel $ch_xmltv_id\n";

    #my $url = $HEAD->{"source-data-url"};
    my $url = $HEAD->{"source-info-url"};
    $url .= "/Televisao/ListaProgramas.aspx?";
    t $url;
    use LWP::UserAgent;

    my $ua = LWP::UserAgent->new;
    $ua->timeout(30);

    $ch_xmltv_id =~ /(.+?)\.tvcabo\.pt/;

    my @all_programs = ();

    for ( my $k = $first_day ; $k <= $last_day ; $k++ ) {
        my @program_data = ();
        my $ch_url_id    = $ch_xmltv_id;

        $ch_url_id =~ s/\+/%2B/g;
        $ch_url_id =~ s/\s/%20/g;

        my $ch_url =
            $url
          . "cat=&channelSigla="
          . encode_entities($ch_url_id) . "&dia="
          . ( today_offset() + $k )
          . "&package=9";

        my $r = $ua->get($ch_url);

        if ( !$r->is_success ) {
            die "Could not fetch $url ($ch_xmltv_id): " . $r->status_line;
        }

        my $data = tidy( $r->content );

        if ( not defined $data ) {
            die "could not fetch $url, aborting\n";
        }

        local $SIG{__WARN__} = sub {
            warn "$url: $_[0]";
        };

        # parse the page to a document object
        my $tree = HTML::TreeBuilder->new();
        $tree->parse($data) or die "cannot parse content of $url\n";
        $tree->eof;

        my %month_conv = (
            'Janeiro'   => 'Jan',
            'Fevereiro' => 'Feb',
            'Março'    => 'Mar',
            'Abril'     => 'Apr',
            'Maio'      => 'May',
            'Junho'     => 'Jun',
            'Julho'     => 'Jul',
            'Agosto'    => 'Aug',
            'Setembro'  => 'Sep',
            'Outubro'   => 'Oct',
            'Novembro'  => 'Nov',
            'Dezembro'  => 'Dec',
        );

        #lets get the day
        my $span = $tree->find_by_tag_name( '_tag' => 'strong' );
        next unless $span;    # ??

        my $date = trim( $span->as_trimmed_text() );

        print STDERR "$date\n" unless $opt_quiet;

        $date =~ /(\d+) (.+)$/;
        my ( $day, $month ) = ( $1, $2 );
        $month = $month_conv{$month};

        #lets calc the year
        my @local_time    = localtime( time() );
        my $current_year  = $local_time[5] + 1900;
        my $current_month = $local_time[4] + 1;
        my $year          = $current_year;
        if ( $month eq 'Jan' && $current_month != 1 ) {
            $year++;
        }

        my $parsed_date =
          ParseDate( sprintf( "%02d %3s %04d", $day, $month, $year ) );
        die "Parse error, could not parse date" unless $parsed_date;
        my $f_date = $parsed_date;
        $f_date =~ s/^(\d{4})(\d{2})(\d{2}).*/$1$2$3/;

        my $epgdiv = $tree->find_by_attribute( 'id',    'epgFrame' );
        my $table  = $epgdiv->find_by_tag_name( '_tag', 'table' );

        # Actually time and title are required, but we don't check that.
        if ($table) {
            my @trs = $table->find_by_tag_name( "_tag" => "tr" );

            # Store these for later, if we want full descs
            my @inputs = $tree->find_by_tag_name( '_tag', 'input' );
            my %descform;
            foreach my $input (@inputs) {
                next unless defined $input->attr('name');
                next if $input->attr('name') eq 'programId';
                $descform{ $input->attr('name') } = $input->attr('value');
            }

            foreach my $tr (@trs) {
                my @tds = $tr->find_by_tag_name( "_tag" => "td" );
                last unless ( $tds[0] && $tds[1] );

                my $time = trim( $tds[0]->as_trimmed_text() );
                my $title =
                  trim( $tds[1]->find_by_tag_name( "_tag" => "a" )
                      ->as_trimmed_text() );
                my $sub_t    = "";
                my $episode  = "";
                my $category = "";
                my $desc     = "";

                #Try to find if they included the subtitle in the title
                if ( $title =~ /^(.*?)\:\#\#(.*?)$/ ) {
                    $title = $1;
                    $sub_t = $2;
                    $sub_t =~ s/^\s*?(.*?)/$1/;
                    $title =~ s/(.*?)\s*?$/$1/;
                }
                elsif ( $title =~ /^(.*?)\#\#(.*?)$/ ) {
                    $title = $1;
                    $sub_t = $2;
                    $sub_t =~ s/^\s*?(.*?)/$1/;
                    $title =~ s/(.*?)\s*?$/$1/;
                }

                #lets get the description
                my $link =
                  $tds[1]->find_by_tag_name( "_tag" => "a" )->attr('href');
                my ($program_id) = ( $link =~ /\"(\d+)\"/ );

                my %credits;
                my $year;

                if ( $program_id && $opt_fullinfo ) {

                    my ( $base_url, undef ) = split( /\?/, $ch_url );

                    $descform{programId} = $program_id;
                    my $r_desc = $ua->post( $base_url, \%descform );

                    my $descr = $r_desc->content();

                    if ( $descr =~
/<\/h4>.*<strong>De:<\/strong>(.*?)<\/p>\s*\n\s*<p><strong>Com:<\/strong>(.*?)<\/p>\s*\n\s*<p>(.*?)<\/p>/ms
                      )
                    {
                        my $director = $1;
                        my $actors   = $2;
                        $desc = $3;

                        $director =~ s/&nbsp;//g;
                        my @directors = split( /,/, $director );
                        $actors =~ s/&nbsp;//g;
                        my @actors = split( /,/, $actors );
                        $credits{actor}    = \@actors;
                        $credits{director} = \@directors;

                    }
                    elsif ( $descr =~ /<\/h4>\s*\n\s*<p>(.*?)<\/p>/ms ) {
                        $desc = $1;
                    }

                    if ( $desc =~ /\sREPLAY:\s.*tecla verde.*/ ) {
                        $desc =~ s/\sREPLAY:\s.*//g;
                    }
                    if ( $desc =~ /\s*Ano:\s*(\d{4})\.\s*$/ ) {
                        $year = $1;
                    }
                }

                #try to find a matching category trough css classes
                my %map = (
                    epgFilm => 'Film',
                    epgSer  => 'Series',
                    epgDoc  => 'Documentary',
                    epgDesp => 'Sports',
                    epgNot  => 'News',
                );
                my $cat_class = $tds[1]->attr('class');
                if ( $cat_class && exists( $map{$cat_class} ) ) {
                    $category = $map{$cat_class};
                }

                $time =~ s/://g;
                $time = sprintf( "%04d", $time );

                for ($title) { s/^\s+$//; s/\s+$// }
                my %h = (
                    time          => $time,
                    title         => $title,
                    date          => $f_date,
                    "episode-num" => $episode,
                    subtitle      => $sub_t,
                    category      => $category,
                    desc          => $desc
                );

                $h{credits} = \%credits if %credits;
                $h{year}    = $year     if $year;

                push @program_data, \%h;
            }
        }

        if ( not @program_data ) {
            warn "$url ($ch_xmltv_id) : no programmes found\n";
            last;
        }
        @all_programs = ( @all_programs, @program_data );
        $tree->delete;
    }

    return () unless scalar(@all_programs);

    my $first = $all_programs[0];
    my @r;
    foreach my $p (@all_programs) {
        push @r, make_programme_hash( $ch_xmltv_id, $p, $first );
    }
    return @r;
}

sub make_programme_hash {
    my ( $ch_xmltv_id, $cur, $first ) = @_;

    my $date = $cur->{date};

    my %prog;

    $prog{channel} = "$ch_xmltv_id.$DOMAIN";

    #$prog{channel} =~ s/\s/_/g;

    $prog{title} = [ [ $cur->{title}, $LANG ] ];
    $prog{"sub-title"} = [ [ $cur->{subtitle}, $LANG ] ] if $cur->{subtitle};
    $prog{category} = [ [ $cur->{category}, $LANG ] ] if $cur->{category};
    $prog{"episode-num"} = $cur->{"episode-num"} if $cur->{"episode-num"};

    if (   ( $cur->{time} < $first->{time} )
        && ( $cur->{date} == $first->{date} ) )
    {
        t "Jumping for next day of ("
          . $cur->{time} . ","
          . $first->{time}
          . ") $date...";
        $date = nextday($date);
        t "Got $date\n";
    }

    my $time = $date . $cur->{time} . "00";

    #print STDERR "Date built = $time\n";

    $prog{start} = utc_offset( $time, '+0000' );
    t "...got $prog{start}";
    unless ( $prog{start} ) {
        warn "bad time string: $cur->{time}";
        return undef;
    }

    $prog{desc} = [ [ $cur->{desc}, $LANG ] ] if $cur->{desc};

    $prog{credits} = $cur->{credits} if $cur->{credits};
    $prog{date}    = $cur->{year}    if $cur->{year};

    return \%prog;
}


# as_trimmed_text() doesn't deal with ASCII 160, non-breaking space.
sub trim( $ ) {
    for (my $tmp = shift) {
	#tr/\240/ /;
	s/^\s+//;
	s/\s+$//;
	return $_;
    }
}

# get channel listing
sub get_channels {
    my $bar = new XMLTV::ProgressBar( 'getting list of channels', 1 )
      if not $opt_quiet;
    my %channels;
    my $url = $HEAD->{'source-data-url'};
    t $url;

    my $tree  = get_nice_tree $url;
    my @menus = $tree->find_by_tag_name( "_tag" => "select" );
    my $uref  = new Unicode::UTF8simple;

    foreach my $elem (@menus) {
        my $cname = $elem->attr('name');
        next unless $cname eq 'channels';
        my @ocanals = $elem->find_by_tag_name( "_tag" => "option" );
        my $chanpos = 0;
        foreach my $sigla (@ocanals) {
            $channelnumbers{ $sigla->attr('value') } = $chanpos;
            $chanpos++;
        }
        @ocanals = sort @ocanals;
        foreach my $opt (@ocanals) {
            my $channel_id = $opt->attr('value');

            next if $channel_id eq 'ALL';

            my $channel_name = trim( $opt->content->[0] );
            $channel_name = $uref->toUTF8( "iso-8859-1", $channel_name );
            $channels{$channel_id} = $channel_name;
            push @ch_all,
              {
                'display-name' => [ [ $channel_name, $LANG ] ],
                'id' => "$channel_id"
              };
        }    #foreach
    }    #while
    die "no channels could be found" if not keys %channels;

    #we need a token
    my $t = $tree->find_by_attribute( 'name', '__VIEWSTATE' );
    $token = $t->attr('value');
    die "Could not get the token\n" unless $token;

    update $bar if not $opt_quiet;
    $tree->delete;
    return %channels;
}

sub nextday {
    my $d = shift;
    my $p = ParseDate($d);
    my $n = DateCalc($p, '+ 1 day');
    return UnixDate($n, '%Q');
}

sub get_icons {
    my %icons;
    my $url= $HEAD->{"source-info-url"}."/Televisao/ListaProgramas.aspx?dia=0&package=9&cat=&channelSigla=";
    my $chan;
    my $tag;
    my $addr;

    my $bar = new XMLTV::ProgressBar('grabbing icons', scalar(keys(%channels)))
      if not $opt_quiet;

    foreach (keys %channels) {
        my $tb = get_nice_tree $url.encode_entities($_), \&tidy;

        $tag = $tb->look_down('_tag' => 'img',
        sub {
            return ($_[0]->attr('src') =~ m/Shared\/img\/televisao\/BackofficeImages\//);
        });
        update $bar if not $opt_quiet;

        unless(ref($tag) eq "HTML::Element") {
                $tb->delete;
                next;
        };

        $icons{$_} = $tag->attr('src');
	$icons{$_} =~ s/\.\./$HEAD->{'source-info-url'}/;

        $tb->delete;
    }
    $bar->finish() if not $opt_quiet;

    return %icons;
}

