#!/usr/bin/perl -w
#
# Copyright G. Westcott - February 2013
#
# This code is distributed under the GNU General Public License v2 (GPLv2) .
#
#   For extended help information run
#         tv_grab_uk_tvguide  --info
#

eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}'
    if 0; # not running under some shell

use Data::Dumper;

use strict;
use warnings;
use XMLTV 1.0.1;
use XMLTV::ProgressBar;
use XMLTV::Options qw/ParseOptions/;
use XMLTV::Configure::Writer;
use XMLTV::Supplement 0.005065 qw/SetSupplementRoot GetSupplementDir GetSupplementLines/;
use XMLTV::Get_nice 0.005065 qw/get_nice_tree/;

use File::Path;
use POSIX qw(strftime);
use DateTime;
use Date::Parse;
#v1.3: use DateTime::Format::DateParse;
use Encode;
use URI::Escape;
use LWP::UserAgent;
use HTTP::Cookies;
use HTML::TreeBuilder;
use HTTP::Cache::Transparent;


#require HTTP::Cookies;
#my $cookies = HTTP::Cookies->new;
#$XMLTV::Get_nice::ua->cookie_jar($cookies);


# Although we use HTTP::Cache::Transparent, this undocumented --cache
# option for debugging is still useful since it will _always_ use a
# cached copy of a page, without contacting the server at all.
#
use XMLTV::Memoize; XMLTV::Memoize::check_argv('XMLTV::Get_nice::get_nice_aux');

use subs qw(debug warning);
my $warnings = 0;


use XMLTV::Usage <<END;
tv_grab_uk_tvguide: Get UK television listings in XMLTV format
Version: $XMLTV::VERSION
To configure: $0 --configure [--config-file FILE] [--gui OPTION]
To grab listings: $0 [--config-file FILE] [--output FILE] [--quiet] [--offset DAYS] [--days DAYS] [--nodetailspage] [--legacychannels]
To list available channels: $0 --list-channels [--output FILE]
END


# ------------------------------------------------------------------------------------------------------------------------------------- #
# Grabber details
my $VERSION				= "$XMLTV::VERSION";
my $GRABBER_NAME		= 'tv_grab_uk_tvguide';
my $GRABBER_DESC		= 'UK - TV Guide (tvguide.co.uk)';
my $GRABBER_URL			= 'http://wiki.xmltv.org/index.php/XMLTVProject';
my $ROOT_URL			= 'https://www.tvguide.co.uk/';
my $SOURCE_NAME			= 'TV Guide UK';
my $SOURCE_URL			= 'https://www.tvguide.co.uk/';
#
my $generator_info_name	= $GRABBER_NAME;
my $generator_info_url	= $GRABBER_URL;
my $source_info_name	= $SOURCE_NAME;
my $source_info_url		= $SOURCE_URL;
#
my $grabberid = '2023-05-02.0921';

# ------------------------------------------------------------------------------------------------------------------------------------- #
# Use XMLTV::Options::ParseOptions to parse the options and take care of the basic capabilities that a tv_grabber should have

our ($opt, $conf) = ParseOptions({
			grabber_name		=> $GRABBER_NAME,
			capabilities		=> [qw/baseline manualconfig apiconfig cache tkconfig/],		# apiconfig is needed for --list-channels (bug)
			stage_sub			=> \&config_stage,
			listchannels_sub	=> \&fetch_channels,
			version				=> $VERSION,
			description			=> $GRABBER_DESC,
			extra_options		=> [qw/nodetailspage legacychannels method:i makeignorelist:s useignorelist:s usage test:i/],
});
 
usage(1) if $opt->{usage};

#print Dumper($conf); exit;

# any overrides?
if (defined( $conf->{'generator-info-name'} ))	{ $generator_info_name 	= $conf->{'generator-info-name'}->[0]; }
if (defined( $conf->{'generator-info-url'} ))	{ $generator_info_url  	= $conf->{'generator-info-url'}->[0]; }
if (defined( $conf->{'source-info-name'} )) 	{ $source_info_name 	= $conf->{'source-info-name'}->[0]; }
if (defined( $conf->{'source-info-url'} ))  	{ $source_info_url  	= $conf->{'source-info-url'}->[0]; }


# initialise the gui
XMLTV::Ask::init($opt->{gui});


# ------------------------------------------------------------------------------------------------------------------------------------- #
# Initialise the web page cache
init_cachedir( $conf->{cachedir}->[0] );
HTTP::Cache::Transparent::init( {
	BasePath => $conf->{cachedir}->[0],
	NoUpdate => 60*60,			# cache time in seconds
	MaxAge => 24,				# flush time in hours
	Verbose => $opt->{debug},
} );


# ------------------------------------------------------------------------------------------------------------------------------------- #
# Check we have all our required conf params
config_check();

# Load the conf file containing mapped channels and categories information
my %mapchannelhash;
my %mapcategoryhash;
loadmapconf();
#print Dumper(\%mapchannelhash, \%mapcategoryhash); exit;



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Progress Bar :)
my $bar = new XMLTV::ProgressBar({
		name => "Fetching listings",
		count => (scalar @{$conf->{channel}}) * ($opt->{days} + 1)		# +1 added for the extra day necessary for <06:00 programmes
}) unless ($opt->{quiet} || $opt->{debug});



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Data store before being written as XML
my $programmes = ();
my $channels = ();

# Store channel names during fetch
my $channames = undef;

# Cache for alternative ID lookups
my $channels_cache = {};
my $channels_alt = {};
my $channels_alt_found = {};

# Get the schedule(s) from TV Guide
fetch_listings();

# print Dumper($programmes);

# Progress Bar
$bar->finish() && undef $bar if defined $bar;



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Filter out programmes outside of requested period (see man page)
my %w_args;
if (($opt->{offset} != 0) || ($opt->{days} != -999)) {
	$w_args{offset} = $opt->{offset};
	$w_args{days}   = ($opt->{days} == -999) ? 100 : $opt->{days};
	$w_args{cutoff} = '000000';			# e.g. '060000'
}



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Generate the XML
my $encoding = 'UTF-8';
my $credits = { 'generator-info-name' 	=> $generator_info_name,
				'generator-info-url' 	=> $generator_info_url,
				'source-info-name' 		=> $source_info_name,
				'source-info-url' 		=> $source_info_url };

XMLTV::write_data([ $encoding, $credits, $channels, $programmes ], %w_args);
# Finished!



# ------------------------------------------------------------------------------------------------------------------------------------- #
# Signal that something went wrong if there were warnings.
exit(1) if $warnings;

# All data fetched ok.
debug "Exiting without warnings.";
exit(0);


# #############################################################################
# # THE MEAT #####################################################################
# ------------------------------------------------------------------------------------------------------------------------------------- #

sub fetch_listings {
	# Fetch listings per channel

	foreach my $channel_id (@{$conf->{channel}}) {
		# Officially:
		# http://www.tvguide.co.uk/channellisting.asp?cTime=3%2F19%2F2013+06%3A00%3A00+&ch=857&go=go
		# But this works too:
		# http://www.tvguide.co.uk/channellisting.asp?ch=86&cTime=3/18/2013

		my $baseurl = $ROOT_URL.'channellistings.asp';

		debug('+' x 80);

		# Now grab listings for each channel on each day, according to the options in $opt
		#
		# tvguide runs from 06:00 so we need to get the previous day as well just for any programmes after midnight
		#
		for (my $i=($opt->{offset} -1); $i < ($opt->{offset} + $opt->{days}); $i++) {

			# Inner loop allows us to push alternative IDs (if found) into @alts when no schedule is found for the user selected ID
			my @alts = int($channel_id);

			# Have we already found a working alternative ID for the user's selected ID
			if (defined $channels_alt_found->{$channel_id}) {
				@alts = $channels_alt_found->{$channel_id};
				debug "Using alternative ID: ".$channels_alt_found->{$channel_id}.' for '.$channel_id;
			}

			foreach my $alt_channel_id (@alts) {

				my $alt_success = 0;

				my $theday = DateTime->today->add (days => $i)->set_time_zone('Europe/London');

				# Construct the listings url
				my $url = $baseurl . '?ch=' . $alt_channel_id . '&cTime=' . uri_escape( $theday->strftime('%m/%d/%Y 00:00:00') );
				#debug "Fetching: $url";

				# If we need to map the fetched channel_id to a different value
				my $xmlchannel_id = $channel_id;
				$xmlchannel_id .= '.tvguide.co.uk' unless $opt->{legacychannels};		# make channel RFC2838 compliant
				if (defined(&map_channel_id)) { $xmlchannel_id = map_channel_id($xmlchannel_id); }

				# Fetch the page
				#   my $tree = XMLTV::Get_nice::get_nice_tree($url);
				my $tree = fetch_url($url);
				# $tree->dump; exit;

				# Scrub the page
				if ($tree) {
					my $channelname = undef;

					# Store the channel ids in a list (do this only once per program run)
					if (!defined $channames) {
						#debug 'fetching options tags';
						my $choptions = $tree->look_down('_tag' => 'select', 'name' => 'ch');
						if (defined $choptions) {
							my @choptionslist = $choptions->look_down('_tag' => 'option');
							if (@choptionslist) {
								foreach my $choption (@choptionslist) {
									$channames->{$choption->attr('value')} = $choption->as_text;
								}
							}
						}
					}

					$channelname = $channames->{$alt_channel_id} if $channames;

					# Try a fallback method if the form options are missing [Credit mkbloke]
					if (!defined $channelname) {
						#debug 'using fallback method';
						my $fallback = $tree->look_down('_tag' => 'input', 'name' => 'cTime');
						$fallback = $fallback->look_up('_tag', 'tr') if $fallback;
						$channelname = $fallback->look_down('_tag' => 'span', 'class' => 'programmeheading') if $fallback;
						$channelname = $channelname->as_text if $channelname;
					}

					#debug 'found channel name: '.$channelname;

					# tvguide website can be very slow - try to avoid barfing when no response
					# if no channelname then assume we got no response from website
					if (!defined $channelname) {
						warning "Unable to retrieve web page for $alt_channel_id";
						next;
					}

					# 	<table border="0" cellpadding="0" style="background:black;border-collapse: collapse;background-image: url(http://i.g8.tv/HighlightImages/Large/);background-repeat: no-repeat;" width="677">
					#
					my @shows = $tree->look_down('_tag' => 'table', 'border' => '0', 'cellpadding' => '0', 'style' => qr/background:\s*black;border-collapse:\s*collapse;/);

					if (@shows) {
						my $count = 0;

						foreach my $show (@shows) {
							#	$show->dump;
							$count++;

							# are we processing yesterday's schedule? (see above)
							if ($i == ($opt->{offset} -1)) {
								my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
								my ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								# 2014-04-02  see note below
								if (!defined $a) {
									$showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
									($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								}
								if ($a eq 'am' && ($h < 6 || $h == 12)) {
									next if $count == 1;	# we don't want first programme in file if it overlaps 6am boundary
									# continue processing of pre-6am programme
								} else {
									next;
								}
								$showstart = $h = $i = $a = undef;
							}

							my %prog = ();

							my $showtime;

							# see if we have a details page
							#		<a href="javascript:popup(151361219);" ...
							#		http://www.tvguide.co.uk/detail.asp?id=151451760
							# 2013-12-14 site changed
							#    <a href="javascript:popupshow('http://www.tvguide.co.uk/detail/1889599/94819598/saturday-kitchen-live');" target="_blank" ...
							# 2014-04-02 site changed
							#    <a href="http://www.tvguide.co.uk/detail/138990373/88745969/breakfast" target="_blank"...
							# 2014-11-26 site changed
							#    <a href="http://watch.tvguide.co.uk/engage/2057116/103685857-fake_britain" target="_blank"...
							# 2014-12-03 site changed
							#  my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/javascript:popup/);
							# 2014-12-03 The new website seems a bit flakey with these details pages, often returning a 500 Server Error
							#    Here's an option to disable the details pages  (  --nodetailspage )
							# 2014-12-24 site changed
							#  my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/\/engage\//);
							#
							if (!$opt->{nodetailspage}) {

								my $webdetails = $show->look_down('_tag' => 'a', 'href' => qr/\/detail\//);
								my $href = $webdetails->attr('href');
								#    my ($id) = $href =~ /javascript:popup\((\d*)\);/;
								#    $url = $ROOT_URL . 'detail.asp?id=' . $id;
								#    my ($url) = $href =~ /javascript:popupshow\('(.*)'\);/;
								my ($url) = $href;
								#debug "Fetching: $url";

								# Fetch the page
								#   my $showdetail = XMLTV::Get_nice::get_nice_tree($url);
								my $showdetail = fetch_url($url);
								#	$showdetail->dump;

								if ($showdetail) {
									# Details page contains Director names and a better list of Actors

									# Get the cast and extract them into a new tree
									my @lis = $showdetail->look_down('_tag' => 'div', 'class' => 'cast-entry');

									LOOP:
									foreach my $person (@lis) {
										#
										# 30/6/16
										#   <div class="cast-entry">
										#       <span class="role">Margaret Sellinger</span>
										#       <a href="http://www.tvguide.co.uk/actor.asp?actor=Lesley-Anne Down" target="_blank">
										#          <span itemprop="actor" itemscope itemtype="http://schema.org/Person">
										#             <span class="actor" itemprop="name">Lesley-Anne Down</span>
										#          </span>
										#       </a>
										#       <a target="_blank" href="http://uk.imdb.com/find?s=nm&q=Lesley-Anne+Down"><span class="actor">(IMDB)</span></a>
										#   </span>
										#   </div>

										my ($name, $role);
										if ( my ($_name) = $person->look_down('_tag' => 'span', 'itemprop' => 'name') ) {
											$name = $_name->as_text;
										}
										if ( my ($_role) = $person->look_down('_tag' => 'span', 'class' => 'role') ) {
											$role = $_role->as_text;
										}
										#	drop the "Executive Director" & "Executive Producer"   - any others we should drop?
										next LOOP  if ( $role =~ /^(Executive Director|Executive Producer)/ );

										# map the website role to an xmltv role
										my %xmltvroles = ( 'Director'=>'director', 'Producer'=>'producer', 'Series Producer'=>'producer', 'Writer'=>'writer', 'Co-Director'=>'director', 'Presenter'=>'presenter', 'Commentator'=>'commentator', 'Guest'=>'guest' );

										my $credit;
										if (exists $xmltvroles{$role}) {
											$credit = $xmltvroles{$role};
										} else {
											$credit = 'actor';
										}

										if ($credit eq 'actor' && defined $role) {
											push @{$prog{'credits'}{$credit}}, [ encode('utf-8', $name), encode('utf-8', $role) ];
										} else {
											push @{$prog{'credits'}{$credit}}, encode('utf-8', $name);
										}

									}

									undef @lis;


									# Get the "Left Panel" which contains the programme times and attributes
									my $lhs = $showdetail->look_down('_tag' => 'div', 'class' => qr/divLHS-section-2/);


									# Get the programme's "attributes" e.g. "Certificate"
									if ($lhs) {
										my @attrs = $lhs->look_down('_tag' => 'span', 'class' => 'LHS-attribute');
										if (@attrs) {
											foreach my $attr (@attrs) {
												# $attr->dump;
												if ( my $showattr = $attr->as_text() ) {
													if ( $showattr =~ /^Certificate\s:\s(.*)\s*$/ ) { $prog{'rating'} = [[ $1, 'BBFC' ]] if $1; }
												}
											}
										}
									}


									# start time, and stop time (actually an optional DTD element)
									#		<span class="datetime">10:00am-11:50am <span class=programmetext> (1 hour 50 minutes)</span> Wed 20 Mar</span>
									# (use the Date provided to avoid issues with the site running from 06:00-06:00)
									#
									# Note site displays stop time wrong on GMT/BST changeover, e.g.:
									#			12:45am-1:10am (25 minutes) Sun 31 Mar
									# 	this should be 12:45am-2:10am (BST)
									# 	this makes $showtime->set barf on "invalid local time for date in timezone"
									#
									#  1/Jan/17 times are now in the left panel. 'datetime' is used for user comments!
									#     my $showtimes = $showdetail->look_down('_tag' => 'span', 'class' => 'datetime');
									#
									#  25/Mar/2023 now displays the correct time on GMT/BST changeover...but the duration is wrong!
									#			Sun 26 Mar 12:40am-2:40am (2 hours)
									#	this should be '1 hour'
									#
									if ($lhs) {

										# Unfortunately the div with the date doesn't have any safe identifier. There are several ways we could remove the
										#  cruft from the container but the following, although clunky, is probably the safest
										my ($dt, $h, $i, $a, $h2, $i2, $a2) = $lhs->as_text =~ /((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun|Christmas\s(?:Eve|Day)|Boxing\sDay|New\sYears\s(?:Eve|Day))[\s<].*?)(\d*):(\d*)(am|pm)(?:-(\d*):(\d*)(am|pm))?/;
										# print STDERR $dt."\n";

										if ($dt && $dt !~ /\D\D\D\s\d\d?\s\D\D\D/) {
											my @thedt = localtime(time);	# ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)
											my ($yr1, $yr2) = ($thedt[5]+1900, $thedt[5]+1900);
											if ($thedt[4] == 11) { $yr2++; }
											if ($thedt[4] == 0) { $yr1--; }
											SWITCH: {
												$dt =~ 'Christmas\s+Eve' 		&& do { $dt = '24 Dec '.$yr1; last SWITCH; };
												$dt =~ 'Christmas\s+Day' 		&& do { $dt = '25 Dec '.$yr1; last SWITCH; };
												$dt =~ 'Boxing\s+Day' 			&& do { $dt = '26 Dec '.$yr1; last SWITCH; };
												$dt =~ 'New\s+Years\s+Eve' 		&& do { $dt = '31 Dec '.$yr1; last SWITCH; };
												$dt =~ 'New\s+Years\s+Day' 		&& do { $dt = '1 Jan ' .$yr2; last SWITCH; };
												undef $dt;
											}
										}
										if ($dt) {
											 #v1.3: $showtime = DateTime::Format::DateParse->parse_datetime( $dt, 'Europe/London' );

											 # workaround for bug in Date::Time::str2time() which generates wrong dates for future months when no year is given
											 #		https://rt.cpan.org/Public/Bug/Display.html?id=92611
											 $dt .= ' '.( $theday->year() )  if $dt !~ /(19|20)\d\d/;
											 $showtime = DateTime->from_epoch( epoch=>str2time( $dt, 'GMT' ) )->set_time_zone('Europe/London');
										} else {
											 $showtime = $theday->clone;
										}
										$h  += 12 if $a  eq 'pm' && $h   < 12;		# e.g. 12:30pm means 12:30 !
										$h  -= 12 if $a  eq 'am' && $h  == 12;		# e.g. 12:30am means 00:30 !
										$h2 += 12 if $a2 eq 'pm' && $h2  < 12;
										$h2 -= 12 if $a2 eq 'am' && $h2 == 12;
										$showtime->set(hour => $h, minute => $i, second => 0);
										$prog{'start'} = $showtime->strftime("%Y%m%d%H%M%S %z");
										my $showtime_ = $showtime->clone;
										if (defined $h2 && $h2 >= 0) {
											$showtime->add (days => 1) if $h2 < $h;
											# see note above re errors with GMT/BST transition
											eval {				# try
												$showtime->set(hour => $h2, minute => $i2, second => 0);
												$prog{'stop'} = $showtime->strftime("%Y%m%d%H%M%S %z");
											} or do {			# catch
												# let's see if we can get a duration
												my ($durh, $durm) = $lhs->as_text =~ /\((?:(\d*)\shours?)?\s?(?:(\d*)\sminutes?)?\)/;
												if (defined $durh || defined $durm) {
													$durh = 0 if !defined $durh; $durm = 0 if !defined $durm;
													$showtime_->set_time_zone('UTC')->add( hours => $durh, minutes => $durm )->set_time_zone('Europe/London');
													$prog{'stop'} = $showtime_->strftime("%Y%m%d%H%M%S %z");
												} else {
													# no output prog 'stop' time
												}
											}
										} else {
											# no output prog 'stop' time
										}

										# if the first programme starts before 06:00 we should ignore it as it will create a duplicate programme
										if ($h < 6) {
											# skip this prog as we have already retrieved it with 'yesterday's schedule
											next if ($count == 1);
										}

									}


									# programme image
									my $c = $showdetail->look_down('_tag' => 'div', 'id' => 'divCentre');
									if ($c) {
										my $img = $c->look_down('_tag' => 'div', 'id' => 'headerImage');
										if ($img) {
											my $showattr = $img->attr('style');
											(my $showimage) = $showattr =~ /background-image:\s*url\((.*?\.jpg)\)/;
											if ($showimage) {
												$prog{'image'} = [[ $showimage, { 'system'=>'tvguide', 'type'=>'backdrop' } ]];
											}
										}
									}

								}	# end showdetail

								$showdetail->delete() if $showdetail;

							}	# end nodetailspage


							# channel
							$prog{'channel'} = $xmlchannel_id;

							# title (mandatory)
							#		<span class="programmeheading">Baywatch</span>
							my $showtitle = $show->look_down('_tag' => 'span', 'class' => 'programmeheading');
							$prog{'title'} = [[ encode('utf-8', $showtitle->as_text), 'en' ]];
							$showtitle->detach;


							# Note: <span class="tvchannel"> is used by StartTime then SubTitle then Category then Subtitles/B&W/etc

							# start (mandatory)
							#		<span class="tvchannel">3:00pm </span>
							# (don't add it even we already have it from the detail page but we still need to delete it from the tree)

							# @ 2014-04-02 the site has changed to
							#                         <span class="season">6:00 am </span>
							# but this just doesn't sound right to me (i.e. I think it might change again), so let's try both ways
							#
							# @2016-08-05 looks like this is permanent
							# my $showstart = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
							my $showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
							if (!$prog{'start'}) {
								my ($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								if (!defined $a) {
									$showstart = $show->look_down('_tag' => 'span', 'class' => 'season');
									($h, $i, $a) = $showstart->as_text =~ /(\d*):(\d*)\s*(am|pm)/;
								}
								$h += 12 if $a eq 'pm' && $h  < 12;		# e.g. 12:30pm means 12:30 !
								$h -= 12 if $a eq 'am' && $h == 12;		# e.g. 12:30am means 00:30 !
								$showtime = $theday->clone;

								# @ 2023-03-25 The assumption that the site works from 06:00 - 06:00 breaks when the first programme overlaps 06:00
								#   so it seems the rule is the 'first' programme ends *after* 6:00 am
								#       Saturday, March 25, 2023  ITVBe
								#       1:00am  Teleshopping
								#       7:00am  Sam Faiers: The Mummy Diaries
								#
								#       Saturday, March 25, 2023  BBC Two HD
								#       3:45am  This Is BBC Two
								#       6:35am  Hey Duggee
								# Only way I can think of to handle this is to tag the first programme on a day
								# In addition, this programme should be ignored to avoid duplicates
								#
								if ($h < 6) {
									# skip this prog as we have already retrieved it with 'yesterday's schedule
									next if ($count == 1);
									$showtime->add(days => 1) if ($h < 6);		# site runs from 06:00-06:00 so anything <06:00 is for tomorrow
								}

								$showtime->set(hour => $h, minute => $i, second => 0);
								$prog{'start'} = $showtime->strftime("%Y%m%d%H%M%S %z");
								# no prog 'stop' time available
							}
							$showstart->detach;

							# category
							#		<span class="tvchannel">Category </span><span class="programmetext">General Movie/Drama</span>
							my $showcategory = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /^Category\s*$/ } );
							if ($showcategory) {
								$showcategory = $showcategory->right;
								my @showcategory = split(/\//, $showcategory->as_text);
								my @showcategories = ();
								foreach my $category (@showcategory) {
									# category translation?
									if (defined(&map_category)) { $category = map_category($category); }
									if ($category =~ /\|/) {
										foreach my $cat (split(/\|/, $category)) { push @showcategories, $cat unless grep(/$cat/, @showcategories); }
									} elsif ($category ne '') {
										push @showcategories, $category unless grep(/$category/, @showcategories);
									}
								}
								foreach my $category (@showcategories) {
									push @{$prog{'category'}}, [ encode('utf-8', $category), 'en' ];
								}
								$showcategory->left->detach;
							}

							# desc
							#		<span class="programmetext">Dissolving bikinis cause a stir on the beach</span>
							my $showdesc = $show->look_down('_tag' => 'span', 'class' => 'programmetext');
							if ($showdesc) {
								$showdesc = $showdesc->as_text;
								$showdesc .= '.' if ( (length $showdesc) && ((substr $showdesc,-1,1) ne '.') );	# append a fullstop
								if (length $showdesc) {
									$prog{'desc'} = [[ encode('utf-8', $showdesc), 'en' ]];
								}
							}

							# year
							# 	strip this off the title e.g. "A Useful Life (2010)"
							my ($showyear) = $prog{'title'}->[0][0] =~ /.*\((\d\d\d\d)\)$/;
							if ($showyear) {
								$prog{'date'} = $showyear;
								# assume anything with a year is a film - add Films category group
								push @{$prog{'category'}}, [ 'Films', 'en' ];
							}

							# flags
							# 		<span class='tvchannel'>(Subtitles)</span> <span class='tvchannel'>(Black &amp; White)</span>
							my $showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Subtitles/ } );
							if ($showflags) {
								push @{$prog{'subtitles'}}, {'type' => 'teletext'};
								$showflags->detach;
							}
							$showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Audio Described/ } );
							if ($showflags) {
							#	push @{$prog{'subtitles'}}, {'type' => 'deaf-signed'};    <-- Audio Described is not deaf-signed
								$showflags->detach;
							}
							$showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Repeat/ } );
							if ($showflags) {
								#	push @{$prog{'previously-shown'}}, {};
								$prog{'previously-shown'} = {};
								$showflags->detach;
							}
							my $showvideo = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Black & White/ } );
							if ($showvideo) {
								$prog{'video'}->{'colour'} = '0';
								$showvideo->detach;
							}
							#if ($showflags && $showflags->as_text =~ '\[REP\]') {
							#	push @{$prog{'previously-shown'}}, {};
							#}
							$showflags = $show->look_down('_tag' => 'span', 'class' => 'tvchannel', sub { $_[0]->as_text =~ /Interactive/ } );
							if ($showflags) {
								# no flag in DTD for this
								$showflags->detach;
							}


							# episode number
							# 	<span class="season">Season 2 </span> <span class="season">Episode 3 of 22</span>
							my @showepisode = $show->look_down('_tag' => 'span', 'class' => 'season');
							my $showepisode;
							foreach my $el (@showepisode) {
								$showepisode .= $el->as_text;
							}
							if ($showepisode) {
								my ($showsxx, $showexx, $showeof) = ( $showepisode =~ /^(?:(?:Series|Season) (\d+)(?:[., :]+)?)?(?:Episode (\d+)(?: of (\d+))?)?/ );
								# scan the description for any "Part x of x." info
								my ($showpxx, $showpof) = ('', '');
								($showpxx, $showpof) = ( $showdesc =~ /Part (one|two|three|four|five|six|seven|eight|nine|\d+)(?: of (one|two|three|four|five|six|seven|eight|nine|\d+))?/ ) if ($showdesc);
								my $showepnum = make_ns_epnum($showsxx, $showexx, $showeof, $showpxx, $showpof);
								if ($showepnum && $showepnum ne '...') {
									$prog{'episode-num'} = [[ $showepnum, 'xmltv_ns' ]];
								}
								#debug "--$showepnum-- ".$showepisode->as_text;
							}

							# episode title
							# 	<span class="tvchannel">The Fabulous Buchannon Boys</span>
							my $showeptitle = $show->look_down('_tag' => 'span', 'class' => 'tvchannel');
							if ($showeptitle) {
							  if ($showeptitle->as_text =~ /\(?Premiere\)?/) {
									$prog{'premiere'} = [];
								} else {
									$prog{'sub-title'} = [[ encode('utf-8', $showeptitle->as_text), 'en' ]];
								}
								$showeptitle->detach;
							}

							# director
							# never seen one but let's assume they're in the description
							if (!$prog{'credits'}->{'director'}) {
								if ($showdesc) {
									my ($directors) = ( $showdesc =~ /(?:Directed by|Director) ([^\.]*)\.?/ );
									if ($directors) {
										$directors =~ s/ (with|and) /,/ig;
										$directors =~ s/ (singer|actor|actress) //ig;			# strip these words
										$directors =~ s/,,/,/g;	# delete empties
										$directors = encode('utf-8', $directors);	# encode names into utf-8
										#debug $directors;
										my @directors = split(/,/, $directors);
										s{^\s+|\s+$}{}g foreach @directors;	# strip leading & trailing spaces
										$prog{'credits'}->{'director'} = \@directors  if (scalar @directors > 0);
									}
								}
							}

							# actors
							# these are buried in the description  :-(
							if (!$prog{'credits'}->{'actor'}) {
								if ($showdesc) {
									my ($actors) = ( $showdesc =~ /(?:starring)([^\.]*)\.?/i );
									if ($actors) {
										$actors =~ s/ (also|starring|with|and) /,/ig;		# may be used to separate names
										$actors =~ s/ (singer|actor|actress) //ig;			# strip these words
										$actors =~ s/,,/,/g;	# delete empties
										$actors = encode('utf-8', $actors);	# encode names into utf-8
										#debug $actors;
										my @actors = split(/,/, $actors);
										s{^\s+|\s+$}{}g foreach @actors;	# strip leading & trailing spaces
										$prog{'credits'}->{'actor'} = \@actors  if (scalar @actors > 0);
									}
								}
							}

							# rating
							#		<span class="programmetext">Rating<br></span><span class="programmeheading">3.9</span>
							my $showrating = $show->look_down('_tag' => 'span', 'class' => 'programmetext', sub { $_[0]->as_trimmed_text =~ /^Rating$/ } );
							if ($showrating) {
								$showrating = $showrating->right;
								$showrating = $showrating->right if ($showrating->tag eq 'br');
								if ($showrating->tag eq 'span' && $showrating->attr('class') eq 'programmeheading') {
									if ($showrating->as_text) {
										$prog{'star-rating'} =  [ $showrating->as_text . '/10' ];
									}
								}
							}

							# programme url
							# 	<a href="https://www.tvguide.co.uk/detail/4206442/61651117/homes-under-the-hammer" title="Click to rate and review">
							my $showurl = $show->look_down('_tag' => 'a', 'title' => 'Click to rate and review');
							if ($showurl) {
								$prog{'url'} = [ encode( 'utf-8', $showurl->attr('href') ) ];
							}

							# programme image
							#	<table border="0" cellpadding="0" width="677" style="background:black;border-collapse: collapse;background-image: url(https://cdn.tvguide.co.uk/HighlightImages/Large/Homes-Under.jpg);background-repeat: no-repeat;">
							my $showattr = $show->attr('style');
							(my $showimage) = $showattr =~ /background-image:\s*url\((.*?\.jpg)\)/;
							if ($showimage) {
								$prog{'image'} = [] if not defined $prog{'image'} or not @{$prog{'image'}};
								push @{$prog{'image'}}, [ $showimage, { 'system'=>'tvguide', 'type'=>'backdrop' } ];
							}


							# debug Dumper \%prog;
							push(@{$programmes}, \%prog);

							$alt_success = 1;

						}

						# if this is an alternative id then remember it
						if ($alt_success) {
							if ($alt_channel_id != $channel_id) {
								$channels_alt_found->{$channel_id} = $alt_channel_id;
								debug('Found working alternative ID '.$alt_channel_id.' for '.$channel_id);
							}
						}


					} else {
						# no schedule found
						debug "No schedule found for channel ID: $alt_channel_id";
						# append alternative channel numbers to list
						if (scalar @alts == 1) {
							push @alts, get_alt_channel_ids($channel_id, $channelname);
							debug "Found alternative IDs: @alts[1..$#alts]" if (scalar @alts > 1);
						}
						# issue warning only when all alternatives have been tried
						warning 'No schedule found' if ($alt_channel_id == $alts[$#alts]);
					}

					undef @shows;

					# Add to the channels hash
					$channels->{$channel_id} = { 'id'=> $xmlchannel_id , 'display-name' => [[ encode('utf-8', $channelname), 'en' ]]  };

					$tree->delete();

				} else {
					# tree conversion failed
					warning 'Could not parse the page';
				}

				last if $alt_success;
			}

			$bar->update if defined $bar;

			debug('-' x 30);
		}
	}
}


# #############################################################################
# # THE VEG ######################################################################
# ------------------------------------------------------------------------------------------------------------------------------------- #

sub make_ns_epnum {
		# Convert an episode number to its xmltv_ns compatible - i.e. reset the base to zero
		# Input = series number, episode number, total episodes,  part number, total parts,
		#  e.g. "1, 3, 6, 2, 4" >> "0.2/6.1/4",    "3, 4" >> "2.3."
		#
		my ($s, $e, $e_of, $p, $p_of) = @_;
		#	debug Dumper(@_);

		# "Part x of x" may contain integers or words (e.g. "Part 1 of 2", or "Part one")
		$p = text_to_num($p) if defined $p;
		$p_of = text_to_num($p_of) if defined $p_of;

		# re-base the series/episode/part numbers
		$s-- if (defined $s && $s > 0);
		$e-- if (defined $e && $e > 0);
		$p-- if (defined $p && $p && $p=~/^\d+$/ && $p > 0);

		# make the xmltv_ns compliant episode-num
		my $episode_ns = '';
		$episode_ns .= $s if defined $s;
		$episode_ns .= '.';
		$episode_ns .= $e if defined $e;
		$episode_ns .= '/'.$e_of if defined $e_of;
		$episode_ns .= '.';
		$episode_ns .= $p if $p;
		$episode_ns .= '/'.$p_of if $p_of;

		#debug "--$episode_ns--";
		return $episode_ns;
}

sub text_to_num {
		# Convert a word number to int e.g. 'one' >> '1'
		#
		my ($text) = @_;
		if ($text !~ /^[+-]?\d+$/) {	# standard test for an int
			my %nums = (one => 1, two => 2, three => 3, four => 4, five => 5, six => 6, seven => 7, eight => 8, nine => 9);
			return $nums{$text} if exists $nums{$text};
		}
		return $text
}

sub map_channel_id {
		# Map the fetched channel_id to a different value (e.g. our PVR needs specific channel ids)
		# mapped channels should be stored in a file called  tv_grab_uk_tvguide.map.conf
		# containing lines of the form:  map==fromchan==tochan  e.g. 'map==109==BBC4'
		#
		my ($channel_id) = @_;
		my $mapchannels = \%mapchannelhash;
		if (%mapchannelhash && exists $mapchannels->{$channel_id}) {
			return $mapchannels->{$channel_id} ;
		}
		return $channel_id;
}

sub map_category {
		# Map the fetched category to a different value (e.g. our PVR needs specific genres)
		# mapped categories should be stored in a file called  tv_grab_uk_guardian.map.conf
		# containing lines of the form:  cat==fromcategory==tocategory  e.g. 'cat==General Movie==Film'
		#
		my ($category) = @_;
		my $mapcategories = \%mapcategoryhash;
		if (%mapcategoryhash && exists $mapcategories->{$category}) {
			return $mapcategories->{$category} ;
		}
		return $category;
}

sub loadmapconf {
		# Load the conf file containing mapped channels and categories information
		#
		# This file contains 2 record types:
		# 	lines starting with "map" are used to 'translate' the incoming channel id to those required by your PVR
		#			e.g. 	map==dave==DAVE     will output "DAVE" in your XML file instead of "dave"
		# 	lines starting with "cat" are used to translate categories (genres) in the incoming data to those required by your PVR
		# 		e.g.  cat==Science Fiction==Sci-fi			will output "Sci-Fi" in your XML file instead of "Science Fiction"
		#
		my $mapchannels = \%mapchannelhash;
		my $mapcategories = \%mapcategoryhash;
		#
		my $supplementdir = $ENV{XMLTV_SUPPLEMENT} || GetSupplementDir();
		# get default file from supplement.xmltv.org if local file not exist 
		if (-f File::Spec->catfile( $supplementdir, $GRABBER_NAME, $GRABBER_NAME . '.map.conf' ) ) {
			SetSupplementRoot($supplementdir);
		}
		my $lines = GetSupplementLines($GRABBER_NAME, $GRABBER_NAME . '.map.conf');
		foreach my $line (@$lines) {
			my ($type, $mapfrom, $mapto, $trash) = $line =~ /^(.*)==(.*)==(.*?)([\s\t]*#.*)?$/;
			SWITCH: {
					lc($type) eq 'map' && do { $mapchannels->{$mapfrom} = $mapto; last SWITCH; };
					lc($type) eq 'cat' && do { $mapcategories->{$mapfrom} = $mapto; last SWITCH; };
					warning "Unknown type in map file: \n $line";
			}
		}
		# debug Dumper ($mapchannels, $mapcategories);
}

sub fetch_all_channel_ids {
	# Fetch all channel IDs with method 1, used for channel list creation and alternative ID searches
	#
	my $channels = {};

	my $tree = fetch_url('https://www.tvguide.co.uk/mychannels.asp?gw=1242', 'post', [
		thisDay => '',
		thisTime => '',
		gridSpan => '',
		emailaddress => '',
		regionid => 1,
		systemid => 5,
		xn => 'Show me the channels'
	]);

	my @c = $tree->look_down('_tag' => qr/table|tr/, 'class' => qr/^tr[XC]/);

	my $j = 0 if $opt->{test};	# --test is an undocumented (private) option

	foreach (@c) {
		my ($ch, $id, $l, $t);

		if ($_->id =~ /^trX?\d+/) {
			($id) = $_->id =~ /^trX?(\d+)/;
			($ch) = $ROOT_URL.'channellistings.asp?ch='.$id;
			($l) = $_->as_HTML =~ /background-image:url\(([^)]+)\)/;
			($t) = $_->as_text;
		}

		$channels->{$id} = {id => $id . (!$opt->{'list-channels'}?"   # ".encode('utf-8', $t):(!$opt->{legacychannels}?'.tvguide.co.uk':'')),
							'display-name' => [[ encode('utf-8', $t), 'en' ]],
							icon => [{ 'src'=>$l }],
							url => [ $ch ],
							}
							if $id;

		debug $id if $opt->{test};
		last if $opt->{test} and (++$j >= $opt->{test});  # limit during testing
	}
	return $channels;
}

sub get_alt_channel_ids ($$) {
	# Find alternate IDs for the same exact channel name
	#
	my ($channel_id, $channel_name) = @_;

	if (defined $channels_alt->{$channel_id}) {
		return @{$channels_alt->{$channel_id}};
	}

	$channels_cache = fetch_all_channel_ids() if (scalar keys %$channels_cache == 0);

	my @alts;

	foreach my $id (keys %$channels_cache) {
		next if $id == $channel_id;
		push @alts, $id if ($channels_cache->{$id}->{'display-name'}[0][0] eq encode('utf-8', $channel_name) );
	}

	$channels_alt->{$channel_id} = \@alts;
	return @{$channels_alt->{$channel_id}};
}

sub fetch_channels {
    # ParseOptions() handles --configure and --list-channels internally without returning,
	#  so we do not have global $opt available during --configure
	#  unless we copy vars to main package. (Note package variable must be declared with 'our')
	#
	($main::conf, $main::opt) = @_;

	my $channels = {};


	# Get the list of available channels
	#-------------------------------------------------------------------------------------------

	# preferred method (uses a html select list of channels)
	#
	if ((scalar keys %$channels == 0) && (!defined $opt->{method} || $opt->{method} == 0)) {

		my $bar = new XMLTV::ProgressBar({
			name => "Fetching channels",
			count => 1
		}) unless ($opt->{quiet} || $opt->{debug});

		# Fetch channels via a dummy call to BBC1 listings
		#
		my $channel_list = $ROOT_URL.'channellistings.asp?ch=74&cTime=';

		my @channels;

		my $tree = XMLTV::Get_nice::get_nice_tree($channel_list);
		#	$tree->dump;
		my $_channels = $tree->look_down('_tag' => 'select', 'name' => 'ch');
		if (defined $_channels) {
			@channels = $_channels->look_down('_tag' => 'option');
			#		debug $_channels->as_HTML;
			#		foreach  my $xchannel (@channels) { debug $xchannel->as_HTML; }
		}

		$bar->update() if defined $bar; $bar->finish() && undef $bar if defined $bar;

		if (scalar @channels > 0) {

			$bar = new XMLTV::ProgressBar({
				name => "Parsing result",
				count => scalar @channels
			}) unless ($opt->{quiet} || $opt->{debug});

			# Browse through the downloaded list of channels and map them to a hash XMLTV::Writer would understand
			foreach my $channel (@channels) {
				if ($channel->as_text) {
					my ($id) = $channel->attr('value');
					my ($url) = 'channellistings.asp?ch=' . $channel->attr('value');
					my ($name) = $channel->as_text;

						$channels->{$id} = {
							id => $id . (!$opt->{'list-channels'}?"   # ".encode('utf-8', $name):(!$opt->{legacychannels}?'.tvguide.co.uk':'')),
							'display-name' => [[ encode('utf-8', $name), 'en' ]],
							url => [ $ROOT_URL.$url ]
					};

				}

				$bar->update() if defined $bar;
			}

			$bar->finish() && undef $bar if defined $bar;

		}

		warning "No channels found in TVGuide" if (scalar keys %$channels == 0);

	}


	#-------------------------------------------------------------------------------------------

	# workaround for broken TVGuide website [2022-01-26]
	#

	# alternative Method 1 (credit: mkbloke )
	#
	#
	if ((scalar keys %$channels == 0) && (!defined $opt->{method} || $opt->{method} == 1)) {

		# alternative method 1
		# fetches channels from the website's "mychannels" page
		#   (preferred method found 899, this method finds 939 )
		#   (as at 2023-01-25 it finds 1207 channels, but many are duplicates with no data)

		warning "Trying alternative method 1";

		my $bar = new XMLTV::ProgressBar({
			name => "Fetching channels",
			count => 1
		}) unless ($opt->{quiet} || $opt->{debug});

		$channels = fetch_all_channel_ids();

		$bar->update() if defined $bar; $bar->finish() && undef $bar if defined $bar;

		warning "Found ".(scalar keys %$channels)." channels";

	}


	#-------------------------------------------------------------------------------------------

	# alternative Method 2
	#
	#
	if ((scalar keys %$channels == 0) && (!defined $opt->{method} || $opt->{method} == 2)) {

		# alternative method 2
		# fetches channels from the website's mobile-friendly page
		#   (preferred method found 899, this method finds 387 )

		warning "Trying alternative method 2";

		my $bar = new XMLTV::ProgressBar({
			name => "Fetching channels",
			count => 12
		}) unless ($opt->{quiet} || $opt->{debug});

		foreach (qw/ 7 3 12 5 25 8 22 19 10 29 18 23 /) {
			my $url = 'https://www.tvguide.co.uk/mobile/?systemid='.$_;
			#debug "Fetching: $url";
			my $tree = XMLTV::Get_nice::get_nice_tree($url);

			my @c = $tree->look_down('_tag' => 'div', 'class' => 'div-channel-progs');

			foreach (@c) {
				my ($ch, $id, $l, $t);
				$ch = $_->look_down('_tag' => 'a')->attr('href');
				($id) = $ch =~ m/\?ch=(\d+)$/;
				my $_dl = $_->look_down('_tag' => 'div', 'class' => 'div-channel-logo');
				if ($_dl) {		# don't chain the look_downs (for better robustness)
					my $_il = $_dl->look_down('_tag' => 'img', 'class' => 'img-channel-logo');
					if ($_il) {
						$l = $_il->attr('src');
						$t = $_il->attr('alt');
						$t =~ s/ TV Listings$//;
					}
				}
				$ch =~ s/mobile\/channellisting\.asp/channellistings\.asp/;
				$channels->{$id} = {id => $id . (!$opt->{'list-channels'}?"   # ".encode('utf-8', $t):(!$opt->{legacychannels}?'.tvguide.co.uk':'')),
									'display-name' => [[ encode('utf-8', $t), 'en' ]],
									icon => [{ 'src'=>$l }],
									url => [ $ch ],
									}
									if $id;
			}

			$bar->update() if defined $bar;
		}

		$bar->finish() && undef $bar if defined $bar;

		warning "Found ".(scalar keys %$channels)." channels \n";

	}



	#-------------------------------------------------------------------------------------------

	# does user want to process the list of channels to:
	# 1) make a list of channels without programme schedule data on tvg website
	# 2) remove these channels from their configuration list

	if (defined $opt->{makeignorelist}) {
		makeignorelist($channels);
	}

	if (defined $opt->{useignorelist}) {
		useignorelist($channels);
		warning "Remaining ".(scalar keys %$channels)." channels \n";
	}

	#debug Dumper $channels;


	#-------------------------------------------------------------------------------------------

	# Format & write out the config file

	# Notifying the user :)
	#$bar = new XMLTV::ProgressBar({
	#	name => "Reformatting",
	#	count => 1
	#}) unless ($opt->{quiet} || $opt->{debug});

	if (scalar keys %$channels == 0) {
		warning "No channels found in TVGuide \n";
		exit 1;
	}

	# Let XMLTV::Writer format the results as a valid xmltv file
	my $result;
	my $writer = new XMLTV::Writer(OUTPUT => \$result, encoding => 'utf-8');
	$writer->start({'generator-info-name' => $generator_info_name});
	#
	# this writes the channels sorted by 'id' but the TVG id has no relation to the actual channel number,
	#  and makes it harder to select the channels to fetch
	###  $writer->write_channels($channels);
	# so let's write them by name
	foreach (sort { $channels->{$a}->{'display-name'}[0][0] cmp $channels->{$b}->{'display-name'}[0][0] } keys %$channels) {
		$writer->write_channel($channels->{$_});
	}
	#
	$writer->end();

	#$bar->update() if defined $bar; $bar->finish() && undef $bar if defined $bar;

	return $result;
}



sub makeignorelist ($) {
	my $channels = shift;

	if (defined $opt->{makeignorelist}) {

		#-------------------------------------------------------------------------------------------#
		# this fetch pulls in a lot of 'duplicate' channels (16x Sky Sports Golf !) which, when you look at them
		#   are devoid of listings.
		# let's see if we can root those out at this stage and not present them to the poor wee user (who doesn't know
		#   which one to pick during '--configure' )
		# get the listings page for this channel and see if it has data
		# this reduces to 1x Sky Sports Golf  :-)
		#
		# reduced total to 494 channels (from 1207 in method 1)
		#
		# doing this for each channel will, of course, slow down the channel fetch 
		#   - so we only do it when asked with --makeignorelist
		#
		#-------------------------------------------------------------------------------------------#

		warning "Checking channels for listings data";

		warning "Creating the check list will take about an hour.\n You don't need to do this every time - unless your check list \n is very out-of-date you can simply use your existing check list. \n (Ctrl+C to abort)";

		my $bar = new XMLTV::ProgressBar({
			name => "Checking channels",
			count => (scalar keys %$channels)
		}) unless ($opt->{quiet} || $opt->{debug});


		my $skipchannels = [];
		my $i = 0 if $opt->{test};

		my @channels = keys %$channels;
		for my $channel (@channels) {

			last if $opt->{test} and (++$i >= $opt->{test});  # limit during testing

			(my $id) = $channels->{$channel}->{id} =~ /^(\d+)/;
			next if !$id;
			#debug "Checking: $id";

			# are there any listings for tomorrow, for this channel?
			my $url = $ROOT_URL.'channellistings.asp' . '?ch=' . $id . '&cTime=' . uri_escape( DateTime->today->add(days => 1)->set_time_zone('Europe/London')->strftime('%m/%d/%Y 00:00:00') );
			#debug "Fetching: $url";

			my $tree = fetch_url($url);
			if ($tree) {
				# 	(see fetch_listings for details)
				my @shows = $tree->look_down('_tag' => 'table', 'border' => '0', 'cellpadding' => '0', 'style' => qr/background:\s*black;border-collapse:\s*collapse;/);

				if (scalar @shows == 0) {			# empty listings schedule
					push @$skipchannels, $id;
					debug "Channel ".$channels->{$channel}->{id}." - SKIPPED";
				}
			}

			$bar->update() if defined $bar;
		}

		$bar->finish() && undef $bar if defined $bar;


		# write the ignore list
		my $fn = ($opt->{makeignorelist} || 'uktvguideignorelist');
		debug "Writing $fn";
		open my $fh, '>', $fn or die "Can't open file $!";
		print $fh '# LIST OF IGNORED CHANNELS'."\n";
		print $fh '# These channels contain no data, so it is safe to '."\n";
		print $fh '#  remove them from the configuration file.'."\n";
		foreach (@$skipchannels) {
			print $fh 'channel='.$_."\n";
		}

		close $fh;

	}

}


sub useignorelist ($) {
	my $channels = shift;

	if (defined $opt->{useignorelist}) {

		warning "Removing ignored channels";

		# read the ignore list
		my $fn = ($opt->{useignorelist} || 'uktvguideignorelist');
		debug "Reading $fn";
		open my $fh, '<', $fn or die "Can't open file $!";

		while (my $r = <$fh>) {
			chomp $r;
			next if $r =~ /^\s*#/;
			(my $id) = $r =~ /^channel=(\d+)$/;
			next if !$id;
			# remove from channellist
			delete $channels->{$id} if ($channels->{$id});
		}

		close $fh;

	}

}


sub config_stage {
	my( $stage, $conf ) = @_;
	die "Unknown stage $stage" if $stage ne "start";

	my $result;
	my $writer = new XMLTV::Configure::Writer( OUTPUT => \$result, encoding => 'utf-8' );
	$writer->start( { grabber => $GRABBER_NAME } );
	$writer->write_string({
			id => 'cachedir',
			title => [ [ 'Directory to store the cache in', 'en' ] ],
			description => [
			 [ $GRABBER_NAME.' uses a cache with files that it has already '.
				 'downloaded. Please specify where the cache shall be stored. ',
				 'en' ] ],
			default => get_default_cachedir(),
	 });

	$writer->end( 'select-channels' );

	return $result;
}

sub config_check {
	if (not defined( $conf->{cachedir} )) {
			print STDERR "No cachedir defined in configfile " .
									 $opt->{'config-file'} . "\n" .
									 "Please run the grabber with --configure.\n";
			exit 1;
	}

	if (not defined( $conf->{'channel'} )) {
			print STDERR "No channels selected in configfile " .
									 $opt->{'config-file'} . "\n" .
									 "Please run the grabber with --configure.\n";
			exit 1;
	}
}

sub fetch_url ($;$$) {
	# fetch a url with up to 5 retries
	my ($url, $method, $varhash) = @_;
	$XMLTV::Get_nice::FailOnError = 0;
	my $content;
	my $maxretry = 5;
	my $retry = 0;
	if (defined $method && lc($method) eq 'post') {

		my $ua = initialise_ua();

		while ( (not defined($content)) || (length($content) == 0) ) {
			my $r = $ua->post($url, $varhash);
			$content = $r->content;
			if ( $r->is_error || (length($content) == 0) ) {
				print STDERR "HTTP error: ".$r->status_line."\n";
				$retry++;
				return undef if $retry > $maxretry;
				print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n";
			}
		}

	} else {

		while ( (not defined($content = XMLTV::Get_nice::get_nice($url))) || (length($content) == 0) ) {
			my $r = $XMLTV::Get_nice::Response;
			print STDERR "HTTP error: ".$r->status_line."\n";
			$retry++;
			return undef if $retry > $maxretry;
			print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n";
		}

	}

	$content = decode('UTF-8', $content);
	my $t = HTML::TreeBuilder->new();
	$t->parse($content) or die "cannot parse content of $url\n";
	$t->eof;
	return $t;
}

sub get_default_dir {
	my $winhome = $ENV{HOMEDRIVE} . $ENV{HOMEPATH}
			if defined( $ENV{HOMEDRIVE} )
					and defined( $ENV{HOMEPATH} );

	my $home = $ENV{HOME} || $winhome || ".";
	return $home;
}

sub get_default_cachedir {
	return get_default_dir() . "/.xmltv/cache";
}

sub init_cachedir {
	my( $path ) = @_;
	if( not -d $path ) {
		mkpath( $path ) or die "Failed to create cache-directory $path: $@";
	}
}

sub debug ( $$ ) {
	my( $message, $nonewline ) = @_;
	print STDERR $message if $opt->{debug};
	print STDERR "\n" if $opt->{debug} && (!defined $nonewline || $nonewline != 1);
}

sub warning ( $ ) {
	my( $message ) = @_;
	print STDERR $message . "\n";
	$warnings++;
}

sub initialise_ua {
	my $cookies = HTTP::Cookies->new;
	#my $ua = LWP::UserAgent->new(keep_alive => 1);
	my $ua = LWP::UserAgent->new;
	# Cookies
	$ua->cookie_jar($cookies);
	# Define user agent type
	$ua->agent('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US)');
	# Define timouts
	$ua->timeout(240);
	# Use proxy if set in http_proxy etc.
	$ua->env_proxy;

	return $ua;
}

# #############################################################################

__END__

=pod

=head1 NAME

B<tv_grab_uk_tvguide> - Grab TV listings for UK from the TVGuide website.

=head1 SYNOPSIS

tv_grab_uk_tvguide --usage

tv_grab_uk_tvguide --version

tv_grab_uk_tvguide --configure [--config-file FILE] [--gui OPTION] [--method N]
                   [--makeignorelist FILE] [--useignorelist FILE] [--debug]

tv_grab_uk_tvguide [--config-file FILE] [--output FILE] [--days N]
                   [--offset N] [--nodetailspage]  [--legacychannels]
                   [--quiet] [--debug]

tv_grab_uk_tvguide --list-channels [--output FILE] [--method N] [--debug]

=head1 DESCRIPTION

Output TV listings in XMLTV format for many channels available in UK.
The data come from tvguide.co.uk

First you must run B<tv_grab_uk_tvguide --configure> to choose which channels
you want to receive.

Then running B<tv_grab_uk_tvguide> with no arguments will get programme listings 
in XML format for the channels you chose, for available days including today.

=head1 OPTIONS

B<--configure> Prompt for which channels to fetch the schedule for, 
and write the configuration file.

B<--config-file FILE> Set the name of the configuration file, the default 
is B<~/.xmltv/tv_grab_uk_tvguide.conf>.  This is the file written by
--configure and read when grabbing.

B<--gui OPTION> Use this option to enable a graphical interface to be used.
OPTION may be 'Tk'. 
Additional allowed values of OPTION are 'Term' for normal terminal output (default)
 and 'TermNoProgressBar' to disable the use of Term::ProgressBar.
(May not work on Windows OS.)
	
B<--days N> Grab N days. The default is 5 days.

B<--offset N> Start N days in the future.  The default is to start
from today.

B<--nodetailspage> Only fetch summary information for each programme. (See discussion below).

B<--legacychannels> Channel ids were made compliant with the XMLTV specification 
in December 2020. Use --legacychannels to output channel ids in the previous format
 (i.e. number only).

B<--output FILE> Write to FILE rather than standard output.

B<--method N> This program has three methods for fetching the list of
channels available. The preferred method can be overridden with an option of
either --method 1 or --method 2 to use one of the two alternative methods.

If no --method parameter is supplied then the various methods will be tried in 
sequence until a channels list is obtained. A parameter of --method 0 will run 
the preferred method only.

Normally you should omit this parameter.

B<--makeignorelist FILE> The source channel list contains many channels which
have no programme listings available. By using a local file of channel ids
these can be filtered out during the configure process. This option creates 
a list of channel ids which can be excluded from your config file

makeignorelist can take over an hour to run but you only need run it 
infrequently (not every time you run --configure).

B<--useignorelist FILE> Specify the file containing a list of channel ids 
to be removed during the configure stage.

This may be the file just created with --makeignorelist
(e.g. --configure --makeignorelist myfile --useignorelist myfile )

B<--quiet> Suppress the progress messages normally written to the console.

B<--debug> Provide more information on progress to standard error to help in
debugging.

B<--list-channels> Output a list (in xmltv format) of all channels that can be fetched.

B<--version> Show the version of the grabber.

B<--usage> Print a help message and exit.

=head1 INSTALLATION

The file F<tv_grab_uk_tvguide.map.conf> has two purposes.  Firstly you can map the channel ids used by the site into something more meaningful to your PVR. E.g.

      map==74==BBC 1

will change "74" to "BBC 1" in the output XML.

Note: the lines are of the form "map=={channel id}=={my name}".

The second purpose is to likewise translate genre names.  So if your PVR doesn"t have a category for "Science Fiction" but uses "Sci-fi" instead, then you can specify

      cat==Science Fiction==Sci-fi

and the output XML will have "Sci-fi".


IMPORTANT: the downloaded "tv_grab_uk_tvguide.map.conf" contains example lines to illustrate the format - you should edit this file to suit your own purposes!

=head1 ERROR HANDLING

If the grabber fails to download data for some channel on a specific day,
it will print an errormessage to STDERR and then continue with the other
channels and days. The grabber will exit with a status code of 1 to indicate
that the data is incomplete.

=head1 ENVIRONMENT VARIABLES

The environment variable HOME can be set to change where configuration
files are stored. All configuration is stored in $HOME/.xmltv/. On Windows,
it might be necessary to set HOME to a path without spaces in it.

=head1 SUPPORTED CHANNELS

For information on supported channels, see http://tvguide.co.uk/

=head1 XMLTV VALIDATION

B<tv_validate_grabber> may report an error similar to:

      "Line 5 Invalid channel-id BBC 1"

This is a because ValidateFile.pm insists the channel-id adheres to RFC2838 despite the xmltv.dtd only saying "preferably" not "SHOULD".
(Having channel ids of the form "bbc1.bbc.co.uk" will be rejected by many PVRs since they require the data to match their own list.)

It may also report:

      "tv_sort failed on the concatenated data. Probably due to overlapping data between days."

Both these errors can be ignored.

=head1 USING --nodetailspage

This option may be useful if you have problems accessing the tvguide website:
it will considerably speed up your grabbing, but at the expense of data richness.
The details page has a better list of actors, as well as director's names, 
film classifications, and background images.

More significantly, the details page includes programme duration and programme end time. If you don't include the details page then your output programmes will not have stop times. Although stop times are an optional XMLTV element, many downstream programs expect them and break without them.

Fortunately, these can be added by piping your xml file through the tv_sort filter: this will add stop times to all programmes except for the last programme on every channel. For example:

      tv_grab_uk_tvguide --days 3 --nodetailspage | tv_sort --by-channel --output myprogrammes.xml

B<NOTE:> As at January 2022 it seems the actor/director names and film classification
is no longer present in the website, although it may be there for some channels(?). 
Therefore you may find the --nodetailspage option useful to significantly reduce your run time.
B<However>, the details page does include background image(s) of the programme, which is
not present in your xml file when using --nodetailspage

=head1 BE KIND

If using the details page from the website then your grabbing might benefit from a more targeted strategy, rather than blithely getting all days for all channels. 

Since the published schedule rarely changes, a strategy of grabbing the next 3 days plus the 1 newest day would give you any new programmes as well as any last minute changes. Simply fetch "--offset 0 --days 3" and concatenate it with a separate fetch of "--offset 7 --days 1".

For example:

=over 6

( tv_grab_uk_tvguide --days 3 --offset 0 --output temp.xml ) && ( tv_grab_uk_tvguide --days 1 --offset 7 | tv_cat --output myprogrammes.xml temp.xml - ) && ( rm -f temp.xml )

=back

A similar strategy could be employed when using --nodetailspage if you have a lot of channels:

=over 6

( tv_grab_uk_tvguide --days 3 --offset 0 --nodetailspage | tv_sort --by-channel >temp.xml ) && ( tv_grab_uk_tvguide --days 1 --offset 7 --nodetailspage | tv_sort --by-channel | tv_cat --output myprogrammes.xml temp.xml - ) && ( rm -f temp.xml )

=back

This avoids overloading the TVGuide website, and significantly reduces your runtime with minimal impact on your viewing schedule.

TVGuide provide this data for free, so let's not abuse their generosity.

=head1 DISCLAIMER

The TVGuide website's license for these data does not allow non-personal use.

Certainly, any commercial use of listings data obtained by using this grabber will breach copyright law, but if you are just using the data for your own personal use then you are probably fine.

By using this grabber you aver you are using the listings data for your own personal use only and you absolve the author(s) from any liability under copyright law or otherwise.

=head1 AUTHOR

Geoff Westcott. This documentation and parts of the code
based on various other tv_grabbers from the XMLTV-project.

=head1 SEE ALSO

L<xmltv(5)>.

=cut


To Do
=====

1.  Improve the progress bar update frequency
2.  Add actor 'character' attribute  DONE 30/6/16
3.  Currently only does Actor, Director, Producer, Writer - does anyone actually use any of the others present in the DTD?
