File Coverage

blib/lib/IMDB/Local.pm

Criterion	Covered	Total	%
statement	14	17	82.3
branch			n/a
condition			n/a
subroutine	5	5	100.0
pod			n/a
total	19	22	86.3

package IMDB::Local;

#
# Suggestions for improvements
# -
#
#

use 5.006;
use strict;
use warnings;

=head1 NAME

IMDB::Local - The great new IMDB::Local!

=head1 VERSION

Version 0.01

=cut

our $VERSION = '1.00';


=head1 SYNOPSIS

Quick summary of what the module does.

Perhaps a little code snippet.

    use IMDB::Local;

    my $foo = IMDB::Local->new();
    ...

=head1 EXPORT

A list of functions that can be exported.  You can delete this section
if you don't export anything, such as for a purely object-oriented module.

=head1 SUBROUTINES/METHODS

=cut

# Use Term::ProgressBar if installed.
use constant Have_bar => eval {
    require Term::ProgressBar;
    $Term::ProgressBar::VERSION >= 2;
};

use IMDB::Local::DB;

=head2 new

Create new IMDB::Local object.

Arguments:

   imdbDir - required or die

   verbose - optional, default is 0.

   listsDir - folder where list files exist (see IMDB::Local::Download).

   showProgressBar - if non-zero and Term::ProgressBar is available progress bars in import methods will be displayed. Ignored if Term::ProgressBar is not available.

=cut

sub new
{
    my ($type) = shift;
    my $self={ @_ };            # remaining args become attributes

    for ('imdbDir', 'verbose') {
	die "invalid usage - no $_" if ( !defined($self->{$_}));
    }

    #$self->{stages} = { 1=>'movies', 2=>'directors', 3=>'actors', 4=>'actresses', 5=>'genres', 6=>'ratings', 7=>'keywords', 8=>'plot' };
    #$self->{optionalStages} = { 'keywords' => 7, 'plot' => 8 };     # list of optional stages - no need to download files for these

    $self->{moviedbInfo}="$self->{imdbDir}/moviedb.info";
    $self->{moviedbOffline}="$self->{imdbDir}/moviedb.offline";

    if ( defined($self->{listsDir}) ) {
	$self->{listFiles}=new IMDB::Local::ListFiles(listsDir=>$self->{listsDir});
    }

    # only leave progress bar on if its available
    if ( !Have_bar ) {
	$self->{showProgressBar}=0;
    }

    bless($self, $type);
    return($self);
}

=head2 listTypes


Returns an array of list files supported (currently 'movies', 'directors', 'actors', 'actresses', 'genres', 'ratings', 'keywords', 'plot')

=cut

sub listTypes($)
{
    my $self=shift;

    return( $self->{listFiles}->types() );
}


sub error($$)
{
    my $self=shift;
    if ( defined($self->{logfd}) ) {
	print {$self->{logfd}} $_[0]."\n";
	$self->{errorCountInLog}++;
    }
    else {
	print STDERR $_[0]."\n";
    }
}

sub status($$)
{
    my $self=shift;

    if ( $self->{verbose} ) {
	print STDERR $_[0]."\n";
    }
}

sub withThousands ($)
{
    my ($val) = @_;
    $val =~ s/(\d{1,3}?)(?=(\d{3})+$)/$1,/g;
    return $val;
}

use XMLTV::Gunzip;
use IO::File;

sub openMaybeGunzip($)
{
    for ( shift ) {
	return gunzip_open($_) if m/\.gz$/;
	return new IO::File("< $_");
    }
}

sub closeMaybeGunzip($$)
{
    if ( $_[0]=~m/\.gz$/o ) {
	# Would close($fh) but that causes segfaults on my system.
	# Investigating, but in the meantime just leave it open.
	#
	#return gunzip_close($_[1]);
    }

    # Apparently this can also segfault (wtf?).
    #return close($_[1]);
}


# Convert a title into a searchtitle by lowercasing,
# making it ASCII and removing punctuation.
#
sub MakeSearchtitle($;$;$) {
  my ($DB, $str, $debug) = @_;
  return lc RemovePunctuation($DB, lc( CharsetMap( $str, $debug ) ), $debug );
}


use Text::Unidecode;

#   All characters outside the ASCII range (0x00-0x7F) are replaced by ASCII equivalents,
#    using function Text::Unidecode::unidecode
#
sub CharsetMap($) {
  my ($str, $debug) = @_;

  # do replacements that unidecode doesn't know about (or does wrong)
  ### IT WOULD BE NICE IF THESE WERE IN A TABLE
  $str =~ s/\x{0133}/ij/g;     # 'ij' -> ij             ("" in unidecode)
  $str =~ s/\x{20ac}/EUR/g;    # euro symbol -> EUR     (EU in unidecode)
  $str =~ s/\x{2122}/TM/g;     # trademark symbol -> TM ("" in unidecode)
  $str =~ s/\x{a3}/GBP/g;      # pound sign -> GBP (PS in unidecode)

  # now do the real decode
  $str = unidecode($str);

  #print "[$str]\n" if ($debug);

  return ($str);
}


my @punctuation;

# Function that removes all punctuation and whitespace from a string.
#  '&' is converted to 'and' along the way
sub RemovePunctuation($;$;$) {
    my ($DB, $str, $debug) = @_;

    # Load the array of hashes that contain the punctuation
    # replacements in priority order
    if ( !@punctuation ) {
        my @plist = @{$DB->select2Matrix("select priority,pattern,replacement from Punctuation order by priority")};
        my $cnt = 0;
        foreach my $p (@plist) {
            my $pattern = $p->[1];
            my $compiled = qr/$pattern/i;
            $punctuation[$cnt]{origpattern} = $pattern;
            $punctuation[$cnt]{pattern}     = $compiled;
            $punctuation[$cnt]{replacement} = $p->[2];
            $cnt++;
        }
    }

    foreach my $ref (@punctuation) {
        #print "[$str] $ref->{origpattern} " if ($debug);
        $str =~ s/$ref->{pattern}/$ref->{replacement}/g;
        #print "[$str]\n" if ($debug);
    }
    #print "<$str>\n--------\n" if ($debug);
    return ($str);
}

sub decodeImdbKey($$$)
{
    my ($self, $DB, $dbkey, $year, $titleID)=@_;

    my %hash;

    $hash{parentId}=0;
    $hash{series}=0;
    $hash{episode}=0;


    # drop episode information - ex: "Studio One" (1948) {Twelve Angry Men (#7.1)}
    if ( $dbkey=~s/\s*\{([^\}]+)\}//o ) {
	my $s=$1;
	if ( $s=~s/\s*\(\#(\d+)\.(\d+)\)$// ) {
	    $hash{series}=$1;
	    $hash{episode}=$2;
	    $hash{title}=$s;

	    # attempt to locate parentId matching series title
	    my $parentKey=$dbkey;
	    $parentKey=~s/^\"//o;
	    $parentKey=~s/\" \(/ \(/o;

	    if ( defined($self->{seriesKeys}->{$parentKey}) ) {
		#if ( $parentKey=~m/24 Hour Design/o ) {
		#warn(" lok: $parentKey");
		#}
		$hash{parentId}=$self->{seriesKeys}->{$parentKey};
	    }
	}
    }

    # todo - this would make things easier
    # change double-quotes around title to be (made-for-tv) suffix instead
    if ( $dbkey=~s/^\"//o && $dbkey=~s/\" \(/ \(/o) {
	if ( $dbkey=~s/\s+\(mini\)$//o ) {
	    if ( $hash{parentId} == 0 ) {
		$hash{qualifier}=$self->{QualifierTypeIDs}->{"tv_mini_series"};
		$self->{seriesKeys}->{$dbkey}=$titleID;
	    }
	    else {
		$hash{qualifier}=$self->{QualifierTypeIDs}->{"episode_of_tv_mini_series"};
	    }
	}
	else {
	    if ( $hash{parentId} == 0 ) {
		$hash{qualifier}=$self->{QualifierTypeIDs}->{"tv_series"};
		#if ( $dbkey=~m/24 Hour Design/o ) {
		    #warn(" key: $dbkey");
		#}
		$self->{seriesKeys}->{$dbkey}=$titleID;
	    }
	    else {
		$hash{qualifier}=$self->{QualifierTypeIDs}->{"episode_of_tv_series"};
	    }
	}
    }
    elsif ( $dbkey=~s/\s+\(TV\)$//o ) {
	# how rude, some entries have (TV) appearing more than once.
	#$dbkey=~s/\s*\(TV\)$//o;
	$hash{qualifier}=$self->{QualifierTypeIDs}->{"tv_movie"};
    }
    elsif ( $dbkey=~s/\s+\(V\)$//o ) {
	$hash{qualifier}=$self->{QualifierTypeIDs}->{"video_movie"};
    }
    elsif ( $dbkey=~s/\s+\(VG\)$//o ) {
	$hash{qualifier}=$self->{QualifierTypeIDs}->{"video_game"};
    }
    else {
	$hash{qualifier}=$self->{QualifierTypeIDs}->{"movie"};
    }

    #if ( $dbkey=~s/\s+\((tv_series|tv_mini_series|tv_movie|video_movie|video_game)\)$//o ) {
    #   $qualifier=$1;
    #}
    $hash{dbkey}=$dbkey;

    my $title=$dbkey;

    # todo - this is the wrong year for episode titles
    if ( $title=~m/^\"/o && $title=~m/\"\s*\(/o ) { #"
	$title=~s/^\"//o; #"
	$title=~s/\"(\s*\()/$1/o; #"
    }

    if ( $title=~s/\s+\((\d\d\d\d)\)$//o ||
	 $title=~s/\s+\((\d\d\d\d)\/[IVX]+\)$//o ) {
	# over-ride with what is given
	if ( defined($year) ) {
	    $hash{year}=$year;
	}
	else {
	    $hash{year}=$1;
	}
    }
    elsif ( $title=~s/\s+\((\?\?\?\?)\)$//o ||
	    $title=~s/\s+\((\?\?\?\?)\/[IVX]+\)$//o ) {
	# over-ride with what is given
	if ( defined($year) ) {
	    $hash{year}=$year;
	}
	else {
	    $hash{year}="";
	}
    }
    else {
	$self->error("movie list format failed to decode year from title '$title'");
	# over-ride with what is given
	if ( defined($year) ) {
	    $hash{year}=$year;
	}
	else {
	    $hash{year}="";
	}
    }
    $title=~s/(.*),\s*(The|A|Une|Las|Les|Los|L\'|Le|La|El|Das|De|Het|Een)$/$2 $1/og;

    if ( !defined($hash{title}) ) {
	$hash{title}=$title;
	$hash{searchTitle}=MakeSearchtitle($DB, $hash{title}, 0);

	# todo - is this more useful ?
	#$hash{searchTitleWithYear}=MakeSearchtitle($DB, $title."(".$hash{year}.")", 0);
    }
    else {
	$hash{searchTitle}=MakeSearchtitle($DB, $title."(".$hash{year}.")", 0);

	# todo - is this more useful ?
	#$hash{searchTitleWithYear}=MakeSearchtitle($DB, $title."(".$hash{year}.")", 0);
    }

    return(\%hash);
}

sub importMovies($$$$)
{
    my ($self, $countEstimate, $file, $DB)=@_;
    my $startTime=time();
    my $lineCount=0;

    my $fh = openMaybeGunzip($file) || return(-2);
    while(<$fh>) {
	$lineCount++;
	if ( m/^MOVIES LIST/o ) {
	    if ( !($_=<$fh>) || !m/^===========/o ) {
		$self->error("missing ======= after 'MOVIES LIST' at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^\s*$/o ) {
		$self->error("missing empty line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    last;
	}
	elsif ( $lineCount > 1000 ) {
	    $self->error("$file: stopping at line $lineCount, didn't see \"MOVIES LIST\" line");
	    closeMaybeGunzip($file, $fh);
	    return(-1);
	}
    }

    my $progress=Term::ProgressBar->new({name  => "importing Movies",
					 count => $countEstimate,
					 ETA   => 'linear'})
	if ( $self->{showProgressBar} );

    $progress->minor(0) if ($self->{showProgressBar});
    $progress->max_update_rate(1) if ($self->{showProgressBar});
    my $next_update=0;

    # preload qualifier types
    $self->{QualifierTypeIDs}=$DB->select2Hash("select Name, QualifierTypeID from QualifierTypes");

    #$DB->runSQL("BEGIN TRANSACTION");

    my $count=0;
    my $tableInsert_sth=$DB->prepare('INSERT INTO Titles VALUES (?,?,?,?,?,?,?,?)');

    while(<$fh>) {
	$lineCount++;
	my $line=$_;
	#print "read line $lineCount:$line\n";

	# end is line consisting of only '-'
	last if ( $line=~m/^\-\-\-\-\-\-\-+/o );

	next if ( $line=~m/\{\{SUSPENDED\}\}/o );

	$line=~s/\n$//o;

	my $tab=index($line, "\t");
	if ( $tab != -1 ) {
	    my $ykey=substr($line, $tab+1);
	    if ( $ykey=m/\s+(\d\d\d\d)$/ ) {
		$ykey=$1;
	    }
	    elsif ( $ykey=m/\s+(\?\?\?\?)$/ ) {
		$ykey=undef;
	    }
	    elsif ( $ykey=m/\s+(\d\d\d\d)\-(\?\?\?\?)$/ ) {
		$ykey=$1;
	    }
	    elsif ( $ykey=m/\s+(\d\d\d\d)\-(\d\d\d\d)$/ ) {
		$ykey=$1;
	    }
	    else {
		warn("invalid year ($ykey) - $line");
		#$ykey=undef;
	    }

	    my $mkey=substr($line, 0, $tab);

	    # lets not import video games
	    #if ( $decoded->{qualifier} != $self->{QualifierTypeIDs}->{'video_game'} ) {
	    # returned count is number of titles found
	    $count++;

	    my $decoded=$self->decodeImdbKey($DB, $mkey, $ykey, $count);

	    $tableInsert_sth->execute($count,
				      $decoded->{searchTitle},
				      $decoded->{title},
				      $decoded->{qualifier},
				      $decoded->{year},
				      $decoded->{parentId},
				      $decoded->{series},
				      $decoded->{episode});
	    if ( $mkey=~m/24 Hour Design/o ) {
		warn("",join(',', $count,
			     $decoded->{searchTitle},
			     $decoded->{title},
			     $decoded->{qualifier},
			     $decoded->{year},
			     $decoded->{parentId},
			     $decoded->{series},
			     $decoded->{episode}));
	    }

	    #$DB->insert_db_raw();
	    #my $titleID=$DB->insert_db('Titles', 'TitleID', IMDBKey=>$mkey,
	    #Title=>$decoded->{title},
	    #	       Episode=>$decoded->{episode},
	    #		       Year=>$decoded->{year},
	    #		       QualifierTypeID=>$decoded->{qualifier});
	    $self->{imdbMovie2DBKey}->{$mkey}=$count;

	    #if ( ($count % 50000) == 0 ) {
	    #$DB->commit();
	    #}
	    #}

	    if ( $self->{showProgressBar} ) {
		# re-adjust target so progress bar doesn't seem too wonky
		if ( $count > $countEstimate ) {
		    $countEstimate = $progress->target($count+1000);
		    $next_update=$progress->update($count);
		}
		elsif ( $count > $next_update ) {
		    $next_update=$progress->update($count);
		}
	    }
	}
	else {
	    $self->error("$file:$lineCount: unrecognized format (missing tab)");
	    $next_update=$progress->update($count) if ($self->{showProgressBar});
	}
    }
    #$DB->runSQL("END TRANSACTION");

    $progress->update($countEstimate) if ($self->{showProgressBar});

    $self->status(sprintf("importing Movies found ".withThousands($count)." titles in ".
			  withThousands($lineCount)." lines in %d seconds",time()-$startTime));

    closeMaybeGunzip($file, $fh);
    return($count);
}

sub importGenres($$$$)
{
    my ($self, $countEstimate, $file, $DB)=@_;
    my $startTime=time();
    my $lineCount=0;

    my $fh = openMaybeGunzip($file) || return(-2);
    while(<$fh>) {
	$lineCount++;
	if ( m/^8: THE GENRES LIST/o ) {
	    if ( !($_=<$fh>) || !m/^===========/o ) {
		$self->error("missing ======= after 'THE GENRES LIST' at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^\s*$/o ) {
		$self->error("missing empty line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    last;
	}
	elsif ( $lineCount > 1000 ) {
	    $self->error("$file: stopping at line $lineCount, didn't see \"THE GENRES LIST\" line");
	    closeMaybeGunzip($file, $fh);
	    return(-1);
	}
    }

    my $progress=Term::ProgressBar->new({name  => "importing Genres",
					 count => $countEstimate,
					 ETA   => 'linear'})
	if ( $self->{showProgressBar} );

    $progress->minor(0) if ($self->{showProgressBar});
    $progress->max_update_rate(1) if ($self->{showProgressBar});
    my $next_update=0;

    # preload qualifier types
    $self->{QualifierTypeIDs}=$DB->select2Hash("select Name, QualifierTypeID from QualifierTypes");

    #$DB->runSQL("BEGIN TRANSACTION");

    my $count=0;
    my $tableInsert_sth=$DB->prepare('INSERT INTO Titles2Genres VALUES (?,?)');

    while(<$fh>) {
	$lineCount++;
	my $line=$_;
	#print "read line $lineCount:$line\n";

	# end is line consisting of only '-'
	last if ( $line=~m/^\-\-\-\-\-\-\-+/o );

	$line=~s/\n$//o;

	my $tab=index($line, "\t");
	if ( $tab != -1 ) {
	    my $mkey=substr($line, 0, $tab);

	    next if ($mkey=~m/\s*\{\{SUSPENDED\}\}/o);

	    # Genres
	    # don't see what these are...?
	    # ignore {{SUSPENDED}}
	    $mkey=~s/\s*\{\{SUSPENDED\}\}//o;

	    # ignore {Twelve Angry Men (1954)}
	    # TODO - do we want this ?
	    #$mkey=~s/\s*\{[^\}]+\}//go;

	    # skip enties that have {} in them since they're tv episodes
	    #next if ( $mkey=~s/\s*\{[^\}]+\}$//o );

	    my $genre=substr($line, $tab);

	    # genres sometimes has more than one tab
	    $genre=~s/^\t+//og;

	    if ( $self->{imdbMovie2DBKey}->{$mkey} ) {
		# insert into db as discovered
		if ( ! defined($self->{GenreID}->{$genre}) ) {
		    $self->{GenreID}->{$genre}=$DB->insert_db('Genres', 'GenreID', Name=>$genre);
		}
		$tableInsert_sth->execute($self->{imdbMovie2DBKey}->{$mkey},
					  $self->{GenreID}->{$genre});

		# returned count is number of titles found
		$count++;

		if ( ($count % 50000) ==0 ) {
		    $DB->commit();
		}
	    }

	    if ( $self->{showProgressBar} ) {
		# re-adjust target so progress bar doesn't seem too wonky
		if ( $count > $countEstimate ) {
		    $countEstimate = $progress->target($count+1000);
		    $next_update=$progress->update($count);
		}
		elsif ( $count > $next_update ) {
		    $next_update=$progress->update($count);
		}
	    }
	}
	else {
	    $self->error("$file:$lineCount: unrecognized format (missing tab)");
	    $next_update=$progress->update($count) if ($self->{showProgressBar});
	}
    }
    #$DB->runSQL("END TRANSACTION");

    $progress->update($countEstimate) if ($self->{showProgressBar});

    $self->status(sprintf("importing Genres found ".withThousands($count)." titles in ".
			  withThousands($lineCount)." lines in %d seconds",time()-$startTime));

    closeMaybeGunzip($file, $fh);
    return($count);
}

sub importActors($$$$)
{
    my ($self, $whichCastOrDirector, $castCountEstimate, $file, $DB)=@_;
    my $startTime=time();

    if ( $whichCastOrDirector eq "Actors" ) {
	if ( $DB->table_row_count('Actors') > 0 ||
	     $DB->table_row_count('Titles2Actors') > 0 ) {
	    $self->status("clearing previously loaded data..");
	    $DB->runSQL("DELETE from Actors");
	    $DB->runSQL("DELETE from Titles2Actors");
	}
    }

    my $header;
    my $whatAreWeParsing;
    my $lineCount=0;

    if ( $whichCastOrDirector eq "Actors" ) {
	$header="THE ACTORS LIST";
	$whatAreWeParsing=1;
    }
    elsif ( $whichCastOrDirector eq "Actresses" ) {
	$header="THE ACTRESSES LIST";
	$whatAreWeParsing=2;
    }
    else {
	die "why are we here ?";
    }

    my $fh = openMaybeGunzip($file) || return(-2);
    my $progress=Term::ProgressBar->new({name  => "importing $whichCastOrDirector",
					 count => $castCountEstimate,
					 ETA   => 'linear'})
      if ($self->{showProgressBar});
    $progress->minor(0) if ($self->{showProgressBar});
    $progress->max_update_rate(1) if ($self->{showProgressBar});
    my $next_update=0;
    while(<$fh>) {
	$lineCount++;
	if ( m/^$header/ ) {
	    if ( !($_=<$fh>) || !m/^===========/o ) {
		$self->error("missing ======= after $header at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^\s*$/o ) {
		$self->error("missing empty line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^Name\s+Titles\s*$/o ) {
		$self->error("missing name/titles line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^[\s\-]+$/o ) {
		$self->error("missing name/titles suffix line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    last;
	}
	elsif ( $lineCount > 1000 ) {
	    $self->error("$file: stopping at line $lineCount, didn't see \"$header\" line");
	    closeMaybeGunzip($file, $fh);
	    return(-1);
	}
    }

    my $cur_name;
    my $count=0;
    my $castNames=0;
    my $tableInsert_sth=$DB->prepare('INSERT INTO Actors VALUES (?,?,?)');
    my $tableInsert_sth2=$DB->prepare('INSERT INTO Titles2Actors VALUES (?,?,?,?,?)');
    my $cur_actorId=$DB->select2Scalar('select count(1) from Actors');

    while(<$fh>) {
	$lineCount++;
	my $line=$_;
	$line=~s/\n$//o;
	#$self->status("read line $lineCount:$line");

	# end is line consisting of only '-'
	last if ( $line=~m/^\-\-\-\-\-\-\-+/o );

	next if ( length($line) == 0 );

	# try ignoring these
	next if ($line=~m/\s*\{\{SUSPENDED\}\}/o);

	my $billing=0;

	# actors or actresses
	if ( $line=~s/\s*<(\d+)>//o ) {
	    $billing=int($1);
	    next if ( $billing >3 );
	}

	if ( $line=~s/^([^\t]+)\t+//o ) {
	    $cur_name=$1;
	    $castNames++;

	    $cur_actorId++;
	    $tableInsert_sth->execute($cur_actorId, MakeSearchtitle($DB, $cur_name, 0), $cur_name);

	    if ( $self->{showProgressBar} ) {
		# re-adjust target so progress bar doesn't seem too wonky
		if ( $castNames > $castCountEstimate ) {
		    $castCountEstimate = $progress->target($castNames+100);
		    $next_update=$progress->update($castNames);
		}
		elsif ( $castNames > $next_update ) {
		    $next_update=$progress->update($castNames);
		}
	    }
	}

	my $isHost=0;
	my $isNarrator=0;
	if ( (my $start=index($line, " [")) != -1 ) {
	    #my $end=rindex($line, "]");
	    my $ex=substr($line, $start+1);

	    if ( $ex=~s/Host//o ) {
		$isHost=1;
	    }
	    if ( $ex=~s/Narrator//o ) {
		$isNarrator=1;
	    }
	    $line=substr($line, 0, $start);
	    # ignore character name
	}

	# BUG
  # [honir] this is wrong - this puts cast from all the episodes as though they are in the entire series!
	# ##ignore {Twelve Angry Men (1954)}
	# TODO - I REMOVED THIS, should we ?
	#$line=~s/\s*\{[^\}]+\}//o;

	if ( $line=~s/\s*\(aka ([^\)]+)\).*$//o ) {
	    #$attrs=$1;
	}
	# TODO - what are we ignoring here ?
	if ( $line=~s/  (\(.*)$//o ) {
	    #$attrs=$1;
	}
	$line=~s/^\s+//og;
	$line=~s/\s+$//og;

	if ( $line=~s/\s+Narrator$//o ) {
	    # TODO - do we want to store this ? Does it actually occur ?
	    # ignore
	}

	#if ( $line=~s/\s*\([A-Z]+\)$//o ) {
	#}

	if ( $self->{imdbMovie2DBKey}->{$line} ) {

	    $tableInsert_sth2->execute($self->{imdbMovie2DBKey}->{$line}, $cur_actorId, $billing, $isHost, $isNarrator);
	    $count++;

	    if ( ($count % 50000) == 0 ) {
		$DB->commit();
	    }
	}
	else {
	    #warn($line);
	}
    }
    $progress->update($castCountEstimate) if ($self->{showProgressBar});

    $self->status(sprintf("importing $whichCastOrDirector found ".withThousands($castNames)." names, ".
			  withThousands($count)." titles in ".withThousands($lineCount)." lines in %d seconds",time()-$startTime));

    closeMaybeGunzip($file, $fh);

    return($castNames);
}

sub importDirectors($$$)
{
    my ($self, $castCountEstimate, $file, $DB)=@_;
    my $startTime=time();

    my $lineCount=0;

    if ( $DB->table_row_count('Directors') > 0 ||
	 $DB->table_row_count('Titles2Directors') > 0 ) {
	$self->status("clearing previously loaded data..");
	$DB->runSQL("DELETE from Directors");
	$DB->runSQL("DELETE from Titles2Directors");
    }

    my $fh = openMaybeGunzip($file) || return(-2);
    my $progress=Term::ProgressBar->new({name  => "importing Directors",
					 count => $castCountEstimate,
					 ETA   => 'linear'})
      if ($self->{showProgressBar});
    $progress->minor(0) if ($self->{showProgressBar});
    $progress->max_update_rate(1) if ($self->{showProgressBar});
    my $next_update=0;
    while(<$fh>) {
	$lineCount++;
	if ( m/^THE DIRECTORS LIST/ ) {
	    if ( !($_=<$fh>) || !m/^===========/o ) {
		$self->error("missing ======= after THE DIRECTORS LIST at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^\s*$/o ) {
		$self->error("missing empty line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^Name\s+Titles\s*$/o ) {
		$self->error("missing name/titles line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^[\s\-]+$/o ) {
		$self->error("missing name/titles suffix line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    last;
	}
	elsif ( $lineCount > 1000 ) {
	    $self->error("$file: stopping at line $lineCount, didn't see \"THE DIRECTORS LIST\" line");
	    closeMaybeGunzip($file, $fh);
	    return(-1);
	}
    }

    my $cur_name;
    my $count=0;
    my $castNames=0;
    my %found;
    my $directorCount=0;

    my $tableInsert_sth=$DB->prepare('INSERT INTO Directors VALUES (?,?,?)');
    my $tableInsert_sth2=$DB->prepare('INSERT INTO Titles2Directors VALUES (?,?)');
    while(<$fh>) {
	$lineCount++;
	my $line=$_;
	$line=~s/\n$//o;
	#$self->status("read line $lineCount:$line");

	# end is line consisting of only '-'
	last if ( $line=~m/^\-\-\-\-\-\-\-+/o );

	next if ( length($line) == 0 );

	if ( $line=~s/^([^\t]+)\t+//o ) {
	    $cur_name=$1;
	    $castNames++;

	    if ( $self->{showProgressBar} ) {
		# re-adjust target so progress bar doesn't seem too wonky
		if ( $castNames > $castCountEstimate ) {
		    $castCountEstimate = $progress->target($castNames+100);
		    $next_update=$progress->update($castNames);
		}
		elsif ( $castNames > $next_update ) {
		    $next_update=$progress->update($castNames);
		}
	    }
	}

	# try ignoring these
	next if ($line=~m/\s*\{\{SUSPENDED\}\}/o);

	# BUG
  # [honir] this is wrong - this puts cast from all the episodes as though they are in the entire series!
	# ##ignore {Twelve Angry Men (1954)}
	$line=~s/\s*\{[^\}]+\}//o;

	# TODO - what are we ignoring here ?
	if ( $line=~s/  (\(.*)$//o ) {
	    # $attrs=$1;
	}
	$line=~s/^\s+//og;
	$line=~s/\s+$//og;

	if ( $self->{imdbMovie2DBKey}->{$line} ) {

	    if ( !defined($found{$cur_name}) ) {
		$found{$cur_name}=$directorCount++;
		$tableInsert_sth->execute($directorCount, MakeSearchtitle($DB, $cur_name, 0), $cur_name);
	    }

	    $tableInsert_sth2->execute($self->{imdbMovie2DBKey}->{$line}, $found{$cur_name});
	    $count++;
	    if ( ($count % 50000) == 0 ) {
		$DB->commit();
	    }
	}
    }
    $progress->update($castCountEstimate) if ($self->{showProgressBar});

    $self->status(sprintf("importing Directors found ".withThousands($castNames)." names, ".
			  withThousands($count)." titles in ".withThousands($lineCount)." lines in %d seconds",time()-$startTime));

    closeMaybeGunzip($file, $fh);

    return($castNames);
}

sub importRatings($$)
{
    my ($self, $countEstimate, $file, $DB)=@_;
    my $startTime=time();
    my $lineCount=0;

    if ( $DB->table_row_count('Ratings') > 0 ) {
	$self->status("clearing previously loaded data..");
	$DB->runSQL("DELETE from Ratings");
    }

    my $fh = openMaybeGunzip($file) || return(-2);
    while(<$fh>) {
	$lineCount++;
	if ( m/^MOVIE RATINGS REPORT/o ) {
	    if ( !($_=<$fh>) || !m/^\s*$/o) {
		$self->error("missing empty line after \"MOVIE RATINGS REPORT\" at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^New  Distribution  Votes  Rank  Title/o ) {
		$self->error("missing \"New  Distribution  Votes  Rank  Title\" at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    last;
	}
	elsif ( $lineCount > 1000 ) {
	    $self->error("$file: stopping at line $lineCount, didn't see \"MOVIE RATINGS REPORT\" line");
	    closeMaybeGunzip($file, $fh);
	    return(-1);
	}
    }

    my $progress=Term::ProgressBar->new({name  => "importing Ratings",
					 count => $countEstimate,
					 ETA   => 'linear'})
      if ($self->{showProgressBar});

    $progress->minor(0) if ($self->{showProgressBar});
    $progress->max_update_rate(1) if ($self->{showProgressBar});
    my $next_update=0;

    my $countImported=0;
    my $count=0;
    my $tableInsert_sth=$DB->prepare('INSERT INTO Ratings VALUES (?,?,?,?)');
    while(<$fh>) {
	$lineCount++;
	my $line=$_;
	#print "read line $lineCount:$line";

	$line=~s/\n$//o;

	# skip empty lines (only really appear right before last line ending with ----
	next if ( $line=~m/^\s*$/o );
	# end is line consisting of only '-'
	last if ( $line=~m/^\-\-\-\-\-\-\-+/o );

        # e.g. New  Distribution  Votes  Rank  Title
        #            0000000133  225568   8.9  12 Angry Men (1957)
	if ( $line=~m/^\s+([\.|\*|\d]+)\s+(\d+)\s+(\d+\.\d+)\s+(.+)$/o ) {

	    if ( $self->{imdbMovie2DBKey}->{$4} ) {
		$tableInsert_sth->execute($self->{imdbMovie2DBKey}->{$4}, $1, $2, $3);
		$countImported++;
		if ( ($countImported % 50000) == 0 ) {
		    $DB->commit();
		}
	    }

	}
	$count++;

	#$self->{movies}{$line}=[$1,$2,"$3.$4"];
	if ( $self->{showProgressBar} ) {
	    # re-adjust target so progress bar doesn't seem too wonky
	    if ( $count > $countEstimate ) {
		$countEstimate = $progress->target($count+1000);
		$next_update=$progress->update($count);
	    }
	    elsif ( $count > $next_update ) {
		$next_update=$progress->update($count);
	    }
	}
	else {
	    $self->error("$file:$lineCount: unrecognized format");
	    $next_update=$progress->update($count) if ($self->{showProgressBar});
	}
    }
    $progress->update($countEstimate) if ($self->{showProgressBar});

    $self->status(sprintf("importing Ratings found ".withThousands($count)." titles in ".
			  withThousands($lineCount)." lines in %d seconds",time()-$startTime));

    closeMaybeGunzip($file, $fh);
    return($count);
}

sub importKeywords($$$$)
{
    my ($self, $countEstimate, $file, $DB)=@_;
    my $startTime=time();
    my $lineCount=0;

    if ( $DB->table_row_count('Keywords') > 0 ) {
	$self->status("clearing previously loaded data..");
	$DB->runSQL("DELETE from Keywords");
    }

    my $fh = openMaybeGunzip($file) || return(-2);
    while(<$fh>) {
	$lineCount++;

	if ( m/THE KEYWORDS LIST/ ) {
	    if ( !($_=<$fh>) || !m/^===========/o ) {
		$self->error("missing ======= after \"THE KEYWORDS LIST\" at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^\s*$/o ) {
		$self->error("missing empty line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    last;
	}
	elsif ( $lineCount > 70000 ) {
	    $self->error("$file: stopping at line $lineCount, didn't see \"THE KEYWORDS LIST\" line");
	    closeMaybeGunzip($file, $fh);
	    return(-1);
	}
    }

    my $progress=Term::ProgressBar->new({name  => "importing Keywords",
					 count => $countEstimate,
					 ETA   => 'linear'})
      if ($self->{showProgressBar});

    $progress->minor(0) if ($self->{showProgressBar});
    $progress->max_update_rate(1) if ($self->{showProgressBar});
    my $next_update=0;

    my $count=0;
    my $countImported=0;
    my %found;
    my $tableInsert_sth=$DB->prepare('INSERT INTO Keywords VALUES (?,?)');
    my $tableInsert_sth2=$DB->prepare('INSERT INTO Titles2Keywords VALUES (?,?)');
    my $keywordCount=0;
    while(<$fh>) {
	$lineCount++;
	my $line=$_;
	chomp($line);
	next if ($line =~ m/^\s*$/);
	my ($title, $keyword) = ($line =~ m/^(.*)\s+(\S+)\s*$/);
	if ( defined($title) and defined($keyword) ) {

            my ($episode) = $title =~ m/\s+(\{.*\})$/o;

            # ignore anything which is an episode (e.g. "{Doctor Who (#10.22)}" )
            #if ( !defined $episode || $episode eq '' )
            #{
		if ( $self->{imdbMovie2DBKey}->{$title} ) {
		    if ( !defined($found{$keyword}) ) {
			$found{$keyword}=$keywordCount++;
			$tableInsert_sth->execute($keywordCount, $keyword);
			#=$DB->insert_db('Keywords', 'KeywordID', Name=>$keyword);
		    }
		    $tableInsert_sth2->execute($self->{imdbMovie2DBKey}->{$title}, $found{$keyword});

		    #$DB->insert_db('Titles2Keywords', undef, TitleID=>$self->{imdbMovie2DBKey}->{$title}, KeywordID=>$found{$keyword});
		    $countImported++;
		    if ( ($countImported % 50000) == 0 ) {
			$DB->commit();
		    }
		}
		$count++;
	    #}
	    if ( $self->{showProgressBar} ) {
		# re-adjust target so progress bar doesn't seem too wonky
		if ( $count > $countEstimate ) {
		    $countEstimate = $progress->target($count+1000);
		    $next_update=$progress->update($count);
		}
		elsif ( $count > $next_update ) {
		    $next_update=$progress->update($count);
		}
	    }
        } else {
	    #$self->error("$file:$lineCount: unrecognized format \"$line\"");
	    $next_update=$progress->update($count) if ($self->{showProgressBar});
	}

    }
    $progress->update($countEstimate) if ($self->{showProgressBar});

    $self->status(sprintf("importing Keywords found ".withThousands($count)." titles in ".
			  withThousands($lineCount)." lines in %d seconds",time()-$startTime));

    closeMaybeGunzip($file, $fh);
    return($count);
}

sub importPlots($$$$)
{
    my ($self, $countEstimate, $file, $DB)=@_;
    my $startTime=time();
    my $lineCount=0;

    if ( $DB->table_row_count('Plots') > 0 ) {
	$self->status("clearing previously loaded data..");
	$DB->runSQL("DELETE from Plots");
    }

    my $fh = openMaybeGunzip($file) || return(-2);
    while(<$fh>) {
	$lineCount++;

	if ( m/PLOT SUMMARIES LIST/ ) {
	    if ( !($_=<$fh>) || !m/^===========/o ) {
		$self->error("missing ======= after \"PLOT SUMMARIES LIST\" at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    if ( !($_=<$fh>) || !m/^-----------/o ) {
		$self->error("missing ------- line after ======= at line $lineCount");
		closeMaybeGunzip($file, $fh);
		return(-1);
	    }
	    last;
	}
	elsif ( $lineCount > 500 ) {
	    $self->error("$file: stopping at line $lineCount, didn't see \"PLOT SUMMARIES LIST\" line");
	    closeMaybeGunzip($file, $fh);
	    return(-1);
	}
    }

    my $progress=Term::ProgressBar->new({name  => "importing Plots",
					 count => $countEstimate,
					 ETA   => 'linear'})
      if ($self->{showProgressBar});

    $progress->minor(0) if ($self->{showProgressBar});
    $progress->max_update_rate(1) if ($self->{showProgressBar});
    my $next_update=0;

    my $count=0;
    my $tableInsert_sth=$DB->prepare('INSERT INTO Plots VALUES (?,?)');
    while(<$fh>) {
	$lineCount++;
	my $line=$_;
	chomp($line);
	next if ($line =~ m/^\s*$/);
	next if ($line=~m/\s*\{\{SUSPENDED\}\}/o);

	my ($title, $episode) = ($line =~ m/^MV:\s(.*?)\s?(\{.*\})?$/);
	if ( defined($title) ) {

	    $line =~s/^MV:\s*//;

	    my $plot = '';
	  LOOP:
	    while (1) {
		if ( my $l = <$fh> ) {
		    $lineCount++;
		    chomp($l);
		    next if ($l =~ m/^\s*$/);
		    if ( $l =~ m/PL:\s(.*)$/ ) {     # plot summary is a number of lines starting "PL:"
			$plot .= ($plot ne ''?' ':'') . $1;
		    }
		    last LOOP if ( $l =~ m/BY:\s(.*)$/ );     # the author line "BY:" signals the end of the plot summary
		} else {
		    last LOOP;
		}
	    }

	    if ( $self->{imdbMovie2DBKey}->{$line} ) {
		$tableInsert_sth->execute($self->{imdbMovie2DBKey}->{$line}, $plot);
		$count++;
		if ( ($count % 50000) == 0 ) {
		    $DB->commit();
		}
	    }
	    else {
		#warn "$line";
	    }

	    if ( $self->{showProgressBar} ) {
		# re-adjust target so progress bar doesn't seem too wonky
		if ( $count > $countEstimate ) {
		    $countEstimate = $progress->target($count+1000);
		    $next_update=$progress->update($count);
		}
		elsif ( $count > $next_update ) {
		    $next_update=$progress->update($count);
		}
	    }
        } else {
            # skip lines up to the next "MV:"
            if ($line !~ m/^(---|PL:|BY:)/ ) {
                $self->error("$file:$lineCount: unrecognized format \"$line\"");
            }
	    $next_update=$progress->update($count) if ($self->{showProgressBar});
	}
    }
    $progress->update($countEstimate) if ($self->{showProgressBar});

    $self->status(sprintf("importing Plots found $count ".withThousands($count)." in ".
			  withThousands($lineCount)." lines in %d seconds",time()-$startTime));

    closeMaybeGunzip($file, $fh);
    return($count);
}

sub loadDBInfo($)
{
    my $file=shift;
    my $info;

    open(INFO, "< $file") || return("imdbDir index file \"$file\":$!");
    while() {
	chop();
	if ( s/^([^:]+)://o ) {
	    $info->{$1}=$_;
	}
    }
    close(INFO);
    return($info);
}

sub dbinfoLoad($)
{
    my $self=shift;

    my $info=loadDBInfo($self->{moviedbInfo});
    if ( ref $info ne 'HASH' ) {
	return(1);
    }
    $self->{dbinfo}=$info;
    return(undef);
}

sub dbinfoAdd($$$)
{
    my ($self, $key, $value)=@_;
    $self->{dbinfo}->{$key}=$value;
}

sub dbinfoGet($$$)
{
    my ($self, $key, $defaultValue)=@_;
    if ( defined($self->{dbinfo}->{$key}) ) {
	return($self->{dbinfo}->{$key});
    }
    return($defaultValue);
}

sub dbinfoSave($)
{
    my $self=shift;
    open(INFO, "> $self->{moviedbInfo}") || return(1);
    for (sort keys %{$self->{dbinfo}}) {
	print INFO "".$_.":".$self->{dbinfo}->{$_}."\n";
    }
    close(INFO);
    return(0);
}

sub dbinfoGetFileSize($$)
{
    my ($self, $key)=@_;


    if ( !defined($self->{listFiles}->paths_isset($key) ) ) {
	die ("invalid call for $key");
    }
    my $filePath=$self->{listFiles}->paths_index($key);
    my $fileSize=int(-s $filePath);

    # if compressed, then attempt to run gzip -l
    if ( $filePath=~m/.gz$/) {
	if ( open(my $fd, "gzip -l $filePath |") ) {
	    # if parse fails, then defalt to wild ass guess of compression of 65%
	    $fileSize=int(($fileSize*100)/(100-65));

	    while(<$fd>) {
		if ( m/^\s*\d+\s+(\d+)/ ) {
		    $fileSize=$1;
		}
	    }
	    close($fd);
	}
	else {
	    # wild ass guess of compression of 65%
	    $fileSize=int(($fileSize*100)/(100-65));
	}
    }
    return($fileSize);
}

sub _redirect($$)
{
    my ($self, $file)=@_;

    if ( defined($file) ) {
	if ( !open($self->{logfd}, "> $file") ) {
	    print STDERR "$file:$!\n";
	    return(0);
	}
	$self->{errorCountInLog}=0;
    }
    else {
	close($self->{logfd});
	$self->{logfd}=undef;
    }
    return(1);
}

=head2 importListComplete

Check to see if spcified list file has been successfully imported

=cut

sub importListComplete($)
{
    my ($self, $type)=@_;

    if ( -f "$self->{imdbDir}/stage-$type.log" ) {
	return(1);
    }
    return(0);
}

sub _prepStage
{
    my ($self, $type)=@_;

    my $DB=new IMDB::Local::DB(database=>"$self->{imdbDir}/imdb.db");

    # if we're restarting, lets start fresh
    if ( $type eq 'movies' ) {
	#warn("recreating db ".$DB->database());
	$DB->delete();

	for my $type ( $self->listTypes() ) {
	    unlink("$self->{imdbDir}/stage-$type.log");
	}

    }

    if ( !$self->_redirect(sprintf("%s/stage-$type.log", $self->{imdbDir})) ) {
	return(1);
    }

    if ( !$DB->connect() ) {
	die "moviedb connect failed:$DBI::errstr";
    }
    $DB->runSQL("PRAGMA synchronous = OFF");
    return($DB);

}

sub _unprepStage
{
    my ($self, $db)=@_;

    $db->commit();
    $db->disconnect();

    $self->_redirect(undef);
}

sub importListFile($$$)
{
    my ($self, $DB, $type)=@_;


    if ( !grep(/^$type$/, $self->listTypes()) ) {
	die "invalid type $type";
    }

    my $dbinfoCalcEstimate=sub {
	my ($self, $key)=@_;

	my %estimateSizePerEntry=(movies=>47,
				  directors=>258,
				  actors=>695,
				  actresses=>779,
				  genres=>38,
				  ratings=>68,
				  keywords=>47,
				  plot=>804);
	my $fileSize=$self->dbinfoGetFileSize($key);

	my $countEstimate=int($fileSize/$estimateSizePerEntry{$key});

	my $filePath=$self->{listFiles}->paths_index($key);


	$self->dbinfoAdd($key."_list_file", $filePath);
	$self->dbinfoAdd($key."_list_file_size", "".int(-s $filePath));
	$self->dbinfoAdd($key."_list_file_size_uncompressed", $fileSize);
	$self->dbinfoAdd($key."_list_count_estimate", $countEstimate);
	return($countEstimate);
    };

    my $dbinfoCalcBytesPerEntry = sub {
	my ($self, $key, $calcActualForThisNumber)=@_;
	my $fileSize=$self->dbinfoGetFileSize($key);
	return(int($fileSize/$calcActualForThisNumber));
    };


    if ( $type eq 'movies') {
	my $countEstimate=&$dbinfoCalcEstimate($self, "movies");

	my $num=$self->importMovies($countEstimate, $self->{listFiles}->paths_index('movies'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('movies')." from ftp.imdb.com");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.10 ) {
	    my $better=&$dbinfoCalcBytesPerEntry($self, "movies", $num);
	    $self->status("ARG estimate of $countEstimate for movies needs updating, found $num ($better bytes/entry)");
	}

	open(OUT, "> $self->{imdbDir}/titles.tsv") || die "$self->{imdbDir}/titles.tsv:$!";
	for my $mkey (sort keys %{$self->{imdbMovie2DBKey}}) {
	    print OUT "".$self->{imdbMovie2DBKey}->{$mkey}."\t".$mkey."\n";
	}
	close(OUT);

	$self->dbinfoAdd("db_stat_movie_count", "$num");
	return(0);
    }

    # read in keys so we have them for follow-up stages
    if ( !defined($self->{imdbMovie2DBKey}) ) {
	#$self->{imdbMovie2DBKey}=$DB->select2Hash("select IMDBKey, TitleID from Titles");

	if ( 1 ) {
	    open(IN, "< $self->{imdbDir}/titles.tsv") || die "$self->{imdbDir}/titles.tsv:$!";
	    while () {
		chomp();
		if ( m/^(\d+)\t(.+)/o ) {
		    $self->{imdbMovie2DBKey}->{$2}=$1;
		}
	    }
	    close(IN);
	}
    }

    # need to read-movie kesy
    if ( $type eq 'directors') {

	my $countEstimate=&$dbinfoCalcEstimate($self, "directors");

	my $num=$self->importDirectors($countEstimate, $self->{listFiles}->paths_index('directors'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('directors')." from ftp.imdb.com (see http://www.imdb.com/interfaces)");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.10 ) {
	    my $better=&$dbinfoCalcBytesPerEntry($self, "directors", $num);
	    $self->status("ARG estimate of $countEstimate for directors needs updating, found $num ($better bytes/entry)");
	}
	$self->dbinfoAdd("db_stat_director_count", "$num");
	return(0);
    }

    if ( $type eq 'actors') {

	#print "re-reading movies into memory for reverse lookup..\n";
	my $countEstimate=&$dbinfoCalcEstimate($self, "actors");

	#my $num=$self->readCast("Actors", $countEstimate, "$self->{imdbListFiles}->{actors}");
	my $num=$self->importActors("Actors", $countEstimate, $self->{listFiles}->paths_index('actors'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('actors')." from ftp.imdb.com (see http://www.imdb.com/interfaces)");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.10 ) {
	    my $better=&$dbinfoCalcBytesPerEntry($self, "actors", $num);
	    $self->status("ARG estimate of $countEstimate for actors needs updating, found $num ($better bytes/entry)");
	}
	$self->dbinfoAdd("db_stat_actor_count", "$num");

	return(0);
    }

    if ( $type eq 'actresses') {

	my $countEstimate=&$dbinfoCalcEstimate($self, "actresses");
	my $num=$self->importActors("Actresses", $countEstimate, $self->{listFiles}->paths_index('actresses'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('actresses')." from ftp.imdb.com (see http://www.imdb.com/interfaces)");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.10 ) {
	    my $better=&$dbinfoCalcBytesPerEntry($self, "actresses", $num);
	    $self->status("ARG estimate of $countEstimate for actresses needs updating, found $num ($better bytes/entry)");
	}
	$self->dbinfoAdd("db_stat_actress_count", "$num");

	return(0);
    }

    if ( $type eq 'genres') {
	my $countEstimate=&$dbinfoCalcEstimate($self, "genres");

	my $num=$self->importGenres($countEstimate, $self->{listFiles}->paths_index('genres'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('genres')." from ftp.imdb.com");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.10 ) {
	    my $better=&$dbinfoCalcBytesPerEntry($self, "genres", $num);
	    $self->status("ARG estimate of $countEstimate for genres needs updating, found $num ($better bytes/entry)");
	}
	$self->dbinfoAdd("db_stat_genres_count", "$num");

	return(0);
    }

    if ( $type eq 'ratings') {
	my $countEstimate=&$dbinfoCalcEstimate($self, "ratings");

	my $num=$self->importRatings($countEstimate, $self->{listFiles}->paths_index('ratings'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('ratings')." from ftp.imdb.com");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.10 ) {
	    my $better=&$dbinfoCalcBytesPerEntry($self, "ratings", $num);
	    $self->status("ARG estimate of $countEstimate for ratings needs updating, found $num ($better bytes/entry)");
	}
	$self->dbinfoAdd("db_stat_ratings_count", "$num");

	return(0);
    }

    if ( $type eq 'keywords') {

	if ( ! -f $self->{listFiles}->paths_index('keywords') ) {
	    $self->status("no keywords file downloaded, see --with-keywords details in documentation");
	    return(0);
	}
	my $countEstimate=&$dbinfoCalcEstimate($self, "keywords");
	#my $countEstimate=5554178;

	my $num=$self->importKeywords($countEstimate, $self->{listFiles}->paths_index('keywords'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('keywords')." from ftp.imdb.com");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.05 ) {
	    $self->status("ARG estimate of $countEstimate for keywords needs updating, found $num");
	}
	$self->dbinfoAdd("keywords_list_file",         $self->{listFiles}->paths_index('keywords'));
	$self->dbinfoAdd("keywords_list_file_size", -s $self->{listFiles}->paths_index('keywords'));
	$self->dbinfoAdd("db_stat_keywords_count", "$num");

	return(0);
    }

    if ( $type eq 'plot') {

	if ( ! -f $self->{listFiles}->paths_index('plot') ) {
	    $self->status("no plot file downloaded, see --with-plot details in documentation");
	    return(0);
	}

	my $countEstimate=&$dbinfoCalcEstimate($self, "plot");
	my $num=$self->importPlots($countEstimate, $self->{listFiles}->paths_index('plot'), $DB);
	if ( $num < 0 ) {
	    if ( $num == -2 ) {
		$self->error("you need to download ".$self->{listFiles}->paths_index('plot')." from ftp.imdb.com");
	    }
	    return(1);
	}
	elsif ( abs($num - $countEstimate) > $countEstimate*.05 ) {
	    $self->status("ARG estimate of $countEstimate for plots needs updating, found $num");
	}
	$self->dbinfoAdd("plots_list_file",         $self->{listFiles}->paths_index('plot'));
	$self->dbinfoAdd("plots_list_file_size", -s $self->{listFiles}->paths_index('plot'));
	$self->dbinfoAdd("db_stat_plots_count", "$num");

	return(0);
    }

    $self->error("invalid type $type");
    return(1);
}

=head2 importList

Import a list file from 'listsDir' into the IMDB::Local Database.

=cut

sub importList($$)
{
    my ($self, $type)=@_;

    my $DB=$self->_prepStage($type);

    # lets load our stats
    $self->dbinfoLoad();

    my $startTime=time();
    if ( $self->importListFile($DB, $type) != 0 ) {
	$DB->disconnect();
	return(1);
    }

    $self->dbinfoAdd("seconds_to_complete_prep_stage_$type", (time()-$startTime));
    $self->dbinfoSave();

    $self->_unprepStage($DB);
    return(0);
}


sub _NOT_USED_checkSantity($)
{
    my ($self)=@_;

    $self->dbinfoAdd("db_version", $IMDB::Local::VERSION);

    if ( $self->dbinfoSave() ) {
	$self->error("$self->{moviedbInfo}:$!");
	return(1);
    }

    $self->status("running quick sanity check on database indexes...");
    my $imdb=new IMDB::Local('imdbDir' => $self->{imdbDir},
			     'verbose' => $self->{verbose});

    if ( -e "$self->{moviedbOffline}" ) {
	unlink("$self->{moviedbOffline}");
    }

    if ( my $errline=$imdb->sanityCheckDatabase() ) {
	open(OFF, "> $self->{moviedbOffline}") || die "$self->{moviedbOffline}:$!";
	print OFF $errline."\n";
	print OFF "one of the prep stages' must have produced corrupt data\n";
	print OFF "report the following details to xmltv-devel\@lists.sf.net\n";

	my $info=loadDBInfo($self->{moviedbInfo});
	if ( ref $info eq 'HASH' ) {
	    for my $key (sort keys %{$info}) {
		print OFF "\t$key:$info->{$key}\n";
	    }
	}
	else {
	    print OFF "\tdbinfo file corrupt\n";
	    print OFF "\t$info";
	}
	print OFF "database taken offline\n";
	close(OFF);
	open(OFF, "< $self->{moviedbOffline}") || die "$self->{moviedbOffline}:$!";
	while() {
	    chop();
	    $self->error($_);
	}
	close(OFF);
	return(1);
    }
    $self->status("sanity intact :)");
    return(0);
}

=head1 AUTHOR

jerryv, C<<  >>

=head1 BUGS

Please report any bugs or feature requests to C, or through
the web interface at L.  I will be notified, and then you'll
automatically be notified of progress on your bug as I make changes.




=head1 SUPPORT

You can find documentation for this module with the perldoc command.

    perldoc IMDB::Local


You can also look for information at:

=over 4

=item * RT: CPAN's request tracker (report bugs here)

L

=item * AnnoCPAN: Annotated CPAN documentation

L

=item * CPAN Ratings

L

=item * Search CPAN

L

=back


=head1 ACKNOWLEDGEMENTS


=head1 LICENSE AND COPYRIGHT

Copyright 2015 jerryv.

This program is free software; you can redistribute it and/or modify it
under the terms of the the Artistic License (2.0). You may obtain a
copy of the full license at:

L

Any use, modification, and distribution of the Standard or Modified
Versions is governed by this Artistic License. By using, modifying or
distributing the Package, you accept this license. Do not use, modify,
or distribute the Package, if you do not accept this license.

If your Modified Version has been derived from a Modified Version made
by someone other than you, you are nevertheless required to ensure that
your Modified Version complies with the requirements of this license.

This license does not grant you the right to use any trademark, service
mark, tradename, or logo of the Copyright Holder.

This license includes the non-exclusive, worldwide, free-of-charge
patent license to make, have made, use, offer to sell, sell, import and
otherwise transfer the Package with respect to any patent claims
licensable by the Copyright Holder that are necessarily infringed by the
Package. If you institute patent litigation (including a cross-claim or
counterclaim) against any party alleging that the Package constitutes
direct or contributory patent infringement, then this Artistic License
to you shall terminate on the date that such litigation is filed.

Disclaimer of Warranty: THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER
AND CONTRIBUTORS "AS IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES.
THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
PURPOSE, OR NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY
YOUR LOCAL LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR
CONTRIBUTOR WILL BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR
CONSEQUENTIAL DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


=cut

1; # End of IMDB::Local