File Coverage

blib/lib/Lingua/DetectCyrillic.pm

Criterion	Covered	Total	%
statement	165	272	60.6
branch	59	128	46.0
condition	4	45	8.8
subroutine	16	16	100.0
pod	0	6	0.0
total	244	467	52.2

package Lingua::DetectCyrillic;
# Íóæíî äëÿ ïåðåêîäèðîâêè. Áóäåò çàìåíåíî â ñëåäóþùèõ âûïóñêàõ
use Unicode::Map8;
use Unicode::String;

use Exporter ();
@ISA = qw ( Exporter );
@EXPORT_OK = qw ( &toLowerCyr &toUpperCyr &TranslateCyr %RusCharset );


# Óâåëè÷åíèå ñëîâàðÿ ñ 300 äî 3000 ñëîâ ïðàêòè÷åñêè íè÷åãî íå äàëî:
# ïðè ðàñïîçíàâàíèè koi8/windows ðàçíèöà áûëà ïîðÿäêà 30.
# Àêòèâèçèðóåì ñëîâàðè è õýøè
  use Lingua::DetectCyrillic::DictRus;
  use Lingua::DetectCyrillic::DictUkr;

# Õåøèðîâàíèå ïî 3 ïðèøëîñü ñíÿòü - îíî äàåò â ñðåäíåì â 5 ðàç
# ëó÷øèé ðåçóëüòàò, íî õýø íóæåí â òå æå 5-7 ðàç áîëüøå.
  use Lingua::DetectCyrillic::WordHash2Rus;
  use Lingua::DetectCyrillic::WordHash2Ukr;

$VERSION = "0.02";

# Ãëîáàëüíûå ïåðåìåííûå
$FullStat=0;

######## Ýêñïîðòèðóåìûå ïåðåìåííûå
$RusCharset{'Upper'} = "ÀÁÂÃÄŨÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞߥª²¯¡";
$RusCharset{'Lower'} = "àáâãäå¸æçèéêëìíîïðñòóôõö÷øùúûüýþÿ´º³¿¢";
$RusCharset{'All'}="ÀÁÂÃÄŨÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäå¸æçèéêëìíîïðñòóôõö÷øùúûüýþÿ¥´ªº²³¯¿¡¢«»“”–—¹";
$RusCharset{'Ukrainian'}= "¥´ªº²³¯¿";
$RusCharset{'Punctuation'}="«»“”–—¹";
### Êîíåö ýêñïîðòèðóåìûõ ïåðåìåííûõ

### Ýêñïîðòèðóåìûå ôóíêöèè

sub TranslateCyr {
my ($CodingIn, $CodingOut, $String) = @_;
my ($MapIn, $MapOut);

$CodingIn=lc ($CodingIn);
$CodingOut=lc ($CodingOut);
# Åñëè âõîäíàÿ è âûõîäíàÿ êîäèðîâêè ñîâïàäàþò, âåðíóòü íåèçìåíåííóþ ñòðîêó
if ( $CodingIn eq $CodingOut ) { return $String; }

if ($CodingIn =~ /(1251|win)/) { $MapIn = Unicode::Map8 -> new("cp1251"); $CodingIn="cp1251"; } elsif
($CodingIn =~ /(koi8u|koi8-u)/) { $MapIn = Unicode::Map8 -> new("koi8-u"); $CodingIn="koi8-u"; } elsif
($CodingIn =~ /koi/) { $MapIn = Unicode::Map8 -> new("koi8-r"); $CodingIn="koi8-r"; } elsif
($CodingIn =~ /(dos|866|alt)/) { $MapIn = Unicode::Map8 -> new("cp866"); $CodingIn="cp866"; } elsif
($CodingIn =~ /(iso|8859-5)/) { $MapIn = Unicode::Map8 -> new("ISO_8859-5"); $CodingIn="ISO_8859-5"; } elsif
($CodingIn =~ /(mac|10007)/) { $MapIn = Unicode::Map8 -> new("cp10007"); $CodingIn="cp10007"; } elsif
($CodingIn =~ /(utf|uni)/) { $CodingIn="utf-8"; } else
{ return ""; } # Åñëè íå îïðåäåëèëè âõîäíóþ êîäèðîâêó - âûéòè

if ($CodingOut =~ /(1251|win)/) { $MapOut = Unicode::Map8 -> new("cp1251"); $CodingOut="cp1251"; } elsif
($CodingOut =~ /(koi8u|koi8-u)/) { $MapOut = Unicode::Map8 -> new("koi8-u"); $CodingOut="koi8-u"; } elsif
($CodingOut =~ /koi/) { $MapOut = Unicode::Map8 -> new("koi8-r"); $CodingOut="koi8-r"; } elsif
($CodingOut =~ /(dos|866|alt)/) { $MapOut = Unicode::Map8 -> new("cp866"); $CodingOut="cp866";} elsif
($CodingOut =~ /(iso|8859-5)/) { $MapOut = Unicode::Map8 -> new("ISO_8859-5"); $CodingOut="ISO_8859-5";} elsif
($CodingOut =~ /(mac|10007)/) { $MapOut = Unicode::Map8 -> new("cp10007"); $CodingOut="cp10007"; } elsif
($CodingOut =~ /(utf|uni)/) { $CodingOut="utf-8"; } else
{ return ""; } # Åñëè íå îïðåäåëèëè âûõîäíóþ êîäèðîâêó - âûéòè

# Èç UTF-8 â 8-áèòîâóþ êîäèðîâêó
if ( $CodingIn eq "utf-8" ) {
$s=Unicode::String::utf8($String)->ucs2;
if ( $CodingOut eq "utf-8" ) { return $String; } else { return $MapOut->to8($s); }
}
# Èç 8-áèòîâîé êîäèðîâêè â UTF-8
if ( $CodingOut eq "utf-8" ) { return $MapIn->tou($String)->utf8; }
# Åñëè ýòî íå utf-8, òî ïåðåêîäèðóåì èç 8-áèòîâãî íàáîðà â 8-áèòîâûé æå
  return $MapIn->recode8($MapOut,$String);
}

sub toLowerCyr {
	my ($s, $SourceCoding ) = @_;
  if ( $SourceCoding ) { $s = TranslateCyr ( $SourceCoding, "win", $s ) }
	eval ("\$s =~ tr/A-Z$RusCharset{'Upper'}/a-z$RusCharset{'Lower'}/");
  if ( $SourceCoding ) { $s = TranslateCyr ( "win", $SourceCoding, $s ) }
  $s;
}
sub toUpperCyr {
	my ($s, $SourceCoding ) = @_;
  if ( $SourceCoding ) { $s = TranslateCyr ( $SourceCoding, "win", $s ) }
	eval ("\$s =~ tr/a-z$RusCharset{'Lower'}/A-Z$RusCharset{'Upper'}/");
  if ( $SourceCoding ) { $s = TranslateCyr ( "win", $SourceCoding, $s ) }
  $s;
}

######## Êîíåö ýêñïîðòèðóåìûõ ïåðåìåííûõ è ôóíêöèé

sub new
{
# Çàïîëíèëè äàííûå ïî óìîë÷àíèþ
%Args = ( MaxTokens => 100,	DetectAllLang => 0 );
my $self = {};
shift;
# Çà÷èòàëè àðãóìåíòû - ýòî ãëîáàëüíûé õýø
  %Args = @_;

@Codings = ( "win1251", "koi8r", "cp866", "utf", "iso", "mac" );
if ( $Args{DetectAllLang} ) {push @Codings, koi8u};
@InputData="";

return bless ($self);
}


sub LogWrite {
shift;
my $Outfile=shift;

########### Ôîðìàò îò÷åòà ##########
format STAT =
@<<<<<<@######@######@######@######@######@##########@##########@######@######
$key,%Stat->{$key}{GoodTokensChars},%Stat->{$key}{GoodTokensCount},%Stat->{$key}{AllTokensChars},%Stat->{$key}{AllTokensCount},%Stat->{$key}{CharsUkr},%Stat->{$key}{HashScore2Rus},%Stat->{$key}{HashScore2Ukr},%Stat->{$key}{WordsRus},%Stat->{$key}{WordsUkr}
.

# Âûâîäèì îò÷åò. Åñëè íàçâàíèÿ ôàéëà íåò, èëè ýòî stdout - âûâîäèì íà ýêðàí, èíà÷å â ôàéë
if (!$Outfile or uc ($Outfile) eq "STDOUT") {
    $OUT=\*STDOUT;
            } else {
    open $OUT, ">$Outfile";
}

   select $OUT;   $~="STAT";
  print "Coding: $Coding    Language: $Language     Algorithm: $Algorithm \n\n";
  print  "         GdChr  GdCnt AllChr AllCnt ChrUkr    HashRus   HashUkr  WRus  WUkr\n";
  print  "*" x 78 ."\n";
   foreach $key (keys %Stat) { write ; }
  print "*" x 78 ."\n";
  print "Time: " .localtime() ."\n";
  print <
 GoodTokensChars - number of characters in pure Cyrillic tokens with correct
capitalization.
 GoodTokensCount - number of pure Cyrillic tokens with correct capitalization.
 AllTokensChars - number of characters in tokens containing Cyrillica.
 AllTokensCount - their number
 CharsUkr - number of Ukrainian letters
 HashScore2Rus (Ukr) - hits on 2-letter hash
 WordsRus (Ukr) - hits on frequency dictionary
POD

   select STDOUT;
   if ( $OUT ne \*STDOUT ) { close $OUT; }

} # LogWrite()


sub Detect {
shift;
# Ïåðåãðóæàåì â ãëîáàëüíûé ìàññèâ @InputData, îí ìîæåò ïîíàäîáèòüñÿ åùå ðàç
# ïðè ïîëó÷åíèè ðàñøèðåííîé ñòàòèñòèêè.

@InputData = @_;
######## Îòëàäî÷íî ñëèâàåì ââîä â ôàéë
#$Outfile = "C:\\test_out3.txt";
#open OUT, ">$Outfile";
#for (@_) { print OUT; }
#close OUT;
########


_GetStat();

# Òåïåðü âûÿñíÿåì êîäèðîâêó
$Language = ""; $Coding="";
$Algorithm=0; # Ýòî ïðèìåíåííàÿ ñõåìà îïðåäåëåíèÿ êîäèðîâêè
$MaxCharsProcessed =0;
_AnalyzeStat();
if ( $Coding eq "win1251" ) { $Coding = "windows-1251" }
elsif ( $Coding eq "koi8r" ) { $Coding = "koi8-r" }
elsif ( $Coding eq "koi8u" ) { $Coding = "koi8-u" }
elsif ( $Coding eq "iso" ) { $Coding = "iso-8859-5" }
elsif ( $Coding eq "cp866" ) { $Coding = "cp866" }
elsif ( $Coding eq "utf" ) { $Coding = "utf-8" }
elsif ( $Coding eq "mac" ) { $Coding = "x-mac-cyrillic" }
else  { $Coding = "iso-8859-1" }

return ( $Coding, $Language, $MaxCharsProcessed, $Algorithm );

}

sub _GetStat {
# Èíèöèàëèçèðóåì ñòðóêòóðó õýøåé (îïèñàíèå ñì. â êîíöå)

for (@Codings) {
if ( $FullStat ) {
  $Stat{$_}{AllTokensChars} = 0;
  $Stat{$_}{AllTokensCount} = 0;
  $Stat{$_}{CharsUkr} = 0;
  $Stat{$_}{HashScore2Rus} = 0;
  $Stat{$_}{HashScore2Ukr} = 0;
  $Stat{$_}{WordsRus} = 0;
  $Stat{$_}{WordsUkr} = 0;

    } else { # $FullStat

  $Stat{$_}{GoodTokensChars} = 0;
  $Stat{$_}{GoodTokensCount} = 0;
  $Stat{$_}{CharsUkr} = 0;

} # $FullStat
} # end for


# Ïîëó÷àåì ñòàòèñòèêó äëÿ êàæäîé ñòðîêè
$EnoughTokens=0;
for ( @InputData ) {
my $String=$_;

for (@Codings) {
    _ParseString ($_,$String,%Stat->{$_});
  # Âûõîäèì, åñëè õîòü ïî îäíîé èç êîäèðîâîê íàáðàëè ìàêñèìàëüíîå ÷èñëî òîêåíîâ
  if (%Stat->{$_}{GoodTokensCount} > $Args{MaxTokens} ) { $EnoughTokens=1; }
}

if ( $EnoughTokens ) { last; }
} # Êîíåö ïîëó÷åíèÿ ñòàòèñòèêè


}

sub _AnalyzeStat {

# Ñíà÷àëà àíàëèçèðóåì ñîîòíîøåíèå áóêâ â ÷èñòî êèðèëëè÷åñêèõ òîêåíàõ
# ñ ïðàâèëüíîé êàïèòàëèçàöèåé


## Àíàëèçèðóåì ôîðìàëüíóþ ñòàòèñòèêó ïî òîêåíàì ñ êèðèëëèöåé
# Ìèíèìàëüíîå ñîîòíîøåíèå òîêåíîâ ñ ïðàâèëüíîé êàïèòàëèçàöèåé, ïðè êîòîðîì
# ðàçíèöó ñ÷èòàåì çíà÷èìîé äëÿ âû÷èñëåíèÿ ðåçóëüòàòà. Ïîêà íå èñïîëüçóåòñÿ.
# my $TokensRatio=0.95;
# Ìèíèìàëüíûé ïðîöåíò òîêåíîâ ñ óêðàèíñêèìè ñèìâîëàìè, ÷òîáû òåêñò ñ÷èòàëñÿ óêðàèíñêèì
my $UkrTokensShare=0.01;

## Àíàëèçèðóåì ÷èñòî êèðèëëè÷åñêèå òîêåíû ñ ïðàâèëüíîé êàïèòàëèçàöèåé.
my @CyrCharRating;
for ( @Codings ) { push @CyrCharRating,[$_, %Stat->{$_}{GoodTokensChars}]; }
@CyrCharRating = sort { $b->[1] <=> $a->[1] } @CyrCharRating;

$MaxCharsProcessed = $CyrCharRating[0]->[1];

# Ïîñëå ñîðòèðîâêè ïîëó÷àåì ñïèñîê íàèáîëåå âåðîÿòíûõ êîäèðîâîê
# Îíè ñîäåðæàò ìàêñèìàëüíîå ÷èñëî "ïðàâèëüíûõ" êèðèëëè÷åñêèõ ñëîâ â äàííîé êîäèðîâêå
# Âûÿñíÿåì, ìåæäó ñêîëüêèìè êîäèðîâêàìè íóæíî ïðîâåñòè ðàçëè÷èå
my @BestCodings;
for $arrayref ( @CyrCharRating ) {
       if  ( $arrayref ->[1] == $CyrCharRating[0]->[1] ) {
        push @BestCodings, $arrayref ->[0] ;
       }
}

# Åñëè ïåðâàÿ âîçìîæíàÿ êîäèðîâêà ñîäåðæèò áîëüøå ïðàâèëüíûõ ñèìâîëîâ,
# ÷åì ëþáàÿ èíàÿ, ñ÷èòàåì, ÷òî äåëî ñäåëàíî. Âîîáùå çäåñü ëó÷øå ââåñòè
# îïðåäåëåíèå ìèíèìàëüíî íåîáõîäèìîãî ïðåèìóùåñòâà, ñêàæåì, 10% èëè ÷òî-òî âðîäå.

if ( scalar(@BestCodings) == 1 ) {
  $Coding = $CyrCharRating[0]->[0];

# Îïðåäåëÿåì ÿçûê. Ñìîòðèì, íåò ëè óêðàèíñêèõ òîêåíîâ. Åñëè îíè ïðèñóòñòâóþò
# â êîëè÷åñòâå íå ìåíåå $UkrTokensShare, ñ÷èòàåì ÿçûê óêðàèíñêèì, èíà÷å - ðóññêèì.
if ( %Stat->{$Coding}{CharsUkr} / %Stat->{$Coding}{GoodTokensCount} > $UkrTokensShare )
{ $Language = "Ukr"; } else { $Language = "Rus"; }
$Algorithm = 11;
return;
} # Êîíåö ðàçáîðà îäíîé êîäèðîâêè

# Ñëåäóþùèé âàðèàíò: îäèíàêîâîå ÷èñëî áàëëîâ íàáðàëè ðîâíî äâå êîäèðîâêè.
# Íå èñêëþ÷åíî, ÷òî ýòî ëèáî koi ðóññêèé è óêðàèíñêèé, ëèáî win1251 è ìàñ.
# Òîãäà ìû èõ ìîæåì ðàçëè÷èòü ïî ôîðìàëüíûì ïàðàìåòðàì.
if ( scalar(@BestCodings) == 2 ) {

$BestCoding1 = $CyrCharRating[0]->[0];
$BestCoding2 = $CyrCharRating[1]->[0];
# Ïåðâûé âàðèàíò - ýòî êîäèðîâêè koi8u è koi8r.
if ( $BestCoding1 =~ /koi/ && $BestCoding2 =~ /koi/ ) {
# Îïðåäåëÿåì ÿçûê è íà ýòîì îñíîâàíèè - êîäèðîâêó
if (%Stat->{$Coding}{GoodTokensCount} > 0 &&  %Stat->{$Coding}{CharsUkr} / %Stat->{$Coding}{GoodTokensCount} > $UkrTokensShare )
{ $Coding = "koi8u"; $Language = "Ukr"; } else { $Coding = "koi8r"; $Language = "Rus"; }
$Algorithm = 21;
return;
} # Êîíåö 1-ãî âàðèàíòà

# Âòîðîé âàðèàíò - ýòî êîäèðîâêè win1251 è mac. Òî åñòü âåñü òåêñò çàïèñàí
# ñòðî÷íûìè áóêâàìè áåç Þ è Ý. Ïðåäïî÷èòàåì îäíîçíà÷íî win1251
if ( $BestCoding1 =~ /(win1251|mac)/ && $BestCoding2 =~ /(win1251|mac)/ ) {
$Coding="win1251";
if (%Stat->{$Coding}{GoodTokensCount} > 0 && %Stat->{$Coding}{CharsUkr} / %Stat->{$Coding}{GoodTokensCount} > $UkrTokensShare )
{ $Language = "Ukr"; } else { $Language = "Rus"; }
$Algorithm = 22;
return;
} # Êîíåö 2-ãî âàðèàíòà

} # Êîíåö ðàçáîðà äâóõ êîäèðîâîê ïðè êîòîðîì êîãäà ìû åùå ìîæåì îáîéòèñü òîëüêî àíàëèçîì
# "ïðàâèëüíûõ" ñèìâîëîâ,  áåç ïðèâëå÷åíèÿ ðàñøèðåííîé ñòàòèñòèêè

# Èòàê, êîäèðîâêó ñ õîäó íå óäàëîñü îïðåäåëèòü ïî ñòàòèñòèêå ñèìâîëîâ ñ ïðàâèëüíîé
# êàïèòàëèçàöèåé. Òîãäà óñòàíàâëèâàåì ôëàã $FullStat è åùå ðàç ïîëó÷àåì ñòàòèñòèêó ïî
# ñòðîêàì - íà ýòîò ðàç ñ õýøåì è ñëîâàðåì
$FullStat = 1;
_GetStat();

# Ïðîâåðÿåì, à åñòü ëè êèðèëëèöà â òåêñòå âîîáùå.
for ( @BestCodings ) { push @CyrCharRating,[$_, %Stat->{$_}{AllTokensChars}]; }
@CyrCharRating = sort { $b->[1] <=> $a->[1] } @CyrCharRating;

$MaxCharsProcessed = $CyrCharRating[0]->[1];

# Âûéòè, åñëè íå áûëî íè îäíîãî êèðèëëè÷åñêîãî ñèìâîëà
if ( $MaxCharsProcessed == 0 ) { $Coding = "iso-8859-1"; $Language = "NoLang"; $Algorithm = 100; return; }


# Äåëàåì ñëåäóþùèå äâà øàãà. Ñíà÷àëà ñîçäàåì ìàññèâ èç êîìáèíàöèé ÿçûêà è êîäèðîâêè
# äëÿ ïîäñ÷åòà ñëîâ èç ñëîâàðÿ, çàòåì îñòàâëÿåì òîëüêî êîìáèíàöèè ñ ìàêñèìàëüíûì çíà÷åíèåì,
# ò.å. ñóæàåì ñïèñîê ïîòåíöèàëüíûõ êîìáèíàöèé. Åñëè ýòèõ êîìáèíàöèé áîëüøå îäíîé,
# ïåðåõîäèì êî âòîðîìó øàãó - ñîçäàåì àíàëîãè÷íûé ìàññèâ äëÿ õýøåé è ñíîâà îòáèðàåì
# êîìáèíàöèè ñ ìàêñèìàëüíûì çíà÷åíèåì. Åñëè ñíîâà íå óäàëîñü âûäåëèòü åäèíñòâåííîãî
# "ïîáåäèòåëÿ", ïðåäïî÷èòàåì ðóññêèé ÿçûê óêðàèíñêîìó, êîäèðîâêó windows - ìàêèíòîøó.

# Øàã 1. Èùåì ìàêñèìàëüíûé ðåéòèíã ñëîâ èç ÷àñòîòíîãî ñëîâàðÿ
my @WordsRating;
for ( @BestCodings ) {
  push @WordsRating, [$_,"Rus", %Stat->{$_}{WordsRus}];
  push @WordsRating, [$_,"Ukr", %Stat->{$_}{WordsUkr}];
}
@WordsRating = sort { $b->[2] <=> $a->[2] } @WordsRating;

#print "WordsRating: \n";
#for $arrayref (@WordsRating) {
#  print "  " . $arrayref ->[0] . " " .$arrayref ->[1] ." ".$arrayref ->[2] ."\n"; }


# Åñëè îáíàðóæèëè â òåêñòå õîòÿ áû îäíî ñëîâî èç ñëîâàðÿ, è íåò àëüòåðíàòèâ,
# òî ñ÷èòàåì, ÷òî îïðåäåëåíèå ÿçûêà/êîäèðîâêè ïðîèçîøëî
if ( $WordsRating[0]->[2] > 0 && $WordsRating[0]->[2] > $WordsRating[1]->[2] ) {
 $Coding = $WordsRating[0]->[0];
 $Language = $WordsRating[0]->[1];
 $Algorithm = 31;
 return;
}

# Ëèáî ñëîâà èç ÷àñòîòíîãî ñëîâàðÿ âîîáùå íå áûëè îáíàðóæåíû,
# ëèáî èìååì ñîâïàäåíèå ÷èñëà ñëîâ äëÿ íåñêîëüêèõ êîìáèíàöèé ÿçûê/êîäèðîâêà
#  Øàã 2. Îáðàùàåìñÿ ê õýøó è åùå áîëüøå ñóæàåì àðåàë ïîèñêà.

my @BestWordsRating;
for $arrayref ( @WordsRating ) {
       if  ( $arrayref ->[2] == $WordsRating[0]->[2] ) {
        push @BestWordsRating, [ $arrayref ->[0],$arrayref ->[1],$arrayref ->[2] ] ;
       }
}
#print "BestWordsRating: \n";
#for $arrayref (@BestWordsRating) {
#  print "  " . $arrayref ->[0] . " " .$arrayref ->[1] ." ".$arrayref ->[2] ."\n"; }


my @HashRating;
for $arrayref ( @BestWordsRating ) {

if ( $arrayref->[1] eq "Rus" ) {
  push @HashRating, [$arrayref->[0],"Rus", %Stat->{$arrayref->[0]}{HashScore2Rus}]; }
if ( $arrayref->[1] eq "Ukr" ) {
  push @HashRating, [$arrayref->[0],"Ukr", %Stat->{$arrayref->[0]}{HashScore2Ukr}]; }

}
@HashRating = sort { $b->[2] <=> $a->[2] } @HashRating;

#for $arrayref (@HashRating) {
#  print "  " .$arrayref ->[0] . " " .$arrayref ->[1] ." ".$arrayref ->[2] ."\n"; }

# Åñëè îáíàðóæèëè â òåêñòå õîòÿ áû îäèí ðåàëüíûé õýø, è íåò àëüòåðíàòèâ,
# òî ñ÷èòàåì, ÷òî îïðåäåëåíèå ÿçûêà/êîäèðîâêè ïðîèçîøëî
if ( $HashRating[0]->[2] > 0 && $HashRating[0]->[2] > $HashRating[1]->[2] ) {
 $Coding = $HashRating[0]->[0];
 $Language = $HashRating[0]->[1];
 $Algorithm = 32;
 return;
}


# Ëèáî õýø íå îáíàðóæåí, ëèáî èìååì ñîâïàäåíèå ÷èñëà ñëîâ äëÿ íåñêîëüêèõ êîìáèíàöèé
# ÿçûê/êîäèðîâêà
# Øàã 3. Îñòàâëÿåì òîëüêî òå êîìáèíàöèè ÿçûê/êîäèðîâêà, êîòîðûå ñîäåðæàò íàèáîëüøåå ÷èñëî
# ïîïàäàíèé â õýø.

my @BestHashRating;
for $arrayref ( @HashRating ) {
       if  ( $arrayref ->[2] == $HashRating[0]->[2] ) {
        push @BestHashRating, [ $arrayref ->[0],$arrayref ->[1] ] ;
       }
}
# for $arrayref (@BestHashRating) {
#  print "  " .$arrayref ->[0] . " " .$arrayref ->[1] ." ".$arrayref ->[2] ."\n"; }


# Òåïåðü íàñòóïèëè òÿæåëûå âðåìåíà. ;-)) Îñòàëèñü òîëüêî òå êîìáèíàöèè êîäèðîâêà/ÿçûê,
# äëÿ êîòîðûõ ïîëíîñòüþ ñîâïàäàþò äàííûå è ïî ÷àñòîòíîìó ñëîâàðþ, è ïî õýøó.
# Ýòî ìîæåò ñëó÷èòüñÿ ðîâíî â äâóõ ñëó÷àÿõ. Ïåðâûé - âåñü òåêñò íàáðàí ñòðî÷íûìè áóêâàìè.
# Òîãäà ñìåøèâàþòñÿ Mac/Win. Ïðåäïî÷èòàåì Win.
# Âòîðîé - òåêñò â koi íàáðàí áåç óêðàèíñêèõ áóêâ. Òîãäà ñìåøèâàþòñÿ koi8-r è koi8-u.
# Ïðåäïî÷èòàåì koi8-r (âïðî÷åì, ðàçíèöû â äàííîì ñëó÷àå íèêàêîé).


for $arrayref (@BestHashRating) {
  if ( $arrayref ->[0] =~ /win/ ) {
   $Coding = "win1251";
    if (%Stat->{$Coding}{GoodTokensCount} > 0 && %Stat->{$Coding}{CharsUkr} / %Stat->{$Coding}{GoodTokensCount} > $UkrTokensShare )
    { $Language = "Ukr"; } else { $Language = "Rus"; }
      $Algorithm = 33;
      return;
  }
}

for $arrayref (@BestHashRating) {
  if ( $arrayref ->[0] =~ /koi/ ) {
   $Coding = "koi8-r";
   $Language = "Rus";
   $Algorithm = 34;
      return;
  }
}

# Íè÷åãî íå ïîäîøëî. Óñòàíàâëèâàåì ïåðâóþ ïîáåäèâøóþ êîäèðîâêó è ÿçûê.
   $Coding = $BestHashRating[0]->[0];
   $Language = $BestHashRating[0]->[1];;
   $Algorithm = 40;


return;
} #end _AnalyzeStat()




sub _ParseString {
my ($Coding, $String, $Hash) = @_;
# Ïåðåâåëè ñòðîêó â êîäèðîâêó win1251 è óáðàëè çíàêè íîâîé ñòðîêè
$String = TranslateCyr($Coding,"win1251",$String);
$String =~ s/[\n\r]//go;

 ## Ðàçáèòèå íà ñëîâà
 ## \xAB\xBB - ïîëèãðàôè÷åñêèå êàâû÷êè, \x93\x94 - êàâû÷êè-"ëàïêè",
 ## \xB9 - çíàê íîìåðà, \x96\x97 - ïîëèãðàôè÷åñêèå òèðå
for (split (/[\xAB\xBB\x93\x94\xB9\x96\x97\.\,\-\s\:\;\?\!\"\(\)\d<>]+/o, $String)) {
    s/^\'+(.*)\'+$/$1/; # Óáðàëè íà÷àëüíûå è êîíå÷íûå àïîñòðîôû

if ( !$FullStat ) {

# Îïðåäåëÿåì, "ïðàâèëüíûé" ëè ýòî òîêåí, ò.å. ñîäåðæèò òîëüêî êèðèëëèöó
# è ëèáî ñòðî÷íûå áóêâû, ëèáî ÏÐÎÏÈÑÍÛÅ, ëèáî íà÷èíàåòñÿ ñ Ïðîïèñíîé.
if  (/^[$RusCharset{'Lower'}]+$/ || /^[$RusCharset{'Upper'}]{1}[$RusCharset{'Lower'}]+$/ || /^[$RusCharset{'Upper'}]+$/ ) {
  $$Hash{GoodTokensChars}+=length();
  # Äëÿ UTF óìíîæàåì ÷èñëî êèðèëëè÷åñêèõ ñèìâîëîâ íà äâà.
  if ( $Coding eq "utf" ) { $$Hash{GoodTokensChars}+=length(); }
  $$Hash{GoodTokensCount}++;
  # Åñëè òîêåí ñîäåðæèò óêðàèíñêèå ñèìâîëû, óâåëè÷èòü ñ÷åò÷èê óêðàèíñêèõ òîêåíîâ
  if ( $Args{DetectAllLang} && /[$RusCharset{'Ukrainian'}]/ ) { $$Hash{CharsUkr}++;}

}

  } else { # !$FullStat

# Îïðåäåëÿåì, ìîæíî ëè âîîáùå ïðîâîäèòü íàä ýòèì òîêåíîì êàêèå-ëèáî äåéñòâèÿ.
# Äëÿ ýòîãî îí äîëæåí ñîäåðæàòü õîòÿ áû îäíó ïðàâèëüíóþ êèðèëëè÷åñêóþ áóêâó,
# àíãëèéñêèå áóêâû è öèôðû â ëþáîé ñìåñè.
if (/[$RusCharset{'All'}]/ && /^[\w\d$RusCharset{'All'}]+$/) {
  $$Hash{AllTokensChars}+=length();
  # Äëÿ UTF óìíîæàåì ÷èñëî ñèìâîëîâ íà äâà.
  if ( $Coding eq "utf" ) { $$Hash{AllTokensChars}+=length(); }
  # Åñëè òîêåí ñîäåðæèò óêðàèíñêèå ñèìâîëû, óâåëè÷èòü ñ÷åò÷èê óêðàèíñêèõ òîêåíîâ
  if ( $Args{DetectAllLang} && /[$RusCharset{'Ukrainian'}]/ ) { $$Hash{CharsUkr}++; }

  # Òåïåðü ïðèñòóïàåì ê îáðàáîòêå õýøà è ñëîâàðåé
# Ïåðåâîäèì òîêåí â íèæíèé ðåãèñòð - è ñëîâàðü, è õýøè ó íàñ â íèæíåì ðåãèñòðå
  $_=toLowerCyr($_);

  if ($DictRus{$_}) { $$Hash{WordsRus}++;  }
  if ($Args{DetectAllLang} && $DictUkr{$_}) { $$Hash{WordsUkr}++;  }

for $i (0..length()-1) {
  if ( $WordHash2Rus{substr($_,$i,2)} ) { $$Hash{HashScore2Rus}++; }
  if ( $Args{DetectAllLang} && $WordHash2Ukr{substr($_,$i,2)} ) { $$Hash{HashScore2Ukr}++; }
} # end for

} # end  if (/^[\w\d$RusCharset{'All'}]+$/)

} # !$FullStat

	} # end	for (split...


} # end routine



1;

__END__