line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
# $Id: Segment.pm 1211 2005-03-10 14:10:14Z clsung $ |
2
|
|
|
|
|
|
|
|
3
|
|
|
|
|
|
|
package Lingua::ZH::Segment; |
4
|
3
|
|
|
3
|
|
75167
|
use strict; |
|
3
|
|
|
|
|
8
|
|
|
3
|
|
|
|
|
114
|
|
5
|
|
|
|
|
|
|
|
6
|
3
|
|
|
3
|
|
3123
|
use Encode::Guess; |
|
3
|
|
|
|
|
99669
|
|
|
3
|
|
|
|
|
25
|
|
7
|
|
|
|
|
|
|
our @ISA = qw(Exporter); |
8
|
|
|
|
|
|
|
our @EXPORT = qw(segment); |
9
|
|
|
|
|
|
|
our $VERSION = '0.02'; |
10
|
|
|
|
|
|
|
|
11
|
|
|
|
|
|
|
=head1 NAME |
12
|
|
|
|
|
|
|
|
13
|
|
|
|
|
|
|
Lingua::ZH::Segment - Chinese Text Segmentation |
14
|
|
|
|
|
|
|
|
15
|
|
|
|
|
|
|
=head1 VERSION |
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
This document describes version 0.01 of Lingua::ZH::Segment, released |
18
|
|
|
|
|
|
|
March 10, 2005. |
19
|
|
|
|
|
|
|
|
20
|
|
|
|
|
|
|
=head1 SYNOPSIS |
21
|
|
|
|
|
|
|
|
22
|
|
|
|
|
|
|
use Lingua::ZH::Segment; |
23
|
|
|
|
|
|
|
|
24
|
|
|
|
|
|
|
print segment('降龍18掌'); # 降 龍 18 掌 |
25
|
|
|
|
|
|
|
|
26
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
=head1 DESCRIPTION |
28
|
|
|
|
|
|
|
|
29
|
|
|
|
|
|
|
This module currently only break chinese text into single |
30
|
|
|
|
|
|
|
character (Chinese word), it will not break up any alphabet. |
31
|
|
|
|
|
|
|
|
32
|
|
|
|
|
|
|
=head1 METHODS |
33
|
|
|
|
|
|
|
|
34
|
|
|
|
|
|
|
Currently, only C is available. |
35
|
|
|
|
|
|
|
|
36
|
|
|
|
|
|
|
=cut |
37
|
|
|
|
|
|
|
|
38
|
|
|
|
|
|
|
sub segment { |
39
|
2
|
|
|
2
|
0
|
20
|
my $word = shift; |
40
|
2
|
|
|
|
|
17
|
my $decoder = guess_encoding ($word, qw/ utf8 big5 /); |
41
|
2
|
|
|
|
|
10404
|
$word = $decoder->decode($word); |
42
|
2
|
|
|
|
|
51
|
my @segs = split /([A-z|\d]+|\S)/, $word; |
43
|
2
|
|
|
|
|
9
|
$word = join " ",@segs; |
44
|
2
|
|
|
|
|
18
|
$word =~ s/\s{2,}/ /g; |
45
|
2
|
|
|
|
|
15
|
$word =~ s/(^\s|\s$)//g; |
46
|
2
|
|
|
|
|
18
|
$word = $decoder->encode($word); |
47
|
2
|
|
|
|
|
12
|
return $word; |
48
|
|
|
|
|
|
|
} |
49
|
|
|
|
|
|
|
|
50
|
0
|
|
|
0
|
|
|
sub CLONE { } |
51
|
0
|
|
|
0
|
|
|
sub DESTROY { } |
52
|
|
|
|
|
|
|
|
53
|
|
|
|
|
|
|
1; |
54
|
|
|
|
|
|
|
|
55
|
|
|
|
|
|
|
=head1 SEE ALSO |
56
|
|
|
|
|
|
|
|
57
|
|
|
|
|
|
|
L |
58
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
=head1 AUTHORS |
60
|
|
|
|
|
|
|
|
61
|
|
|
|
|
|
|
Cheng-Lung Sung Eclsung@tw.freebsd.orgE |
62
|
|
|
|
|
|
|
|
63
|
|
|
|
|
|
|
=head1 KUDOS |
64
|
|
|
|
|
|
|
|
65
|
|
|
|
|
|
|
Hsin-Chan Chien for inspiring me about L. |
66
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
=head1 COPYRIGHT |
68
|
|
|
|
|
|
|
|
69
|
|
|
|
|
|
|
Copyright 2005 by Cheng-Lung Sung Eclsung@tw.freebsd.orgE |
70
|
|
|
|
|
|
|
|
71
|
|
|
|
|
|
|
This program is free software; you can redistribute it and/or modify it |
72
|
|
|
|
|
|
|
under the same terms as Perl itself. |
73
|
|
|
|
|
|
|
|
74
|
|
|
|
|
|
|
See L |
75
|
|
|
|
|
|
|
|
76
|
|
|
|
|
|
|
=cut |