line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
# ========================================================================== # |
2
|
|
|
|
|
|
|
# Text/FixedWidth/Parser.pm - This module used to read the FixedWidth files |
3
|
|
|
|
|
|
|
# ========================================================================== # |
4
|
|
|
|
|
|
|
|
5
|
|
|
|
|
|
|
package Text::FixedWidth::Parser; |
6
|
|
|
|
|
|
|
|
7
|
1
|
|
|
1
|
|
2836
|
use Moose; |
|
1
|
|
|
|
|
394842
|
|
|
1
|
|
|
|
|
7
|
|
8
|
1
|
|
|
1
|
|
7666
|
use Math::Expression; |
|
1
|
|
|
|
|
11271
|
|
|
1
|
|
|
|
|
168
|
|
9
|
1
|
|
|
1
|
|
507
|
use DateTime::Format::Strptime; |
|
1
|
|
|
|
|
428699
|
|
|
1
|
|
|
|
|
5
|
|
10
|
|
|
|
|
|
|
|
11
|
|
|
|
|
|
|
our $VERSION = '0.4'; |
12
|
|
|
|
|
|
|
|
13
|
|
|
|
|
|
|
# ========================================================================== # |
14
|
|
|
|
|
|
|
|
15
|
|
|
|
|
|
|
=pod |
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
=encoding UTF-8 |
18
|
|
|
|
|
|
|
|
19
|
|
|
|
|
|
|
=head1 NAME |
20
|
|
|
|
|
|
|
|
21
|
|
|
|
|
|
|
Text::FixedWidth::Parser - Used to parse the fixed width text file |
22
|
|
|
|
|
|
|
|
23
|
|
|
|
|
|
|
=head1 DESCRIPTION |
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
The Text::FixedWidth::Parser module allows you to read fixed width text file by specifying string mapper |
26
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
|
28
|
|
|
|
|
|
|
=head1 SYNOPSIS |
29
|
|
|
|
|
|
|
|
30
|
|
|
|
|
|
|
use Text::FixedWidth::Parser; |
31
|
|
|
|
|
|
|
|
32
|
|
|
|
|
|
|
FileData |
33
|
|
|
|
|
|
|
~~~~~~~~ |
34
|
|
|
|
|
|
|
ADDRESS001XXXXX YYYYYYY84 SOUTH STREET USA |
35
|
|
|
|
|
|
|
MARK0018286989020140101 |
36
|
|
|
|
|
|
|
ADDRESS002YYYYYYY 69 BELL STREET UK |
37
|
|
|
|
|
|
|
MARK0028869893920140101 |
38
|
|
|
|
|
|
|
|
39
|
|
|
|
|
|
|
my $string_mapper = [ |
40
|
|
|
|
|
|
|
{ |
41
|
|
|
|
|
|
|
Rule => { |
42
|
|
|
|
|
|
|
LinePrefix => [1, 7], |
43
|
|
|
|
|
|
|
Expression => "LinePrefix eq 'ADDRESS'" |
44
|
|
|
|
|
|
|
}, |
45
|
|
|
|
|
|
|
Id => [8, 3], |
46
|
|
|
|
|
|
|
Name => [11, 13], |
47
|
|
|
|
|
|
|
Address => {DoorNo => [24, 2], Street => [26, 14]}, |
48
|
|
|
|
|
|
|
Country => [40, 3] |
49
|
|
|
|
|
|
|
}, |
50
|
|
|
|
|
|
|
{ |
51
|
|
|
|
|
|
|
Rule => { |
52
|
|
|
|
|
|
|
LinePrefix => [1, 4], |
53
|
|
|
|
|
|
|
Expression => "LinePrefix eq 'MARK'" |
54
|
|
|
|
|
|
|
}, |
55
|
|
|
|
|
|
|
Id => [5, 3], |
56
|
|
|
|
|
|
|
Mark1 => [8, 2], |
57
|
|
|
|
|
|
|
Mark2 => [10, 2], |
58
|
|
|
|
|
|
|
Mark3 => [12, 2], |
59
|
|
|
|
|
|
|
Mark4 => [14, 3], |
60
|
|
|
|
|
|
|
ResultDate => [15, 8], |
61
|
|
|
|
|
|
|
ResultDatePattern => '%Y%m%d', |
62
|
|
|
|
|
|
|
ResultDateTimezone => 'America/Chicago' |
63
|
|
|
|
|
|
|
} |
64
|
|
|
|
|
|
|
]; |
65
|
|
|
|
|
|
|
|
66
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
# StringMapper should be passed while creating object |
68
|
|
|
|
|
|
|
my $obj = Text::FixedWidth::Parser->new( |
69
|
|
|
|
|
|
|
{ |
70
|
|
|
|
|
|
|
#Required Params |
71
|
|
|
|
|
|
|
StringMapper => $string_mapper, |
72
|
|
|
|
|
|
|
#optional Params |
73
|
|
|
|
|
|
|
TimestampToEpochFields => ['ResultDate'], |
74
|
|
|
|
|
|
|
DefaultDatePattern => '%Y%m%d', |
75
|
|
|
|
|
|
|
DefaultTimezone => 'GMT', |
76
|
|
|
|
|
|
|
ConcateString => '', |
77
|
|
|
|
|
|
|
EmptyAsUndef => 1 |
78
|
|
|
|
|
|
|
} |
79
|
|
|
|
|
|
|
); |
80
|
|
|
|
|
|
|
|
81
|
|
|
|
|
|
|
open my $fh, '<', 'filename'; |
82
|
|
|
|
|
|
|
|
83
|
|
|
|
|
|
|
$data = $obj->read($fh); |
84
|
|
|
|
|
|
|
|
85
|
|
|
|
|
|
|
=head1 PARAMS |
86
|
|
|
|
|
|
|
|
87
|
|
|
|
|
|
|
=over 4 |
88
|
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
|
90
|
|
|
|
|
|
|
=item B<StringMapper> |
91
|
|
|
|
|
|
|
|
92
|
|
|
|
|
|
|
* StringMapper can be HASHRef or multiple StringMappers as ARRAY of HASHRefs |
93
|
|
|
|
|
|
|
|
94
|
|
|
|
|
|
|
* If Multiple StringMappers exist, Based on Rule apropriate StringMapper will get selected |
95
|
|
|
|
|
|
|
|
96
|
|
|
|
|
|
|
* In Multiple StringMappers, Its better to place Rule-less mapper after Rule based mappers |
97
|
|
|
|
|
|
|
|
98
|
|
|
|
|
|
|
* Rule-less mapper will picked as soon as its get access in an array |
99
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
* StringMapper fields should be defined as ARRAY, First element as StartingPoint of string and Second element as length of the string |
101
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
* Rule, Expression are keywords, overriding or changing those will affect the functionality |
103
|
|
|
|
|
|
|
|
104
|
|
|
|
|
|
|
=item B<TimestampToEpochFields> |
105
|
|
|
|
|
|
|
|
106
|
|
|
|
|
|
|
* TimestampToEpochFields can have ARRAY of timestamp fields which need to be converted as epoch |
107
|
|
|
|
|
|
|
|
108
|
|
|
|
|
|
|
* TimestampToEpochFields can have Pattern of the timestamp in StringMapper as field name suffixed with Pattern keyword, Which will override L</"DefaultDatePattern"> for that particular field |
109
|
|
|
|
|
|
|
|
110
|
|
|
|
|
|
|
Eg:- FieldName : DOB, DOBPattern => '%Y%m%d' |
111
|
|
|
|
|
|
|
|
112
|
|
|
|
|
|
|
* see L<STRPTIME PATTERN TOKENS|DateTime::Format::Strptime/"STRPTIME PATTERN TOKENS"> section in DateTime::Format::Strptime for more patterns |
113
|
|
|
|
|
|
|
|
114
|
|
|
|
|
|
|
* TimestampToEpochFields can have timezone of the timestamp in StringMapper as field name suffixed with Timezone keyword, Which will override L</"DefaultTimezone"> for that particular field |
115
|
|
|
|
|
|
|
|
116
|
|
|
|
|
|
|
Eg:- FieldName : DOB, DOBTimezone=> 'GMT' |
117
|
|
|
|
|
|
|
|
118
|
|
|
|
|
|
|
=item B<DefaultDatePattern> |
119
|
|
|
|
|
|
|
|
120
|
|
|
|
|
|
|
* DefaultDatePattern can have DatePattern which will be used to convert date to epoch by default |
121
|
|
|
|
|
|
|
|
122
|
|
|
|
|
|
|
=item B<DefaultTimezone> |
123
|
|
|
|
|
|
|
|
124
|
|
|
|
|
|
|
* DefaultTimezone can have timezone which will be used while converting date to epoch |
125
|
|
|
|
|
|
|
|
126
|
|
|
|
|
|
|
|
127
|
|
|
|
|
|
|
=item B<ConcateString> |
128
|
|
|
|
|
|
|
|
129
|
|
|
|
|
|
|
* StringMapper can be defined as {Address => [24, 2, 26, 14]} |
130
|
|
|
|
|
|
|
|
131
|
|
|
|
|
|
|
* This represents, Address field value will be concatenation of two strings, which are Startingpoint 24, Length 2 and Startingpoint 26, Length 14 |
132
|
|
|
|
|
|
|
|
133
|
|
|
|
|
|
|
* While concatenating strings, value of I<ConcateString> will be used |
134
|
|
|
|
|
|
|
|
135
|
|
|
|
|
|
|
Eg: ConcateString = '-'; The Value of Address = 84-SOUTH STREET |
136
|
|
|
|
|
|
|
|
137
|
|
|
|
|
|
|
* Space(' ') is default ConcateString |
138
|
|
|
|
|
|
|
|
139
|
|
|
|
|
|
|
=item B<EmptyAsUndef> |
140
|
|
|
|
|
|
|
|
141
|
|
|
|
|
|
|
* If this flag is enabled, Empty values will be assigned as undef |
142
|
|
|
|
|
|
|
|
143
|
|
|
|
|
|
|
Eg: Name = '', it will be assigned as Name = undef |
144
|
|
|
|
|
|
|
=cut |
145
|
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
=back |
147
|
|
|
|
|
|
|
|
148
|
|
|
|
|
|
|
=head1 METHODS |
149
|
|
|
|
|
|
|
|
150
|
|
|
|
|
|
|
=over 4 |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
=cut |
153
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
# ========================================================================== # |
155
|
|
|
|
|
|
|
has me_obj => ( |
156
|
|
|
|
|
|
|
is => 'ro', |
157
|
|
|
|
|
|
|
isa => 'Math::Expression', |
158
|
|
|
|
|
|
|
default => sub { Math::Expression->new } |
159
|
|
|
|
|
|
|
); |
160
|
|
|
|
|
|
|
|
161
|
|
|
|
|
|
|
# ========================================================================== # |
162
|
|
|
|
|
|
|
|
163
|
|
|
|
|
|
|
=item B<get_string_mapper> |
164
|
|
|
|
|
|
|
|
165
|
|
|
|
|
|
|
Desc : This method will return the StringMapper |
166
|
|
|
|
|
|
|
|
167
|
|
|
|
|
|
|
Params : NONE |
168
|
|
|
|
|
|
|
|
169
|
|
|
|
|
|
|
Returns: HASHRef as Mentioned in the config |
170
|
|
|
|
|
|
|
|
171
|
|
|
|
|
|
|
=cut |
172
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
=item B<set_string_mapper> |
174
|
|
|
|
|
|
|
|
175
|
|
|
|
|
|
|
Desc : This method is used set the StringMapper |
176
|
|
|
|
|
|
|
|
177
|
|
|
|
|
|
|
Params : StringMapper |
178
|
|
|
|
|
|
|
|
179
|
|
|
|
|
|
|
Returns: NONE |
180
|
|
|
|
|
|
|
|
181
|
|
|
|
|
|
|
=cut |
182
|
|
|
|
|
|
|
|
183
|
|
|
|
|
|
|
has 'StringMapper' => ( |
184
|
|
|
|
|
|
|
is => 'rw', |
185
|
|
|
|
|
|
|
required => 1, |
186
|
|
|
|
|
|
|
reader => 'get_string_mapper', |
187
|
|
|
|
|
|
|
writer => 'set_string_mapper', |
188
|
|
|
|
|
|
|
documentation => 'This attribute is used to read the file values' |
189
|
|
|
|
|
|
|
); |
190
|
|
|
|
|
|
|
|
191
|
|
|
|
|
|
|
# ========================================================================== # |
192
|
|
|
|
|
|
|
|
193
|
|
|
|
|
|
|
=item B<get_concate_string> |
194
|
|
|
|
|
|
|
|
195
|
|
|
|
|
|
|
Desc : This method will return the ConcateString |
196
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
Params : NONE |
198
|
|
|
|
|
|
|
|
199
|
|
|
|
|
|
|
Returns: ConcateString |
200
|
|
|
|
|
|
|
|
201
|
|
|
|
|
|
|
=cut |
202
|
|
|
|
|
|
|
|
203
|
|
|
|
|
|
|
=item B<set_concate_string> |
204
|
|
|
|
|
|
|
|
205
|
|
|
|
|
|
|
Desc : This method is used to set ConcateString |
206
|
|
|
|
|
|
|
|
207
|
|
|
|
|
|
|
Params : String |
208
|
|
|
|
|
|
|
|
209
|
|
|
|
|
|
|
Returns: NONE |
210
|
|
|
|
|
|
|
|
211
|
|
|
|
|
|
|
=cut |
212
|
|
|
|
|
|
|
|
213
|
|
|
|
|
|
|
has 'ConcateString' => ( |
214
|
|
|
|
|
|
|
is => 'rw', |
215
|
|
|
|
|
|
|
isa => 'Str', |
216
|
|
|
|
|
|
|
default => ' ', |
217
|
|
|
|
|
|
|
reader => 'get_concate_string', |
218
|
|
|
|
|
|
|
writer => 'set_concate_string', |
219
|
|
|
|
|
|
|
documentation => 'This attribute is used to concatenate string with given string. Default value is space' |
220
|
|
|
|
|
|
|
); |
221
|
|
|
|
|
|
|
|
222
|
|
|
|
|
|
|
# ========================================================================== # |
223
|
|
|
|
|
|
|
|
224
|
|
|
|
|
|
|
=item B<is_empty_undef> |
225
|
|
|
|
|
|
|
|
226
|
|
|
|
|
|
|
Desc : This method will indicate is empty flag enabled or disabled |
227
|
|
|
|
|
|
|
|
228
|
|
|
|
|
|
|
Params : NONE |
229
|
|
|
|
|
|
|
|
230
|
|
|
|
|
|
|
Returns: 1 on enabled, 0 on disabled |
231
|
|
|
|
|
|
|
|
232
|
|
|
|
|
|
|
=cut |
233
|
|
|
|
|
|
|
|
234
|
|
|
|
|
|
|
=item B<set_empty_undef> |
235
|
|
|
|
|
|
|
|
236
|
|
|
|
|
|
|
Desc : This method is used to enable or disable EmptyAsUndef flag |
237
|
|
|
|
|
|
|
|
238
|
|
|
|
|
|
|
Params : 1 to enable, 0 to disable |
239
|
|
|
|
|
|
|
|
240
|
|
|
|
|
|
|
Returns: NONE |
241
|
|
|
|
|
|
|
|
242
|
|
|
|
|
|
|
=cut |
243
|
|
|
|
|
|
|
|
244
|
|
|
|
|
|
|
has 'EmptyAsUndef' => ( |
245
|
|
|
|
|
|
|
is => 'rw', |
246
|
|
|
|
|
|
|
isa => 'Bool', |
247
|
|
|
|
|
|
|
default => 0, |
248
|
|
|
|
|
|
|
reader => 'is_empty_undef', |
249
|
|
|
|
|
|
|
writer => 'set_empty_undef', |
250
|
|
|
|
|
|
|
documentation => 'This attribute is used to say set undef where the value is undef' |
251
|
|
|
|
|
|
|
); |
252
|
|
|
|
|
|
|
|
253
|
|
|
|
|
|
|
# ========================================================================== # |
254
|
|
|
|
|
|
|
|
255
|
|
|
|
|
|
|
=item B<set_timestamp_to_epoch_fields> |
256
|
|
|
|
|
|
|
|
257
|
|
|
|
|
|
|
Desc : This method is used to set fields that need to be converted to epoch |
258
|
|
|
|
|
|
|
|
259
|
|
|
|
|
|
|
Params : [FieldName14,..] |
260
|
|
|
|
|
|
|
|
261
|
|
|
|
|
|
|
Returns: NONE |
262
|
|
|
|
|
|
|
|
263
|
|
|
|
|
|
|
=cut |
264
|
|
|
|
|
|
|
|
265
|
|
|
|
|
|
|
has 'TimestampToEpochFields' => ( |
266
|
|
|
|
|
|
|
is => 'rw', |
267
|
|
|
|
|
|
|
default => 0, |
268
|
|
|
|
|
|
|
writer => 'set_timestamp_to_epoch_fields', |
269
|
|
|
|
|
|
|
trigger => sub { |
270
|
|
|
|
|
|
|
my ($self, $new_val, $old_val) = @_; |
271
|
|
|
|
|
|
|
print("Invalid TimestampToEpochFields value, Value should be ARRAYREF\n") and return undef |
272
|
|
|
|
|
|
|
if (ref($new_val) ne 'ARRAY'); |
273
|
|
|
|
|
|
|
my $val; |
274
|
|
|
|
|
|
|
$val->{$_} = 1 for (@$new_val); |
275
|
|
|
|
|
|
|
$self->{time_to_epoch_fields} = $val; |
276
|
|
|
|
|
|
|
}, |
277
|
|
|
|
|
|
|
documentation => 'This attribute is used to configure fileds that need to be convert as epoch' |
278
|
|
|
|
|
|
|
); |
279
|
|
|
|
|
|
|
|
280
|
|
|
|
|
|
|
# ========================================================================== # |
281
|
|
|
|
|
|
|
|
282
|
|
|
|
|
|
|
=item B<add_timestamp_to_epoch_fields> |
283
|
|
|
|
|
|
|
|
284
|
|
|
|
|
|
|
Desc : This method is used to add fields with existing fields that need to be converted to epoch |
285
|
|
|
|
|
|
|
|
286
|
|
|
|
|
|
|
Params : [FieldName14,..] |
287
|
|
|
|
|
|
|
|
288
|
|
|
|
|
|
|
Returns: NONE |
289
|
|
|
|
|
|
|
|
290
|
|
|
|
|
|
|
=cut |
291
|
|
|
|
|
|
|
|
292
|
|
|
|
|
|
|
sub add_timestamp_to_epoch_fields |
293
|
|
|
|
|
|
|
{ |
294
|
0
|
|
|
0
|
1
|
0
|
my ($self, $value) = @_; |
295
|
|
|
|
|
|
|
|
296
|
0
|
0
|
0
|
|
|
0
|
print("Invalid TimestampToEpochFields value, Value should be ARRAYREF\n") and return undef |
297
|
|
|
|
|
|
|
if (ref($value) ne 'ARRAY'); |
298
|
|
|
|
|
|
|
|
299
|
0
|
|
|
|
|
0
|
$self->{time_to_epoch_fields}{$_} = 1 for (@$value); |
300
|
0
|
|
|
|
|
0
|
push(@{$self->{TimestampToEpochFields}}, @$value); |
|
0
|
|
|
|
|
0
|
|
301
|
|
|
|
|
|
|
} |
302
|
|
|
|
|
|
|
|
303
|
|
|
|
|
|
|
# ========================================================================== # |
304
|
|
|
|
|
|
|
|
305
|
|
|
|
|
|
|
=item B<get_timestamp_to_epoch_fields> |
306
|
|
|
|
|
|
|
|
307
|
|
|
|
|
|
|
Desc : This method is used to get fields that will be converted to epoch |
308
|
|
|
|
|
|
|
|
309
|
|
|
|
|
|
|
Params : [FieldName14,..] |
310
|
|
|
|
|
|
|
|
311
|
|
|
|
|
|
|
Returns: NONE |
312
|
|
|
|
|
|
|
|
313
|
|
|
|
|
|
|
=cut |
314
|
|
|
|
|
|
|
|
315
|
|
|
|
|
|
|
sub get_timestamp_to_epoch_fields |
316
|
|
|
|
|
|
|
{ |
317
|
0
|
|
|
0
|
1
|
0
|
my $self = shift; |
318
|
|
|
|
|
|
|
|
319
|
0
|
|
|
|
|
0
|
my @fields; |
320
|
|
|
|
|
|
|
|
321
|
0
|
0
|
|
|
|
0
|
if (defined $self->{time_to_epoch_fields}) { |
322
|
0
|
|
|
|
|
0
|
@fields = keys @{$self->{time_to_epoch_fields}}; |
|
0
|
|
|
|
|
0
|
|
323
|
|
|
|
|
|
|
} |
324
|
|
|
|
|
|
|
|
325
|
0
|
|
|
|
|
0
|
return \@fields; |
326
|
|
|
|
|
|
|
} |
327
|
|
|
|
|
|
|
|
328
|
|
|
|
|
|
|
# ========================================================================== # |
329
|
|
|
|
|
|
|
|
330
|
|
|
|
|
|
|
=item B<set_default_date_pattern> |
331
|
|
|
|
|
|
|
|
332
|
|
|
|
|
|
|
Desc : This method is used to set date format which will be used to convert the date to epoch. '%Y%m%d' is default DatePattern. |
333
|
|
|
|
|
|
|
|
334
|
|
|
|
|
|
|
Params : DatePattern eg:- '%Y%m%d' |
335
|
|
|
|
|
|
|
|
336
|
|
|
|
|
|
|
Returns: NONE |
337
|
|
|
|
|
|
|
|
338
|
|
|
|
|
|
|
=cut |
339
|
|
|
|
|
|
|
|
340
|
|
|
|
|
|
|
=item B<get_default_date_pattern> |
341
|
|
|
|
|
|
|
|
342
|
|
|
|
|
|
|
Desc : This method will return the date format which will be used to convert the date to epoch. |
343
|
|
|
|
|
|
|
|
344
|
|
|
|
|
|
|
Params : NONE |
345
|
|
|
|
|
|
|
|
346
|
|
|
|
|
|
|
Returns: DatePattern eg:- '%Y%m%d' |
347
|
|
|
|
|
|
|
|
348
|
|
|
|
|
|
|
=cut |
349
|
|
|
|
|
|
|
|
350
|
|
|
|
|
|
|
has 'DefaultDatePattern' => ( |
351
|
|
|
|
|
|
|
is => 'rw', |
352
|
|
|
|
|
|
|
isa => 'Str', |
353
|
|
|
|
|
|
|
writer => 'set_default_date_pattern', |
354
|
|
|
|
|
|
|
reader => 'get_default_date_pattern', |
355
|
|
|
|
|
|
|
default => '%Y%m%d', |
356
|
|
|
|
|
|
|
lazy => 1, |
357
|
|
|
|
|
|
|
documentation => |
358
|
|
|
|
|
|
|
'This attribute is used to configure the default date format which will be used to convert date to epoch' |
359
|
|
|
|
|
|
|
); |
360
|
|
|
|
|
|
|
|
361
|
|
|
|
|
|
|
# ========================================================================== # |
362
|
|
|
|
|
|
|
|
363
|
|
|
|
|
|
|
=item B<set_default_timezone> |
364
|
|
|
|
|
|
|
|
365
|
|
|
|
|
|
|
Desc : This method is used to set timezone which will be used while converting date to epoch. GMT is a default timezone. |
366
|
|
|
|
|
|
|
|
367
|
|
|
|
|
|
|
Params : Timezone eg:- 'GMT' |
368
|
|
|
|
|
|
|
|
369
|
|
|
|
|
|
|
Returns: NONE |
370
|
|
|
|
|
|
|
|
371
|
|
|
|
|
|
|
=cut |
372
|
|
|
|
|
|
|
|
373
|
|
|
|
|
|
|
=item B<get_default_timezone> |
374
|
|
|
|
|
|
|
|
375
|
|
|
|
|
|
|
Desc : This method will return timezone which will be used while converting timestamp to epoch |
376
|
|
|
|
|
|
|
|
377
|
|
|
|
|
|
|
Params : NONE |
378
|
|
|
|
|
|
|
|
379
|
|
|
|
|
|
|
Returns: Timezone eg:- 'GMT' |
380
|
|
|
|
|
|
|
|
381
|
|
|
|
|
|
|
=cut |
382
|
|
|
|
|
|
|
|
383
|
|
|
|
|
|
|
has 'DefaultTimezone' => ( |
384
|
|
|
|
|
|
|
is => 'rw', |
385
|
|
|
|
|
|
|
isa => 'Str', |
386
|
|
|
|
|
|
|
writer => 'set_default_timezone', |
387
|
|
|
|
|
|
|
reader => 'get_default_timezone', |
388
|
|
|
|
|
|
|
default => 'GMT', |
389
|
|
|
|
|
|
|
documentation => 'This attribute is used to configure the default timezone' |
390
|
|
|
|
|
|
|
); |
391
|
|
|
|
|
|
|
|
392
|
|
|
|
|
|
|
# ========================================================================== # |
393
|
|
|
|
|
|
|
|
394
|
|
|
|
|
|
|
=item B<read> |
395
|
|
|
|
|
|
|
|
396
|
|
|
|
|
|
|
Desc : This method is used to read the line by line values |
397
|
|
|
|
|
|
|
|
398
|
|
|
|
|
|
|
Params : FileHandle |
399
|
|
|
|
|
|
|
|
400
|
|
|
|
|
|
|
Returns: HASHRef as Mentioned in the StringMapper |
401
|
|
|
|
|
|
|
|
402
|
|
|
|
|
|
|
Eg : { |
403
|
|
|
|
|
|
|
'Address' => { |
404
|
|
|
|
|
|
|
'DoorNo' => '84', |
405
|
|
|
|
|
|
|
'Street' => 'SOUTH STREET' |
406
|
|
|
|
|
|
|
}, |
407
|
|
|
|
|
|
|
'Country' => 'USA', |
408
|
|
|
|
|
|
|
'Id' => '001', |
409
|
|
|
|
|
|
|
'Name' => 'XXXXX YYYYYYY' |
410
|
|
|
|
|
|
|
} |
411
|
|
|
|
|
|
|
=cut |
412
|
|
|
|
|
|
|
|
413
|
|
|
|
|
|
|
sub read |
414
|
|
|
|
|
|
|
{ |
415
|
2
|
|
|
2
|
1
|
1896
|
my ($self, $fh) = @_; |
416
|
|
|
|
|
|
|
|
417
|
2
|
50
|
|
|
|
24
|
return undef if eof $fh; |
418
|
|
|
|
|
|
|
|
419
|
2
|
|
|
|
|
5
|
my $data = {}; |
420
|
|
|
|
|
|
|
|
421
|
2
|
|
|
|
|
8
|
my $line = <$fh>; |
422
|
|
|
|
|
|
|
|
423
|
2
|
50
|
|
|
|
15
|
$data = $self->_read_data($line, $self->_get_config($line)) if ($line !~ /^\s+$/); |
424
|
|
|
|
|
|
|
|
425
|
2
|
|
|
|
|
6
|
return $data; |
426
|
|
|
|
|
|
|
} |
427
|
|
|
|
|
|
|
|
428
|
|
|
|
|
|
|
# This is private method used to read the file and Construct data structure as per StringMapper |
429
|
|
|
|
|
|
|
sub _read_data |
430
|
|
|
|
|
|
|
{ |
431
|
|
|
|
|
|
|
|
432
|
22
|
|
|
22
|
|
2717
|
my ($self, $line, $string_mapper) = @_; |
433
|
|
|
|
|
|
|
|
434
|
22
|
100
|
66
|
|
|
97
|
return undef if ((not defined $line) or (not defined $string_mapper)); |
435
|
|
|
|
|
|
|
|
436
|
18
|
|
|
|
|
448
|
my $concate_string = $self->get_concate_string; |
437
|
|
|
|
|
|
|
|
438
|
|
|
|
|
|
|
#Match empty value with or without space |
439
|
18
|
|
|
|
|
55
|
my $is_empty = qr/^\s*$/; |
440
|
|
|
|
|
|
|
|
441
|
18
|
|
|
|
|
26
|
my $data; |
442
|
|
|
|
|
|
|
|
443
|
18
|
|
|
|
|
24
|
foreach my $field (keys %{$string_mapper}) { |
|
18
|
|
|
|
|
56
|
|
444
|
|
|
|
|
|
|
|
445
|
190
|
|
|
|
|
255
|
my $field_map = $string_mapper->{$field}; |
446
|
|
|
|
|
|
|
|
447
|
190
|
100
|
100
|
|
|
426
|
if (ref($field_map) eq 'HASH' && $field ne 'Rule') { |
448
|
4
|
|
|
|
|
12
|
$data->{$field} = $self->_read_data($line, $field_map); |
449
|
4
|
|
|
|
|
8
|
next; |
450
|
|
|
|
|
|
|
} |
451
|
|
|
|
|
|
|
|
452
|
186
|
100
|
|
|
|
337
|
next if (ref($field_map) ne 'ARRAY'); |
453
|
|
|
|
|
|
|
|
454
|
160
|
|
|
|
|
165
|
my $map_count = @{$field_map} / 2; |
|
160
|
|
|
|
|
269
|
|
455
|
160
|
|
|
|
|
187
|
my $column_val; |
456
|
|
|
|
|
|
|
|
457
|
160
|
|
|
|
|
238
|
foreach my $count (1 .. $map_count) { |
458
|
|
|
|
|
|
|
|
459
|
|
|
|
|
|
|
#start_index decremented by one to match substr postion |
460
|
|
|
|
|
|
|
#substr() start_index always one char before the string |
461
|
160
|
|
|
|
|
210
|
my $start_index = $field_map->[$count - 1] - 1; |
462
|
160
|
|
|
|
|
193
|
my $length = $field_map->[$count]; |
463
|
|
|
|
|
|
|
|
464
|
160
|
|
|
|
|
263
|
my $extracted_value = substr($line, $start_index, $length); |
465
|
|
|
|
|
|
|
|
466
|
|
|
|
|
|
|
# To Remove the space before and after string |
467
|
160
|
|
|
|
|
434
|
$extracted_value =~ s/^\s+|\s+$//g; |
468
|
|
|
|
|
|
|
|
469
|
160
|
100
|
66
|
|
|
613
|
if ($self->{time_to_epoch_fields} && $self->{time_to_epoch_fields}{$field}) { |
470
|
|
|
|
|
|
|
|
471
|
|
|
|
|
|
|
my $format = |
472
|
|
|
|
|
|
|
(defined $string_mapper->{"${field}Pattern"}) |
473
|
10
|
50
|
|
|
|
28
|
? $string_mapper->{"${field}Pattern"} |
474
|
|
|
|
|
|
|
: $self->get_default_date_format; |
475
|
10
|
|
|
|
|
14
|
my $tz = $string_mapper->{"${field}Timezone"}; |
476
|
10
|
|
|
|
|
21
|
$extracted_value = $self->_get_time_stamp_to_epoch($extracted_value, $format, $tz); |
477
|
|
|
|
|
|
|
} |
478
|
|
|
|
|
|
|
|
479
|
|
|
|
|
|
|
# Adding ConcateString between the strings while concatenate |
480
|
160
|
50
|
|
|
|
356
|
defined $column_val ? $column_val .= "$concate_string$extracted_value" : $column_val = $extracted_value; |
481
|
|
|
|
|
|
|
} |
482
|
|
|
|
|
|
|
|
483
|
160
|
100
|
66
|
|
|
1165
|
$column_val = undef if (((not defined $column_val) || $column_val =~ $is_empty) && $self->is_empty_undef); |
|
|
|
100
|
|
|
|
|
484
|
|
|
|
|
|
|
|
485
|
160
|
|
|
|
|
339
|
$data->{$field} = $column_val; |
486
|
|
|
|
|
|
|
} |
487
|
|
|
|
|
|
|
|
488
|
18
|
|
|
|
|
49
|
return $data; |
489
|
|
|
|
|
|
|
} |
490
|
|
|
|
|
|
|
|
491
|
|
|
|
|
|
|
# ========================================================================== # |
492
|
|
|
|
|
|
|
|
493
|
|
|
|
|
|
|
=item B<read_all> |
494
|
|
|
|
|
|
|
|
495
|
|
|
|
|
|
|
Desc : This method is used to read complete file |
496
|
|
|
|
|
|
|
|
497
|
|
|
|
|
|
|
Params : FileHandle |
498
|
|
|
|
|
|
|
|
499
|
|
|
|
|
|
|
Returns: HASHRef as Mentioned in the StringMapper |
500
|
|
|
|
|
|
|
|
501
|
|
|
|
|
|
|
Eg : [ |
502
|
|
|
|
|
|
|
{ |
503
|
|
|
|
|
|
|
'Address' => { |
504
|
|
|
|
|
|
|
'DoorNo' => '84', |
505
|
|
|
|
|
|
|
'Street' => 'SOUTH STREET' |
506
|
|
|
|
|
|
|
}, |
507
|
|
|
|
|
|
|
'Country' => 'USA', |
508
|
|
|
|
|
|
|
'Id' => '001', |
509
|
|
|
|
|
|
|
'Name' => 'XXXXX YYYYYYY' |
510
|
|
|
|
|
|
|
}, |
511
|
|
|
|
|
|
|
{ |
512
|
|
|
|
|
|
|
'Id' => '001', |
513
|
|
|
|
|
|
|
'Mark1' => '82', |
514
|
|
|
|
|
|
|
'Mark2' => '86', |
515
|
|
|
|
|
|
|
'Mark3' => '98', |
516
|
|
|
|
|
|
|
'Mark4' => '90' |
517
|
|
|
|
|
|
|
}, |
518
|
|
|
|
|
|
|
{ |
519
|
|
|
|
|
|
|
'Address' => { |
520
|
|
|
|
|
|
|
'DoorNo' => '69', |
521
|
|
|
|
|
|
|
'Street' => 'BELL STREET' |
522
|
|
|
|
|
|
|
}, |
523
|
|
|
|
|
|
|
'Country' => 'UK', |
524
|
|
|
|
|
|
|
'Id' => '002', |
525
|
|
|
|
|
|
|
'Name' => 'YYYYYYY' |
526
|
|
|
|
|
|
|
}, |
527
|
|
|
|
|
|
|
{ |
528
|
|
|
|
|
|
|
'Id' => '002', |
529
|
|
|
|
|
|
|
'Mark1' => '88', |
530
|
|
|
|
|
|
|
'Mark2' => '69', |
531
|
|
|
|
|
|
|
'Mark3' => '89', |
532
|
|
|
|
|
|
|
'Mark4' => '39' |
533
|
|
|
|
|
|
|
} |
534
|
|
|
|
|
|
|
] |
535
|
|
|
|
|
|
|
|
536
|
|
|
|
|
|
|
|
537
|
|
|
|
|
|
|
=cut |
538
|
|
|
|
|
|
|
|
539
|
|
|
|
|
|
|
sub read_all |
540
|
|
|
|
|
|
|
{ |
541
|
|
|
|
|
|
|
|
542
|
3
|
|
|
3
|
1
|
1578
|
my ($self, $fh) = @_; |
543
|
|
|
|
|
|
|
|
544
|
3
|
|
|
|
|
4
|
my $data; |
545
|
|
|
|
|
|
|
|
546
|
3
|
|
|
|
|
41
|
while (my $line = <$fh>) { |
547
|
16
|
50
|
|
|
|
58
|
next if ($line =~ /^\s+$/); |
548
|
|
|
|
|
|
|
|
549
|
16
|
|
|
|
|
34
|
my $extracted_value = $self->_read_data($line, $self->_get_config($line)); |
550
|
|
|
|
|
|
|
|
551
|
16
|
100
|
|
|
|
92
|
push(@$data, $extracted_value) if ($extracted_value); |
552
|
|
|
|
|
|
|
} |
553
|
|
|
|
|
|
|
|
554
|
3
|
|
|
|
|
10
|
return $data; |
555
|
|
|
|
|
|
|
} |
556
|
|
|
|
|
|
|
|
557
|
|
|
|
|
|
|
# ========================================================================== # |
558
|
|
|
|
|
|
|
|
559
|
|
|
|
|
|
|
# This method will return the config based on rule. If rule does not exist, it will return the base config. |
560
|
|
|
|
|
|
|
|
561
|
|
|
|
|
|
|
sub _get_config |
562
|
|
|
|
|
|
|
{ |
563
|
18
|
|
|
18
|
|
34
|
my ($self, $line) = @_; |
564
|
|
|
|
|
|
|
|
565
|
18
|
|
|
|
|
454
|
my $config_set = $self->get_string_mapper; |
566
|
|
|
|
|
|
|
|
567
|
18
|
100
|
|
|
|
60
|
$config_set = [$config_set] if (ref($config_set) ne 'ARRAY'); |
568
|
|
|
|
|
|
|
|
569
|
18
|
|
|
|
|
37
|
my $me_obj = $self->{me_obj}; |
570
|
|
|
|
|
|
|
|
571
|
18
|
|
|
|
|
23
|
foreach my $config (@{$config_set}) { |
|
18
|
|
|
|
|
32
|
|
572
|
|
|
|
|
|
|
|
573
|
20
|
|
|
|
|
725
|
my $rule = $config->{Rule}; |
574
|
|
|
|
|
|
|
|
575
|
20
|
100
|
|
|
|
42
|
if ($rule) { |
576
|
|
|
|
|
|
|
|
577
|
12
|
|
|
|
|
13
|
foreach my $rule_key (keys %{$rule}) { |
|
12
|
|
|
|
|
28
|
|
578
|
|
|
|
|
|
|
|
579
|
24
|
100
|
|
|
|
215
|
next if (ref($rule->{$rule_key}) ne 'ARRAY'); |
580
|
|
|
|
|
|
|
|
581
|
12
|
|
|
|
|
18
|
my $start_index = $rule->{$rule_key}[0] - 1; |
582
|
12
|
|
|
|
|
15
|
my $length = $rule->{$rule_key}[1]; |
583
|
|
|
|
|
|
|
|
584
|
12
|
|
|
|
|
22
|
my $extracted_value = substr($line, $start_index, $length); |
585
|
|
|
|
|
|
|
|
586
|
12
|
|
|
|
|
44
|
$extracted_value =~ s/^\s+|\s+$//g; |
587
|
|
|
|
|
|
|
|
588
|
12
|
|
|
|
|
39
|
$me_obj->VarSetScalar($rule_key, $extracted_value); |
589
|
|
|
|
|
|
|
} |
590
|
|
|
|
|
|
|
|
591
|
12
|
|
|
|
|
46
|
my $expression = $rule->{Expression}; |
592
|
|
|
|
|
|
|
|
593
|
12
|
100
|
|
|
|
26
|
return $config if ($me_obj->ParseToScalar($expression)); |
594
|
|
|
|
|
|
|
|
595
|
|
|
|
|
|
|
} |
596
|
|
|
|
|
|
|
else { |
597
|
8
|
|
|
|
|
27
|
return $config; |
598
|
|
|
|
|
|
|
} |
599
|
|
|
|
|
|
|
} |
600
|
|
|
|
|
|
|
|
601
|
4
|
|
|
|
|
3044
|
return undef; |
602
|
|
|
|
|
|
|
} |
603
|
|
|
|
|
|
|
|
604
|
|
|
|
|
|
|
# ========================================================================== # |
605
|
|
|
|
|
|
|
|
606
|
|
|
|
|
|
|
# This method is used to convert the date to epoch using date format |
607
|
|
|
|
|
|
|
|
608
|
|
|
|
|
|
|
sub _get_time_stamp_to_epoch |
609
|
|
|
|
|
|
|
{ |
610
|
10
|
|
|
10
|
|
20
|
my ($self, $date, $format, $tz) = @_; |
611
|
|
|
|
|
|
|
|
612
|
10
|
|
33
|
|
|
21
|
my $time_zone = $tz || $self->get_default_timezone; |
613
|
10
|
|
|
|
|
12
|
my $pattern = $format; |
614
|
|
|
|
|
|
|
|
615
|
10
|
|
|
|
|
42
|
my $strp = DateTime::Format::Strptime->new( |
616
|
|
|
|
|
|
|
pattern => $pattern, |
617
|
|
|
|
|
|
|
time_zone => $time_zone |
618
|
|
|
|
|
|
|
); |
619
|
|
|
|
|
|
|
|
620
|
10
|
|
|
|
|
23209
|
my $dt = $strp->parse_datetime($date); |
621
|
|
|
|
|
|
|
|
622
|
10
|
|
|
|
|
9034
|
my $epoch; |
623
|
|
|
|
|
|
|
|
624
|
10
|
50
|
|
|
|
28
|
if (defined $dt) { |
625
|
|
|
|
|
|
|
|
626
|
10
|
|
|
|
|
28
|
$epoch = $dt->epoch; |
627
|
|
|
|
|
|
|
|
628
|
10
|
50
|
|
|
|
111
|
if ($epoch !~ /^\d{1,10}$/o) { |
629
|
0
|
|
|
|
|
0
|
return undef; |
630
|
|
|
|
|
|
|
} |
631
|
|
|
|
|
|
|
} |
632
|
10
|
|
|
|
|
318
|
return $epoch; |
633
|
|
|
|
|
|
|
} |
634
|
|
|
|
|
|
|
|
635
|
|
|
|
|
|
|
1; |
636
|
|
|
|
|
|
|
|
637
|
|
|
|
|
|
|
__END__ |
638
|
|
|
|
|
|
|
|
639
|
|
|
|
|
|
|
=back |
640
|
|
|
|
|
|
|
|
641
|
|
|
|
|
|
|
=head1 LICENSE |
642
|
|
|
|
|
|
|
|
643
|
|
|
|
|
|
|
This library is free software; you can redistribute and/or modify it under the same terms as Perl itself. |
644
|
|
|
|
|
|
|
|
645
|
|
|
|
|
|
|
=head1 AUTHORS |
646
|
|
|
|
|
|
|
|
647
|
|
|
|
|
|
|
Venkatesan Narayanan, <venkatesanmusiri@gmail.com> |
648
|
|
|
|
|
|
|
|
649
|
|
|
|
|
|
|
=cut |
650
|
|
|
|
|
|
|
|
651
|
|
|
|
|
|
|
# vim: ts=4 |
652
|
|
|
|
|
|
|
# vim600: fdm=marker fdl=0 fdc=3 |
653
|
|
|
|
|
|
|
|