line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Data::Chronicle::Reader; |
2
|
|
|
|
|
|
|
|
3
|
1
|
|
|
1
|
|
18
|
use 5.014; |
|
1
|
|
|
|
|
2
|
|
4
|
1
|
|
|
1
|
|
4
|
use strict; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
28
|
|
5
|
1
|
|
|
1
|
|
6
|
use warnings; |
|
1
|
|
|
|
|
1
|
|
|
1
|
|
|
|
|
51
|
|
6
|
1
|
|
|
1
|
|
411
|
use Data::Chronicle; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
61
|
|
7
|
|
|
|
|
|
|
|
8
|
|
|
|
|
|
|
=head1 NAME |
9
|
|
|
|
|
|
|
|
10
|
|
|
|
|
|
|
Data::Chronicle::Reader - Provides reading from an efficient data storage for volatile and time-based data |
11
|
|
|
|
|
|
|
|
12
|
|
|
|
|
|
|
=cut |
13
|
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
our $VERSION = '0.15'; ## VERSION |
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
=head1 DESCRIPTION |
17
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
This module contains helper methods which can be used to store and retrieve information |
19
|
|
|
|
|
|
|
on an efficient storage with below properties: |
20
|
|
|
|
|
|
|
|
21
|
|
|
|
|
|
|
=over 4 |
22
|
|
|
|
|
|
|
|
23
|
|
|
|
|
|
|
=item B<Timeliness> |
24
|
|
|
|
|
|
|
|
25
|
|
|
|
|
|
|
It is assumed that data to be stored are time-based meaning they change over time and the latest version is most important for us. |
26
|
|
|
|
|
|
|
|
27
|
|
|
|
|
|
|
=item B<Efficient> |
28
|
|
|
|
|
|
|
|
29
|
|
|
|
|
|
|
The module uses Redis cache to provide efficient data storage and retrieval. |
30
|
|
|
|
|
|
|
|
31
|
|
|
|
|
|
|
=item B<Persistent> |
32
|
|
|
|
|
|
|
|
33
|
|
|
|
|
|
|
In addition to caching every incoming data, it is also stored in PostgreSQL for future retrieval. |
34
|
|
|
|
|
|
|
|
35
|
|
|
|
|
|
|
=item B<Transparent> |
36
|
|
|
|
|
|
|
|
37
|
|
|
|
|
|
|
This modules hides all the details about distribution, caching, database structure and ... from developer. He only needs to call a method |
38
|
|
|
|
|
|
|
to save data and another method to retrieve it. All the underlying complexities are handled by the module. |
39
|
|
|
|
|
|
|
|
40
|
|
|
|
|
|
|
=back |
41
|
|
|
|
|
|
|
|
42
|
|
|
|
|
|
|
There are three important methods this module provides: |
43
|
|
|
|
|
|
|
|
44
|
|
|
|
|
|
|
=over 4 |
45
|
|
|
|
|
|
|
|
46
|
|
|
|
|
|
|
=item C<set> |
47
|
|
|
|
|
|
|
|
48
|
|
|
|
|
|
|
Given a category, name and value stores the JSONified value in Redis and PostgreSQL database under "category::name" group and also stores current |
49
|
|
|
|
|
|
|
system time as the timestamp for the data (Which can be used for future retrieval if we want to get data as of a specific time). Note that the value |
50
|
|
|
|
|
|
|
MUST be either hash-ref or array-ref. |
51
|
|
|
|
|
|
|
|
52
|
|
|
|
|
|
|
=item C<get> |
53
|
|
|
|
|
|
|
|
54
|
|
|
|
|
|
|
Given a category and name returns the latest version of the data according to current Redis cache |
55
|
|
|
|
|
|
|
|
56
|
|
|
|
|
|
|
=item C<get_for> |
57
|
|
|
|
|
|
|
|
58
|
|
|
|
|
|
|
Given a category, name and timestamp returns version of data under "category::name" as of the given date (using a DB lookup). |
59
|
|
|
|
|
|
|
|
60
|
|
|
|
|
|
|
=back |
61
|
|
|
|
|
|
|
|
62
|
|
|
|
|
|
|
=head1 Example |
63
|
|
|
|
|
|
|
|
64
|
|
|
|
|
|
|
my $d = get_some_log_data(); |
65
|
|
|
|
|
|
|
|
66
|
|
|
|
|
|
|
my $chronicle_w = Data::Chronicle::Writer->new( |
67
|
|
|
|
|
|
|
cache_writer => $writer, |
68
|
|
|
|
|
|
|
db_handle => $dbh); |
69
|
|
|
|
|
|
|
|
70
|
|
|
|
|
|
|
my $chronicle_r = Data::Chronicle::Reader->new( |
71
|
|
|
|
|
|
|
cache_reader => $reader, |
72
|
|
|
|
|
|
|
db_handle => $dbh); |
73
|
|
|
|
|
|
|
|
74
|
|
|
|
|
|
|
my $chronicle_r2 = Data::Chronicle::Reader->new( |
75
|
|
|
|
|
|
|
cache_reader => $hash_ref); |
76
|
|
|
|
|
|
|
|
77
|
|
|
|
|
|
|
#store data into Chronicle - each time we call `set` it will also store |
78
|
|
|
|
|
|
|
#a copy of the data for historical data retrieval |
79
|
|
|
|
|
|
|
$chronicle_w->set("log_files", "syslog", $d); |
80
|
|
|
|
|
|
|
|
81
|
|
|
|
|
|
|
#retrieve latest data stored for syslog under log_files category |
82
|
|
|
|
|
|
|
my $dt = $chronicle_r->get("log_files", "syslog"); |
83
|
|
|
|
|
|
|
|
84
|
|
|
|
|
|
|
#find historical data for `syslog` at given point in time |
85
|
|
|
|
|
|
|
my $some_old_data = $chronicle_r->get_for("log_files", "syslog", $epoch1); |
86
|
|
|
|
|
|
|
|
87
|
|
|
|
|
|
|
=cut |
88
|
|
|
|
|
|
|
|
89
|
1
|
|
|
1
|
|
570
|
use JSON; |
|
1
|
|
|
|
|
8563
|
|
|
1
|
|
|
|
|
3
|
|
90
|
1
|
|
|
1
|
|
539
|
use Date::Utility; |
|
1
|
|
|
|
|
811714
|
|
|
1
|
|
|
|
|
43
|
|
91
|
1
|
|
|
1
|
|
12
|
use Moose; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
5
|
|
92
|
|
|
|
|
|
|
|
93
|
|
|
|
|
|
|
=head2 cache_reader |
94
|
|
|
|
|
|
|
|
95
|
|
|
|
|
|
|
cahce_reader can be an object which has `get` method used to fetch data. |
96
|
|
|
|
|
|
|
or it can be a plain hash-ref. |
97
|
|
|
|
|
|
|
|
98
|
|
|
|
|
|
|
=cut |
99
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
|
has [qw(cache_reader db_handle)] => ( |
101
|
|
|
|
|
|
|
is => 'ro', |
102
|
|
|
|
|
|
|
default => undef, |
103
|
|
|
|
|
|
|
); |
104
|
|
|
|
|
|
|
|
105
|
|
|
|
|
|
|
=head3 C<< my $data = get("category1", "name1") >> |
106
|
|
|
|
|
|
|
|
107
|
|
|
|
|
|
|
Query for the latest data under "category1::name1" from the cache reader. |
108
|
|
|
|
|
|
|
Will return `undef` if the data does not exist. |
109
|
|
|
|
|
|
|
|
110
|
|
|
|
|
|
|
=cut |
111
|
|
|
|
|
|
|
|
112
|
|
|
|
|
|
|
sub get { |
113
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
114
|
0
|
|
|
|
|
|
my $category = shift; |
115
|
0
|
|
|
|
|
|
my $name = shift; |
116
|
|
|
|
|
|
|
|
117
|
0
|
|
|
|
|
|
my $key = $category . '::' . $name; |
118
|
|
|
|
|
|
|
|
119
|
0
|
0
|
|
|
|
|
if (blessed($self->cache_reader)) { |
120
|
0
|
|
|
|
|
|
my $cached_data = $self->cache_reader->get($key); |
121
|
0
|
0
|
|
|
|
|
return JSON::from_json($cached_data) if defined $cached_data; |
122
|
|
|
|
|
|
|
} else { |
123
|
0
|
|
|
|
|
|
return $self->cache_reader->{$key}; |
124
|
|
|
|
|
|
|
} |
125
|
|
|
|
|
|
|
|
126
|
0
|
|
|
|
|
|
return undef; |
127
|
|
|
|
|
|
|
} |
128
|
|
|
|
|
|
|
|
129
|
|
|
|
|
|
|
=head3 C<< my $data = get_for("category1", "name1", 1447401505) >> |
130
|
|
|
|
|
|
|
|
131
|
|
|
|
|
|
|
Query Pg archive for the data under "category1::name1" at or exactly before the given epoch/Date::Utility. |
132
|
|
|
|
|
|
|
|
133
|
|
|
|
|
|
|
=cut |
134
|
|
|
|
|
|
|
|
135
|
|
|
|
|
|
|
sub get_for { |
136
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
137
|
0
|
|
|
|
|
|
my $category = shift; |
138
|
0
|
|
|
|
|
|
my $name = shift; |
139
|
0
|
|
|
|
|
|
my $date_for = shift; #epoch or Date::Utility |
140
|
|
|
|
|
|
|
|
141
|
0
|
|
|
|
|
|
my $db_timestamp = Date::Utility->new($date_for)->db_timestamp; |
142
|
|
|
|
|
|
|
|
143
|
0
|
0
|
|
|
|
|
die "Requesting for historical data without a valid DB connection [$category,$name,$date_for]" if not defined $self->db_handle; |
144
|
|
|
|
|
|
|
|
145
|
0
|
|
|
|
|
|
my $db_data = |
146
|
|
|
|
|
|
|
$self->db_handle->selectall_hashref(q{SELECT * FROM chronicle where category=? and name=? and timestamp<=? order by timestamp desc limit 1}, |
147
|
|
|
|
|
|
|
'id', {}, $category, $name, $db_timestamp); |
148
|
|
|
|
|
|
|
|
149
|
0
|
0
|
|
|
|
|
return if not %$db_data; |
150
|
|
|
|
|
|
|
|
151
|
0
|
|
|
|
|
|
my $id_value = (sort keys %{$db_data})[0]; |
|
0
|
|
|
|
|
|
|
152
|
0
|
|
|
|
|
|
my $db_value = $db_data->{$id_value}->{value}; |
153
|
|
|
|
|
|
|
|
154
|
0
|
|
|
|
|
|
return JSON::from_json($db_value); |
155
|
|
|
|
|
|
|
} |
156
|
|
|
|
|
|
|
|
157
|
|
|
|
|
|
|
=head3 C<< my $data = get_for_period("category1", "name1", 1447401505, 1447401900) >> |
158
|
|
|
|
|
|
|
|
159
|
|
|
|
|
|
|
Query Pg historical data and return records whose date is between given period. |
160
|
|
|
|
|
|
|
|
161
|
|
|
|
|
|
|
=cut |
162
|
|
|
|
|
|
|
|
163
|
|
|
|
|
|
|
sub get_for_period { |
164
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
165
|
0
|
|
|
|
|
|
my $category = shift; |
166
|
0
|
|
|
|
|
|
my $name = shift; |
167
|
0
|
|
|
|
|
|
my $start = shift; #epoch or Date::Utility |
168
|
0
|
|
|
|
|
|
my $end = shift; #epoch or Date::Utility |
169
|
|
|
|
|
|
|
|
170
|
0
|
|
|
|
|
|
my $start_timestamp = Date::Utility->new($start)->db_timestamp; |
171
|
0
|
|
|
|
|
|
my $end_timestamp = Date::Utility->new($end)->db_timestamp; |
172
|
|
|
|
|
|
|
|
173
|
0
|
0
|
|
|
|
|
die "Requesting for historical period data without a valid DB connection [$category,$name]" if not defined $self->db_handle; |
174
|
|
|
|
|
|
|
|
175
|
0
|
|
|
|
|
|
my $db_data = |
176
|
|
|
|
|
|
|
$self->db_handle->selectall_hashref( |
177
|
|
|
|
|
|
|
q{SELECT * FROM chronicle where category=? and name=? and timestamp<=? AND timestamp >=? order by timestamp desc}, |
178
|
|
|
|
|
|
|
'id', {}, $category, $name, $end_timestamp, $start_timestamp); |
179
|
|
|
|
|
|
|
|
180
|
0
|
0
|
|
|
|
|
return if not %$db_data; |
181
|
|
|
|
|
|
|
|
182
|
0
|
|
|
|
|
|
my @result; |
183
|
|
|
|
|
|
|
|
184
|
0
|
|
|
|
|
|
for my $id_value (keys %$db_data) { |
185
|
0
|
|
|
|
|
|
my $db_value = $db_data->{$id_value}->{value}; |
186
|
|
|
|
|
|
|
|
187
|
0
|
|
|
|
|
|
push @result, JSON::from_json($db_value); |
188
|
|
|
|
|
|
|
} |
189
|
|
|
|
|
|
|
|
190
|
0
|
|
|
|
|
|
return \@result; |
191
|
|
|
|
|
|
|
} |
192
|
|
|
|
|
|
|
|
193
|
1
|
|
|
1
|
|
5278
|
no Moose; |
|
1
|
|
|
|
|
2
|
|
|
1
|
|
|
|
|
4
|
|
194
|
|
|
|
|
|
|
|
195
|
|
|
|
|
|
|
=head1 AUTHOR |
196
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
Binary.com, C<< <support at binary.com> >> |
198
|
|
|
|
|
|
|
|
199
|
|
|
|
|
|
|
=head1 BUGS |
200
|
|
|
|
|
|
|
|
201
|
|
|
|
|
|
|
Please report any bugs or feature requests to C<bug-data-chronicle at rt.cpan.org>, or through |
202
|
|
|
|
|
|
|
the web interface at L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=Data-Chronicle>. I will be notified, and then you'll |
203
|
|
|
|
|
|
|
automatically be notified of progress on your bug as I make changes. |
204
|
|
|
|
|
|
|
|
205
|
|
|
|
|
|
|
|
206
|
|
|
|
|
|
|
|
207
|
|
|
|
|
|
|
|
208
|
|
|
|
|
|
|
=head1 SUPPORT |
209
|
|
|
|
|
|
|
|
210
|
|
|
|
|
|
|
You can find documentation for this module with the perldoc command. |
211
|
|
|
|
|
|
|
|
212
|
|
|
|
|
|
|
perldoc Data::Chronicle::Reader |
213
|
|
|
|
|
|
|
|
214
|
|
|
|
|
|
|
|
215
|
|
|
|
|
|
|
You can also look for information at: |
216
|
|
|
|
|
|
|
|
217
|
|
|
|
|
|
|
=over 4 |
218
|
|
|
|
|
|
|
|
219
|
|
|
|
|
|
|
=item * RT: CPAN's request tracker (report bugs here) |
220
|
|
|
|
|
|
|
|
221
|
|
|
|
|
|
|
L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=Data-Chronicle> |
222
|
|
|
|
|
|
|
|
223
|
|
|
|
|
|
|
=item * AnnoCPAN: Annotated CPAN documentation |
224
|
|
|
|
|
|
|
|
225
|
|
|
|
|
|
|
L<http://annocpan.org/dist/Data-Chronicle> |
226
|
|
|
|
|
|
|
|
227
|
|
|
|
|
|
|
=item * CPAN Ratings |
228
|
|
|
|
|
|
|
|
229
|
|
|
|
|
|
|
L<http://cpanratings.perl.org/d/Data-Chronicle> |
230
|
|
|
|
|
|
|
|
231
|
|
|
|
|
|
|
=item * Search CPAN |
232
|
|
|
|
|
|
|
|
233
|
|
|
|
|
|
|
L<http://search.cpan.org/dist/Data-Chronicle/> |
234
|
|
|
|
|
|
|
|
235
|
|
|
|
|
|
|
=back |
236
|
|
|
|
|
|
|
|
237
|
|
|
|
|
|
|
|
238
|
|
|
|
|
|
|
=head1 ACKNOWLEDGEMENTS |
239
|
|
|
|
|
|
|
|
240
|
|
|
|
|
|
|
=cut |
241
|
|
|
|
|
|
|
|
242
|
|
|
|
|
|
|
1; |