| line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
|
1
|
|
|
|
|
|
|
package Datahub::Factory; |
|
2
|
|
|
|
|
|
|
|
|
3
|
1
|
|
|
1
|
|
143314
|
use Datahub::Factory::Sane; |
|
|
1
|
|
|
|
|
3
|
|
|
|
1
|
|
|
|
|
7
|
|
|
4
|
|
|
|
|
|
|
|
|
5
|
|
|
|
|
|
|
our $VERSION = '1.75'; |
|
6
|
|
|
|
|
|
|
|
|
7
|
1
|
|
|
1
|
|
458
|
use Datahub::Factory::Env; |
|
|
1
|
|
|
|
|
3
|
|
|
|
1
|
|
|
|
|
30
|
|
|
8
|
1
|
|
|
1
|
|
404
|
use Datahub::Factory::Config; |
|
|
1
|
|
|
|
|
4
|
|
|
|
1
|
|
|
|
|
30
|
|
|
9
|
1
|
|
|
1
|
|
419
|
use Datahub::Factory::Pipeline; |
|
|
1
|
|
|
|
|
4
|
|
|
|
1
|
|
|
|
|
33
|
|
|
10
|
1
|
|
|
1
|
|
5
|
use File::Spec; |
|
|
1
|
|
|
|
|
2
|
|
|
|
1
|
|
|
|
|
25
|
|
|
11
|
1
|
|
|
1
|
|
5
|
use Sub::Exporter::Util qw(curry_method); |
|
|
1
|
|
|
|
|
2
|
|
|
|
1
|
|
|
|
|
10
|
|
|
12
|
1
|
|
|
|
|
6
|
use Sub::Exporter -setup => { |
|
13
|
|
|
|
|
|
|
exports => [ |
|
14
|
|
|
|
|
|
|
log => curry_method, |
|
15
|
|
|
|
|
|
|
cfg => curry_method, |
|
16
|
|
|
|
|
|
|
importer => curry_method, |
|
17
|
|
|
|
|
|
|
fixer => curry_method, |
|
18
|
|
|
|
|
|
|
store => curry_method, |
|
19
|
|
|
|
|
|
|
exporter => curry_method, |
|
20
|
|
|
|
|
|
|
indexer => curry_method, |
|
21
|
|
|
|
|
|
|
pipeline => curry_method, |
|
22
|
|
|
|
|
|
|
module => curry_method, |
|
23
|
|
|
|
|
|
|
], |
|
24
|
|
|
|
|
|
|
collectors => {'-load' => \'_import_load', ':load' => \'_import_load'}, |
|
25
|
1
|
|
|
1
|
|
245
|
}; |
|
|
1
|
|
|
|
|
1
|
|
|
26
|
1
|
|
|
1
|
|
657
|
use namespace::clean; |
|
|
1
|
|
|
|
|
1
|
|
|
|
1
|
|
|
|
|
4
|
|
|
27
|
|
|
|
|
|
|
|
|
28
|
|
|
|
|
|
|
sub _import_load { |
|
29
|
0
|
|
|
0
|
|
|
my ($self, $value, $data) = @_; |
|
30
|
|
|
|
|
|
|
|
|
31
|
0
|
0
|
|
|
|
|
if (is_array_ref $value) { |
|
32
|
0
|
|
|
|
|
|
self->load(@$value); |
|
33
|
|
|
|
|
|
|
} |
|
34
|
|
|
|
|
|
|
else { |
|
35
|
0
|
|
|
|
|
|
$self->load; |
|
36
|
|
|
|
|
|
|
} |
|
37
|
|
|
|
|
|
|
|
|
38
|
0
|
|
|
|
|
|
1; |
|
39
|
|
|
|
|
|
|
} |
|
40
|
|
|
|
|
|
|
|
|
41
|
|
|
|
|
|
|
sub load { |
|
42
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
|
43
|
0
|
|
|
|
|
|
my $env = Datahub::Factory::Env->new(); |
|
44
|
0
|
|
|
|
|
|
$class->_env($env); |
|
45
|
0
|
|
|
|
|
|
$class; |
|
46
|
|
|
|
|
|
|
} |
|
47
|
|
|
|
|
|
|
|
|
48
|
|
|
|
|
|
|
sub _env { |
|
49
|
0
|
|
|
0
|
|
|
my ($class, $env) = @_; |
|
50
|
0
|
|
|
|
|
|
state $loaded_env; |
|
51
|
0
|
0
|
|
|
|
|
$loaded_env = $env if defined $env; |
|
52
|
0
|
|
0
|
|
|
|
$loaded_env |
|
53
|
|
|
|
|
|
|
||= Datahub::Factory::Env->new(load_paths => $class->default_load_path); |
|
54
|
|
|
|
|
|
|
} |
|
55
|
|
|
|
|
|
|
|
|
56
|
|
|
|
|
|
|
sub default_load_path { # TODO move to Catmandu::Env |
|
57
|
0
|
|
|
0
|
0
|
|
my ($class, $path) = @_; |
|
58
|
0
|
|
|
|
|
|
state $default_path; |
|
59
|
0
|
0
|
|
|
|
|
$default_path = $path if defined $path; |
|
60
|
0
|
|
0
|
|
|
|
$default_path //= do { |
|
61
|
0
|
|
|
|
|
|
my $script = File::Spec->rel2abs($0); |
|
62
|
0
|
|
|
|
|
|
my ($script_vol, $script_path, $script_name) |
|
63
|
|
|
|
|
|
|
= File::Spec->splitpath($script); |
|
64
|
0
|
|
|
|
|
|
my @dirs = grep length, File::Spec->splitdir($script_path); |
|
65
|
0
|
0
|
|
|
|
|
if ($dirs[-1] eq 'bin') { |
|
66
|
0
|
|
|
|
|
|
pop @dirs; |
|
67
|
0
|
|
|
|
|
|
File::Spec->catdir(File::Spec->rootdir, @dirs); |
|
68
|
|
|
|
|
|
|
} |
|
69
|
|
|
|
|
|
|
else { |
|
70
|
0
|
|
|
|
|
|
$script_path; |
|
71
|
|
|
|
|
|
|
} |
|
72
|
|
|
|
|
|
|
}; |
|
73
|
|
|
|
|
|
|
} |
|
74
|
|
|
|
|
|
|
|
|
75
|
|
|
|
|
|
|
sub config { |
|
76
|
0
|
|
|
0
|
0
|
|
my ($class, $config) = @_; |
|
77
|
|
|
|
|
|
|
|
|
78
|
0
|
0
|
|
|
|
|
if ($config) { |
|
79
|
0
|
|
|
|
|
|
my $env = Datahub::Factory::Env->new(load_paths => $class->_env->load_paths); |
|
80
|
0
|
|
|
|
|
|
$env->_set_config($config); |
|
81
|
0
|
|
|
|
|
|
$class->_env($env); |
|
82
|
|
|
|
|
|
|
} |
|
83
|
|
|
|
|
|
|
|
|
84
|
0
|
|
|
|
|
|
$class->_env->config; |
|
85
|
|
|
|
|
|
|
} |
|
86
|
|
|
|
|
|
|
|
|
87
|
|
|
|
|
|
|
sub importer { |
|
88
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
|
89
|
0
|
|
|
|
|
|
$class->_env->importer(@_); |
|
90
|
|
|
|
|
|
|
} |
|
91
|
|
|
|
|
|
|
|
|
92
|
|
|
|
|
|
|
sub fixer { |
|
93
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
|
94
|
0
|
|
|
|
|
|
$class->_env->fixer(@_); |
|
95
|
|
|
|
|
|
|
} |
|
96
|
|
|
|
|
|
|
|
|
97
|
|
|
|
|
|
|
sub exporter { |
|
98
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
|
99
|
0
|
|
|
|
|
|
$class->_env->exporter(@_); |
|
100
|
|
|
|
|
|
|
} |
|
101
|
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
sub indexer { |
|
103
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
|
104
|
0
|
|
|
|
|
|
$class->_env->indexer(@_); |
|
105
|
|
|
|
|
|
|
} |
|
106
|
|
|
|
|
|
|
|
|
107
|
|
|
|
|
|
|
sub log { |
|
108
|
0
|
|
|
0
|
0
|
|
$_[0]->_env->log; |
|
109
|
|
|
|
|
|
|
} |
|
110
|
|
|
|
|
|
|
|
|
111
|
|
|
|
|
|
|
sub cfg { |
|
112
|
0
|
|
|
0
|
0
|
|
my $cfg = Datahub::Factory::Config->new(); |
|
113
|
0
|
|
|
|
|
|
return $cfg->config; |
|
114
|
|
|
|
|
|
|
} |
|
115
|
|
|
|
|
|
|
|
|
116
|
|
|
|
|
|
|
sub pipeline { |
|
117
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
|
118
|
0
|
|
|
|
|
|
return $class->_env->pipeline(@_); |
|
119
|
|
|
|
|
|
|
} |
|
120
|
|
|
|
|
|
|
|
|
121
|
|
|
|
|
|
|
sub module { |
|
122
|
0
|
|
|
0
|
0
|
|
my $class = shift; |
|
123
|
0
|
|
|
|
|
|
return $class->_env->module(@_); |
|
124
|
|
|
|
|
|
|
} |
|
125
|
|
|
|
|
|
|
|
|
126
|
|
|
|
|
|
|
1; |
|
127
|
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
__END__ |
|
129
|
|
|
|
|
|
|
|
|
130
|
|
|
|
|
|
|
=pod |
|
131
|
|
|
|
|
|
|
|
|
132
|
|
|
|
|
|
|
=head1 NAME |
|
133
|
|
|
|
|
|
|
|
|
134
|
|
|
|
|
|
|
Datahub::Factory - A conveyor belt which transports data from a data source to a data sink. |
|
135
|
|
|
|
|
|
|
|
|
136
|
|
|
|
|
|
|
=head1 SYNOPSIS |
|
137
|
|
|
|
|
|
|
|
|
138
|
|
|
|
|
|
|
dhconveyor command OPTIONS |
|
139
|
|
|
|
|
|
|
|
|
140
|
|
|
|
|
|
|
=head1 DESCRIPTION |
|
141
|
|
|
|
|
|
|
|
|
142
|
|
|
|
|
|
|
Datahub::Factory is a command line conveyor belt which automates three tasks: |
|
143
|
|
|
|
|
|
|
|
|
144
|
|
|
|
|
|
|
=over |
|
145
|
|
|
|
|
|
|
|
|
146
|
|
|
|
|
|
|
=item Data is fetched automatically from a local or remote data source. |
|
147
|
|
|
|
|
|
|
|
|
148
|
|
|
|
|
|
|
=item Data is converted to an exchange format. |
|
149
|
|
|
|
|
|
|
|
|
150
|
|
|
|
|
|
|
=item The output is pushed to a data sink. |
|
151
|
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
=back |
|
153
|
|
|
|
|
|
|
|
|
154
|
|
|
|
|
|
|
Datahub::Factory fetches data from several sources as specified by the Importer settings, |
|
155
|
|
|
|
|
|
|
executes a L<Catmandu::Fix> and sends it to a data sink, set via an Exporter. |
|
156
|
|
|
|
|
|
|
Several importer and exporter modules are provided, but developers can extend the functionality with their own modules. |
|
157
|
|
|
|
|
|
|
|
|
158
|
|
|
|
|
|
|
Datahub::Factory contains Log4perl support. |
|
159
|
|
|
|
|
|
|
|
|
160
|
|
|
|
|
|
|
=head1 CONFIGURATION |
|
161
|
|
|
|
|
|
|
|
|
162
|
|
|
|
|
|
|
=head2 Command line options |
|
163
|
|
|
|
|
|
|
|
|
164
|
|
|
|
|
|
|
All commands share the following switches: |
|
165
|
|
|
|
|
|
|
|
|
166
|
|
|
|
|
|
|
=over |
|
167
|
|
|
|
|
|
|
|
|
168
|
|
|
|
|
|
|
=item --log_level |
|
169
|
|
|
|
|
|
|
|
|
170
|
|
|
|
|
|
|
Set the log_level. Takes a numeric parameter. Supported levels are: 1 (WARN), |
|
171
|
|
|
|
|
|
|
2 (INFO), 3 (DEBUG). WARN (1) is the default. |
|
172
|
|
|
|
|
|
|
|
|
173
|
|
|
|
|
|
|
=item --log_output |
|
174
|
|
|
|
|
|
|
|
|
175
|
|
|
|
|
|
|
Selects an output for the log messages. By default, it will send them to STDERR |
|
176
|
|
|
|
|
|
|
(pass STDERR as parameter), but STDOUT (STDOUT) and a log file. |
|
177
|
|
|
|
|
|
|
|
|
178
|
|
|
|
|
|
|
=item --verbose |
|
179
|
|
|
|
|
|
|
|
|
180
|
|
|
|
|
|
|
Set verbosity. Invoking the command with the --verbose, -v flag will render |
|
181
|
|
|
|
|
|
|
verbose output to the terminal. |
|
182
|
|
|
|
|
|
|
|
|
183
|
|
|
|
|
|
|
=back |
|
184
|
|
|
|
|
|
|
|
|
185
|
|
|
|
|
|
|
=head1 COMMANDS |
|
186
|
|
|
|
|
|
|
|
|
187
|
|
|
|
|
|
|
=head2 help COMMAND |
|
188
|
|
|
|
|
|
|
|
|
189
|
|
|
|
|
|
|
Documentation about command line options. |
|
190
|
|
|
|
|
|
|
|
|
191
|
|
|
|
|
|
|
=head2 L<transport OPTIONS|https://metacpan.org/pod/Datahub::Factory::Command::transport> |
|
192
|
|
|
|
|
|
|
|
|
193
|
|
|
|
|
|
|
Fetch data from a local or remote source, convert it to an exchange format and export the data. |
|
194
|
|
|
|
|
|
|
|
|
195
|
|
|
|
|
|
|
=head2 L<transport OPTIONS|https://metacpan.org/pod/Datahub::Factory::Command::index> |
|
196
|
|
|
|
|
|
|
|
|
197
|
|
|
|
|
|
|
Fetch data from a local source, and push it to an enterprise search engine via a bulk API. |
|
198
|
|
|
|
|
|
|
|
|
199
|
|
|
|
|
|
|
|
|
200
|
|
|
|
|
|
|
=head1 API |
|
201
|
|
|
|
|
|
|
|
|
202
|
|
|
|
|
|
|
Datahub::Factory uses a plugin-based architecture, making it easy to extend with new functionality. |
|
203
|
|
|
|
|
|
|
|
|
204
|
|
|
|
|
|
|
New commands can be added by creating a Perl module that contains a `command_name.pm` file in the `lib/Datahub/Factory/Command` path. |
|
205
|
|
|
|
|
|
|
Datahub::Factory uses the L<Datahub::Factory::Command> namespace and leverages L<App::Cmd> internally. |
|
206
|
|
|
|
|
|
|
|
|
207
|
|
|
|
|
|
|
New L<Datahub::Factory::Importer>, L<Datahub::Factory::Exporter> and L<Datahub::Factory::Fixer> plugins can be added in the same way. |
|
208
|
|
|
|
|
|
|
|
|
209
|
|
|
|
|
|
|
=head1 AUTHORS |
|
210
|
|
|
|
|
|
|
|
|
211
|
|
|
|
|
|
|
Matthias Vandermaesen <matthias.vandermaesen@vlaamsekunstcollectie.be> |
|
212
|
|
|
|
|
|
|
Pieter De Praetere <pieter@packed.be> |
|
213
|
|
|
|
|
|
|
|
|
214
|
|
|
|
|
|
|
=head1 COPYRIGHT AND LICENSE |
|
215
|
|
|
|
|
|
|
|
|
216
|
|
|
|
|
|
|
This software is copyright (c) 2016 by PACKED, vzw, Vlaamse Kunstcollectie, vzw. |
|
217
|
|
|
|
|
|
|
|
|
218
|
|
|
|
|
|
|
This is free software; you can redistribute it and/or modify it under the terms |
|
219
|
|
|
|
|
|
|
of the GNU General Public License, Version 3, June 2007. |
|
220
|
|
|
|
|
|
|
|
|
221
|
|
|
|
|
|
|
=cut |