line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
package Paws::DataPipeline; |
2
|
1
|
|
|
1
|
|
4971
|
use Moose; |
|
1
|
|
|
1
|
|
6
|
|
|
1
|
|
|
|
|
12
|
|
|
1
|
|
|
|
|
1610
|
|
|
1
|
|
|
|
|
3
|
|
|
1
|
|
|
|
|
13
|
|
3
|
1
|
|
|
1
|
0
|
4
|
sub service { 'datapipeline' } |
4
|
0
|
|
|
0
|
0
|
|
sub version { '2012-10-29' } |
5
|
0
|
|
|
0
|
0
|
|
sub target_prefix { 'DataPipeline' } |
6
|
0
|
|
|
0
|
0
|
|
sub json_version { "1.1" } |
7
|
|
|
|
|
|
|
has max_attempts => (is => 'ro', isa => 'Int', default => 5); |
8
|
|
|
|
|
|
|
has retry => (is => 'ro', isa => 'HashRef', default => sub { |
9
|
|
|
|
|
|
|
{ base => 'rand', type => 'exponential', growth_factor => 2 } |
10
|
|
|
|
|
|
|
}); |
11
|
|
|
|
|
|
|
has retriables => (is => 'ro', isa => 'ArrayRef', default => sub { [ |
12
|
|
|
|
|
|
|
] }); |
13
|
|
|
|
|
|
|
|
14
|
|
|
|
|
|
|
with 'Paws::API::Caller', 'Paws::API::EndpointResolver', 'Paws::Net::V4Signature', 'Paws::Net::JsonCaller', 'Paws::Net::JsonResponse'; |
15
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
|
17
|
|
|
|
|
|
|
sub ActivatePipeline { |
18
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
19
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::ActivatePipeline', @_); |
20
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
21
|
|
|
|
|
|
|
} |
22
|
|
|
|
|
|
|
sub AddTags { |
23
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
24
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::AddTags', @_); |
25
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
26
|
|
|
|
|
|
|
} |
27
|
|
|
|
|
|
|
sub CreatePipeline { |
28
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
29
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::CreatePipeline', @_); |
30
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
31
|
|
|
|
|
|
|
} |
32
|
|
|
|
|
|
|
sub DeactivatePipeline { |
33
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
34
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::DeactivatePipeline', @_); |
35
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
36
|
|
|
|
|
|
|
} |
37
|
|
|
|
|
|
|
sub DeletePipeline { |
38
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
39
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::DeletePipeline', @_); |
40
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
41
|
|
|
|
|
|
|
} |
42
|
|
|
|
|
|
|
sub DescribeObjects { |
43
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
44
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::DescribeObjects', @_); |
45
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
46
|
|
|
|
|
|
|
} |
47
|
|
|
|
|
|
|
sub DescribePipelines { |
48
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
49
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::DescribePipelines', @_); |
50
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
51
|
|
|
|
|
|
|
} |
52
|
|
|
|
|
|
|
sub EvaluateExpression { |
53
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
54
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::EvaluateExpression', @_); |
55
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
56
|
|
|
|
|
|
|
} |
57
|
|
|
|
|
|
|
sub GetPipelineDefinition { |
58
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
59
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::GetPipelineDefinition', @_); |
60
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
61
|
|
|
|
|
|
|
} |
62
|
|
|
|
|
|
|
sub ListPipelines { |
63
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
64
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::ListPipelines', @_); |
65
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
66
|
|
|
|
|
|
|
} |
67
|
|
|
|
|
|
|
sub PollForTask { |
68
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
69
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::PollForTask', @_); |
70
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
71
|
|
|
|
|
|
|
} |
72
|
|
|
|
|
|
|
sub PutPipelineDefinition { |
73
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
74
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::PutPipelineDefinition', @_); |
75
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
76
|
|
|
|
|
|
|
} |
77
|
|
|
|
|
|
|
sub QueryObjects { |
78
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
79
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::QueryObjects', @_); |
80
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
81
|
|
|
|
|
|
|
} |
82
|
|
|
|
|
|
|
sub RemoveTags { |
83
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
84
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::RemoveTags', @_); |
85
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
86
|
|
|
|
|
|
|
} |
87
|
|
|
|
|
|
|
sub ReportTaskProgress { |
88
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
89
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::ReportTaskProgress', @_); |
90
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
91
|
|
|
|
|
|
|
} |
92
|
|
|
|
|
|
|
sub ReportTaskRunnerHeartbeat { |
93
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
94
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::ReportTaskRunnerHeartbeat', @_); |
95
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
96
|
|
|
|
|
|
|
} |
97
|
|
|
|
|
|
|
sub SetStatus { |
98
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
99
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::SetStatus', @_); |
100
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
101
|
|
|
|
|
|
|
} |
102
|
|
|
|
|
|
|
sub SetTaskStatus { |
103
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
104
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::SetTaskStatus', @_); |
105
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
106
|
|
|
|
|
|
|
} |
107
|
|
|
|
|
|
|
sub ValidatePipelineDefinition { |
108
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
109
|
0
|
|
|
|
|
|
my $call_object = $self->new_with_coercions('Paws::DataPipeline::ValidatePipelineDefinition', @_); |
110
|
0
|
|
|
|
|
|
return $self->caller->do_call($self, $call_object); |
111
|
|
|
|
|
|
|
} |
112
|
|
|
|
|
|
|
|
113
|
|
|
|
|
|
|
sub DescribeAllObjects { |
114
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
115
|
|
|
|
|
|
|
|
116
|
0
|
0
|
|
|
|
|
my $callback = shift @_ if (ref($_[0]) eq 'CODE'); |
117
|
0
|
|
|
|
|
|
my $result = $self->DescribeObjects(@_); |
118
|
0
|
|
|
|
|
|
my $next_result = $result; |
119
|
|
|
|
|
|
|
|
120
|
0
|
0
|
|
|
|
|
if (not defined $callback) { |
121
|
0
|
|
|
|
|
|
while ($next_result->hasMoreResults) { |
122
|
0
|
|
|
|
|
|
$next_result = $self->DescribeObjects(@_, marker => $next_result->marker); |
123
|
0
|
|
|
|
|
|
push @{ $result->pipelineObjects }, @{ $next_result->pipelineObjects }; |
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
124
|
|
|
|
|
|
|
} |
125
|
0
|
|
|
|
|
|
return $result; |
126
|
|
|
|
|
|
|
} else { |
127
|
0
|
|
|
|
|
|
while ($result->hasMoreResults) { |
128
|
0
|
|
|
|
|
|
$callback->($_ => 'pipelineObjects') foreach (@{ $result->pipelineObjects }); |
|
0
|
|
|
|
|
|
|
129
|
0
|
|
|
|
|
|
$result = $self->DescribeObjects(@_, marker => $result->marker); |
130
|
|
|
|
|
|
|
} |
131
|
0
|
|
|
|
|
|
$callback->($_ => 'pipelineObjects') foreach (@{ $result->pipelineObjects }); |
|
0
|
|
|
|
|
|
|
132
|
|
|
|
|
|
|
} |
133
|
|
|
|
|
|
|
|
134
|
|
|
|
|
|
|
return undef |
135
|
0
|
|
|
|
|
|
} |
136
|
|
|
|
|
|
|
sub ListAllPipelines { |
137
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
138
|
|
|
|
|
|
|
|
139
|
0
|
0
|
|
|
|
|
my $callback = shift @_ if (ref($_[0]) eq 'CODE'); |
140
|
0
|
|
|
|
|
|
my $result = $self->ListPipelines(@_); |
141
|
0
|
|
|
|
|
|
my $next_result = $result; |
142
|
|
|
|
|
|
|
|
143
|
0
|
0
|
|
|
|
|
if (not defined $callback) { |
144
|
0
|
|
|
|
|
|
while ($next_result->hasMoreResults) { |
145
|
0
|
|
|
|
|
|
$next_result = $self->ListPipelines(@_, marker => $next_result->marker); |
146
|
0
|
|
|
|
|
|
push @{ $result->pipelineIdList }, @{ $next_result->pipelineIdList }; |
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
147
|
|
|
|
|
|
|
} |
148
|
0
|
|
|
|
|
|
return $result; |
149
|
|
|
|
|
|
|
} else { |
150
|
0
|
|
|
|
|
|
while ($result->hasMoreResults) { |
151
|
0
|
|
|
|
|
|
$callback->($_ => 'pipelineIdList') foreach (@{ $result->pipelineIdList }); |
|
0
|
|
|
|
|
|
|
152
|
0
|
|
|
|
|
|
$result = $self->ListPipelines(@_, marker => $result->marker); |
153
|
|
|
|
|
|
|
} |
154
|
0
|
|
|
|
|
|
$callback->($_ => 'pipelineIdList') foreach (@{ $result->pipelineIdList }); |
|
0
|
|
|
|
|
|
|
155
|
|
|
|
|
|
|
} |
156
|
|
|
|
|
|
|
|
157
|
|
|
|
|
|
|
return undef |
158
|
0
|
|
|
|
|
|
} |
159
|
|
|
|
|
|
|
sub QueryAllObjects { |
160
|
0
|
|
|
0
|
1
|
|
my $self = shift; |
161
|
|
|
|
|
|
|
|
162
|
0
|
0
|
|
|
|
|
my $callback = shift @_ if (ref($_[0]) eq 'CODE'); |
163
|
0
|
|
|
|
|
|
my $result = $self->QueryObjects(@_); |
164
|
0
|
|
|
|
|
|
my $next_result = $result; |
165
|
|
|
|
|
|
|
|
166
|
0
|
0
|
|
|
|
|
if (not defined $callback) { |
167
|
0
|
|
|
|
|
|
while ($next_result->hasMoreResults) { |
168
|
0
|
|
|
|
|
|
$next_result = $self->QueryObjects(@_, marker => $next_result->marker); |
169
|
0
|
|
|
|
|
|
push @{ $result->ids }, @{ $next_result->ids }; |
|
0
|
|
|
|
|
|
|
|
0
|
|
|
|
|
|
|
170
|
|
|
|
|
|
|
} |
171
|
0
|
|
|
|
|
|
return $result; |
172
|
|
|
|
|
|
|
} else { |
173
|
0
|
|
|
|
|
|
while ($result->hasMoreResults) { |
174
|
0
|
|
|
|
|
|
$callback->($_ => 'ids') foreach (@{ $result->ids }); |
|
0
|
|
|
|
|
|
|
175
|
0
|
|
|
|
|
|
$result = $self->QueryObjects(@_, marker => $result->marker); |
176
|
|
|
|
|
|
|
} |
177
|
0
|
|
|
|
|
|
$callback->($_ => 'ids') foreach (@{ $result->ids }); |
|
0
|
|
|
|
|
|
|
178
|
|
|
|
|
|
|
} |
179
|
|
|
|
|
|
|
|
180
|
|
|
|
|
|
|
return undef |
181
|
0
|
|
|
|
|
|
} |
182
|
|
|
|
|
|
|
|
183
|
|
|
|
|
|
|
|
184
|
0
|
|
|
0
|
0
|
|
sub operations { qw/ActivatePipeline AddTags CreatePipeline DeactivatePipeline DeletePipeline DescribeObjects DescribePipelines EvaluateExpression GetPipelineDefinition ListPipelines PollForTask PutPipelineDefinition QueryObjects RemoveTags ReportTaskProgress ReportTaskRunnerHeartbeat SetStatus SetTaskStatus ValidatePipelineDefinition / } |
185
|
|
|
|
|
|
|
|
186
|
|
|
|
|
|
|
1; |
187
|
|
|
|
|
|
|
|
188
|
|
|
|
|
|
|
### main pod documentation begin ### |
189
|
|
|
|
|
|
|
|
190
|
|
|
|
|
|
|
=head1 NAME |
191
|
|
|
|
|
|
|
|
192
|
|
|
|
|
|
|
Paws::DataPipeline - Perl Interface to AWS AWS Data Pipeline |
193
|
|
|
|
|
|
|
|
194
|
|
|
|
|
|
|
=head1 SYNOPSIS |
195
|
|
|
|
|
|
|
|
196
|
|
|
|
|
|
|
use Paws; |
197
|
|
|
|
|
|
|
|
198
|
|
|
|
|
|
|
my $obj = Paws->service('DataPipeline'); |
199
|
|
|
|
|
|
|
my $res = $obj->Method( |
200
|
|
|
|
|
|
|
Arg1 => $val1, |
201
|
|
|
|
|
|
|
Arg2 => [ 'V1', 'V2' ], |
202
|
|
|
|
|
|
|
# if Arg3 is an object, the HashRef will be used as arguments to the constructor |
203
|
|
|
|
|
|
|
# of the arguments type |
204
|
|
|
|
|
|
|
Arg3 => { Att1 => 'Val1' }, |
205
|
|
|
|
|
|
|
# if Arg4 is an array of objects, the HashRefs will be passed as arguments to |
206
|
|
|
|
|
|
|
# the constructor of the arguments type |
207
|
|
|
|
|
|
|
Arg4 => [ { Att1 => 'Val1' }, { Att1 => 'Val2' } ], |
208
|
|
|
|
|
|
|
); |
209
|
|
|
|
|
|
|
|
210
|
|
|
|
|
|
|
=head1 DESCRIPTION |
211
|
|
|
|
|
|
|
|
212
|
|
|
|
|
|
|
AWS Data Pipeline configures and manages a data-driven workflow called |
213
|
|
|
|
|
|
|
a pipeline. AWS Data Pipeline handles the details of scheduling and |
214
|
|
|
|
|
|
|
ensuring that data dependencies are met so that your application can |
215
|
|
|
|
|
|
|
focus on processing the data. |
216
|
|
|
|
|
|
|
|
217
|
|
|
|
|
|
|
AWS Data Pipeline provides a JAR implementation of a task runner called |
218
|
|
|
|
|
|
|
AWS Data Pipeline Task Runner. AWS Data Pipeline Task Runner provides |
219
|
|
|
|
|
|
|
logic for common data management scenarios, such as performing database |
220
|
|
|
|
|
|
|
queries and running data analysis using Amazon Elastic MapReduce |
221
|
|
|
|
|
|
|
(Amazon EMR). You can use AWS Data Pipeline Task Runner as your task |
222
|
|
|
|
|
|
|
runner, or you can write your own task runner to provide custom data |
223
|
|
|
|
|
|
|
management. |
224
|
|
|
|
|
|
|
|
225
|
|
|
|
|
|
|
AWS Data Pipeline implements two main sets of functionality. Use the |
226
|
|
|
|
|
|
|
first set to create a pipeline and define data sources, schedules, |
227
|
|
|
|
|
|
|
dependencies, and the transforms to be performed on the data. Use the |
228
|
|
|
|
|
|
|
second set in your task runner application to receive the next task |
229
|
|
|
|
|
|
|
ready for processing. The logic for performing the task, such as |
230
|
|
|
|
|
|
|
querying the data, running data analysis, or converting the data from |
231
|
|
|
|
|
|
|
one format to another, is contained within the task runner. The task |
232
|
|
|
|
|
|
|
runner performs the task assigned to it by the web service, reporting |
233
|
|
|
|
|
|
|
progress to the web service as it does so. When the task is done, the |
234
|
|
|
|
|
|
|
task runner reports the final success or failure of the task to the web |
235
|
|
|
|
|
|
|
service. |
236
|
|
|
|
|
|
|
|
237
|
|
|
|
|
|
|
=head1 METHODS |
238
|
|
|
|
|
|
|
|
239
|
|
|
|
|
|
|
=head2 ActivatePipeline(PipelineId => Str, [ParameterValues => ArrayRef[L<Paws::DataPipeline::ParameterValue>], StartTimestamp => Str]) |
240
|
|
|
|
|
|
|
|
241
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::ActivatePipeline> |
242
|
|
|
|
|
|
|
|
243
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::ActivatePipelineOutput> instance |
244
|
|
|
|
|
|
|
|
245
|
|
|
|
|
|
|
Validates the specified pipeline and starts processing pipeline tasks. |
246
|
|
|
|
|
|
|
If the pipeline does not pass validation, activation fails. |
247
|
|
|
|
|
|
|
|
248
|
|
|
|
|
|
|
If you need to pause the pipeline to investigate an issue with a |
249
|
|
|
|
|
|
|
component, such as a data source or script, call DeactivatePipeline. |
250
|
|
|
|
|
|
|
|
251
|
|
|
|
|
|
|
To activate a finished pipeline, modify the end date for the pipeline |
252
|
|
|
|
|
|
|
and then activate it. |
253
|
|
|
|
|
|
|
|
254
|
|
|
|
|
|
|
|
255
|
|
|
|
|
|
|
=head2 AddTags(PipelineId => Str, Tags => ArrayRef[L<Paws::DataPipeline::Tag>]) |
256
|
|
|
|
|
|
|
|
257
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::AddTags> |
258
|
|
|
|
|
|
|
|
259
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::AddTagsOutput> instance |
260
|
|
|
|
|
|
|
|
261
|
|
|
|
|
|
|
Adds or modifies tags for the specified pipeline. |
262
|
|
|
|
|
|
|
|
263
|
|
|
|
|
|
|
|
264
|
|
|
|
|
|
|
=head2 CreatePipeline(Name => Str, UniqueId => Str, [Description => Str, Tags => ArrayRef[L<Paws::DataPipeline::Tag>]]) |
265
|
|
|
|
|
|
|
|
266
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::CreatePipeline> |
267
|
|
|
|
|
|
|
|
268
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::CreatePipelineOutput> instance |
269
|
|
|
|
|
|
|
|
270
|
|
|
|
|
|
|
Creates a new, empty pipeline. Use PutPipelineDefinition to populate |
271
|
|
|
|
|
|
|
the pipeline. |
272
|
|
|
|
|
|
|
|
273
|
|
|
|
|
|
|
|
274
|
|
|
|
|
|
|
=head2 DeactivatePipeline(PipelineId => Str, [CancelActive => Bool]) |
275
|
|
|
|
|
|
|
|
276
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::DeactivatePipeline> |
277
|
|
|
|
|
|
|
|
278
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::DeactivatePipelineOutput> instance |
279
|
|
|
|
|
|
|
|
280
|
|
|
|
|
|
|
Deactivates the specified running pipeline. The pipeline is set to the |
281
|
|
|
|
|
|
|
C<DEACTIVATING> state until the deactivation process completes. |
282
|
|
|
|
|
|
|
|
283
|
|
|
|
|
|
|
To resume a deactivated pipeline, use ActivatePipeline. By default, the |
284
|
|
|
|
|
|
|
pipeline resumes from the last completed execution. Optionally, you can |
285
|
|
|
|
|
|
|
specify the date and time to resume the pipeline. |
286
|
|
|
|
|
|
|
|
287
|
|
|
|
|
|
|
|
288
|
|
|
|
|
|
|
=head2 DeletePipeline(PipelineId => Str) |
289
|
|
|
|
|
|
|
|
290
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::DeletePipeline> |
291
|
|
|
|
|
|
|
|
292
|
|
|
|
|
|
|
Returns: nothing |
293
|
|
|
|
|
|
|
|
294
|
|
|
|
|
|
|
Deletes a pipeline, its pipeline definition, and its run history. AWS |
295
|
|
|
|
|
|
|
Data Pipeline attempts to cancel instances associated with the pipeline |
296
|
|
|
|
|
|
|
that are currently being processed by task runners. |
297
|
|
|
|
|
|
|
|
298
|
|
|
|
|
|
|
Deleting a pipeline cannot be undone. You cannot query or restore a |
299
|
|
|
|
|
|
|
deleted pipeline. To temporarily pause a pipeline instead of deleting |
300
|
|
|
|
|
|
|
it, call SetStatus with the status set to C<PAUSE> on individual |
301
|
|
|
|
|
|
|
components. Components that are paused by SetStatus can be resumed. |
302
|
|
|
|
|
|
|
|
303
|
|
|
|
|
|
|
|
304
|
|
|
|
|
|
|
=head2 DescribeObjects(ObjectIds => ArrayRef[Str|Undef], PipelineId => Str, [EvaluateExpressions => Bool, Marker => Str]) |
305
|
|
|
|
|
|
|
|
306
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::DescribeObjects> |
307
|
|
|
|
|
|
|
|
308
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::DescribeObjectsOutput> instance |
309
|
|
|
|
|
|
|
|
310
|
|
|
|
|
|
|
Gets the object definitions for a set of objects associated with the |
311
|
|
|
|
|
|
|
pipeline. Object definitions are composed of a set of fields that |
312
|
|
|
|
|
|
|
define the properties of the object. |
313
|
|
|
|
|
|
|
|
314
|
|
|
|
|
|
|
|
315
|
|
|
|
|
|
|
=head2 DescribePipelines(PipelineIds => ArrayRef[Str|Undef]) |
316
|
|
|
|
|
|
|
|
317
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::DescribePipelines> |
318
|
|
|
|
|
|
|
|
319
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::DescribePipelinesOutput> instance |
320
|
|
|
|
|
|
|
|
321
|
|
|
|
|
|
|
Retrieves metadata about one or more pipelines. The information |
322
|
|
|
|
|
|
|
retrieved includes the name of the pipeline, the pipeline identifier, |
323
|
|
|
|
|
|
|
its current state, and the user account that owns the pipeline. Using |
324
|
|
|
|
|
|
|
account credentials, you can retrieve metadata about pipelines that you |
325
|
|
|
|
|
|
|
or your IAM users have created. If you are using an IAM user account, |
326
|
|
|
|
|
|
|
you can retrieve metadata about only those pipelines for which you have |
327
|
|
|
|
|
|
|
read permissions. |
328
|
|
|
|
|
|
|
|
329
|
|
|
|
|
|
|
To retrieve the full pipeline definition instead of metadata about the |
330
|
|
|
|
|
|
|
pipeline, call GetPipelineDefinition. |
331
|
|
|
|
|
|
|
|
332
|
|
|
|
|
|
|
|
333
|
|
|
|
|
|
|
=head2 EvaluateExpression(Expression => Str, ObjectId => Str, PipelineId => Str) |
334
|
|
|
|
|
|
|
|
335
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::EvaluateExpression> |
336
|
|
|
|
|
|
|
|
337
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::EvaluateExpressionOutput> instance |
338
|
|
|
|
|
|
|
|
339
|
|
|
|
|
|
|
Task runners call C<EvaluateExpression> to evaluate a string in the |
340
|
|
|
|
|
|
|
context of the specified object. For example, a task runner can |
341
|
|
|
|
|
|
|
evaluate SQL queries stored in Amazon S3. |
342
|
|
|
|
|
|
|
|
343
|
|
|
|
|
|
|
|
344
|
|
|
|
|
|
|
=head2 GetPipelineDefinition(PipelineId => Str, [Version => Str]) |
345
|
|
|
|
|
|
|
|
346
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::GetPipelineDefinition> |
347
|
|
|
|
|
|
|
|
348
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::GetPipelineDefinitionOutput> instance |
349
|
|
|
|
|
|
|
|
350
|
|
|
|
|
|
|
Gets the definition of the specified pipeline. You can call |
351
|
|
|
|
|
|
|
C<GetPipelineDefinition> to retrieve the pipeline definition that you |
352
|
|
|
|
|
|
|
provided using PutPipelineDefinition. |
353
|
|
|
|
|
|
|
|
354
|
|
|
|
|
|
|
|
355
|
|
|
|
|
|
|
=head2 ListPipelines([Marker => Str]) |
356
|
|
|
|
|
|
|
|
357
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::ListPipelines> |
358
|
|
|
|
|
|
|
|
359
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::ListPipelinesOutput> instance |
360
|
|
|
|
|
|
|
|
361
|
|
|
|
|
|
|
Lists the pipeline identifiers for all active pipelines that you have |
362
|
|
|
|
|
|
|
permission to access. |
363
|
|
|
|
|
|
|
|
364
|
|
|
|
|
|
|
|
365
|
|
|
|
|
|
|
=head2 PollForTask(WorkerGroup => Str, [Hostname => Str, InstanceIdentity => L<Paws::DataPipeline::InstanceIdentity>]) |
366
|
|
|
|
|
|
|
|
367
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::PollForTask> |
368
|
|
|
|
|
|
|
|
369
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::PollForTaskOutput> instance |
370
|
|
|
|
|
|
|
|
371
|
|
|
|
|
|
|
Task runners call C<PollForTask> to receive a task to perform from AWS |
372
|
|
|
|
|
|
|
Data Pipeline. The task runner specifies which tasks it can perform by |
373
|
|
|
|
|
|
|
setting a value for the C<workerGroup> parameter. The task returned can |
374
|
|
|
|
|
|
|
come from any of the pipelines that match the C<workerGroup> value |
375
|
|
|
|
|
|
|
passed in by the task runner and that was launched using the IAM user |
376
|
|
|
|
|
|
|
credentials specified by the task runner. |
377
|
|
|
|
|
|
|
|
378
|
|
|
|
|
|
|
If tasks are ready in the work queue, C<PollForTask> returns a response |
379
|
|
|
|
|
|
|
immediately. If no tasks are available in the queue, C<PollForTask> |
380
|
|
|
|
|
|
|
uses long-polling and holds on to a poll connection for up to a 90 |
381
|
|
|
|
|
|
|
seconds, during which time the first newly scheduled task is handed to |
382
|
|
|
|
|
|
|
the task runner. To accomodate this, set the socket timeout in your |
383
|
|
|
|
|
|
|
task runner to 90 seconds. The task runner should not call |
384
|
|
|
|
|
|
|
C<PollForTask> again on the same C<workerGroup> until it receives a |
385
|
|
|
|
|
|
|
response, and this can take up to 90 seconds. |
386
|
|
|
|
|
|
|
|
387
|
|
|
|
|
|
|
|
388
|
|
|
|
|
|
|
=head2 PutPipelineDefinition(PipelineId => Str, PipelineObjects => ArrayRef[L<Paws::DataPipeline::PipelineObject>], [ParameterObjects => ArrayRef[L<Paws::DataPipeline::ParameterObject>], ParameterValues => ArrayRef[L<Paws::DataPipeline::ParameterValue>]]) |
389
|
|
|
|
|
|
|
|
390
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::PutPipelineDefinition> |
391
|
|
|
|
|
|
|
|
392
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::PutPipelineDefinitionOutput> instance |
393
|
|
|
|
|
|
|
|
394
|
|
|
|
|
|
|
Adds tasks, schedules, and preconditions to the specified pipeline. You |
395
|
|
|
|
|
|
|
can use C<PutPipelineDefinition> to populate a new pipeline. |
396
|
|
|
|
|
|
|
|
397
|
|
|
|
|
|
|
C<PutPipelineDefinition> also validates the configuration as it adds it |
398
|
|
|
|
|
|
|
to the pipeline. Changes to the pipeline are saved unless one of the |
399
|
|
|
|
|
|
|
following three validation errors exists in the pipeline. |
400
|
|
|
|
|
|
|
|
401
|
|
|
|
|
|
|
=over |
402
|
|
|
|
|
|
|
|
403
|
|
|
|
|
|
|
=item 1. An object is missing a name or identifier field. |
404
|
|
|
|
|
|
|
|
405
|
|
|
|
|
|
|
=item 2. A string or reference field is empty. |
406
|
|
|
|
|
|
|
|
407
|
|
|
|
|
|
|
=item 3. The number of objects in the pipeline exceeds the maximum |
408
|
|
|
|
|
|
|
allowed objects. |
409
|
|
|
|
|
|
|
|
410
|
|
|
|
|
|
|
=item 4. The pipeline is in a FINISHED state. |
411
|
|
|
|
|
|
|
|
412
|
|
|
|
|
|
|
=back |
413
|
|
|
|
|
|
|
|
414
|
|
|
|
|
|
|
Pipeline object definitions are passed to the C<PutPipelineDefinition> |
415
|
|
|
|
|
|
|
action and returned by the GetPipelineDefinition action. |
416
|
|
|
|
|
|
|
|
417
|
|
|
|
|
|
|
|
418
|
|
|
|
|
|
|
=head2 QueryObjects(PipelineId => Str, Sphere => Str, [Limit => Int, Marker => Str, Query => L<Paws::DataPipeline::Query>]) |
419
|
|
|
|
|
|
|
|
420
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::QueryObjects> |
421
|
|
|
|
|
|
|
|
422
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::QueryObjectsOutput> instance |
423
|
|
|
|
|
|
|
|
424
|
|
|
|
|
|
|
Queries the specified pipeline for the names of objects that match the |
425
|
|
|
|
|
|
|
specified set of conditions. |
426
|
|
|
|
|
|
|
|
427
|
|
|
|
|
|
|
|
428
|
|
|
|
|
|
|
=head2 RemoveTags(PipelineId => Str, TagKeys => ArrayRef[Str|Undef]) |
429
|
|
|
|
|
|
|
|
430
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::RemoveTags> |
431
|
|
|
|
|
|
|
|
432
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::RemoveTagsOutput> instance |
433
|
|
|
|
|
|
|
|
434
|
|
|
|
|
|
|
Removes existing tags from the specified pipeline. |
435
|
|
|
|
|
|
|
|
436
|
|
|
|
|
|
|
|
437
|
|
|
|
|
|
|
=head2 ReportTaskProgress(TaskId => Str, [Fields => ArrayRef[L<Paws::DataPipeline::Field>]]) |
438
|
|
|
|
|
|
|
|
439
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::ReportTaskProgress> |
440
|
|
|
|
|
|
|
|
441
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::ReportTaskProgressOutput> instance |
442
|
|
|
|
|
|
|
|
443
|
|
|
|
|
|
|
Task runners call C<ReportTaskProgress> when assigned a task to |
444
|
|
|
|
|
|
|
acknowledge that it has the task. If the web service does not receive |
445
|
|
|
|
|
|
|
this acknowledgement within 2 minutes, it assigns the task in a |
446
|
|
|
|
|
|
|
subsequent PollForTask call. After this initial acknowledgement, the |
447
|
|
|
|
|
|
|
task runner only needs to report progress every 15 minutes to maintain |
448
|
|
|
|
|
|
|
its ownership of the task. You can change this reporting time from 15 |
449
|
|
|
|
|
|
|
minutes by specifying a C<reportProgressTimeout> field in your |
450
|
|
|
|
|
|
|
pipeline. |
451
|
|
|
|
|
|
|
|
452
|
|
|
|
|
|
|
If a task runner does not report its status after 5 minutes, AWS Data |
453
|
|
|
|
|
|
|
Pipeline assumes that the task runner is unable to process the task and |
454
|
|
|
|
|
|
|
reassigns the task in a subsequent response to PollForTask. Task |
455
|
|
|
|
|
|
|
runners should call C<ReportTaskProgress> every 60 seconds. |
456
|
|
|
|
|
|
|
|
457
|
|
|
|
|
|
|
|
458
|
|
|
|
|
|
|
=head2 ReportTaskRunnerHeartbeat(TaskrunnerId => Str, [Hostname => Str, WorkerGroup => Str]) |
459
|
|
|
|
|
|
|
|
460
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::ReportTaskRunnerHeartbeat> |
461
|
|
|
|
|
|
|
|
462
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::ReportTaskRunnerHeartbeatOutput> instance |
463
|
|
|
|
|
|
|
|
464
|
|
|
|
|
|
|
Task runners call C<ReportTaskRunnerHeartbeat> every 15 minutes to |
465
|
|
|
|
|
|
|
indicate that they are operational. If the AWS Data Pipeline Task |
466
|
|
|
|
|
|
|
Runner is launched on a resource managed by AWS Data Pipeline, the web |
467
|
|
|
|
|
|
|
service can use this call to detect when the task runner application |
468
|
|
|
|
|
|
|
has failed and restart a new instance. |
469
|
|
|
|
|
|
|
|
470
|
|
|
|
|
|
|
|
471
|
|
|
|
|
|
|
=head2 SetStatus(ObjectIds => ArrayRef[Str|Undef], PipelineId => Str, Status => Str) |
472
|
|
|
|
|
|
|
|
473
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::SetStatus> |
474
|
|
|
|
|
|
|
|
475
|
|
|
|
|
|
|
Returns: nothing |
476
|
|
|
|
|
|
|
|
477
|
|
|
|
|
|
|
Requests that the status of the specified physical or logical pipeline |
478
|
|
|
|
|
|
|
objects be updated in the specified pipeline. This update might not |
479
|
|
|
|
|
|
|
occur immediately, but is eventually consistent. The status that can be |
480
|
|
|
|
|
|
|
set depends on the type of object (for example, DataNode or Activity). |
481
|
|
|
|
|
|
|
You cannot perform this operation on C<FINISHED> pipelines and |
482
|
|
|
|
|
|
|
attempting to do so returns C<InvalidRequestException>. |
483
|
|
|
|
|
|
|
|
484
|
|
|
|
|
|
|
|
485
|
|
|
|
|
|
|
=head2 SetTaskStatus(TaskId => Str, TaskStatus => Str, [ErrorId => Str, ErrorMessage => Str, ErrorStackTrace => Str]) |
486
|
|
|
|
|
|
|
|
487
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::SetTaskStatus> |
488
|
|
|
|
|
|
|
|
489
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::SetTaskStatusOutput> instance |
490
|
|
|
|
|
|
|
|
491
|
|
|
|
|
|
|
Task runners call C<SetTaskStatus> to notify AWS Data Pipeline that a |
492
|
|
|
|
|
|
|
task is completed and provide information about the final status. A |
493
|
|
|
|
|
|
|
task runner makes this call regardless of whether the task was |
494
|
|
|
|
|
|
|
sucessful. A task runner does not need to call C<SetTaskStatus> for |
495
|
|
|
|
|
|
|
tasks that are canceled by the web service during a call to |
496
|
|
|
|
|
|
|
ReportTaskProgress. |
497
|
|
|
|
|
|
|
|
498
|
|
|
|
|
|
|
|
499
|
|
|
|
|
|
|
=head2 ValidatePipelineDefinition(PipelineId => Str, PipelineObjects => ArrayRef[L<Paws::DataPipeline::PipelineObject>], [ParameterObjects => ArrayRef[L<Paws::DataPipeline::ParameterObject>], ParameterValues => ArrayRef[L<Paws::DataPipeline::ParameterValue>]]) |
500
|
|
|
|
|
|
|
|
501
|
|
|
|
|
|
|
Each argument is described in detail in: L<Paws::DataPipeline::ValidatePipelineDefinition> |
502
|
|
|
|
|
|
|
|
503
|
|
|
|
|
|
|
Returns: a L<Paws::DataPipeline::ValidatePipelineDefinitionOutput> instance |
504
|
|
|
|
|
|
|
|
505
|
|
|
|
|
|
|
Validates the specified pipeline definition to ensure that it is well |
506
|
|
|
|
|
|
|
formed and can be run without error. |
507
|
|
|
|
|
|
|
|
508
|
|
|
|
|
|
|
|
509
|
|
|
|
|
|
|
|
510
|
|
|
|
|
|
|
|
511
|
|
|
|
|
|
|
=head1 PAGINATORS |
512
|
|
|
|
|
|
|
|
513
|
|
|
|
|
|
|
Paginator methods are helpers that repetively call methods that return partial results |
514
|
|
|
|
|
|
|
|
515
|
|
|
|
|
|
|
=head2 DescribeAllObjects(sub { },ObjectIds => ArrayRef[Str|Undef], PipelineId => Str, [EvaluateExpressions => Bool, Marker => Str]) |
516
|
|
|
|
|
|
|
|
517
|
|
|
|
|
|
|
=head2 DescribeAllObjects(ObjectIds => ArrayRef[Str|Undef], PipelineId => Str, [EvaluateExpressions => Bool, Marker => Str]) |
518
|
|
|
|
|
|
|
|
519
|
|
|
|
|
|
|
|
520
|
|
|
|
|
|
|
If passed a sub as first parameter, it will call the sub for each element found in : |
521
|
|
|
|
|
|
|
|
522
|
|
|
|
|
|
|
- pipelineObjects, passing the object as the first parameter, and the string 'pipelineObjects' as the second parameter |
523
|
|
|
|
|
|
|
|
524
|
|
|
|
|
|
|
If not, it will return a a L<Paws::DataPipeline::DescribeObjectsOutput> instance with all the C<param>s; from all the responses. Please take into account that this mode can potentially consume vasts ammounts of memory. |
525
|
|
|
|
|
|
|
|
526
|
|
|
|
|
|
|
|
527
|
|
|
|
|
|
|
=head2 ListAllPipelines(sub { },[Marker => Str]) |
528
|
|
|
|
|
|
|
|
529
|
|
|
|
|
|
|
=head2 ListAllPipelines([Marker => Str]) |
530
|
|
|
|
|
|
|
|
531
|
|
|
|
|
|
|
|
532
|
|
|
|
|
|
|
If passed a sub as first parameter, it will call the sub for each element found in : |
533
|
|
|
|
|
|
|
|
534
|
|
|
|
|
|
|
- pipelineIdList, passing the object as the first parameter, and the string 'pipelineIdList' as the second parameter |
535
|
|
|
|
|
|
|
|
536
|
|
|
|
|
|
|
If not, it will return a a L<Paws::DataPipeline::ListPipelinesOutput> instance with all the C<param>s; from all the responses. Please take into account that this mode can potentially consume vasts ammounts of memory. |
537
|
|
|
|
|
|
|
|
538
|
|
|
|
|
|
|
|
539
|
|
|
|
|
|
|
=head2 QueryAllObjects(sub { },PipelineId => Str, Sphere => Str, [Limit => Int, Marker => Str, Query => L<Paws::DataPipeline::Query>]) |
540
|
|
|
|
|
|
|
|
541
|
|
|
|
|
|
|
=head2 QueryAllObjects(PipelineId => Str, Sphere => Str, [Limit => Int, Marker => Str, Query => L<Paws::DataPipeline::Query>]) |
542
|
|
|
|
|
|
|
|
543
|
|
|
|
|
|
|
|
544
|
|
|
|
|
|
|
If passed a sub as first parameter, it will call the sub for each element found in : |
545
|
|
|
|
|
|
|
|
546
|
|
|
|
|
|
|
- ids, passing the object as the first parameter, and the string 'ids' as the second parameter |
547
|
|
|
|
|
|
|
|
548
|
|
|
|
|
|
|
If not, it will return a a L<Paws::DataPipeline::QueryObjectsOutput> instance with all the C<param>s; from all the responses. Please take into account that this mode can potentially consume vasts ammounts of memory. |
549
|
|
|
|
|
|
|
|
550
|
|
|
|
|
|
|
|
551
|
|
|
|
|
|
|
|
552
|
|
|
|
|
|
|
|
553
|
|
|
|
|
|
|
|
554
|
|
|
|
|
|
|
=head1 SEE ALSO |
555
|
|
|
|
|
|
|
|
556
|
|
|
|
|
|
|
This service class forms part of L<Paws> |
557
|
|
|
|
|
|
|
|
558
|
|
|
|
|
|
|
=head1 BUGS and CONTRIBUTIONS |
559
|
|
|
|
|
|
|
|
560
|
|
|
|
|
|
|
The source code is located here: https://github.com/pplu/aws-sdk-perl |
561
|
|
|
|
|
|
|
|
562
|
|
|
|
|
|
|
Please report bugs to: https://github.com/pplu/aws-sdk-perl/issues |
563
|
|
|
|
|
|
|
|
564
|
|
|
|
|
|
|
=cut |
565
|
|
|
|
|
|
|
|