File Coverage

blib/lib/Python/File.pm
Criterion Covered Total %
statement 180 641 28.0
branch 43 224 19.2
condition 37 183 20.2
subroutine 27 30 90.0
pod 0 6 0.0
total 287 1084 26.4


line stmt bran cond sub pod time code
1             #
2             # This file is part of App-PythonToPerl
3             #
4             # This software is Copyright (c) 2023 by Auto-Parallel Technologies, Inc.
5             #
6             # This is free software, licensed under:
7             #
8             # The GNU General Public License, Version 3, June 2007
9             #
10             # [[[ HEADER ]]]
11             #use RPerl;
12             package Python::File;
13 1     1   251857 use strict;
  1         9  
  1         29  
14 1     1   5 use warnings;
  1         2  
  1         42  
15             our $VERSION = 0.022_000;
16              
17             # [[[ OO INHERITANCE ]]]
18 1     1   6 use parent qw(Python::Component);
  1         2  
  1         12  
19 1     1   77 use Python::Component;
  1         3  
  1         53  
20              
21             # [[[ CRITICS ]]]
22             ## no critic qw(ProhibitUselessNoCritic ProhibitMagicNumbers RequireCheckedSyscalls) # USER DEFAULT 1: allow numeric values & print op
23             ## no critic qw(RequireInterpolationOfMetachars) # USER DEFAULT 2: allow single-quoted control characters & sigils
24             ## no critic qw(ProhibitConstantPragma ProhibitMagicNumbers) # USER DEFAULT 3: allow constants
25              
26             # [[[ INCLUDES ]]]
27             # system includes
28 1     1   7 use Perl::Types;
  1         2  
  1         452  
29             #use re 'debugcolor'; # output regex debugging info
30 1     1   8 use File::Spec; # for splitpath() and catpath()
  1         5  
  1         34  
31 1     1   5 use Fcntl qw( :mode ); # for S_IXUSR etc; https://perldoc.perl.org/perlfunc#stat
  1         3  
  1         248  
32              
33             # internal includes
34 1     1   523 use Champ;
  1         5  
  1         62  
35 1     1   442 use Python::Shebang;
  1         5  
  1         30  
36 1     1   412 use Python::Blank;
  1         13  
  1         34  
37 1     1   406 use Python::Whitespace;
  1         2  
  1         35  
38 1     1   440 use Python::Comment;
  1         9  
  1         40  
39 1     1   428 use Python::CommentSingleQuotes;
  1         3  
  1         34  
40 1     1   440 use Python::CommentDoubleQuotes;
  1         14  
  1         33  
41 1     1   491 use Python::Include;
  1         3  
  1         37  
42 1     1   437 use Python::FunctionDecorator;
  1         2  
  1         38  
43 1     1   511 use Python::Function;
  1         3  
  1         33  
44 1     1   439 use Python::InnerFunction;
  1         2  
  1         35  
45 1     1   406 use Python::Method;
  1         2  
  1         39  
46 1     1   499 use Python::Class;
  1         2  
  1         39  
47 1     1   409 use Python::InnerClass;
  1         14  
  1         43  
48 1     1   413 use Python::LocalClass;
  1         3  
  1         44  
49 1     1   494 use Python::Unknown;
  1         3  
  1         31  
50              
51             # external (3rd party) includes
52 1     1   6 use OpenAI::API;
  1         3  
  1         10388  
53              
54             # [[[ ADDITIONAL CLASSES ]]]
55             package Python::File::arrayref; 1;
56             package Python::File::hashref; 1;
57              
58             package Python::File;
59              
60             # [[[ OO PROPERTIES ]]]
61             our hashref $properties = {
62             component_type => my string $TYPED_component_type = 'Python::File',
63             python_file_path => my string $TYPED_python_file_path = undef,
64             python_source_code => my string $TYPED_python_source_code = undef, # original source code, read from the Python input file
65             python_source_code_full => my string $TYPED_python_source_code_full = undef, # de-parsed source code, made from pre-parsed components
66             python_preparsed => my Python::Component::arrayref $TYPED_python_preparsed = undef, # source code, pre-parsed into Perl data structures
67             perl_file_path => my string $TYPED_perl_file_path = undef,
68             # perl_source_code => my string $TYPED_perl_source_code = undef, # in Python::File, perl_source_code is same as perl_source_code_full
69             perl_source_code_full => my string $TYPED_perl_source_code_full = undef,
70             };
71              
72             # [[[ SUBROUTINES & OO METHODS ]]]
73              
74             # PYFI05x
75             sub python_file_to_perl_file {
76             # translate Python file into Perl and save Perl file
77             # translate a file of Python source code into Perl, and save the new Perl file
78 0     0 0 0 { my string $RETURN_TYPE };
  0         0  
79 0         0 ( my Python::File $self,
80             my OpenAI::API $openai,
81             my Python::Include::hashref $python_includes,
82             my Python::Function::hashref $python_functions,
83             my Python::Class::hashref $python_classes
84             ) = @ARG;
85              
86             # error if no OpenAI API
87 0 0       0 if (not defined $openai) {
88 0         0 croak 'ERROR EPYFI050: undefined OpenAI API, croaking';
89             }
90              
91             # DEV NOTE, PYFI051: $self->{python_source_code} not used in this translation, no need to error check
92              
93             # PRE-PARSE PYTHON FILE
94             #print 'in python_file_to_perl_file(), about to call python_file_to_python_preparsed()...', "\n";
95             # DEV NOTE: capturing the return value below is purely optional,
96             # the pre-parsed data structures output will also be stored in the Python::File object's python_preparsed property
97             # my Python::Component::arrayref $python_preparsed =
98 0         0 $self->python_file_to_python_preparsed( $python_includes, $python_functions, $python_classes);
99             #print 'in python_file_to_perl_file(), ret from python_file_to_python_preparsed(), have $self = ', Dumper($self), "\n";
100             #print 'in python_file_to_perl_file(), ret from python_file_to_python_preparsed(), have $python_files = ', Dumper($python_files), "\n";
101 0         0 print 'in python_file_to_perl_file(), ret from python_file_to_python_preparsed(), have $self->{python_preparsed} = ', Dumper($self->{python_preparsed}), "\n";
102             #print 'in python_file_to_perl_file(), have $python_includes = ', Dumper($python_includes), "\n";
103             #print 'in python_file_to_perl_file(), have $python_functions = ', Dumper($python_functions), "\n";
104             #print 'in python_file_to_perl_file(), have $python_classes = ', Dumper($python_classes), "\n";
105              
106             # error or warning if no pre-parsed components
107 0 0 0     0 if ((not exists $self->{python_preparsed}) or
    0          
108             (not defined $self->{python_preparsed})) {
109 0         0 croak 'ERROR EPYFI052: non-existent or undefined Python pre-parsed components, croaking';
110             }
111 0         0 elsif ((scalar @{$self->{python_preparsed}}) == 0) {
112 0         0 carp 'WARNING WPYFI052: empty Python pre-parsed components, carping';
113             }
114              
115             # DE-PARSE PYTHON FILE BACK INTO PYTHON AKA ROUND-TRIPPING
116 0         0 print 'in python_file_to_perl_file(), about to call python_preparsed_to_python_source()...', "\n";
117             # DEV NOTE: capturing the return value below is purely optional,
118             # the source code output will also be stored in the Python::File object's python_source_code_full property
119             # my string $python_source_code =
120 0         0 $self->python_preparsed_to_python_source();
121 0         0 print 'in python_file_to_perl_file(), ret from python_preparsed_to_python_source(), have $self->{python_source_code_full} = ', "\n", $self->{python_source_code_full}, "\n";
122             #die 'TMP DEBUG, PYTHON ROUND-TRIPPING';
123              
124             # DE-PARSE & TRANSLATE PYTHON FILE INTO PERL SOURCE CODE
125 0         0 print 'in python_file_to_perl_file(), about to call python_preparsed_to_perl_source()...', "\n";
126             # DEV NOTE: capturing the return value below is purely optional,
127             # the source code output will also be stored in the Python::File object's perl_source_code_full property
128             # my string $perl_source_code =
129 0         0 $self->python_preparsed_to_perl_source($openai);
130 0         0 print 'in python_file_to_perl_file(), ret from python_preparsed_to_perl_source(), have $self->{perl_source_code_full} = ', "\n", $self->{perl_source_code_full}, "\n";
131              
132             # SAVE PERL FILE
133 0         0 print 'in python_file_to_perl_file(), about to call python_file_path_to_perl_file_path()...', "\n";
134             # DEV NOTE: capturing the return value below is purely optional,
135             # the file name output will also be stored in the Python::File object's perl_file_path property
136              
137             # generate Perl file path based on Python file path
138             # my string $perl_file_path =
139 0         0 $self->python_file_path_to_perl_file_path();
140 0         0 print 'in python_file_to_perl_file(), ret from python_file_path_to_perl_file_path(), have $self->{perl_file_path} = \'', $self->{perl_file_path}, '\'', "\n";
141              
142             # open new Perl source code file, write all contents to disk, close file
143 0         0 print 'in python_file_to_perl_file(), about to save Perl file...', "\n";
144             open (my filehandleref $PERL_FILE, '>', $self->{perl_file_path})
145             or croak 'ERROR EPYFI05x: failed to open Perl source code file \'', $self->{perl_file_path},
146 0 0       0 '\' for writing, received OS error message \'', $OS_ERROR, '\', croaking'; # DEV NOTE: $OS_ERROR == $!
147 0         0 print {$PERL_FILE} $self->{perl_source_code_full};
  0         0  
148             close($PERL_FILE)
149             or croak 'ERROR EPYFI05y: failed to close Perl source code file \'', $self->{perl_file_path},
150 0 0       0 '\' after writing, received OS error message \'', $OS_ERROR, '\', croaking';
151 0         0 print 'in python_file_to_perl_file(), ret from calls to save Perl file', "\n";
152              
153             # set file executability permission based on presence of shebang;
154             # an executable script has a shebang, a non-executable module AKA library does not have a shebang
155 0 0 0     0 if((defined $self->{python_preparsed}->[0]) and ($self->{python_preparsed}->[0]->isa('Python::Shebang'))) {
156 0         0 print 'in python_file_to_perl_file(), setting executable permission of Perl file', "\n";
157             chmod(S_IXUSR, $self->{perl_file_path})
158             or croak 'ERROR EPYFI05z: failed to chmod u+x Perl source code file \'', $self->{perl_file_path},
159 0 0       0 '\' after closing, received OS error message \'', $OS_ERROR, '\', croaking';
160             }
161              
162 0         0 print 'in python_file_to_perl_file(), about to return $self->{perl_file_path} = \'', $self->{perl_file_path}, '\'', "\n";
163 0         0 return $self->{perl_file_path};
164             }
165              
166              
167             # PYFI04x
168             sub python_file_path_to_perl_file_path {
169             # convert Python file path to Perl file path
170 0     0 0 0 { my string $RETURN_TYPE };
  0         0  
171 0         0 ( my Python::File $self ) = @ARG;
172              
173             # DEV NOTE, PYFI040: $openai not used in this subroutine, no need to error check
174             # DEV NOTE, PYFI041: $self->{python_source_code} not used in this subroutine, no need to error check
175              
176             # error or warning if no pre-parsed components;
177             # must have already pre-parsed to check for shebang
178 0 0 0     0 if ((not exists $self->{python_preparsed}) or
    0          
179             (not defined $self->{python_preparsed})) {
180             # croak 'ERROR EPYFI042a: non-existent or undefined Python pre-parsed components, croaking';
181 0         0 carp 'WARNING WPYFI042a: non-existent or undefined Python pre-parsed components, carping';
182             }
183 0         0 elsif ((scalar @{$self->{python_preparsed}}) == 0) {
184 0         0 carp 'WARNING WPYFI042b: empty Python pre-parsed components, carping';
185             }
186              
187             # error if no file path
188 0 0 0     0 if ((not exists $self->{python_file_path}) or
      0        
189             (not defined $self->{python_file_path}) or
190             ($self->{python_file_path} eq '')) {
191 0         0 croak 'ERROR EPYFI043: non-existent or undefined or empty Python file path, croaking';
192             }
193              
194             # split file path into volume (ignored in Unix operating systems), directory, and file name
195 0         0 my string $python_volume;
196             my string $python_directory;
197 0         0 my string $python_file_name;
198 0         0 ($python_volume, $python_directory, $python_file_name) = File::Spec->splitpath($self->{python_file_path});
199              
200             # ensure file name was split correctly
201 0 0 0     0 if ((not defined $python_file_name) or
202             ($python_file_name eq '')) {
203 0         0 croak 'ERROR EPYFI044: undefined or empty Python file name, croaking';
204             }
205              
206             # always base Perl file name on original Python file name
207 0         0 my string $perl_file_name = $python_file_name;
208              
209             # set file suffix based on presence of shebang;
210             # an executable script has a shebang, a non-executable module AKA library does not have a shebang
211 0         0 my string $perl_file_suffix = 'pl';
212              
213              
214              
215             # NEED FIX: re-enable Perl file suffix selection below
216             # NEED FIX: re-enable Perl file suffix selection below
217             # NEED FIX: re-enable Perl file suffix selection below
218              
219             # if((defined $self->{python_preparsed}) and
220             # (defined $self->{python_preparsed}->[0]) and
221             # ($self->{python_preparsed}->[0]->isa('Python::Shebang'))) {
222             # $perl_file_suffix = 'pl';
223             # }
224             # else {
225             # $perl_file_suffix = 'pm';
226             # }
227              
228              
229              
230             # for simple 'FOO.py' file names, simply replace with 'FOO.p(l|m)'
231 0 0       0 if ($python_file_name =~ m/\.py$/) {
    0          
232 0         0 substr $perl_file_name, -2, 2, $perl_file_suffix;
233             }
234             # for various 'FOO.pyX' file names, emit warning and replace with 'FOO.pyX.p(l|m)'
235             elsif ($python_file_name =~ m/\.py[0-9a-zA-Z]*$/) {
236              
237             # NEED UPGRADE: handle Pyrex 'FOO.pyx' file names
238             # NEED UPGRADE: handle Pyrex 'FOO.pyx' file names
239             # NEED UPGRADE: handle Pyrex 'FOO.pyx' file names
240              
241             # $perl_file_name =~ s/\.py[0-9a-zA-Z]*$/\.pl/gmsx; # rename FOO.pyX to FOO.pl
242 0         0 $perl_file_name = $python_file_name . '.' . $perl_file_suffix; # rename FOO.pyX to FOO.pyX.p(l|m)
243 0         0 carp 'WARNING WPYFI045a: received special Python file name \'', $python_file_name, '\', renaming to standard Perl file name \'', $perl_file_name, '\', carping';
244             }
245             # for unrecognized 'FOO.BAR' file names, emit warning and replace with 'FOO.BAR.p(l|m)'
246             else {
247 0         0 $perl_file_name = $python_file_name . '.' . $perl_file_suffix;
248 0         0 carp 'WARNING WPYFI045b: received unrecognized file name \'', $python_file_name, '\', renaming to appended Perl file name \'', $perl_file_name, '\', carping';
249             }
250              
251             # NEED FIX: accept and utilize Perl directory
252 0         0 $self->{perl_file_path} = File::Spec->catpath($python_volume, $python_directory, $perl_file_name );
253 0         0 print 'in python_file_path_to_perl_file_path(), about to return $self->{perl_file_path} = \'', $self->{perl_file_path}, '\'', "\n";
254              
255 0         0 return $self->{perl_file_path};
256             }
257              
258              
259             # PYFI03x
260             sub python_preparsed_to_perl_source {
261             # return Perl source code
262 0     0 0 0 { my string $RETURN_TYPE };
  0         0  
263 0         0 ( my Python::File $self, my OpenAI::API $openai ) = @ARG;
264              
265             # error if no OpenAI API
266 0 0       0 if (not defined $openai) {
267 0         0 croak 'ERROR EPYFI030: undefined OpenAI API, croaking';
268             }
269              
270             # DEV NOTE, PYFI031: $self->{python_source_code} not used in this translation, no need to error check
271              
272             # error or warning if no pre-parsed components
273 0 0 0     0 if ((not exists $self->{python_preparsed}) or
    0          
274             (not defined $self->{python_preparsed})) {
275 0         0 croak 'ERROR EPYFI032: non-existent or undefined Python pre-parsed components, croaking';
276             }
277 0         0 elsif ((scalar @{$self->{python_preparsed}}) == 0) {
278 0         0 carp 'WARNING WPYFI032: empty Python pre-parsed components, carping';
279             }
280              
281             # initialize property that will store de-parsed & translated source code;
282             # save fully translated Perl source code, to avoid repeated translating
283 0         0 $self->{perl_source_code_full} = '';
284              
285             # de-parse & translate file contents
286 0         0 foreach my Python::Component $python_preparsed_component (@{$self->{python_preparsed}}) {
  0         0  
287             #print 'in Python::File->python_preparsed_to_perl_source(), file \'', $self->{python_file_path}, '\', de-parsing & translating file contents, about to call python_preparsed_to_perl_source()...', "\n";
288 0         0 $self->{perl_source_code_full} .= $python_preparsed_component->python_preparsed_to_perl_source($openai) . "\n";
289             #print 'in Python::File->python_preparsed_to_perl_source(), file \'', $self->{python_file_path}, '\', de-parsing & translating file contents, ret from call python_preparsed_to_perl_source()', "\n";
290             }
291              
292             # remove extra trailing newline, to match original input source code
293 0         0 chomp $self->{perl_source_code_full};
294              
295             # return de-parsed & translated Perl source code
296 0         0 return $self->{perl_source_code_full};
297             }
298              
299              
300             # PYFI02x
301             sub python_preparsed_to_python_source {
302             # return Python source code
303 1     1 0 75144 { my string $RETURN_TYPE };
  1         3  
304 1         3 ( my Python::File $self ) = @ARG;
305              
306             # DEV NOTE, PYFI020: $openai not used in Python-to-Python de-parse round-tripping, no need to error check
307              
308             # DEV NOTE: in Python::File objects, the python_source_code property contains the original source code read from the Python input file,
309             # not the pre-parsed Python source code which can be found in the python_source_code property of other Python::Component classes,
310             # so the python_source_code property is not used as part of the de-parse process and thus does not need to be error-checked here
311             # if ((not exists $self->{python_source_code}) or
312             # (not defined $self->{python_source_code})) {
313             # croak 'ERROR EPYFI021: non-existent or undefined Python source code, croaking';
314             # }
315             # elsif ($self->{python_source_code} eq '') {
316             # carp 'WARNING WPYFI021: empty Python source code, carping';
317             # }
318              
319             # error or warning if no pre-parsed components
320 1 50 33     13 if ((not exists $self->{python_preparsed}) or
    50          
321             (not defined $self->{python_preparsed})) {
322 0         0 croak 'ERROR EPYFI022: non-existent or undefined Python pre-parsed components, croaking';
323             }
324 1         6 elsif ((scalar @{$self->{python_preparsed}}) == 0) {
325 0         0 carp 'WARNING WPYFI022: empty Python pre-parsed components, carping';
326             }
327              
328             # initialize property that will store de-parsed source code;
329             # save fully de-parsed Python source code, to avoid repeated de-parsing
330 1         3 $self->{python_source_code_full} = '';
331              
332             # de-parse file contents
333 1         3 foreach my Python::Component $python_preparsed_component (@{$self->{python_preparsed}}) {
  1         2  
334             #print 'in Python::File->python_preparsed_to_python_source(), file \'', $self->{python_file_path}, '\', de-parsing file contents, about to call python_preparsed_to_python_source()...', "\n";
335 63         126 $self->{python_source_code_full} .= $python_preparsed_component->python_preparsed_to_python_source() . "\n";
336             #print 'in Python::File->python_preparsed_to_python_source(), file \'', $self->{python_file_path}, '\', de-parsing file contents, ret from call python_preparsed_to_python_source()', "\n";
337             }
338              
339             # remove extra trailing newline, to match original input source code
340 1         20 chomp $self->{python_source_code_full};
341              
342             # return de-parsed Python source code
343 1         11 return $self->{python_source_code_full};
344             }
345              
346              
347             # PYFI01x
348             sub python_last_active_character_find {
349             # select last active (non-comment, non-whitespace) character
350 105     105 0 164 { my character $RETURN_TYPE };
  105         156  
351 105         253 ( my Python::File $self,
352             my character $python_last_active_character,
353             my string $python_last_active_characters,
354             ) = @ARG;
355 105 100       672 print 'in python_last_active_character_find(), received $python_last_active_character = \'', ((defined $python_last_active_character) ? $python_last_active_character : '<undef>'), '\'', "\n";
356 105         679 print 'in python_last_active_character_find(), received $python_last_active_characters = \'', $python_last_active_characters, '\'', "\n";
357              
358             # match last non-whitespace (\S) non-comment (\#.*) character in the string
359 105         845 $python_last_active_characters =~ m/(\S)\s*(\#.*)?$/;
360              
361 105 50 33     479 if ((defined $1) and ($1 ne q{})) {
362 105         206 $python_last_active_character = $1;
363 105         636 print 'in python_last_active_character_find(), YES actually updating $python_last_active_character = \'', $python_last_active_character, '\'', "\n";
364             }
365             else {
366 0         0 print 'in python_last_active_character_find(), NOT actually updating $python_last_active_character = \'', $python_last_active_character, '\'', "\n";
367 0         0 carp 'WARNING WPYFI010: no active character found in source code where one was expected, carping';
368             }
369              
370 105         677 print 'in python_last_active_character_find(), about to return $python_last_active_character = \'', $python_last_active_character, '\'', "\n";
371 105         370 return $python_last_active_character;
372             }
373              
374              
375             # PYFI00x
376             sub python_file_to_python_preparsed {
377             # pre-parse a file of Python source code into Perl data structures
378 1     1 0 6761 { my Python::Component::arrayref $RETURN_TYPE };
  1         2  
379 1         4 ( my Python::File $self,
380             my Python::Include::hashref $python_includes,
381             my Python::Function::hashref $python_functions,
382             my Python::Class::hashref $python_classes
383             ) = @ARG;
384 1         14 print 'in python_file_to_python_preparsed(), received $self->{python_file_path} = \'', $self->{python_file_path}, '\'', "\n";
385              
386             # open Python source code file for reading
387             open (my filehandleref $PYTHON_FILE, '<', $self->{python_file_path})
388             or croak 'ERROR EPYFI000a: failed to open Python source code file \'', $self->{python_file_path},
389 1 50       53 '\' for reading, received OS error message \'', $OS_ERROR, '\', croaking'; # DEV NOTE: $OS_ERROR == $!
390              
391             # initialize primary python_preparsed data structure to empty array, will be populated with Python::Component objects
392 1         4 $self->{python_preparsed} = [];
393              
394             # initialize property that will store input source code
395 1         3 $self->{python_source_code} = '';
396              
397             # the changing target for where each new pre-parsed component should be stored, based on the nesting of namespaces;
398             # default value is the primary python_preparsed data structure
399 1         3 my hashref::arrayref $python_preparsed_target = $self->{python_preparsed};
400              
401             # initialize variables used to track components of this Python source code file
402 1         2 my hashref $python_component = undef; # Python component represented as Perl hash data structure, to be blessed into Perl class
403 1         1 my integer $python_line_number = 0; # current line number
404 1         2 my string $python_namespace_name = undef; # fully scoped name of namespace (function or class) currently being parsed
405 1         2 my Python::Component::arrayref $python_namespaces = []; # namespace stack AKA the scope, all namespaces enclosing current line of code
406 1         3 my character $python_last_active_character = undef; # the last non-whitespace non-comment character, for look-back
407              
408             # loop through each line of the Python source code file;
409             # read all file contents into variable in memory and check for includes, functions, and classes;
410 1         52 while (<$PYTHON_FILE>) {
411 120         281 $self->{python_source_code} .= $ARG; # DEV NOTE: $ARG == $_
412 120         161 $python_line_number++;
413 120         320 print 'in python_file_to_python_preparsed(), input line #', $python_line_number, ', have champ($ARG) = \'', champ($ARG), '\'', "\n";
414 120 100       1044 print 'in python_file_to_python_preparsed(), input line #', $python_line_number, ', have $python_last_active_character = \'', ((defined $python_last_active_character) ? $python_last_active_character : '<undef>'), '\'', "\n";
415              
416             # check for shebang on first line only
417 120 50 66     368 if (($python_line_number == 1) and ($ARG =~ m/^#!.*python.*$/)) {
418 0         0 print 'in python_file_to_python_preparsed(), Python shebang detected', "\n";
419             # create new component
420 0         0 push @{$python_preparsed_target},
  0         0  
421             Python::Shebang->new(
422             {
423             component_type => 'Python::Shebang',
424             python_line_number_begin => $python_line_number,
425             python_line_number_end => $python_line_number,
426             python_source_code => $ARG,
427             });
428 0         0 next;
429             }
430              
431             # pre-parse & skip everything inside multi-line '''comment''', except for closing quotes
432 120 50 66     141 if (((scalar @{$python_preparsed_target}) > 0) and
  120 50 33     832  
    100 66        
    50 33        
    50 100        
      100        
      66        
      33        
      66        
      33        
433             $python_preparsed_target->[-1]->isa('Python::CommentSingleQuotes') and
434             ($python_preparsed_target->[-1]->{python_line_number_end} < 0)) {
435 0         0 print 'in python_file_to_python_preparsed(), inside multi-line single-quotes \'\'\'comment\'\'\'', "\n";
436             # accumulate current (possibly last) Python line of multi-line component
437 0         0 chomp $ARG;
438 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
439              
440 0 0       0 if ($ARG =~ m/^(.*)\'\'\'\s*$/) {
    0          
    0          
441             # accumulate last Perl line of multi-line component;
442             # comments are translated during pre-parse phase, save translated Perl source code component after Python component ends;
443              
444 0 0       0 if ($python_preparsed_target->[-1]->{is_actually_string_literal}) {
445 0         0 print 'in python_file_to_python_preparsed(), ending multi-line single-quotes \'\'\'string literal\'\'\'', "\n";
446              
447             # convert Comment object to Unknown object, because this is actually a string literal not a comment
448              
449             # DEV NOTE, CORRELATION PYFI102: all Unknown logic in this if-elsif-else block must be copied
450             # check if previous component was Unknown
451 0 0 0     0 if (((scalar @{$python_preparsed_target}) > 1) and
  0 0 0     0  
      0        
452             $python_preparsed_target->[-2]->isa('Python::Unknown')) {
453 0         0 print 'in python_file_to_python_preparsed(), have previous UNKNOWN line, accumulating', "\n";
454             # accumulate multiple single-line components into a multi-line component
455             $python_preparsed_target->[-2]->{python_source_code} .=
456 0         0 "\n" . $python_preparsed_target->[-1]->{python_source_code};
457             # update ending line number
458 0         0 $python_preparsed_target->[-2]->{python_line_number_end} = $python_line_number;
459             # discard now-redundant String Literal components
460 0         0 pop @{$python_preparsed_target};
  0         0  
461             }
462 0         0 elsif (((scalar @{$python_preparsed_target}) > 2) and
463             $python_preparsed_target->[-2]->isa('Python::Blank') and
464             $python_preparsed_target->[-3]->isa('Python::Unknown')) {
465 0         0 print 'in python_file_to_python_preparsed(), have previous UNKNOWN line preceded by blank line(s) and other UNKNOWN line(s), merging components', "\n";
466              
467             # merge 3 components into a single component;
468             # Unknown + Blank + String Literal = Unknown
469             $python_preparsed_target->[-3]->{python_source_code} .=
470             "\n" . $python_preparsed_target->[-2]->{python_source_code} .
471 0         0 "\n" . $python_preparsed_target->[-1]->{python_source_code};
472             # update ending line number
473 0         0 $python_preparsed_target->[-3]->{python_line_number_end} = $python_line_number;
474             # discard now-redundant Blank and String Literal components
475 0         0 pop @{$python_preparsed_target};
  0         0  
476 0         0 pop @{$python_preparsed_target};
  0         0  
477             }
478             else {
479 0         0 print 'in python_file_to_python_preparsed(), have previous UNKNOWN line, converting', "\n";
480             # discard now-redundant String Literal component,
481             # saving for just long enough to create new Unknown component
482 0         0 my Python::Comment $discarded_string_literal = pop @{$python_preparsed_target};
  0         0  
483             # create new component
484 0         0 push @{$python_preparsed_target},
485             Python::Unknown->new(
486             {
487             component_type => 'Python::Unknown',
488             python_line_number_begin => $discarded_string_literal->{python_line_number_begin},
489             python_line_number_end => $python_line_number,
490             python_source_code => $discarded_string_literal->{python_source_code},
491 0         0 perl_source_code => undef # unknown code is not translated during pre-parse phase
492             });
493             }
494              
495 0         0 print 'in python_file_to_python_preparsed(), ending multi-line single-quotes \'\'\'string literal\'\'\'', "\n";
496             #die 'TMP DEBUG, END MULTI-LINE STRING LITERAL SINGLE QUOTES';
497             }
498             else {
499 0         0 print 'in python_file_to_python_preparsed(), ending multi-line single-quotes \'\'\'comment\'\'\'', "\n";
500             # if ending line contains leading characters (including whitespace),
501             # then append/prepend additional '#' char to ensure leading characters are maintained as comments
502 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n";
503             # indentation whitespace only before closing ''', do NOT prepend additional '#' character
504 0 0       0 if ($ARG =~ m/^(\s+)\'\'\'\s*$/) {
    0          
    0          
505 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= $1;
506             }
507             # indentation whitespace and other characters before closing ''',
508             # append additional '#' character to indentation
509             elsif ($ARG =~ m/^(\s+)(.+)\'\'\'\s*$/) {
510 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= (substr $1, 0, -1) . '#' . $2;
511             }
512             # non-whitespace characters before closing ''',
513             # prepend additional '#' character by shifting all characters to the right
514             elsif ($ARG =~ m/^(.+)\'\'\'\s*$/) {
515 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= '#' . $1;
516             }
517             # else, no characters at all before closing '''
518 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= q{#''};
519              
520             # set ending line number, indicating we are no longer inside this multi-line component
521 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number;
522              
523 0         0 print 'in python_file_to_python_preparsed(), ending multi-line single-quotes \'\'\'comment\'\'\', have $python_preparsed_target->[-1]->{perl_source_code} = ', "\n", $python_preparsed_target->[-1]->{perl_source_code}, "\n";
524             #die 'TMP DEBUG, END MULTI-LINE COMMENT SINGLE QUOTES';
525             }
526              
527 0         0 next;
528             }
529             elsif ($ARG =~ m/\'\'\'/) {
530 0         0 croak 'ERROR EPYFI001: have multi-line single-quotes comment closing, but not at end of line, do not know how to handle, croaking';
531             }
532             elsif ($ARG =~ m/\"\"\"/) {
533 0         0 carp 'WARNING WPYFI001: have multi-line double-quotes comment while currently inside multi-line single-quotes comment, ignoring, carping';
534             }
535              
536             # prepend '#' character for non-blank comments,
537             # either replacing indentation space or shifting all characters to the right
538 0         0 my string $comment = $ARG;
539 0 0 0     0 if (($comment eq '') or
    0          
540             ($python_preparsed_target->[-1]->{is_actually_string_literal}))
541 0         0 { 1; }
542             elsif ($comment =~ m/^(\s+)(.*)$/) {
543             # if indented at least 2 spaces, then we can vertically align all '#' characters
544 0 0 0     0 if (((length $python_preparsed_target->[-1]->{indentation}) >= 2) and
545             ((length $1) >= 2)) {
546 0         0 substr $comment, ((length $python_preparsed_target->[-1]->{indentation}) - 2), 1, '#';
547             }
548 0         0 else { $comment = (substr $1, 0, -2) . '# ' . $2; }
549             }
550 0         0 else { $comment = '#' . $comment; }
551              
552             # accumulate non-last Perl line of multi-line component; copy comments verbatim
553 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n" . $comment;
554              
555             # did not end multi-line component, go on to next line
556 0         0 next;
557             }
558             # pre-parse & skip everything inside multi-line """comment""", except for closing quotes
559 120         579 elsif (((scalar @{$python_preparsed_target}) > 0) and
560             $python_preparsed_target->[-1]->isa('Python::CommentDoubleQuotes') and
561             ($python_preparsed_target->[-1]->{python_line_number_end} < 0)) {
562 0         0 print 'in python_file_to_python_preparsed(), inside multi-line double-quotes \"\"\"comment\"\"\"', "\n";
563             # accumulate current (possibly last) Python line of multi-line component
564 0         0 chomp $ARG;
565 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
566              
567 0 0       0 if ($ARG =~ m/^(.*)\"\"\"\s*$/) {
    0          
    0          
568 0         0 print 'in python_file_to_python_preparsed(), ending multi-line double-quotes \"\"\"comment\"\"\"', "\n";
569             # accumulate last Perl line of multi-line component;
570             # comments are translated during pre-parse phase, save translated Perl source code component after Python component ends;
571              
572              
573              
574 0 0       0 if ($python_preparsed_target->[-1]->{is_actually_string_literal}) {
575 0         0 print 'in python_file_to_python_preparsed(), ending multi-line double-quotes \"\"\"string literal\"\"\"', "\n";
576              
577             # convert Comment object to Unknown object, because this is actually a string literal not a comment
578              
579             # DEV NOTE, CORRELATION PYFI102: all Unknown logic in this if-elsif-else block must be copied
580             # check if previous component was Unknown
581 0 0 0     0 if (((scalar @{$python_preparsed_target}) > 1) and
  0 0 0     0  
      0        
582             $python_preparsed_target->[-2]->isa('Python::Unknown')) {
583 0         0 print 'in python_file_to_python_preparsed(), have previous UNKNOWN line, accumulating', "\n";
584             # accumulate multiple single-line components into a multi-line component
585             $python_preparsed_target->[-2]->{python_source_code} .=
586 0         0 "\n" . $python_preparsed_target->[-1]->{python_source_code};
587             # update ending line number
588 0         0 $python_preparsed_target->[-2]->{python_line_number_end} = $python_line_number;
589             # discard now-redundant String Literal components
590 0         0 pop @{$python_preparsed_target};
  0         0  
591             }
592 0         0 elsif (((scalar @{$python_preparsed_target}) > 2) and
593             $python_preparsed_target->[-2]->isa('Python::Blank') and
594             $python_preparsed_target->[-3]->isa('Python::Unknown')) {
595 0         0 print 'in python_file_to_python_preparsed(), have previous UNKNOWN line preceded by blank line(s) and other UNKNOWN line(s), merging components', "\n";
596              
597             # merge 3 components into a single component;
598             # Unknown + Blank + String Literal = Unknown
599             $python_preparsed_target->[-3]->{python_source_code} .=
600             "\n" . $python_preparsed_target->[-2]->{python_source_code} .
601 0         0 "\n" . $python_preparsed_target->[-1]->{python_source_code};
602             # update ending line number
603 0         0 $python_preparsed_target->[-3]->{python_line_number_end} = $python_line_number;
604             # discard now-redundant Blank and String Literal components
605 0         0 pop @{$python_preparsed_target};
  0         0  
606 0         0 pop @{$python_preparsed_target};
  0         0  
607             }
608             else {
609 0         0 print 'in python_file_to_python_preparsed(), have previous UNKNOWN line, converting', "\n";
610             # discard now-redundant String Literal component,
611             # saving for just long enough to create new Unknown component
612 0         0 my Python::Comment $discarded_string_literal = pop @{$python_preparsed_target};
  0         0  
613             # create new component
614 0         0 push @{$python_preparsed_target},
615             Python::Unknown->new(
616             {
617             component_type => 'Python::Unknown',
618             python_line_number_begin => $discarded_string_literal->{python_line_number_begin},
619             python_line_number_end => $python_line_number,
620             python_source_code => $discarded_string_literal->{python_source_code},
621 0         0 perl_source_code => undef # unknown code is not translated during pre-parse phase
622             });
623             }
624              
625 0         0 print 'in python_file_to_python_preparsed(), ending multi-line double-quotes \"\"\"string literal\"\"\"', "\n";
626             #die 'TMP DEBUG, END MULTI-LINE STRING LITERAL SINGLE QUOTES';
627             }
628             else {
629 0         0 print 'in python_file_to_python_preparsed(), ending multi-line double-quotes \"\"\"comment\"\"\"', "\n";
630             # if ending line contains leading characters (including whitespace),
631             # then append/prepend additional '#' char to ensure leading characters are maintained as comments
632 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n";
633             # indentation whitespace only before closing """, do NOT prepend additional '#' character
634 0 0       0 if ($ARG =~ m/^(\s+)\"\"\"\s*$/) {
    0          
    0          
635 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= $1;
636             }
637             # indentation whitespace and other characters before closing """, append additional '#' character to indentation
638             elsif ($ARG =~ m/^(\s+)(.+)\"\"\"\s*$/) {
639 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= (substr $1, 0, -1) . '#' . $2;
640             }
641             # non-whitespace characters before closing """, prepend additional '#' character by shifting all characters to the right
642             elsif ($ARG =~ m/^(.+)\"\"\"\s*$/) {
643 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= '#' . $1;
644             }
645             # else, no characters at all before closing """
646 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= q{#""};
647              
648             # set ending line number, indicating we are no longer inside this multi-line component
649 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number;
650              
651 0         0 print 'in python_file_to_python_preparsed(), ending multi-line double-quotes \"\"\"comment\"\"\", have $python_preparsed_target->[-1]->{perl_source_code} = ', "\n", $python_preparsed_target->[-1]->{perl_source_code}, "\n";
652             #die 'TMP DEBUG, MULTI-LINE COMMENT DOUBLE QUOTES';
653             }
654              
655 0         0 next;
656             }
657             elsif ($ARG =~ m/\"\"\"/) {
658 0         0 croak 'ERROR EPYFI002: have multi-line double-quotes comment closing, but not at end of line, do not know how to handle, croaking';
659             }
660             elsif ($ARG =~ m/\'\'\'/) {
661 0         0 carp 'WARNING WPYFI002: have multi-line single-quotes comment while currently inside multi-line double-quotes comment, ignoring, carping';
662             }
663              
664             # prepend '#' character for non-blank comments,
665             # either replacing last indentation space or shifting all characters to the right
666 0         0 my string $comment = $ARG;
667 0 0 0     0 if (($comment eq '') or
    0          
668             ($python_preparsed_target->[-1]->{is_actually_string_literal}))
669 0         0 { 1; }
670             elsif ($comment =~ m/^(\s+)(.*)$/) {
671             # if indented at least 2 spaces, then we can vertically align all '#' characters
672 0 0 0     0 if (((length $python_preparsed_target->[-1]->{indentation}) >= 2) and
673             ((length $1) >= 2)) {
674 0         0 substr $comment, ((length $python_preparsed_target->[-1]->{indentation}) - 2), 1, '#';
675             }
676 0         0 else { $comment = (substr $1, 0, -2) . '# ' . $2; }
677             }
678 0         0 else { $comment = '#' . $comment; }
679              
680             # accumulate non-last Perl line of multi-line component; copy comments verbatim
681 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n" . $comment;
682              
683             # did not end multi-line component, go on to next line
684 0         0 next;
685             }
686             # pre-parse & accumulate everything inside multi-line include statement
687 120         598 elsif (((scalar @{$python_preparsed_target}) > 0) and
688             $python_preparsed_target->[-1]->isa('Python::Include') and
689             ($python_preparsed_target->[-1]->{python_line_number_end} < 0)) {
690 57         383 print 'in python_file_to_python_preparsed(), inside multi-line include', "\n";
691             # accumulate current (possibly last) Python line of multi-line component
692 57         144 chomp $ARG;
693 57         170 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
694              
695             # update last active character
696 57         142 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
697 57         343 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
698              
699             # multi-line includes with parentheses end differently than those without parentheses
700 57 100       172 if ($python_preparsed_target->[-1]->{python_has_parentheses}) {
701             # end multi-line include (w/ parentheses) when the last non-whitespace non-comment character is a close parentheses
702 20 100       108 if ($ARG =~ m/^.*\)\s*(?:\#.*)?$/) {
703 5         30 print 'in python_file_to_python_preparsed(), ending multi-line include w/ parentheses', "\n";
704             # set ending line number, indicating we are no longer inside this multi-line component
705 5         16 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number;
706              
707 5         21 next;
708             }
709             }
710             else {
711             # end multi-line include (w/out parentheses) when the last non-whitespace character is not a backslash
712             # if ($ARG =~ m/^.*[^\\]\s*$/) { # does not match correctly?
713 37 100       155 if ($ARG !~ m/^.*\\\s*$/) {
714 11         63 print 'in python_file_to_python_preparsed(), ending multi-line include w/out parentheses', "\n";
715             # set ending line number, indicating we are no longer inside this multi-line component
716 11         27 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number;
717              
718 11         53 next;
719             }
720             }
721              
722             # error if multi-line component invalidly nested inside other multi-line component
723 41 50       116 if ($ARG =~ m/\'\'\'/) {
    50          
724 0         0 croak 'ERROR EPYFI003a: have multi-line single-quotes comment while currently inside multi-line include statement, do not know how to handle, croaking';
725             }
726             elsif ($ARG =~ m/\"\"\"/) {
727 0         0 croak 'ERROR EPYFI003b: have multi-line double-quotes comment while currently inside multi-line include statement, do not know how to handle, croaking';
728             }
729              
730             # did not end multi-line component, go on to next line
731 41         166 next;
732             }
733             # pre-parse & accumulate everything inside multi-line function header
734 63         289 elsif (((scalar @{$python_preparsed_target}) > 0) and
735             $python_preparsed_target->[-1]->isa('Python::Function') and
736             ($python_preparsed_target->[-1]->{python_line_number_end_header} < 0)) {
737 0         0 print 'in python_file_to_python_preparsed(), inside multi-line function header', "\n";
738             # accumulate current (possibly last) Python line of multi-line component
739 0         0 chomp $ARG;
740 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
741              
742             # update last active character
743 0         0 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
744 0         0 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
745              
746             # end multi-line function header when it matches the entire regex;
747              
748             # DEV NOTE, CORRELATION PYFI100: all regex changes must be reflected in both locations,
749             # the only difference should be the optional trailing comment pattern \s*(?:\#.*\n)?\s*
750             # which is not in the header-opening regex and is used twice in the header-closing regex;
751             # DEV NOTE: do NOT join multiple lines into one line for regex match,
752             # need \n characters to detect trailing comments,
753             # \s matches \n so multiple lines do not need to be combined
754             # $1 $2 $3 $4 $5 $6
755 0 0 0     0 if (# Python
756             ($python_preparsed_target->[-1]->{python_source_code} =~
757             m/^(\s*)def\s+(\w+)\s*\(\s*((?:[\w\.\*]+\s*(?::\s*[\w\.]+\s*)?(?:\[.*\]\s*)?(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,\s*(?:\#.*\n)?\s*)*[\w\.\*]+\s*(?::\s*[\w\.]+\s*)?(?:\[.*\]\s*)?(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,?\s*(?:\#.*\n)?)?\s*\)\s*(?:->\s*([\w\.]+\s*(?:\[.*\]\s*)?))?\s*(:)\s*(\#.*)?$/) or
758             # NEED ANSWER: does Pyrex accept both C and Python types? if so, update Pyrex regex below to accept ':str' Python types
759             # NEED ANSWER: does Pyrex accept both C and Python types? if so, update Pyrex regex below to accept ':str' Python types
760             # NEED ANSWER: does Pyrex accept both C and Python types? if so, update Pyrex regex below to accept ':str' Python types
761             # Pyrex
762             ($python_preparsed_target->[-1]->{python_source_code} =~
763             m/^(\s*)def\s+(\w+)\s*\(\s*((?:(?:(?:const\s+)?(?:[\w\.]+\s*(?:\[[\:\d\,\s]+\])?\s+))?[\w\.\*]+(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,\s*(?:\#.*\n)?\s*)*(?:(?:const\s+)?(?:[\w\.]+\s*(?:\[[\:\d\,\s]+\])?\s+))?[\w\.\*]+(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,?\s*(?:\#.*\n)?)?\s*\)\s*(?:->\s*([\w\.]+))?\s*(:)\s*(\#.*)?$/)) {
764              
765              
766              
767              
768 0         0 print 'in python_file_to_python_preparsed(), ending multi-line function header', "\n";
769              
770             # all function header sub-components have been received, so accept them all
771 0 0       0 if (defined $3) { $python_preparsed_target->[-1]->{arguments} = $3; }
  0         0  
772 0 0       0 if (defined $4) { $python_preparsed_target->[-1]->{return_type} = $4; }
  0         0  
773              
774             # set ending line number, indicating we are no longer inside this multi-line component
775 0         0 $python_preparsed_target->[-1]->{python_line_number_end_header} = $python_line_number;
776              
777 0         0 print 'in python_file_to_python_preparsed(), ending multi-line function header, have $python_preparsed_target->[-1] = ', Dumper($python_preparsed_target->[-1]), "\n";
778             #die 'TMP DEBUG, END MULTI-LINE FUNCTION HEADER' if ($python_preparsed_target->[-1]->{symbol} eq '__init__');
779              
780 0         0 next;
781             }
782              
783             # did not end multi-line component, go on to next line
784 0         0 next;
785             }
786             # pre-parse & accumulate everything inside multi-line class header
787 63         300 elsif (((scalar @{$python_preparsed_target}) > 0) and
788             $python_preparsed_target->[-1]->isa('Python::Class') and
789             ($python_preparsed_target->[-1]->{python_line_number_end_header} < 0)) {
790 0         0 print 'in python_file_to_python_preparsed(), inside multi-line class header', "\n";
791             # accumulate current (possibly last) Python line of multi-line component
792 0         0 chomp $ARG;
793 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
794              
795             # update last active character
796 0         0 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
797 0         0 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
798              
799             # end multi-line class header when it matches the entire regex;
800              
801             # DEV NOTE, CORRELATION PYFI101: all regex changes must be reflected in both locations,
802             # the only difference should be the optional trailing comment pattern \s*(?:\#.*\n)?\s*
803             # which is not in the header-opening regex and is used twice in the header-closing regex;
804             # DEV NOTE: do NOT join multiple lines into one line for regex match,
805             # need \n characters to detect trailing comments,
806             # \s matches \n so multiple lines do not need to be combined
807 0 0 0     0 if (# Python
808             ($python_preparsed_target->[-1]->{python_source_code} =~
809             # $1 $2 $3 $4 $5
810             m/^(\s*)class\s+(\w+)\s*(?:\(\s*((?:[\w\.=]+\s*\,\s*(?:\#.*\n)?\s*)*[\w\.=]+\s*\,?\s*(?:\#.*\n)?)?\s*\)\s*)?(:)\s*(\#.*)?$/) or
811             # Pyrex
812             ($python_preparsed_target->[-1]->{python_source_code} =~
813             m/^(\s*)cdef\s+class\s+(\w+)(?:\{\{\w+\}\})?\s*(?:\(\s*((?:[\w\.=]+(?:\{\{\w+\}\})?[\w\.=]*\s*\,\s*(?:\#.*\n)?\s*)*[\w\.=]+(?:\{\{\w+\}\})?[\w\.=]*\s*\,?\s*(?:\#.*\n)?)?\s*\)\s*)?(:)\s*(\#.*)?$/)) {
814 0         0 print 'in python_file_to_python_preparsed(), ending multi-line class header', "\n";
815              
816             # all class header sub-components have been received, so accept them all
817 0 0       0 if (defined $3) { $python_preparsed_target->[-1]->{parents} = $3; }
  0         0  
818              
819             # set ending line number, indicating we are no longer inside this multi-line component
820 0         0 $python_preparsed_target->[-1]->{python_line_number_end_header} = $python_line_number;
821              
822 0         0 print 'in python_file_to_python_preparsed(), ending multi-line class header, have $python_preparsed_target->[-1] = ', Dumper($python_preparsed_target->[-1]), "\n";
823             #die 'TMP DEBUG, END MULTI-LINE CLASS HEADER' if ($python_preparsed_target->[-1]->{symbol} eq '__init__');
824              
825 0         0 next;
826             }
827              
828             # did not end multi-line component, go on to next line
829 0         0 next;
830             }
831              
832             # DEV NOTE: multi-line classes & functions can contain multi-line comments & includes, so break elsif() and start new if();
833             # pre-parse & accumulate everything inside multi-line namespaces (functions & classes)
834 63 50       105 if ((scalar @{$python_namespaces}) > 0) {
  63         114  
835 0         0 print 'in python_file_to_python_preparsed(), inside multi-line namespace', "\n";
836             #print 'in python_file_to_python_preparsed(), have all outer namespaces $python_namespaces = ', Dumper($python_namespaces), "\n";
837             #print 'in python_file_to_python_preparsed(), have next outer namespace $python_namespaces->[-1] = ', Dumper($python_namespaces->[-1]), "\n";
838 0         0 print 'in python_file_to_python_preparsed(), have next outer namespace $python_namespaces->[-1]->{symbol_scoped} = \'', $python_namespaces->[-1]->{symbol_scoped}, '\'', "\n";
839              
840             # end multi-line namespace(s) when the indentation level returns to the same as, or less than, the first line of its definition,
841             # not counting blank (empty) lines or whitespace-only lines
842 0         0 $ARG =~ m/^(\s*)[^\s]/;
843              
844 0 0       0 print 'in python_file_to_python_preparsed(), have current line leading whitespace $1 = \'', (defined($1) ? $1 : '<<<undef>>>'), '\'', "\n";
845 0         0 print 'in python_file_to_python_preparsed(), have next outer namespace $python_namespaces->[-1]->{indentation} = \'', $python_namespaces->[-1]->{indentation}, '\'', "\n";
846              
847             # if regex above does not match, then $1 will be undefined;
848             # this can only happen with blank (empty) lines and whitespace-only lines
849 0 0 0     0 if ((defined $1) and
850             ((length $1) <= (length $python_namespaces->[-1]->{indentation}))) {
851 0         0 print 'in python_file_to_python_preparsed(), ending one or more multi-line namespaces', "\n";
852              
853             # continue removing namespaces from the stack, as long as the stack is not empty and the indentation level is less or equal
854 0   0     0 while (((scalar @{$python_namespaces}) > 0) and
  0         0  
855             ((length $1) <= (length $python_namespaces->[-1]->{indentation}))) {
856 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\'', "\n";
857              
858             # DEV NOTE: namespaces do not end with blank lines;
859             # move all blank lines to end of next-higher namespace if there are any more,
860             # or to $self->{python_preparsed} if this is the last namespace on the stack;
861             # if this is not the last namespace on the stack, then blank line(s) remain in the last active namespace
862 0         0 my Python::Component::arrayref $independent_blank_lines = [];
863 0   0     0 while (((scalar @{$python_namespaces->[-1]->{python_preparsed}}) > 0) and
  0         0  
864             $python_namespaces->[-1]->{python_preparsed}->[-1]->isa('Python::Blank')) {
865 0         0 unshift(@{$independent_blank_lines}, pop(@{$python_namespaces->[-1]->{python_preparsed}}));
  0         0  
  0         0  
866 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\', popped off independent blank line = ', Dumper($independent_blank_lines->[0]), "\n";
867             }
868              
869 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\', have $independent_blank_lines = ', Dumper($independent_blank_lines), "\n";
870              
871 0 0       0 if ((scalar @{$python_namespaces}) > 1) {
  0         0  
872 0         0 push @{$python_namespaces->[-2]->{python_preparsed}}, @{$independent_blank_lines};
  0         0  
  0         0  
873 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\', pushed independent blank lines onto next-higher namespace = \'', $python_namespaces->[-2]->{symbol_scoped}, '\'', "\n";
874             }
875             else {
876 0         0 push @{$self->{python_preparsed}}, @{$independent_blank_lines};
  0         0  
  0         0  
877 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\', pushed independent blank lines onto top-level $self->{python_preparsed}', "\n";
878             }
879              
880             # set ending line number, indicating we are no longer inside this multi-line component;
881             # we are currently on a new line outside the component,
882             # so the multi-line component actually ended on the previous line number if no independent blank lines were moved,
883             # otherwise the multi-line component actually ended on the line before the first blank line started
884 0 0       0 if ((scalar @{$independent_blank_lines}) == 0) {
  0         0  
885 0         0 $python_namespaces->[-1]->{python_line_number_end} = $python_line_number - 1;
886 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\', no independent blank lines found so namespace ended on previous line = ', $python_namespaces->[-1]->{python_line_number_end}, "\n";
887             }
888             else {
889 0         0 $python_namespaces->[-1]->{python_line_number_end} = $independent_blank_lines->[0]->{python_line_number_begin} - 1;
890 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\', yes independent blank lines found so namespace ended on line before first blank line started = ', $python_namespaces->[-1]->{python_line_number_end}, "\n";
891             }
892              
893 0         0 pop @{$python_namespaces};
  0         0  
894             }
895              
896             # update name of current namespace
897 0 0       0 if ((scalar @{$python_namespaces}) > 0) {
  0         0  
898 0         0 $python_namespace_name = $python_namespaces->[-1]->{symbol_scoped};
899             }
900             else {
901 0         0 $python_namespace_name = undef;
902             }
903              
904             # do not skip to next line, continue pre-parsing as-yet-unidentified current line of Python source code in $ARG
905             }
906             }
907              
908             # DEV NOTE: if namespace (function or class) is ended above,
909             # then code is not accumulated and not yet pre-parsed, so use if() not elsif()
910              
911             # determine correct location to push preparsed components;
912 63 50       82 if ((scalar @{$python_namespaces}) > 0) {
  63         119  
913             # if we are inside an enclosing namespace, always push preparsed components into that namespace's object;
914             # the enclosing component is the last item in the namespace stack
915 0         0 $python_preparsed_target = $python_namespaces->[-1]->{python_preparsed};
916             }
917             else {
918             # otherwise we are outside all namespaces, so we push preparsed components into the primary base-level data structure
919 63         107 $python_preparsed_target = $self->{python_preparsed};
920             }
921              
922             # NEED ANSWER: can multi-line includes really include blank (empty) lines?
923             # pre-parse & skip blank (empty) lines; must check after multi-line components, which may contain their own blank (empty) lines
924 63 100 33     959 if ($ARG =~ m/^$/) {
    50 33        
    100 33        
    50 33        
    50 100        
    50 100        
    50 0        
    100 0        
    50 0        
    0 0        
    0 0        
925 7         47 print 'in python_file_to_python_preparsed(), have blank (empty) line', "\n";
926 7         21 chomp $ARG; # trim trailing newline, if present
927              
928             # check if previous component was same type
929 7 50 33     9 if (((scalar @{$python_preparsed_target}) > 0) and
  7         51  
930             $python_preparsed_target->[-1]->isa('Python::Blank')) {
931 0         0 print 'in python_file_to_python_preparsed(), have blank (empty) line, accumulating', "\n";
932             # accumulate multiple single-line components into a multi-line component
933 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number; # update ending line number
934 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
935 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n" . $ARG; # copy blank (empty) lines verbatim
936             }
937             else {
938 7         56 print 'in python_file_to_python_preparsed(), have blank (empty) line, creating', "\n";
939             # create new component
940 7         14 push @{$python_preparsed_target},
  7         52  
941             Python::Blank->new(
942             {
943             component_type => 'Python::Blank',
944             python_line_number_begin => $python_line_number,
945             python_line_number_end => $python_line_number,
946             python_source_code => $ARG,
947             perl_source_code => $ARG # copy blank (empty) lines verbatim
948             });
949             }
950              
951 7         605 next;
952             }
953             # NEED ANSWER: can multi-line includes really include whitespace lines?
954             # pre-parse & skip whitespace lines; must check after multi-line components, which may contain their own whitespace lines
955             elsif ($ARG =~ m/^\s+$/) {
956 0         0 print 'in python_file_to_python_preparsed(), have whitespace line', "\n";
957 0         0 chomp $ARG; # trim trailing newline, if present
958              
959             # check if previous component was same type
960 0 0 0     0 if (((scalar @{$python_preparsed_target}) > 0) and
  0         0  
961             $python_preparsed_target->[-1]->isa('Python::Whitespace')) {
962 0         0 print 'in python_file_to_python_preparsed(), have whitespace line, accumulating', "\n";
963             # accumulate multiple single-line components into a multi-line component
964 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number; # update ending line number
965 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
966 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n" . $ARG; # copy whitespace lines verbatim
967             }
968             else {
969 0         0 print 'in python_file_to_python_preparsed(), have whitespace line, creating', "\n";
970             # create new component
971 0         0 push @{$python_preparsed_target},
  0         0  
972             Python::Whitespace->new(
973             {
974             component_type => 'Python::Whitespace',
975             python_line_number_begin => $python_line_number,
976             python_line_number_end => $python_line_number,
977             python_source_code => $ARG,
978             perl_source_code => $ARG # copy whitespace lines verbatim
979             });
980             }
981              
982 0         0 next;
983             }
984             # pre-parse & skip single-line # comments
985             elsif ($ARG =~ m/^\s*\#/) {
986 8         60 print 'in python_file_to_python_preparsed(), have single-line # comment', "\n";
987 8         23 chomp $ARG; # trim trailing newline, if present
988              
989             # check if previous component was same type
990 8 50 66     10 if (((scalar @{$python_preparsed_target}) > 0) and
  8         55  
991             $python_preparsed_target->[-1]->isa('Python::Comment')) {
992 0         0 print 'in python_file_to_python_preparsed(), have single-line # comment, accumulating', "\n";
993             # accumulate multiple single-line components into a multi-line component
994 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number; # update ending line number
995 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
996 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n" . $ARG; # copy comments verbatim
997             }
998             else {
999 8         50 print 'in python_file_to_python_preparsed(), have single-line # comment, creating', "\n";
1000             # create new component
1001 8         18 push @{$python_preparsed_target},
  8         63  
1002             Python::Comment->new(
1003             {
1004             component_type => 'Python::Comment',
1005             python_line_number_begin => $python_line_number,
1006             python_line_number_end => $python_line_number,
1007             python_source_code => $ARG,
1008             perl_source_code => $ARG # copy comments verbatim
1009             });
1010             }
1011              
1012 8         679 next;
1013             }
1014              
1015             # NEED ANSWER: other than left parentheses and comma, what other characters indicate non-void context???
1016             # NEED ANSWER: other than left parentheses and comma, what other characters indicate non-void context???
1017             # NEED ANSWER: other than left parentheses and comma, what other characters indicate non-void context???
1018              
1019             # pre-parse & skip single-line '''comments''';
1020             # DEV NOTE: if last active character is left parentheses or comma,
1021             # then context is not void and this is not a comment
1022             elsif (($ARG =~ m/^(\s*)\'\'\'(.*)\'\'\'\s*$/) and
1023             ($python_last_active_character ne '(') and
1024             ($python_last_active_character ne ',')) {
1025 0         0 print 'in python_file_to_python_preparsed(), have single-line \'\'\'comment\'\'\'', "\n";
1026 0         0 chomp $ARG; # trim trailing newline, if present
1027              
1028             # update last active character
1029 0         0 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
1030 0         0 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
1031              
1032             # check if previous component was same type
1033 0 0 0     0 if (((scalar @{$python_preparsed_target}) > 0) and
  0         0  
1034             $python_preparsed_target->[-1]->isa('Python::CommentSingleQuotes')) {
1035 0         0 print 'in python_file_to_python_preparsed(), have single-line \'\'\'comment\'\'\', accumulating', "\n";
1036             # accumulate multiple single-line components into a multi-line component
1037 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number; # update ending line number
1038 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
1039 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n" . ($1 . '# ' . $2); # reformat comments & retain spacing
1040             }
1041             else {
1042 0         0 print 'in python_file_to_python_preparsed(), have single-line \'\'\'comment\'\'\', creating', "\n";
1043             # create new component
1044 0         0 push @{$python_preparsed_target},
  0         0  
1045             Python::CommentSingleQuotes->new(
1046             {
1047             component_type => 'Python::CommentSingleQuotes',
1048             python_line_number_begin => $python_line_number,
1049             python_line_number_end => $python_line_number,
1050             python_source_code => $ARG,
1051             perl_source_code => ($1 . '# ' . $2) # reformat comments & retain spacing
1052             });
1053             }
1054              
1055 0         0 next;
1056             }
1057             # pre-parse & skip single-line """comments""";
1058             # DEV NOTE: if last active character is left parentheses or comma,
1059             # then context is not void and this is not a comment
1060             elsif (($ARG =~ m/^(\s*)\"\"\"(.*)\"\"\"\s*$/) and
1061             ($python_last_active_character ne '(') and
1062             ($python_last_active_character ne ',')) {
1063 0         0 print 'in python_file_to_python_preparsed(), have single-line \"\"\"comment\"\"\"', "\n";
1064 0         0 chomp $ARG; # trim trailing newline, if present
1065              
1066             # update last active character
1067 0         0 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
1068 0         0 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
1069              
1070             # check if previous component was same type
1071 0 0 0     0 if (((scalar @{$python_preparsed_target}) > 0) and
  0         0  
1072             $python_preparsed_target->[-1]->isa('Python::CommentDoubleQuotes')) {
1073 0         0 print 'in python_file_to_python_preparsed(), have single-line \"\"\"comment\"\"\", accumulating', "\n";
1074             # accumulate multiple single-line components into a multi-line component
1075 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number; # update ending line number
1076 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
1077 0         0 $python_preparsed_target->[-1]->{perl_source_code} .= "\n" . ($1 . '# ' . $2); # reformat comments & retain spacing
1078             }
1079             else {
1080 0         0 print 'in python_file_to_python_preparsed(), have single-line \"\"\"comment\"\"\", creating', "\n";
1081             # create new component
1082 0         0 push @{$python_preparsed_target},
  0         0  
1083             Python::CommentDoubleQuotes->new(
1084             {
1085             component_type => 'Python::CommentDoubleQuotes',
1086             python_line_number_begin => $python_line_number,
1087             python_line_number_end => $python_line_number,
1088             python_source_code => $ARG,
1089             perl_source_code => ($1 . '# ' . $2) # reformat comments & retain spacing
1090             });
1091             }
1092              
1093 0         0 next;
1094             }
1095             # start multi-line '''comments''';
1096             elsif ($ARG =~ m/^(\s*)\'\'\'(.*)$/) {
1097             # DEV NOTE: if last active character is left parentheses or comma,
1098             # then context is not void and this is not a comment
1099 0         0 my boolean $is_actually_string_literal;
1100 0 0 0     0 if (($python_last_active_character eq '(') or
1101             ($python_last_active_character eq ',')) {
1102 0         0 print 'in python_file_to_python_preparsed(), have multi-line \'\'\'string literal, starting', "\n";
1103 0         0 $is_actually_string_literal = 1;
1104             }
1105             else {
1106 0         0 print 'in python_file_to_python_preparsed(), have multi-line \'\'\'comment, starting', "\n";
1107 0         0 $is_actually_string_literal = 0;
1108             }
1109              
1110 0         0 push @{$python_preparsed_target},
  0         0  
1111             Python::CommentSingleQuotes->new(
1112             {
1113             component_type => 'Python::CommentSingleQuotes',
1114             indentation => $1,
1115             is_actually_string_literal => $is_actually_string_literal,
1116             python_line_number_begin => $python_line_number,
1117             python_line_number_end => -1, # negative value means we are currently inside multi-line component
1118             python_source_code => champ($ARG),
1119             # perl_source_code => champ('=x ' . $1 . $2) # DEV NOTE: don't use POD for multi-line comments, POD parsers are inconsistent
1120             perl_source_code => champ($1 . q{#''} . $2) # reformat comments & retain spacing
1121             });
1122 0         0 next;
1123             }
1124             # start multi-line """comments""";
1125             # DEV NOTE: if last active character is left parentheses or comma,
1126             # then context is not void and this is not a comment
1127             elsif ($ARG =~ m/^(\s*)\"\"\"(.*)$/) {
1128             # DEV NOTE: if last active character is left parentheses or comma,
1129             # then context is not void and this is not a comment
1130 0         0 my boolean $is_actually_string_literal;
1131 0 0 0     0 if (($python_last_active_character eq '(') or
1132             ($python_last_active_character eq ',')) {
1133 0         0 print 'in python_file_to_python_preparsed(), have multi-line \"\"\"string literal, starting', "\n";
1134 0         0 $is_actually_string_literal = 1;
1135             }
1136             else {
1137 0         0 print 'in python_file_to_python_preparsed(), have multi-line \"\"\"comment, starting', "\n";
1138 0         0 $is_actually_string_literal = 0;
1139             }
1140              
1141 0         0 push @{$python_preparsed_target},
  0         0  
1142             Python::CommentDoubleQuotes->new(
1143             {
1144             component_type => 'Python::CommentDoubleQuotes',
1145             indentation => $1,
1146             is_actually_string_literal => $is_actually_string_literal,
1147             python_line_number_begin => $python_line_number,
1148             python_line_number_end => -1, # negative value means we are currently inside multi-line component
1149             python_source_code => champ($ARG),
1150             # perl_source_code => champ('=x ' . $1 . $2) # DEV NOTE: don't use POD for multi-line comments, POD parsers are inconsistent
1151             perl_source_code => champ($1 . q{#""} . $2) # reformat comments & retain spacing
1152             });
1153 0         0 next;
1154             }
1155             # start multi-line include statements, either with enclosing parentheses,
1156             # or with long lines ending in backslash AKA line continuation;
1157             # match any line starting with 'from' and including an open but not close parentheses,
1158             # or starting with either 'from' or 'import', or 'cimport' for Pyrex, and ending in backslash
1159             # https://python-reference.readthedocs.io/en/latest/docs/operators/slash.html
1160             # NEED ANSWER: can the line continuation backslash appear without any preceding whitespace?
1161             elsif (($ARG =~ m/^\s*from\s+.*(\()[^\)]*$/) or
1162             ($ARG =~ m/^\s*from\s+.*\\$/) or
1163             ($ARG =~ m/^\s*c?import\s+.*\\$/)) {
1164 16         122 print 'in python_file_to_python_preparsed(), have multi-line include, starting', "\n";
1165 16         50 chomp $ARG; # trim trailing newline, if present
1166              
1167             # update last active character
1168 16         45 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
1169 16         93 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
1170              
1171 16         144 push @{$python_preparsed_target},
1172             Python::Include->new(
1173             {
1174             component_type => 'Python::Include',
1175             python_file_path => $self->{python_file_path},
1176 16 100       36 python_line_number_begin => $python_line_number,
1177             python_line_number_end => -1, # negative value means we are currently inside multi-line component
1178             python_source_code => $ARG,
1179             python_has_parentheses => (defined $1) ? 1 : 0,
1180             perl_source_code => undef, # includes are not translated during pre-parse phase
1181             });
1182 16         1492 next;
1183             }
1184             # pre-parse single-line include statements;
1185             # 'import', or 'cimport' for Pyrex, must be followed by open parentheses or whitespace
1186             elsif ($ARG =~ m/^\s*(from\s+.+\s+)?c?import[\(\s].+$/) {
1187 32         240 print 'in python_file_to_python_preparsed(), have single-line include', "\n";
1188 32         109 chomp $ARG; # trim trailing newline, if present
1189              
1190             # update last active character
1191 32         91 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
1192 32         189 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
1193              
1194 32         232 push @{$python_preparsed_target},
1195             Python::Include->new(
1196             {
1197             component_type => 'Python::Include',
1198             python_file_path => $self->{python_file_path},
1199 32         66 python_line_number_begin => $python_line_number,
1200             python_line_number_end => $python_line_number,
1201             python_source_code => $ARG,
1202             perl_source_code => undef, # includes are not translated during pre-parse phase
1203             });
1204              
1205 32         2834 next;
1206             }
1207              
1208             # NEED UPGRADE: support single-line Python functions; also, decorators on same line as function definition?
1209             # NEED UPGRADE: support single-line Python functions; also, decorators on same line as function definition?
1210             # NEED UPGRADE: support single-line Python functions; also, decorators on same line as function definition?
1211              
1212             # start function definitions
1213             # DEV NOTE: can have whitespace around commas
1214             # DEV NOTE: can have whitespace but NOT newline in between function name and open parentheses
1215             # DEV NOTE: can have whitespace but NOT newline in between close parentheses and colon
1216             # DEV NOTE: can have trailing comma at the end of function argument list
1217              
1218             # Python function header examples
1219             # def FOO():
1220             # def FOO(BAR):
1221             # def FOO ( BAR, BAT, BAZ, ) :
1222             # def FOO(BAR, BAT, BAZ) -> FooReturnType:
1223             # def __FOO__(self, *, BAR=None, var_BAR=1e-9, force_alpha="warn"):
1224             # def FOO_BAR(code, extra_preargs=[], extra_postargs=[]):
1225             # def FOO (
1226             #
1227             # ) :
1228             # def __FOO__(
1229             # self,
1230             # n_clusters=2,
1231             # *,
1232             # affinity="deprecated", # TODO(1.4): Remove
1233             # metric=None, # TODO(1.4): Set to "euclidean"
1234             # memory=None,
1235             # connectivity=None,
1236             # compute_full_tree="auto",
1237             # linkage="ward",
1238             # distance_threshold=None,
1239             # compute_distances=False,
1240             # damping=0.5,
1241             # eps=np.finfo(np.float64).eps,
1242             # slice_=(slice(70, 195), slice(78, 172)),
1243             # ):
1244             # def FOO(
1245             # BAR: BarType, BAT:BatType="howdy", BAX : BaxType = 23
1246             # ) -> FooReturnType:
1247             # def FOO(
1248             # BAR: int,
1249             # BAT1: Optional [ str ],
1250             # BAT2: typing.Optional[str],
1251             # BAX: float = 1.0,
1252             # BAZ1: Union[int, str],
1253             # BAZ2: typing.Union [ int , str ] = 'howdy',
1254             # ) -> Dict[str, Any]:
1255              
1256             # Pyrex function header examples
1257             # def FOO(
1258             # const cnp.uint8_t[::1] BAR,
1259             # object[:] BAT,
1260             # cnp.npy_intp [::1] BAX
1261             # ):
1262             # def FOO(
1263             # const cnp.float64_t[::1] BAR,
1264             # const cnp.float64_t[:, ::1] BAT,
1265             # const cnp.intp_t[::1] BAX,
1266             # BAZ,
1267             # cnp.float64_t[::1] QUUX
1268             # ):
1269             # def FOO(cnp.intp_t BAR, BAT, cnp.intp_t BAX):
1270              
1271             # DEV NOTE: either match the entire regex for single-line function header,
1272             # or match only the start of the function header 'def FOO' and
1273             # start multi-line component to accumulate source code lines until entire regex can be matched
1274             # DEV NOTE, CORRELATION PYFI100: all regex changes must be reflected in both locations,
1275             # the only difference should be the optional trailing comment pattern \s*(?:\#.*\n)?\s*
1276             # which is not in the header-opening regex and is used twice in the header-closing regex;
1277             elsif ( # Python
1278             # $1 $2 $3 $4 $5 $6
1279             ($ARG =~ m/^(\s*)def\s+(\w+)\s*\(\s*((?:[\w\.\*]+\s*(?::\s*[\w\.]+\s*)?(?:\[.*\]\s*)?(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,\s*(?:\#.*\n)?\s*)*[\w\.\*]+\s*(?::\s*[\w\.]+\s*)?(?:\[.*\]\s*)?(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,?\s*(?:\#.*\n)?)?\s*\)\s*(?:->\s*([\w\.]+\s*(?:\[.*\]\s*)?))?\s*(:)\s*(\#.*)?$/) or
1280             # NEED ANSWER: does Pyrex accept both C and Python types? if so, update Pyrex regex below to accept ':str' Python types
1281             # NEED ANSWER: does Pyrex accept both C and Python types? if so, update Pyrex regex below to accept ':str' Python types
1282             # NEED ANSWER: does Pyrex accept both C and Python types? if so, update Pyrex regex below to accept ':str' Python types
1283             # Pyrex
1284             # $1 $2 $3 $4 $5 $6
1285             ($ARG =~ m/^(\s*)def\s+(\w+)\s*\(\s*((?:(?:(?:const\s+)?(?:[\w\.]+\s*(?:\[[\:\d\,\s]+\])?\s+))?[\w\.\*]+(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,\s*)*(?:(?:const\s+)?(?:[\w\.]+\s*(?:\[[\:\d\,\s]+\])?\s+))?[\w\.\*]+(?:\=\s*(?:(?:\'.*\')|(?:\".*\")|(?:\(.*\))|(?:\[.*\])|[\w\.\-\(\)]+))?\s*\,?)?\s*\)\s*(?:->\s*([\w\.]+))?\s*(:)\s*(\#.*)?$/) or
1286             # Python or Pyrex
1287             ($ARG =~ m/^(\s*)def\s+(\w+)/)) {
1288 0         0 print 'in python_file_to_python_preparsed(), have function, starting header', "\n";
1289             #die 'TMP DEBUG, FUNCTION HEADER';
1290              
1291             # NEED UPGRADE: utilize optional trailing comment $6, include in generated Perl source code if present
1292             # NEED UPGRADE: utilize optional trailing comment $6, include in generated Perl source code if present
1293             # NEED UPGRADE: utilize optional trailing comment $6, include in generated Perl source code if present
1294              
1295 0         0 chomp $ARG; # trim trailing newline, if present
1296              
1297             # update last active character
1298 0         0 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
1299 0         0 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
1300              
1301             # if multi-line function header, we must receive the function name as part of the first line,
1302             # in order to pre-parse correctly below
1303 0         0 $python_namespace_name = $2;
1304              
1305             $python_component =
1306             {
1307             component_type => undef, # set below, either Python::Function or Python::Method or Python::InnerFunction
1308             decorators => '', # empty value means no declared decorators; possibly set below
1309             indentation => $1, # empty match returns empty string '', not undef
1310             symbol => $python_namespace_name, # set now to get non-scoped symbol
1311             symbol_scoped => undef, # set below, possibly-scoped symbol
1312             arguments => '', # empty value means no declared arguments; possibly set below
1313             return_type => '', # empty value means no declared return type; possibly set below
1314             python_file_path => $self->{python_file_path},
1315 0         0 python_line_number_begin => $python_line_number,
1316             python_line_number_end => -1, # negative value means we are currently inside multi-line component
1317             python_line_number_end_header => -1, # negative value means we are currently inside multi-line header; possibly set below
1318             python_source_code => $ARG, # only function header, remaining source code will be nested in python_preparsed below
1319             python_preparsed => [], # nested pre-parsed data structures of all source code inside this function
1320             python_preparsed_decorators => [], # nested pre-parsed data structures of all decorators above this function
1321             perl_source_code => undef # functions are not translated during pre-parse phase
1322             };
1323              
1324             # if all function header sub-components have been received, then accept them all;
1325             # consider final colon ':' captured in $5 to be the ending character of the function header
1326 0 0       0 if (defined $5) {
1327 0         0 print 'in python_file_to_python_preparsed(), have function, ending header', "\n";
1328 0 0       0 if (defined $3) { $python_component->{arguments} = $3; }
  0         0  
1329 0 0       0 if (defined $4) { $python_component->{return_type} = $4; }
  0         0  
1330              
1331             # set header ending line number, indicating this is not a multi-line function header
1332 0         0 $python_component->{python_line_number_end_header} = $python_line_number;
1333             }
1334             else {
1335 0         0 print 'in python_file_to_python_preparsed(), have function, no closing colon found, NOT ending header', "\n";
1336             }
1337             #die 'TMP DEBUG, PARSE FUNCTION HEADER';
1338              
1339             # look-back to capture function decorators (@abstractmethod, @staticmethod, etc) on previous lines
1340 0         0 while ((scalar @{$python_preparsed_target}) > 0) {
  0         0  
1341 0         0 print 'in python_file_to_python_preparsed(), have function, top of look-back loop', "\n";
1342             # pop blank / whitespace / comment lines onto temporary stack,
1343             # to either be captured along with decorators or put back if no decorator encountered
1344 0 0 0     0 if ($python_preparsed_target->[-1]->isa('Python::Blank') or
      0        
1345             $python_preparsed_target->[-1]->isa('Python::Whitespace') or
1346             $python_preparsed_target->[-1]->isa('Python::Comment')) {
1347 0         0 print 'in python_file_to_python_preparsed(), have function, in look-back loop, moving blank / whitespace / comment line to temporary stack', "\n";
1348              
1349             # utilize as-yet-unused function body python_preparsed as temporary stack
1350 0         0 unshift @{$python_component->{python_preparsed}}, (pop @{$python_preparsed_target});
  0         0  
  0         0  
1351 0         0 next;
1352             }
1353              
1354             # capture function decorators
1355             # @FOO
1356             # @FOO.BAR(scope="function")
1357 0 0 0     0 if ($python_preparsed_target->[-1]->isa('Python::Unknown') and
1358             ($python_preparsed_target->[-1]->{python_source_code} =~ m/^\s*@.+$/)) {
1359 0         0 print 'in python_file_to_python_preparsed(), have function, in look-back loop, capturing decorator \'', $python_preparsed_target->[-1]->{python_source_code}, '\' by deleting sleep_seconds & sleep_retry_multiplier & retries_max object properties & re-blessing from Unknown to FunctionDecorator', "\n";
1360              
1361             # NEED REMOVE HIGH-MAGIC: how can we re-bless from Unknown with sleep_seconds etc properties to FunctionDecorator w/out it?
1362             # NEED REMOVE HIGH-MAGIC: how can we re-bless from Unknown with sleep_seconds etc properties to FunctionDecorator w/out it?
1363             # NEED REMOVE HIGH-MAGIC: how can we re-bless from Unknown with sleep_seconds etc properties to FunctionDecorator w/out it?
1364             # re-bless from Unknown to FunctionDecorator
1365 0         0 delete $python_preparsed_target->[-1]->{sleep_seconds}; # HIGH-MAGIC: remove object property
1366 0         0 delete $python_preparsed_target->[-1]->{sleep_retry_multiplier}; # HIGH-MAGIC
1367 0         0 delete $python_preparsed_target->[-1]->{retries_max}; # HIGH-MAGIC
1368 0         0 $python_preparsed_target->[-1] = Python::FunctionDecorator->new($python_preparsed_target->[-1]); # HIGH-MAGIC: re-bless
1369 0         0 $python_preparsed_target->[-1]->{component_type} = 'Python::FunctionDecorator'; # LOW-MAGIC: just a string
1370              
1371             # save decorator source code separately from function header source code,
1372             # to allow for proper output code generation,
1373             # where the function header source code and the function decorators source code
1374             # must be generator separately for correctness
1375             $python_component->{decorators} =
1376 0         0 $python_preparsed_target->[-1]->{python_source_code} . "\n" . $python_component->{decorators};
1377 0         0 chomp $python_component->{decorators};
1378              
1379             # update starting line of function header to be where first function decorator appears
1380 0         0 $python_component->{python_line_number_begin} = $python_preparsed_target->[-1]->{python_line_number_begin};
1381              
1382             # move contents of temporary stack into preparsed decorators
1383 0         0 unshift @{$python_component->{python_preparsed_decorators}}, @{$python_component->{python_preparsed}};
  0         0  
  0         0  
1384 0         0 $python_component->{python_preparsed} = []; # empty temporary stack
1385            
1386             # move formerly-Unknown component into preparsed decorators
1387 0         0 unshift @{$python_component->{python_preparsed_decorators}}, (pop @{$python_preparsed_target});
  0         0  
  0         0  
1388              
1389 0         0 next;
1390             }
1391              
1392 0         0 print 'in python_file_to_python_preparsed(), have function, in look-back loop, leaving loop', "\n";
1393 0         0 last;
1394             }
1395              
1396             # if not blank / whitespace / comment line or function decorator line,
1397             # then stop here and restore stack if needed
1398 0 0       0 if ((scalar @{$python_component->{python_preparsed}}) > 0) {
  0         0  
1399 0         0 print 'in python_file_to_python_preparsed(), have function, after look-back loop, restoring stack', "\n";
1400 0         0 push @{$python_preparsed_target}, @{$python_component->{python_preparsed}};
  0         0  
  0         0  
1401 0         0 $python_component->{python_preparsed} = [];
1402             }
1403              
1404             # determine if function is a normal function, a method, or an inner function
1405 0 0       0 if ((scalar @{$python_namespaces}) > 0) {
  0         0  
1406             # prepend all encompassing namespaces to function name, to create scoped function name;
1407             # immediately enclosing component already has scoped name, no need to loop through entire namespace stack
1408 0         0 $python_namespace_name = $python_namespaces->[-1]->{symbol_scoped} . '.' . $python_namespace_name;
1409 0         0 $python_component->{symbol_scoped} = $python_namespace_name;
1410              
1411 0 0       0 if ($python_namespaces->[-1]->isa('Python::Function')) {
    0          
1412             # a function defined (nested) inside another function is an inner function
1413 0         0 $python_component->{component_type} = 'Python::InnerFunction';
1414 0         0 push @{$python_preparsed_target}, Python::InnerFunction->new($python_component);
  0         0  
1415              
1416 0         0 print 'in python_file_to_python_preparsed(), Python inner function named \'', $python_namespace_name, '\' defined inside outer function named \'', $python_namespaces->[-1]->{symbol_scoped}, '\'', "\n";
1417             }
1418             elsif ($python_namespaces->[-1]->isa('Python::Class')) {
1419             # a function defined inside a class is a method
1420 0         0 $python_component->{component_type} = 'Python::Method';
1421              
1422             # save scope for use when declaring data type (class name) of $self argument
1423 0         0 $python_component->{scope} = $python_namespaces->[-1]->{symbol_scoped};
1424              
1425 0         0 push @{$python_preparsed_target}, Python::Method->new($python_component);
  0         0  
1426              
1427 0         0 print 'in python_file_to_python_preparsed(), Python method named \'', $python_namespace_name, '\' defined inside class named \'', $python_namespaces->[-1]->{symbol_scoped}, '\'', "\n";
1428 0         0 print 'in python_file_to_python_preparsed(), Python method named \'', $python_namespace_name, '\', set $python_component->{scope} = \'', $python_component->{scope}, '\'', "\n";
1429             }
1430             else {
1431 0         0 print 'in python_file_to_python_preparsed(), have enclosing namespace ', Dumper($python_namespaces->[1]), "\n";
1432 0         0 croak 'ERROR EPYFI004a: Unrecognized enclosing namespace, only Functions & Classes accepted; ', Dumper($python_namespaces->[1]), ', croaking';
1433             }
1434             }
1435             else {
1436             # a function defined outside all namespaces (classes or functions) is just a normal function
1437 0         0 $python_component->{symbol_scoped} = $python_namespace_name; # scoped symbol is same as non-scoped for normal functions
1438 0         0 $python_component->{component_type} = 'Python::Function';
1439 0         0 push @{$python_preparsed_target}, Python::Function->new($python_component);
  0         0  
1440 0         0 print 'in python_file_to_python_preparsed(), Python normal function named \'', $python_namespace_name, '\' defined outside all namespaces', "\n";
1441             }
1442              
1443             # can't have the same function declared twice
1444 0 0       0 if (exists $python_functions->{$python_namespace_name}) {
1445 0 0       0 if (ref($python_functions->{$python_namespace_name}) eq 'ARRAY') {
    0          
1446 0         0 push @{$python_functions->{$python_namespace_name}}, $python_preparsed_target->[-1];
  0         0  
1447 0         0 carp 'WARNING WPYFI004b: Python function named \'', $python_namespace_name, '\' already pre-parsed, pushing onto already-created function array, carping';
1448             }
1449             elsif ($python_functions->{$python_namespace_name}->isa('Python::Function')) {
1450 0         0 $python_functions->{$python_namespace_name} = [$python_functions->{$python_namespace_name}, $python_preparsed_target->[-1]];
1451 0         0 carp 'WARNING WPYFI004c: Python function named \'', $python_namespace_name, '\' already pre-parsed, creating function array, carping';
1452             }
1453             else {
1454 0         0 croak 'ERROR EPYFI004d: Python function named \'', $python_namespace_name, '\' already pre-parsed, did not find either function or function array, croaking';
1455             }
1456             }
1457             else {
1458             # save reference to current function along with all other functions, for easy name-based access
1459 0         0 $python_functions->{$python_namespace_name} = $python_preparsed_target->[-1];
1460             }
1461              
1462             # being inside this new function increases the namespace stack (deepens the current scope)
1463 0         0 push @{$python_namespaces}, $python_preparsed_target->[-1];
  0         0  
1464 0         0 next;
1465             }
1466              
1467             # start class definitions
1468             # NEED ANSWER: are all Python classes multi-line?
1469             # DEV NOTE: can have whitespace around commas
1470             # DEV NOTE: can have whitespace but NOT newline in between class name and open parentheses
1471             # DEV NOTE: can have whitespace but NOT newline in between close parentheses and colon
1472             # DEV NOTE: can have trailing comma at the end of parent class list
1473              
1474             # Python class header examples
1475             # class FOO:
1476             # class FOO(BAR):
1477             # class FOO ( B.AR ) :
1478             # class FOO (
1479             #
1480             # ) :
1481             # class FOO(BAR, BAT, BAX):
1482             # class FOO(B.AR, BA.T, BAX):
1483             # class _FOO (BAR, BAT, BAX=BAY) :
1484             # class FOO ( BAR,
1485             # BAT, BAX,
1486             # ) :
1487              
1488             # Pyrex class header examples
1489             # cdef class FOO:
1490             # cdef class FOO(BAR):
1491              
1492             # DEV NOTE: either match the entire regex for single-line class header,
1493             # or match only the start of the class header 'class FOO(' and
1494             # start multi-line component to accumulate source code lines until entire regex can be matched
1495             # DEV NOTE, CORRELATION PYFI101: all regex changes must be reflected in both locations,
1496             # the only difference should be the optional trailing comment pattern \s*(?:\#.*\n)?\s*
1497             # which is not in the header-opening regex and is used twice in the header-closing regex;
1498             # $1 $2 $3 $4 $5
1499             elsif ( # Python
1500             ($ARG =~ m/^(\s*)class\s+(\w+)\s*(?:\(\s*((?:[\w\.=]+\s*\,\s*)*[\w\.=]+\s*\,?)?\s*\)\s*)?(:)\s*(\#.*)?$/) or
1501             ($ARG =~ m/^(\s*)class\s+(\w+)\s*\(/) or
1502             # Pyrex
1503             ($ARG =~ m/^(\s*)cdef\s+class\s+(\w+)\s*(?:\(\s*((?:[\w\.=]+(?:\{\{\w+\}\})?[\w\.=]*\s*\,\s*)*[\w\.=]+(?:\{\{\w+\}\})?[\w\.=]*\s*\,?)?\s*\)\s*)?(:)\s*(\#.*)?$/) or
1504             ($ARG =~ m/^(\s*)cdef\s+class\s+(\w+)\s*\(/)) {
1505              
1506             # NEED UPGRADE: utilize optional trailing comment $5, include in generated Perl source code if present
1507             # NEED UPGRADE: utilize optional trailing comment $5, include in generated Perl source code if present
1508             # NEED UPGRADE: utilize optional trailing comment $5, include in generated Perl source code if present
1509              
1510 0         0 print 'in python_file_to_python_preparsed(), have class, starting header', "\n";
1511              
1512 0         0 chomp $ARG; # trim trailing newline, if present
1513              
1514             # update last active character
1515 0         0 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
1516 0         0 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
1517              
1518             # if multi-line class header, we must receive the class name as part of the first line,
1519             # in order to pre-parse correctly below
1520 0         0 $python_namespace_name = $2;
1521              
1522             $python_component =
1523             {
1524             component_type => undef, # set below, either Python::Class or Python::LocalClass or Python::InnerClass
1525             indentation => $1, # empty match returns empty string '', not undef
1526             symbol => $python_namespace_name,
1527             parents => '', # empty value means no declared parent classes; possibly set below
1528             python_file_path => $self->{python_file_path},
1529 0         0 python_line_number_begin => $python_line_number,
1530             python_line_number_end => -1, # negative value means we are currently inside multi-line component
1531             python_line_number_end_header => -1, # negative value means we are currently inside multi-line header; possibly set below
1532             python_source_code => $ARG, # only class header, remaining source code will be nested in python_preparsed below
1533             python_preparsed => [], # nested pre-parsed data structures of all source code inside this class
1534             perl_source_code => undef # classes are not translated during pre-parse phase
1535             };
1536              
1537             # if all class header sub-components have been received, then accept them all;
1538             # consider final colon ':' captured in $4 to be the ending character of the class header
1539 0 0       0 if (defined $4) {
1540 0         0 print 'in python_file_to_python_preparsed(), have class, ending header', "\n";
1541 0 0       0 if (defined $3) { $python_component->{parents} = $3; }
  0         0  
1542              
1543             # set header ending line number, indicating this is not a multi-line class header
1544 0         0 $python_component->{python_line_number_end_header} = $python_line_number;
1545             }
1546             else {
1547 0         0 print 'in python_file_to_python_preparsed(), have class, no closing colon found, NOT ending header', "\n";
1548             }
1549             #die 'TMP DEBUG, PARSE CLASS HEADER';
1550              
1551             # determine if class is a normal class, a local class, or an inner class
1552 0 0       0 if ((scalar @{$python_namespaces}) > 0) {
  0         0  
1553             # prepend all encompassing namespaces to class name, to create scoped class name;
1554             # immediately enclosing component already has scoped name, no need to loop through entire namespace stack
1555 0         0 $python_namespace_name = $python_namespaces->[-1]->{symbol_scoped} . '.' . $python_namespace_name;
1556 0         0 $python_component->{symbol_scoped} = $python_namespace_name;
1557              
1558 0 0       0 if ($python_namespaces->[-1]->isa('Python::Class')) {
    0          
1559             # a class defined (nested) inside another class is an inner class
1560 0         0 $python_component->{component_type} = 'Python::InnerClass';
1561 0         0 push @{$python_preparsed_target}, Python::InnerClass->new($python_component);
  0         0  
1562              
1563 0         0 print 'in python_file_to_python_preparsed(), Python inner class named \'', $python_namespace_name, '\' defined inside outer class named \'', $python_namespaces->[-1]->{symbol_scoped}, '\'', "\n";
1564             }
1565             elsif ($python_namespaces->[-1]->isa('Python::Function')) {
1566             # a class defined inside a function is a local class
1567 0         0 $python_component->{component_type} = 'Python::LocalClass';
1568 0         0 push @{$python_preparsed_target}, Python::LocalClass->new($python_component);
  0         0  
1569              
1570 0         0 print 'in python_file_to_python_preparsed(), Python local class named \'', $python_namespace_name, '\' defined inside function named \'', $python_namespaces->[-1]->{symbol_scoped}, '\'', "\n";
1571             }
1572             else {
1573 0         0 print 'in python_file_to_python_preparsed(), have enclosing namespace ', Dumper($python_namespaces->[1]), "\n";
1574 0         0 croak 'ERROR EPYFI005a: Unrecognized enclosing namespace, only Functions & Classes accepted; ', Dumper($python_namespaces->[1]), ', croaking';
1575             }
1576             }
1577             else {
1578             # a class defined outside all namespaces (classes or functions) is just a normal class
1579 0         0 $python_component->{symbol_scoped} = $python_namespace_name; # scoped symbol is same as non-scoped for normal classes
1580 0         0 $python_component->{component_type} = 'Python::Class';
1581 0         0 push @{$python_preparsed_target}, Python::Class->new($python_component);
  0         0  
1582             }
1583              
1584             # can't have the same class declared twice
1585 0 0       0 if (exists $python_classes->{$python_namespace_name}) {
1586 0         0 croak 'ERROR EPYFI005b: Python class named \'', $python_namespace_name, '\' already pre-parsed, croaking';
1587             }
1588              
1589             # save reference to current class along with all other classes, for easy name-based access
1590 0         0 $python_classes->{$python_namespace_name} = $python_preparsed_target->[-1];
1591              
1592             # being inside this new class increases the namespace stack (deepens the current scope)
1593 0         0 push @{$python_namespaces}, $python_preparsed_target->[-1];
  0         0  
1594 0         0 next;
1595             }
1596             else {
1597 0         0 print 'in python_file_to_python_preparsed(), have UNKNOWN line of code', "\n";
1598 0         0 chomp $ARG; # trim trailing newline, if present
1599              
1600             # update last active character
1601 0         0 $python_last_active_character = $self->python_last_active_character_find($python_last_active_character, $ARG);
1602 0         0 print 'in python_file_to_python_preparsed(), possibly updated last active character to \'', $python_last_active_character, '\'', "\n";
1603              
1604             # ensure we correctly parse all namespaces (functions & classes)
1605 0 0       0 if ($ARG =~ m/^(\s*)def\s+/) {
    0          
1606 0         0 croak 'ERROR EPYFI006a: Python function with UNKNOWN format, croaking';
1607             }
1608             elsif ($ARG =~ m/^(\s*)class\s+/) {
1609 0         0 croak 'ERROR EPYFI006b: Python class with UNKNOWN format, croaking';
1610             }
1611              
1612             # DEV NOTE, CORRELATION PYFI102: all Unknown logic in this if-elsif-else block must be copied
1613             # check if previous component was same type
1614 0 0 0     0 if (((scalar @{$python_preparsed_target}) > 0) and
  0 0 0     0  
      0        
1615             $python_preparsed_target->[-1]->isa('Python::Unknown')) {
1616 0         0 print 'in python_file_to_python_preparsed(), have UNKNOWN line, accumulating', "\n";
1617             # accumulate multiple single-line components into a multi-line component
1618 0         0 $python_preparsed_target->[-1]->{python_source_code} .= "\n" . $ARG;
1619             # update ending line number
1620 0         0 $python_preparsed_target->[-1]->{python_line_number_end} = $python_line_number;
1621             }
1622             # merge Unknown components when separated by only Blank components
1623 0         0 elsif (((scalar @{$python_preparsed_target}) > 1) and
1624             $python_preparsed_target->[-1]->isa('Python::Blank') and
1625             $python_preparsed_target->[-2]->isa('Python::Unknown')) {
1626 0         0 print 'in python_file_to_python_preparsed(), have UNKNOWN line preceded by blank line(s) and other UNKNOWN line(s), merging components', "\n";
1627              
1628             # merge 3 components into a single component;
1629             # Unknown + Blank + Unknown = Unknown
1630             $python_preparsed_target->[-2]->{python_source_code} .=
1631 0         0 "\n" . $python_preparsed_target->[-1]->{python_source_code} . "\n" . $ARG;
1632             # update ending line number
1633 0         0 $python_preparsed_target->[-2]->{python_line_number_end} = $python_line_number;
1634             # discard now-redundant Blank component
1635 0         0 pop @{$python_preparsed_target};
  0         0  
1636             }
1637             else {
1638 0         0 print 'in python_file_to_python_preparsed(), have UNKNOWN line, creating', "\n";
1639             # create new component
1640 0         0 push @{$python_preparsed_target},
  0         0  
1641             Python::Unknown->new(
1642             {
1643             component_type => 'Python::Unknown',
1644             python_line_number_begin => $python_line_number,
1645             python_line_number_end => $python_line_number,
1646             python_source_code => $ARG,
1647             perl_source_code => undef # unknown code is not translated during pre-parse phase
1648             });
1649             }
1650              
1651 0         0 next;
1652             }
1653             }
1654              
1655 1         13 print 'in python_file_to_python_preparsed(), EOF end of file \'', $self->{python_file_path}, '\'', "\n";
1656              
1657             # close file after reading
1658             close($PYTHON_FILE)
1659             or croak 'ERROR EPYFI000b: failed to close Python source code file \'', $self->{python_file_path},
1660 1 50       16 '\' after reading, received OS error message \'', $OS_ERROR, '\', croaking';
1661              
1662             # ensure we finish parsing all multi-line function headers
1663 1 50 33     4 if (((scalar @{$python_preparsed_target}) > 0) and
  1   33     20  
1664             $python_preparsed_target->[-1]->isa('Python::Function') and
1665             ($python_preparsed_target->[-1]->{python_line_number_end_header} < 0)) {
1666             croak 'ERROR EPYFI007: failed to end header for Python function \'',
1667 0         0 $python_preparsed_target->[-1]->{symbol_scoped}, '\', croaking';
1668             }
1669              
1670             # Python source code file is ended; remove all remaining namespaces from stack
1671 1         2 while ((scalar @{$python_namespaces}) > 0) {
  1         6  
1672 0         0 print 'in python_file_to_python_preparsed(), ending multi-line namespace \'', $python_namespaces->[-1]->{symbol_scoped}, '\'', "\n";
1673             # set ending line number, indicating we are no longer inside this multi-line component;
1674             # the multi-line component actually ended on the last line number
1675 0         0 $python_namespaces->[-1]->{python_line_number_end} = $python_line_number;
1676              
1677 0         0 pop @{$python_namespaces};
  0         0  
1678             }
1679              
1680             # remove extra trailing newline, to match original input source code
1681             # DEV NOTE: while loop causes extraneous newline to be added at end of last line!
1682 1         3 chomp $self->{python_source_code};
1683              
1684 1         10 print 'in python_file_to_python_preparsed(), after reading & closing input file, about to return $self->{python_preparsed} = ', Dumper($self->{python_preparsed}), "\n";
1685              
1686 1         2945 return $self->{python_preparsed};
1687             }
1688              
1689             1;