line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
1
|
|
|
|
|
|
|
/* Copyright 2009 Peter Karman |
2
|
|
|
|
|
|
|
* |
3
|
|
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify |
4
|
|
|
|
|
|
|
* under the same terms as Perl itself. |
5
|
|
|
|
|
|
|
*/ |
6
|
|
|
|
|
|
|
|
7
|
|
|
|
|
|
|
/* |
8
|
|
|
|
|
|
|
* Standard XS greeting. |
9
|
|
|
|
|
|
|
*/ |
10
|
|
|
|
|
|
|
#ifdef __cplusplus |
11
|
|
|
|
|
|
|
extern "C" { |
12
|
|
|
|
|
|
|
#endif |
13
|
|
|
|
|
|
|
#define PERL_NO_GET_CONTEXT |
14
|
|
|
|
|
|
|
#include "EXTERN.h" |
15
|
|
|
|
|
|
|
#include "perl.h" |
16
|
|
|
|
|
|
|
#include "XSUB.h" |
17
|
|
|
|
|
|
|
#include "ppport.h" |
18
|
|
|
|
|
|
|
#ifdef __cplusplus |
19
|
|
|
|
|
|
|
} |
20
|
|
|
|
|
|
|
#endif |
21
|
|
|
|
|
|
|
|
22
|
|
|
|
|
|
|
#ifdef EXTERN |
23
|
|
|
|
|
|
|
#undef EXTERN |
24
|
|
|
|
|
|
|
#endif |
25
|
|
|
|
|
|
|
|
26
|
|
|
|
|
|
|
#define EXTERN static |
27
|
|
|
|
|
|
|
|
28
|
|
|
|
|
|
|
/* pure C helpers */ |
29
|
|
|
|
|
|
|
#include "search-tools.c" |
30
|
|
|
|
|
|
|
|
31
|
|
|
|
|
|
|
/********************************************************************/ |
32
|
|
|
|
|
|
|
|
33
|
|
|
|
|
|
|
MODULE = Search::Tools PACKAGE = Search::Tools |
34
|
|
|
|
|
|
|
|
35
|
|
|
|
|
|
|
PROTOTYPES: enable |
36
|
|
|
|
|
|
|
|
37
|
|
|
|
|
|
|
|
38
|
|
|
|
|
|
|
void |
39
|
|
|
|
|
|
|
describe(thing) |
40
|
|
|
|
|
|
|
SV *thing |
41
|
|
|
|
|
|
|
|
42
|
|
|
|
|
|
|
CODE: |
43
|
0
|
|
|
|
|
|
st_describe_object(thing); |
44
|
0
|
|
|
|
|
|
st_dump_sv(thing); |
45
|
|
|
|
|
|
|
|
46
|
|
|
|
|
|
|
|
47
|
|
|
|
|
|
|
###################################################################### |
48
|
|
|
|
|
|
|
MODULE = Search::Tools PACKAGE = Search::Tools::UTF8 |
49
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
|
PROTOTYPES: enable |
51
|
|
|
|
|
|
|
|
52
|
|
|
|
|
|
|
int |
53
|
|
|
|
|
|
|
byte_length(string) |
54
|
|
|
|
|
|
|
SV* string; |
55
|
|
|
|
|
|
|
|
56
|
|
|
|
|
|
|
PREINIT: |
57
|
|
|
|
|
|
|
STRLEN len; |
58
|
|
|
|
|
|
|
U8 * bytes; |
59
|
|
|
|
|
|
|
|
60
|
|
|
|
|
|
|
CODE: |
61
|
328
|
50
|
|
|
|
|
bytes = (U8*)SvPV(string, len); |
62
|
328
|
|
|
|
|
|
RETVAL = len; |
63
|
|
|
|
|
|
|
|
64
|
|
|
|
|
|
|
OUTPUT: |
65
|
|
|
|
|
|
|
RETVAL |
66
|
|
|
|
|
|
|
|
67
|
|
|
|
|
|
|
|
68
|
|
|
|
|
|
|
int |
69
|
|
|
|
|
|
|
is_perl_utf8_string(string) |
70
|
|
|
|
|
|
|
SV* string; |
71
|
|
|
|
|
|
|
|
72
|
|
|
|
|
|
|
PREINIT: |
73
|
|
|
|
|
|
|
STRLEN len; |
74
|
|
|
|
|
|
|
U8 * bytes; |
75
|
|
|
|
|
|
|
|
76
|
|
|
|
|
|
|
CODE: |
77
|
46
|
50
|
|
|
|
|
bytes = (U8*)SvPV(string, len); |
78
|
46
|
|
|
|
|
|
RETVAL = is_utf8_string(bytes, len); |
79
|
|
|
|
|
|
|
|
80
|
|
|
|
|
|
|
OUTPUT: |
81
|
|
|
|
|
|
|
RETVAL |
82
|
|
|
|
|
|
|
|
83
|
|
|
|
|
|
|
|
84
|
|
|
|
|
|
|
|
85
|
|
|
|
|
|
|
SV* |
86
|
|
|
|
|
|
|
find_bad_utf8(string) |
87
|
|
|
|
|
|
|
SV* string; |
88
|
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
CODE: |
90
|
5
|
|
|
|
|
|
RETVAL = st_find_bad_utf8(string); |
91
|
|
|
|
|
|
|
|
92
|
|
|
|
|
|
|
OUTPUT: |
93
|
|
|
|
|
|
|
RETVAL |
94
|
|
|
|
|
|
|
|
95
|
|
|
|
|
|
|
|
96
|
|
|
|
|
|
|
# benchmarks show these XS versions are 9x faster |
97
|
|
|
|
|
|
|
# than their native Perl regex counterparts |
98
|
|
|
|
|
|
|
boolean |
99
|
|
|
|
|
|
|
is_ascii(string) |
100
|
|
|
|
|
|
|
SV* string; |
101
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
CODE: |
103
|
607
|
|
|
|
|
|
RETVAL = st_is_ascii(string); |
104
|
|
|
|
|
|
|
|
105
|
|
|
|
|
|
|
OUTPUT: |
106
|
|
|
|
|
|
|
RETVAL |
107
|
|
|
|
|
|
|
|
108
|
|
|
|
|
|
|
|
109
|
|
|
|
|
|
|
boolean |
110
|
|
|
|
|
|
|
is_latin1(string) |
111
|
|
|
|
|
|
|
SV* string; |
112
|
|
|
|
|
|
|
|
113
|
|
|
|
|
|
|
PREINIT: |
114
|
|
|
|
|
|
|
STRLEN len; |
115
|
|
|
|
|
|
|
unsigned char* bytes; |
116
|
|
|
|
|
|
|
unsigned int i; |
117
|
|
|
|
|
|
|
|
118
|
|
|
|
|
|
|
CODE: |
119
|
49
|
50
|
|
|
|
|
bytes = (unsigned char*)SvPV(string, len); |
120
|
49
|
|
|
|
|
|
RETVAL = 1; |
121
|
9660
|
100
|
|
|
|
|
for(i=0; i < len; i++) { |
122
|
9643
|
100
|
|
|
|
|
if (bytes[i] > 0x7f && bytes[i] < 0xa0) { |
|
|
100
|
|
|
|
|
|
123
|
32
|
|
|
|
|
|
RETVAL = 0; |
124
|
32
|
|
|
|
|
|
break; |
125
|
|
|
|
|
|
|
} |
126
|
|
|
|
|
|
|
} |
127
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
OUTPUT: |
129
|
|
|
|
|
|
|
RETVAL |
130
|
|
|
|
|
|
|
|
131
|
|
|
|
|
|
|
|
132
|
|
|
|
|
|
|
void |
133
|
|
|
|
|
|
|
debug_bytes(string) |
134
|
|
|
|
|
|
|
SV* string; |
135
|
|
|
|
|
|
|
|
136
|
|
|
|
|
|
|
PREINIT: |
137
|
|
|
|
|
|
|
STRLEN len; |
138
|
|
|
|
|
|
|
unsigned char* bytes; |
139
|
|
|
|
|
|
|
unsigned int i; |
140
|
|
|
|
|
|
|
|
141
|
|
|
|
|
|
|
CODE: |
142
|
0
|
0
|
|
|
|
|
bytes = (unsigned char*)SvPV(string, len); |
143
|
0
|
0
|
|
|
|
|
for(i=0; i < len; i++) { |
144
|
0
|
|
|
|
|
|
warn("'%c' \\x%x \\%d\n", bytes[i], bytes[i], bytes[i]); |
145
|
|
|
|
|
|
|
} |
146
|
|
|
|
|
|
|
|
147
|
|
|
|
|
|
|
|
148
|
|
|
|
|
|
|
IV |
149
|
|
|
|
|
|
|
find_bad_ascii(string) |
150
|
|
|
|
|
|
|
SV* string; |
151
|
|
|
|
|
|
|
|
152
|
|
|
|
|
|
|
PREINIT: |
153
|
|
|
|
|
|
|
STRLEN len; |
154
|
|
|
|
|
|
|
unsigned char* bytes; |
155
|
|
|
|
|
|
|
int i; |
156
|
|
|
|
|
|
|
|
157
|
|
|
|
|
|
|
CODE: |
158
|
1
|
50
|
|
|
|
|
bytes = (unsigned char*)SvPV(string, len); |
159
|
1
|
|
|
|
|
|
RETVAL = -1; |
160
|
1
|
50
|
|
|
|
|
for(i=0; i < len; i++) { |
161
|
1
|
50
|
|
|
|
|
if (bytes[i] >= 0x80) { |
162
|
1
|
|
|
|
|
|
RETVAL = i; |
163
|
1
|
|
|
|
|
|
break; |
164
|
|
|
|
|
|
|
} |
165
|
|
|
|
|
|
|
} |
166
|
|
|
|
|
|
|
|
167
|
|
|
|
|
|
|
OUTPUT: |
168
|
|
|
|
|
|
|
RETVAL |
169
|
|
|
|
|
|
|
|
170
|
|
|
|
|
|
|
int |
171
|
|
|
|
|
|
|
find_bad_latin1(string) |
172
|
|
|
|
|
|
|
SV* string; |
173
|
|
|
|
|
|
|
|
174
|
|
|
|
|
|
|
PREINIT: |
175
|
|
|
|
|
|
|
STRLEN len; |
176
|
|
|
|
|
|
|
unsigned char* bytes; |
177
|
|
|
|
|
|
|
int i; |
178
|
|
|
|
|
|
|
|
179
|
|
|
|
|
|
|
CODE: |
180
|
4
|
50
|
|
|
|
|
bytes = (unsigned char*)SvPV(string, len); |
181
|
4
|
|
|
|
|
|
RETVAL = -1; |
182
|
88
|
100
|
|
|
|
|
for(i=0; i < len; i++) { |
183
|
87
|
100
|
|
|
|
|
if (bytes[i] > 0x7f && bytes[i] < 0xa0) { |
|
|
100
|
|
|
|
|
|
184
|
3
|
|
|
|
|
|
RETVAL = i; |
185
|
3
|
|
|
|
|
|
break; |
186
|
|
|
|
|
|
|
} |
187
|
|
|
|
|
|
|
} |
188
|
|
|
|
|
|
|
|
189
|
|
|
|
|
|
|
OUTPUT: |
190
|
|
|
|
|
|
|
RETVAL |
191
|
|
|
|
|
|
|
|
192
|
|
|
|
|
|
|
|
193
|
|
|
|
|
|
|
|
194
|
|
|
|
|
|
|
############################################################################# |
195
|
|
|
|
|
|
|
|
196
|
|
|
|
|
|
|
MODULE = Search::Tools PACKAGE = Search::Tools::Tokenizer |
197
|
|
|
|
|
|
|
|
198
|
|
|
|
|
|
|
PROTOTYPES: enable |
199
|
|
|
|
|
|
|
|
200
|
|
|
|
|
|
|
SV* |
201
|
|
|
|
|
|
|
tokenize(self, str, ...) |
202
|
|
|
|
|
|
|
SV* self; |
203
|
|
|
|
|
|
|
SV* str; |
204
|
|
|
|
|
|
|
|
205
|
|
|
|
|
|
|
PREINIT: |
206
|
|
|
|
|
|
|
SV* token_re; |
207
|
|
|
|
|
|
|
SV* token_list_sv; |
208
|
|
|
|
|
|
|
STRLEN len; |
209
|
|
|
|
|
|
|
U8* bytes; |
210
|
48
|
|
|
|
|
|
SV* heat_seeker = NULL; |
211
|
|
|
|
|
|
|
IV match_num; |
212
|
|
|
|
|
|
|
|
213
|
|
|
|
|
|
|
CODE: |
214
|
48
|
100
|
|
|
|
|
if (items > 2) { |
215
|
43
|
|
|
|
|
|
heat_seeker = ST(2); |
216
|
|
|
|
|
|
|
} |
217
|
48
|
|
|
|
|
|
match_num = 0; |
218
|
48
|
50
|
|
|
|
|
if (items > 3) { |
219
|
0
|
0
|
|
|
|
|
match_num = SvIV(ST(3)); |
220
|
|
|
|
|
|
|
} |
221
|
|
|
|
|
|
|
|
222
|
|
|
|
|
|
|
/* test if utf8 flag on and make sure it is. |
223
|
|
|
|
|
|
|
* otherwise, regex for \w can fail for multibyte chars. |
224
|
|
|
|
|
|
|
* we do a slight (~7%) optimization for ascii str because |
225
|
|
|
|
|
|
|
* the regex engine is faster for all-ascii texts. |
226
|
|
|
|
|
|
|
* the logic is: |
227
|
|
|
|
|
|
|
* if the flag is on, ok. |
228
|
|
|
|
|
|
|
* else, |
229
|
|
|
|
|
|
|
* if the string is ascii, ok for flag to be off, |
230
|
|
|
|
|
|
|
* but we don't turn it off. |
231
|
|
|
|
|
|
|
* if the string is NOT ascii, make sure it is utf8 |
232
|
|
|
|
|
|
|
* and turn the flag on. |
233
|
|
|
|
|
|
|
*/ |
234
|
48
|
100
|
|
|
|
|
if (!SvUTF8(str)) { |
235
|
15
|
100
|
|
|
|
|
if (!st_is_ascii(str)) { |
236
|
2
|
50
|
|
|
|
|
bytes = (U8*)SvPV(str, len); |
237
|
2
|
50
|
|
|
|
|
if(!is_utf8_string(bytes, len)) { |
238
|
0
|
|
|
|
|
|
croak(ST_BAD_UTF8); |
239
|
|
|
|
|
|
|
} |
240
|
2
|
|
|
|
|
|
SvUTF8_on(str); |
241
|
|
|
|
|
|
|
} |
242
|
|
|
|
|
|
|
} |
243
|
|
|
|
|
|
|
|
244
|
48
|
|
|
|
|
|
token_re = st_hvref_fetch(self, "re"); |
245
|
48
|
|
|
|
|
|
token_list_sv = st_tokenize(str, token_re, heat_seeker, match_num); |
246
|
48
|
|
|
|
|
|
RETVAL = token_list_sv; |
247
|
|
|
|
|
|
|
|
248
|
|
|
|
|
|
|
OUTPUT: |
249
|
|
|
|
|
|
|
RETVAL |
250
|
|
|
|
|
|
|
|
251
|
|
|
|
|
|
|
SV* |
252
|
|
|
|
|
|
|
set_debug(self, val) |
253
|
|
|
|
|
|
|
SV* self; |
254
|
|
|
|
|
|
|
boolean val; |
255
|
|
|
|
|
|
|
|
256
|
|
|
|
|
|
|
CODE: |
257
|
|
|
|
|
|
|
SV* st_debug_var; |
258
|
0
|
|
|
|
|
|
st_debug_var = get_sv("Search::Tools::XS_DEBUG", GV_ADD); |
259
|
|
|
|
|
|
|
//warn(" st_debug_var before = '%s'\n", SvPV_nolen(st_debug_var)); |
260
|
0
|
|
|
|
|
|
SvIV_set(st_debug_var, val); |
261
|
0
|
|
|
|
|
|
SvIOK_on(st_debug_var); |
262
|
|
|
|
|
|
|
//warn("ST_DEBUG set to %d", val); |
263
|
|
|
|
|
|
|
//warn(" st_debug_var set = '%d'\n", ST_DEBUG); |
264
|
0
|
0
|
|
|
|
|
if (SvREFCNT(st_debug_var) == 1) { |
265
|
|
|
|
|
|
|
// IMPORTANT because we access var from Perl and C |
266
|
0
|
|
|
|
|
|
SvREFCNT_inc(st_debug_var); |
267
|
|
|
|
|
|
|
} |
268
|
0
|
|
|
|
|
|
RETVAL = st_debug_var; |
269
|
|
|
|
|
|
|
|
270
|
|
|
|
|
|
|
OUTPUT: |
271
|
|
|
|
|
|
|
RETVAL |
272
|
|
|
|
|
|
|
|
273
|
|
|
|
|
|
|
|
274
|
|
|
|
|
|
|
SV* |
275
|
|
|
|
|
|
|
get_offsets(self, str, regex) |
276
|
|
|
|
|
|
|
SV* self; |
277
|
|
|
|
|
|
|
SV* str; |
278
|
|
|
|
|
|
|
SV* regex; |
279
|
|
|
|
|
|
|
|
280
|
|
|
|
|
|
|
CODE: |
281
|
25
|
|
|
|
|
|
RETVAL = newRV_noinc((SV*)st_heat_seeker_offsets(str, regex)); |
282
|
|
|
|
|
|
|
|
283
|
|
|
|
|
|
|
OUTPUT: |
284
|
|
|
|
|
|
|
RETVAL |
285
|
|
|
|
|
|
|
|
286
|
|
|
|
|
|
|
|
287
|
|
|
|
|
|
|
|
288
|
|
|
|
|
|
|
############################################################################ |
289
|
|
|
|
|
|
|
|
290
|
|
|
|
|
|
|
MODULE = Search::Tools PACKAGE = Search::Tools::TokenList |
291
|
|
|
|
|
|
|
|
292
|
|
|
|
|
|
|
PROTOTYPES: enable |
293
|
|
|
|
|
|
|
|
294
|
|
|
|
|
|
|
void |
295
|
|
|
|
|
|
|
dump(self) |
296
|
|
|
|
|
|
|
st_token_list *self; |
297
|
|
|
|
|
|
|
|
298
|
|
|
|
|
|
|
CODE: |
299
|
0
|
|
|
|
|
|
st_dump_token_list(self); |
300
|
|
|
|
|
|
|
|
301
|
|
|
|
|
|
|
|
302
|
|
|
|
|
|
|
SV* |
303
|
|
|
|
|
|
|
next(self) |
304
|
|
|
|
|
|
|
st_token_list *self; |
305
|
|
|
|
|
|
|
|
306
|
|
|
|
|
|
|
PREINIT: |
307
|
|
|
|
|
|
|
IV len; |
308
|
|
|
|
|
|
|
|
309
|
|
|
|
|
|
|
CODE: |
310
|
349
|
|
|
|
|
|
len = av_len(self->tokens); |
311
|
|
|
|
|
|
|
//warn("len = %d and pos = %d", len, self->pos); |
312
|
|
|
|
|
|
|
|
313
|
349
|
50
|
|
|
|
|
if (len == -1) { |
314
|
|
|
|
|
|
|
// empty list |
315
|
0
|
|
|
|
|
|
RETVAL = &PL_sv_undef; |
316
|
|
|
|
|
|
|
} |
317
|
349
|
100
|
|
|
|
|
else if (self->pos > len) { |
318
|
|
|
|
|
|
|
// exceeded end of list |
319
|
9
|
|
|
|
|
|
RETVAL = &PL_sv_undef; |
320
|
|
|
|
|
|
|
} |
321
|
|
|
|
|
|
|
else { |
322
|
340
|
50
|
|
|
|
|
if (!av_exists(self->tokens, self->pos)) { |
323
|
0
|
|
|
|
|
|
ST_CROAK("no such index at %d", self->pos); |
324
|
|
|
|
|
|
|
} |
325
|
|
|
|
|
|
|
//st_dump_sv( st_av_fetch(self->tokens, self->pos) ); |
326
|
340
|
|
|
|
|
|
RETVAL = SvREFCNT_inc(st_av_fetch(self->tokens, self->pos++)); |
327
|
|
|
|
|
|
|
|
328
|
|
|
|
|
|
|
} |
329
|
|
|
|
|
|
|
|
330
|
|
|
|
|
|
|
|
331
|
|
|
|
|
|
|
OUTPUT: |
332
|
|
|
|
|
|
|
RETVAL |
333
|
|
|
|
|
|
|
|
334
|
|
|
|
|
|
|
|
335
|
|
|
|
|
|
|
SV* |
336
|
|
|
|
|
|
|
prev(self) |
337
|
|
|
|
|
|
|
st_token_list *self; |
338
|
|
|
|
|
|
|
|
339
|
|
|
|
|
|
|
PREINIT: |
340
|
|
|
|
|
|
|
IV len; |
341
|
|
|
|
|
|
|
|
342
|
|
|
|
|
|
|
CODE: |
343
|
6
|
|
|
|
|
|
len = av_len(self->tokens); |
344
|
6
|
50
|
|
|
|
|
if (len == -1) { |
345
|
|
|
|
|
|
|
// empty list |
346
|
0
|
|
|
|
|
|
RETVAL = &PL_sv_undef; |
347
|
|
|
|
|
|
|
} |
348
|
6
|
50
|
|
|
|
|
else if (self->pos < 0) { |
349
|
|
|
|
|
|
|
// exceeded start of list |
350
|
0
|
|
|
|
|
|
RETVAL = &PL_sv_undef; |
351
|
|
|
|
|
|
|
} |
352
|
|
|
|
|
|
|
else { |
353
|
6
|
50
|
|
|
|
|
if (!av_exists(self->tokens, (self->pos-1))) { |
354
|
0
|
|
|
|
|
|
ST_CROAK("no such index at %d", (self->pos-1)); |
355
|
|
|
|
|
|
|
} |
356
|
6
|
|
|
|
|
|
RETVAL = SvREFCNT_inc(st_av_fetch(self->tokens, --(self->pos))); |
357
|
|
|
|
|
|
|
} |
358
|
|
|
|
|
|
|
|
359
|
|
|
|
|
|
|
|
360
|
|
|
|
|
|
|
OUTPUT: |
361
|
|
|
|
|
|
|
RETVAL |
362
|
|
|
|
|
|
|
|
363
|
|
|
|
|
|
|
|
364
|
|
|
|
|
|
|
SV* |
365
|
|
|
|
|
|
|
get_token(self, pos) |
366
|
|
|
|
|
|
|
st_token_list *self; |
367
|
|
|
|
|
|
|
IV pos; |
368
|
|
|
|
|
|
|
|
369
|
|
|
|
|
|
|
CODE: |
370
|
5808
|
50
|
|
|
|
|
if (!av_exists(self->tokens, pos)) { |
371
|
0
|
|
|
|
|
|
RETVAL = &PL_sv_undef; |
372
|
|
|
|
|
|
|
} |
373
|
|
|
|
|
|
|
else { |
374
|
5808
|
|
|
|
|
|
RETVAL = SvREFCNT_inc(st_av_fetch(self->tokens, pos)); |
375
|
|
|
|
|
|
|
} |
376
|
|
|
|
|
|
|
|
377
|
|
|
|
|
|
|
OUTPUT: |
378
|
|
|
|
|
|
|
RETVAL |
379
|
|
|
|
|
|
|
|
380
|
|
|
|
|
|
|
|
381
|
|
|
|
|
|
|
IV |
382
|
|
|
|
|
|
|
set_pos(self, new_pos) |
383
|
|
|
|
|
|
|
st_token_list *self; |
384
|
|
|
|
|
|
|
IV new_pos; |
385
|
|
|
|
|
|
|
|
386
|
|
|
|
|
|
|
CODE: |
387
|
0
|
|
|
|
|
|
RETVAL = self->pos; |
388
|
0
|
|
|
|
|
|
self->pos = new_pos; |
389
|
|
|
|
|
|
|
|
390
|
|
|
|
|
|
|
OUTPUT: |
391
|
|
|
|
|
|
|
RETVAL |
392
|
|
|
|
|
|
|
|
393
|
|
|
|
|
|
|
|
394
|
|
|
|
|
|
|
IV |
395
|
|
|
|
|
|
|
reset(self) |
396
|
|
|
|
|
|
|
st_token_list *self; |
397
|
|
|
|
|
|
|
|
398
|
|
|
|
|
|
|
CODE: |
399
|
0
|
|
|
|
|
|
RETVAL = self->pos; |
400
|
0
|
|
|
|
|
|
self->pos = 0; |
401
|
|
|
|
|
|
|
|
402
|
|
|
|
|
|
|
OUTPUT: |
403
|
|
|
|
|
|
|
RETVAL |
404
|
|
|
|
|
|
|
|
405
|
|
|
|
|
|
|
|
406
|
|
|
|
|
|
|
IV |
407
|
|
|
|
|
|
|
len(self) |
408
|
|
|
|
|
|
|
st_token_list *self; |
409
|
|
|
|
|
|
|
|
410
|
|
|
|
|
|
|
CODE: |
411
|
113
|
|
|
|
|
|
RETVAL = av_len(self->tokens) + 1; |
412
|
|
|
|
|
|
|
|
413
|
|
|
|
|
|
|
OUTPUT: |
414
|
|
|
|
|
|
|
RETVAL |
415
|
|
|
|
|
|
|
|
416
|
|
|
|
|
|
|
|
417
|
|
|
|
|
|
|
IV |
418
|
|
|
|
|
|
|
num(self) |
419
|
|
|
|
|
|
|
st_token_list *self; |
420
|
|
|
|
|
|
|
|
421
|
|
|
|
|
|
|
CODE: |
422
|
28
|
|
|
|
|
|
RETVAL = self->num; |
423
|
|
|
|
|
|
|
|
424
|
|
|
|
|
|
|
OUTPUT: |
425
|
|
|
|
|
|
|
RETVAL |
426
|
|
|
|
|
|
|
|
427
|
|
|
|
|
|
|
|
428
|
|
|
|
|
|
|
IV |
429
|
|
|
|
|
|
|
pos(self) |
430
|
|
|
|
|
|
|
st_token_list *self; |
431
|
|
|
|
|
|
|
|
432
|
|
|
|
|
|
|
CODE: |
433
|
13
|
|
|
|
|
|
RETVAL = self->pos; |
434
|
|
|
|
|
|
|
|
435
|
|
|
|
|
|
|
OUTPUT: |
436
|
|
|
|
|
|
|
RETVAL |
437
|
|
|
|
|
|
|
|
438
|
|
|
|
|
|
|
|
439
|
|
|
|
|
|
|
SV* |
440
|
|
|
|
|
|
|
as_array(self) |
441
|
|
|
|
|
|
|
st_token_list *self; |
442
|
|
|
|
|
|
|
|
443
|
|
|
|
|
|
|
CODE: |
444
|
73
|
|
|
|
|
|
RETVAL = newRV_inc((SV*)self->tokens); |
445
|
|
|
|
|
|
|
|
446
|
|
|
|
|
|
|
OUTPUT: |
447
|
|
|
|
|
|
|
RETVAL |
448
|
|
|
|
|
|
|
|
449
|
|
|
|
|
|
|
|
450
|
|
|
|
|
|
|
SV* |
451
|
|
|
|
|
|
|
get_heat(self) |
452
|
|
|
|
|
|
|
st_token_list *self; |
453
|
|
|
|
|
|
|
|
454
|
|
|
|
|
|
|
PREINIT: |
455
|
|
|
|
|
|
|
AV *heat; |
456
|
|
|
|
|
|
|
IV len; |
457
|
|
|
|
|
|
|
IV pos; |
458
|
|
|
|
|
|
|
SV* h; |
459
|
|
|
|
|
|
|
|
460
|
|
|
|
|
|
|
CODE: |
461
|
60
|
|
|
|
|
|
heat = newAV(); |
462
|
60
|
|
|
|
|
|
pos = 0; |
463
|
60
|
|
|
|
|
|
len = av_len(self->heat)+1; |
464
|
298
|
100
|
|
|
|
|
while (pos < len) { |
465
|
238
|
|
|
|
|
|
h = st_av_fetch(self->heat, pos++); |
466
|
238
|
|
|
|
|
|
av_push(heat, h); |
467
|
|
|
|
|
|
|
} |
468
|
60
|
|
|
|
|
|
RETVAL = newRV((SV*)heat); /* no _inc -- this is a copy */ |
469
|
|
|
|
|
|
|
|
470
|
|
|
|
|
|
|
OUTPUT: |
471
|
|
|
|
|
|
|
RETVAL |
472
|
|
|
|
|
|
|
|
473
|
|
|
|
|
|
|
|
474
|
|
|
|
|
|
|
SV* |
475
|
|
|
|
|
|
|
get_sentence_starts(self) |
476
|
|
|
|
|
|
|
st_token_list *self; |
477
|
|
|
|
|
|
|
|
478
|
|
|
|
|
|
|
PREINIT: |
479
|
|
|
|
|
|
|
AV *starts; |
480
|
|
|
|
|
|
|
IV len; |
481
|
|
|
|
|
|
|
IV pos; |
482
|
|
|
|
|
|
|
SV* sstart; |
483
|
|
|
|
|
|
|
|
484
|
|
|
|
|
|
|
CODE: |
485
|
15
|
|
|
|
|
|
starts = newAV(); |
486
|
15
|
|
|
|
|
|
pos = 0; |
487
|
15
|
|
|
|
|
|
len = av_len(self->sentence_starts)+1; |
488
|
59
|
100
|
|
|
|
|
while (pos < len) { |
489
|
44
|
|
|
|
|
|
sstart = st_av_fetch(self->sentence_starts, pos++); |
490
|
44
|
|
|
|
|
|
av_push(starts, sstart); |
491
|
|
|
|
|
|
|
} |
492
|
15
|
|
|
|
|
|
RETVAL = newRV((SV*)starts); /* no _inc -- this is a copy */ |
493
|
|
|
|
|
|
|
|
494
|
|
|
|
|
|
|
OUTPUT: |
495
|
|
|
|
|
|
|
RETVAL |
496
|
|
|
|
|
|
|
|
497
|
|
|
|
|
|
|
|
498
|
|
|
|
|
|
|
SV* |
499
|
|
|
|
|
|
|
matches(self) |
500
|
|
|
|
|
|
|
st_token_list *self; |
501
|
|
|
|
|
|
|
|
502
|
|
|
|
|
|
|
PREINIT: |
503
|
|
|
|
|
|
|
AV *matches; |
504
|
|
|
|
|
|
|
IV pos; |
505
|
|
|
|
|
|
|
IV len; |
506
|
|
|
|
|
|
|
SV* tok; |
507
|
|
|
|
|
|
|
st_token *token; |
508
|
|
|
|
|
|
|
|
509
|
|
|
|
|
|
|
CODE: |
510
|
7
|
|
|
|
|
|
matches = newAV(); |
511
|
7
|
|
|
|
|
|
pos = 0; |
512
|
7
|
|
|
|
|
|
len = av_len(self->tokens)+1; |
513
|
200
|
100
|
|
|
|
|
while (pos < len) { |
514
|
193
|
|
|
|
|
|
tok = st_av_fetch(self->tokens, pos++); |
515
|
193
|
|
|
|
|
|
token = (st_token*)st_extract_ptr(tok); |
516
|
193
|
100
|
|
|
|
|
if (token->is_match) { |
517
|
3
|
|
|
|
|
|
av_push(matches, tok); |
518
|
|
|
|
|
|
|
} |
519
|
|
|
|
|
|
|
} |
520
|
7
|
|
|
|
|
|
RETVAL = newRV((SV*)matches); /* no _inc -- this is only copy */ |
521
|
|
|
|
|
|
|
|
522
|
|
|
|
|
|
|
OUTPUT: |
523
|
|
|
|
|
|
|
RETVAL |
524
|
|
|
|
|
|
|
|
525
|
|
|
|
|
|
|
|
526
|
|
|
|
|
|
|
IV |
527
|
|
|
|
|
|
|
num_matches(self) |
528
|
|
|
|
|
|
|
st_token_list *self; |
529
|
|
|
|
|
|
|
|
530
|
|
|
|
|
|
|
PREINIT: |
531
|
|
|
|
|
|
|
IV pos; |
532
|
|
|
|
|
|
|
IV len; |
533
|
|
|
|
|
|
|
IV num_matches; |
534
|
|
|
|
|
|
|
st_token *token; |
535
|
|
|
|
|
|
|
|
536
|
|
|
|
|
|
|
CODE: |
537
|
0
|
|
|
|
|
|
num_matches = 0; |
538
|
0
|
|
|
|
|
|
pos = 0; |
539
|
0
|
|
|
|
|
|
len = av_len(self->tokens)+1; |
540
|
0
|
0
|
|
|
|
|
while (pos < len) { |
541
|
0
|
|
|
|
|
|
token = (st_token*)st_av_fetch_ptr(self->tokens, pos++); |
542
|
0
|
0
|
|
|
|
|
if (token->is_match) { |
543
|
0
|
|
|
|
|
|
num_matches++; |
544
|
|
|
|
|
|
|
} |
545
|
|
|
|
|
|
|
} |
546
|
0
|
|
|
|
|
|
RETVAL = num_matches; |
547
|
|
|
|
|
|
|
|
548
|
|
|
|
|
|
|
OUTPUT: |
549
|
|
|
|
|
|
|
RETVAL |
550
|
|
|
|
|
|
|
|
551
|
|
|
|
|
|
|
|
552
|
|
|
|
|
|
|
void |
553
|
|
|
|
|
|
|
DESTROY(self) |
554
|
|
|
|
|
|
|
SV *self; |
555
|
|
|
|
|
|
|
|
556
|
|
|
|
|
|
|
PREINIT: |
557
|
|
|
|
|
|
|
st_token_list *tl; |
558
|
|
|
|
|
|
|
|
559
|
|
|
|
|
|
|
CODE: |
560
|
|
|
|
|
|
|
|
561
|
|
|
|
|
|
|
|
562
|
48
|
|
|
|
|
|
tl = (st_token_list*)st_extract_ptr(self); |
563
|
48
|
|
|
|
|
|
tl->ref_cnt--; |
564
|
48
|
50
|
|
|
|
|
if (ST_DEBUG) { |
|
|
50
|
|
|
|
|
|
565
|
0
|
|
|
|
|
|
warn("............................"); |
566
|
0
|
0
|
|
|
|
|
warn("DESTROY %s [%ld] [0x%lx]\n", |
567
|
0
|
|
|
|
|
|
SvPV_nolen(self), (unsigned long)tl->ref_cnt, (unsigned long)tl); |
568
|
0
|
|
|
|
|
|
st_describe_object(self); |
569
|
0
|
|
|
|
|
|
st_dump_sv((SV*)tl->tokens); |
570
|
|
|
|
|
|
|
} |
571
|
48
|
50
|
|
|
|
|
if (tl->ref_cnt < 1) { |
572
|
48
|
|
|
|
|
|
st_free_token_list(tl); |
573
|
|
|
|
|
|
|
} |
574
|
|
|
|
|
|
|
|
575
|
|
|
|
|
|
|
|
576
|
|
|
|
|
|
|
|
577
|
|
|
|
|
|
|
############################################################################ |
578
|
|
|
|
|
|
|
|
579
|
|
|
|
|
|
|
MODULE = Search::Tools PACKAGE = Search::Tools::Token |
580
|
|
|
|
|
|
|
|
581
|
|
|
|
|
|
|
PROTOTYPES: enable |
582
|
|
|
|
|
|
|
|
583
|
|
|
|
|
|
|
IV |
584
|
|
|
|
|
|
|
pos(self) |
585
|
|
|
|
|
|
|
st_token *self; |
586
|
|
|
|
|
|
|
|
587
|
|
|
|
|
|
|
CODE: |
588
|
3754
|
|
|
|
|
|
RETVAL = self->pos; |
589
|
|
|
|
|
|
|
|
590
|
|
|
|
|
|
|
OUTPUT: |
591
|
|
|
|
|
|
|
RETVAL |
592
|
|
|
|
|
|
|
|
593
|
|
|
|
|
|
|
|
594
|
|
|
|
|
|
|
SV* |
595
|
|
|
|
|
|
|
str(self) |
596
|
|
|
|
|
|
|
st_token *self; |
597
|
|
|
|
|
|
|
|
598
|
|
|
|
|
|
|
CODE: |
599
|
4593
|
|
|
|
|
|
RETVAL = SvREFCNT_inc(self->str); |
600
|
|
|
|
|
|
|
|
601
|
|
|
|
|
|
|
OUTPUT: |
602
|
|
|
|
|
|
|
RETVAL |
603
|
|
|
|
|
|
|
|
604
|
|
|
|
|
|
|
|
605
|
|
|
|
|
|
|
IV |
606
|
|
|
|
|
|
|
len(self) |
607
|
|
|
|
|
|
|
st_token *self; |
608
|
|
|
|
|
|
|
|
609
|
|
|
|
|
|
|
CODE: |
610
|
4787
|
|
|
|
|
|
RETVAL = self->len; |
611
|
|
|
|
|
|
|
|
612
|
|
|
|
|
|
|
OUTPUT: |
613
|
|
|
|
|
|
|
RETVAL |
614
|
|
|
|
|
|
|
|
615
|
|
|
|
|
|
|
|
616
|
|
|
|
|
|
|
IV |
617
|
|
|
|
|
|
|
u8len(self) |
618
|
|
|
|
|
|
|
st_token *self; |
619
|
|
|
|
|
|
|
|
620
|
|
|
|
|
|
|
CODE: |
621
|
188
|
|
|
|
|
|
RETVAL = self->u8len; |
622
|
|
|
|
|
|
|
|
623
|
|
|
|
|
|
|
OUTPUT: |
624
|
|
|
|
|
|
|
RETVAL |
625
|
|
|
|
|
|
|
|
626
|
|
|
|
|
|
|
|
627
|
|
|
|
|
|
|
IV |
628
|
|
|
|
|
|
|
is_hot(self) |
629
|
|
|
|
|
|
|
st_token *self; |
630
|
|
|
|
|
|
|
|
631
|
|
|
|
|
|
|
CODE: |
632
|
187
|
|
|
|
|
|
RETVAL = self->is_hot; |
633
|
|
|
|
|
|
|
|
634
|
|
|
|
|
|
|
OUTPUT: |
635
|
|
|
|
|
|
|
RETVAL |
636
|
|
|
|
|
|
|
|
637
|
|
|
|
|
|
|
|
638
|
|
|
|
|
|
|
IV |
639
|
|
|
|
|
|
|
is_match(self) |
640
|
|
|
|
|
|
|
st_token *self; |
641
|
|
|
|
|
|
|
|
642
|
|
|
|
|
|
|
CODE: |
643
|
340
|
|
|
|
|
|
RETVAL = self->is_match; |
644
|
|
|
|
|
|
|
|
645
|
|
|
|
|
|
|
OUTPUT: |
646
|
|
|
|
|
|
|
RETVAL |
647
|
|
|
|
|
|
|
|
648
|
|
|
|
|
|
|
|
649
|
|
|
|
|
|
|
IV |
650
|
|
|
|
|
|
|
is_sentence_start(self) |
651
|
|
|
|
|
|
|
st_token *self; |
652
|
|
|
|
|
|
|
|
653
|
|
|
|
|
|
|
CODE: |
654
|
220
|
|
|
|
|
|
RETVAL = self->is_sentence_start; |
655
|
|
|
|
|
|
|
|
656
|
|
|
|
|
|
|
OUTPUT: |
657
|
|
|
|
|
|
|
RETVAL |
658
|
|
|
|
|
|
|
|
659
|
|
|
|
|
|
|
|
660
|
|
|
|
|
|
|
IV |
661
|
|
|
|
|
|
|
is_sentence_end(self) |
662
|
|
|
|
|
|
|
st_token *self; |
663
|
|
|
|
|
|
|
|
664
|
|
|
|
|
|
|
CODE: |
665
|
1994
|
|
|
|
|
|
RETVAL = self->is_sentence_end; |
666
|
|
|
|
|
|
|
|
667
|
|
|
|
|
|
|
OUTPUT: |
668
|
|
|
|
|
|
|
RETVAL |
669
|
|
|
|
|
|
|
|
670
|
|
|
|
|
|
|
IV |
671
|
|
|
|
|
|
|
is_abbreviation(self) |
672
|
|
|
|
|
|
|
st_token *self; |
673
|
|
|
|
|
|
|
|
674
|
|
|
|
|
|
|
CODE: |
675
|
0
|
|
|
|
|
|
RETVAL = self->is_abbreviation; |
676
|
|
|
|
|
|
|
|
677
|
|
|
|
|
|
|
OUTPUT: |
678
|
|
|
|
|
|
|
RETVAL |
679
|
|
|
|
|
|
|
|
680
|
|
|
|
|
|
|
IV |
681
|
|
|
|
|
|
|
set_match(self, val) |
682
|
|
|
|
|
|
|
st_token *self; |
683
|
|
|
|
|
|
|
IV val; |
684
|
|
|
|
|
|
|
|
685
|
|
|
|
|
|
|
CODE: |
686
|
188
|
|
|
|
|
|
RETVAL = self->is_match; |
687
|
188
|
|
|
|
|
|
self->is_match = val; |
688
|
|
|
|
|
|
|
|
689
|
|
|
|
|
|
|
OUTPUT: |
690
|
|
|
|
|
|
|
RETVAL |
691
|
|
|
|
|
|
|
|
692
|
|
|
|
|
|
|
|
693
|
|
|
|
|
|
|
IV |
694
|
|
|
|
|
|
|
set_hot(self, val) |
695
|
|
|
|
|
|
|
st_token *self; |
696
|
|
|
|
|
|
|
IV val; |
697
|
|
|
|
|
|
|
|
698
|
|
|
|
|
|
|
CODE: |
699
|
188
|
|
|
|
|
|
RETVAL = self->is_hot; |
700
|
188
|
|
|
|
|
|
self->is_hot = val; |
701
|
|
|
|
|
|
|
|
702
|
|
|
|
|
|
|
OUTPUT: |
703
|
|
|
|
|
|
|
RETVAL |
704
|
|
|
|
|
|
|
|
705
|
|
|
|
|
|
|
|
706
|
|
|
|
|
|
|
void |
707
|
|
|
|
|
|
|
dump(self) |
708
|
|
|
|
|
|
|
st_token *self; |
709
|
|
|
|
|
|
|
|
710
|
|
|
|
|
|
|
CODE: |
711
|
0
|
|
|
|
|
|
st_dump_token(self); |
712
|
|
|
|
|
|
|
|
713
|
|
|
|
|
|
|
|
714
|
|
|
|
|
|
|
void |
715
|
|
|
|
|
|
|
DESTROY(self) |
716
|
|
|
|
|
|
|
SV *self; |
717
|
|
|
|
|
|
|
|
718
|
|
|
|
|
|
|
PREINIT: |
719
|
|
|
|
|
|
|
st_token *tok; |
720
|
|
|
|
|
|
|
|
721
|
|
|
|
|
|
|
CODE: |
722
|
12450
|
|
|
|
|
|
tok = (st_token*)st_extract_ptr(self); |
723
|
12450
|
|
|
|
|
|
tok->ref_cnt--; |
724
|
12450
|
50
|
|
|
|
|
if (ST_DEBUG) { |
|
|
50
|
|
|
|
|
|
725
|
0
|
|
|
|
|
|
warn("............................"); |
726
|
0
|
0
|
|
|
|
|
warn("DESTROY %s [%ld] [0x%lx]\n", |
727
|
0
|
|
|
|
|
|
SvPV_nolen(self), (unsigned long)tok->ref_cnt, (unsigned long)tok); |
728
|
|
|
|
|
|
|
} |
729
|
12450
|
50
|
|
|
|
|
if (tok->ref_cnt < 1) { |
730
|
12450
|
|
|
|
|
|
st_free_token(tok); |
731
|
|
|
|
|
|
|
} |
732
|
|
|
|
|
|
|
|
733
|
|
|
|
|
|
|
|
734
|
|
|
|
|
|
|
############################################################################ |
735
|
|
|
|
|
|
|
|
736
|
|
|
|
|
|
|
MODULE = Search::Tools PACKAGE = Search::Tools::XML |
737
|
|
|
|
|
|
|
|
738
|
|
|
|
|
|
|
PROTOTYPES: enable |
739
|
|
|
|
|
|
|
|
740
|
|
|
|
|
|
|
SV* |
741
|
|
|
|
|
|
|
_escape_xml(text, is_flagged_utf8) |
742
|
|
|
|
|
|
|
char *text; |
743
|
|
|
|
|
|
|
int is_flagged_utf8; |
744
|
|
|
|
|
|
|
|
745
|
|
|
|
|
|
|
CODE: |
746
|
6
|
|
|
|
|
|
RETVAL = st_escape_xml(text); |
747
|
6
|
100
|
|
|
|
|
if (is_flagged_utf8) { |
748
|
1
|
|
|
|
|
|
SvUTF8_on(RETVAL); |
749
|
|
|
|
|
|
|
} |
750
|
|
|
|
|
|
|
|
751
|
|
|
|
|
|
|
OUTPUT: |
752
|
|
|
|
|
|
|
RETVAL |
753
|
|
|
|
|
|
|
|