1 # This file is part of lyx2lyx
2 # -*- coding: utf-8 -*-
3 # Copyright (C) 2002-2011 Dekel Tsur <dekel@lyx.org>,
4 # José Matos <jamatos@lyx.org>, Richard Heck <rgheck@comcast.net>
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License
8 # as published by the Free Software Foundation; either version 2
9 # of the License, or (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 This module offers several free functions to help parse lines.
23 More documentaton is below, but here is a quick guide to what
24 they do. Optional arguments are marked by brackets.
26 find_token(lines, token[, start[, end[, ignorews]]]):
27 Returns the first line i, start <= i < end, on which
28 token is found at the beginning. Returns -1 if not
30 If ignorews is (given and) True, then differences
31 in whitespace do not count, except that there must be no
32 extra whitespace following token itself.
34 find_token_exact(lines, token[, start[, end]]]):
35 As find_token, but with ignorews set to True.
37 find_tokens(lines, tokens[, start[, end[, ignorews]]]):
38 Returns the first line i, start <= i < end, on which
39 one of the tokens in tokens is found at the beginning.
40 Returns -1 if not found.
41 If ignorews is (given and) True, then differences
42 in whitespace do not count, except that there must be no
43 extra whitespace following token itself.
45 find_tokens_exact(lines, token[, start[, end]]):
46 As find_tokens, but with ignorews True.
48 find_token_backwards(lines, token, start):
49 find_tokens_backwards(lines, tokens, start):
50 As before, but look backwards.
52 find_re(lines, rexp, start[, end]):
53 As find_token, but rexp is a regular expression object,
54 so it has to be passed as e.g.: re.compile(r'...').
56 get_value(lines, token[, start[, end[, default[, delete]]]]):
57 Similar to find_token, but it returns what follows the
58 token on the found line. Example:
59 get_value(document.header, "\\use_xetex", 0)
60 will find a line like:
62 and, in that case, return "true". (Note that whitespace
63 is stripped.) The final argument, default, defaults to "",
64 and is what is returned if we do not find anything. So you
65 can use that to set a default.
67 get_quoted_value(lines, token[, start[, end[, default[, delete]]]]):
68 Similar to get_value, but it will strip quotes off the
69 value, if they are present. So use this one for cases
70 where the value is normally quoted.
72 get_option_value(line, option):
73 This assumes we have a line with something like:
75 and returns value. Returns "" if not found.
77 get_bool_value(lines, token[, start[, end[, default, delete]]]]):
78 Like get_value, but returns a boolean.
80 del_token(lines, token, start[, end]):
81 Like find_token, but deletes the line if it finds one.
82 Returns True if a line got deleted, otherwise False.
84 find_beginning_of(lines, i, start_token, end_token):
85 Here, start_token and end_token are meant to be a matching
86 pair, like "\\begin_layout" and "\\end_layout". We look for
87 the start_token that pairs with the end_token that occurs
88 on or after line i. Returns -1 if not found.
89 So, in the layout case, this would find the \\begin_layout
90 for the layout line i is in.
92 ec = find_token(document.body, "</cell", i)
93 bc = find_beginning_of(document.body, ec, \
95 Now, assuming no -1s, bc-ec wraps the cell for line i.
97 find_end_of(lines, i, start_token, end_token):
98 Like find_beginning_of, but looking for the matching
99 end_token. This might look like:
100 bc = find_token_(document.body, "<cell", i)
101 ec = find_end_of(document.body, bc, "<cell", "</cell")
102 Now, assuming no -1s, bc-ec wrap the next cell.
104 find_end_of_inset(lines, i):
105 Specialization of find_end_of for insets.
107 find_end_of_layout(lines, i):
108 Specialization of find_end_of for layouts.
110 find_end_of_sequence(lines, i):
111 Find the end of the sequence of layouts of the same kind.
112 Considers nesting. If the last paragraph in sequence is nested,
113 the position of the last \end_deeper is returned, else
114 the position of the last \end_layout.
116 is_in_inset(lines, i, inset, default=(-1,-1)):
117 Check if line i is in an inset of the given type.
118 If so, returns starting and ending lines. Otherwise,
121 is_in_inset(document.body, i, "\\begin_inset Tabular")
122 returns (-1,-1) unless i is within a table. If it is, then
123 it returns the line on which the table begins and the one
124 on which it ends. Note that this pair will evaulate to
126 if is_in_inset(..., default=False):
127 will do what you expect.
129 get_containing_inset(lines, i):
130 Finds out what kind of inset line i is within. Returns a
131 list containing what follows \begin_inset on the line
132 on which the inset begins, plus the starting and ending line.
133 Returns False on any kind of error or if it isn't in an inset.
134 So get_containing_inset(document.body, i) might return:
135 ("CommandInset ref", 300, 306)
136 if i is within an InsetRef beginning on line 300 and ending
139 get_containing_layout(lines, i):
140 As get_containing_inset, but for layout. Additionally returns the
141 position of real paragraph start (after par params) as 4th value.
143 find_nonempty_line(lines, start[, end):
144 Finds the next non-empty line.
146 check_token(line, token):
147 Does line begin with token?
149 is_nonempty_line(line):
150 Does line contain something besides whitespace?
152 count_pars_in_inset(lines, i):
153 Counts the paragraphs inside an inset.
159 # Utilities for one line
160 def check_token(line, token):
161 """ check_token(line, token) -> bool
163 Return True if token is present in line and is the first element
166 Deprecated. Use line.startswith(token).
168 return line.startswith(token)
171 def is_nonempty_line(line):
172 """ is_nonempty_line(line) -> bool
174 Return False if line is either empty or it has only whitespaces,
176 return bool(line.strip())
179 # Utilities for a list of lines
180 def find_token(lines, token, start=0, end=0, ignorews=False):
181 """ find_token(lines, token, start[[, end], ignorews]) -> int
183 Return the lowest line where token is found, and is the first
184 element, in lines[start, end].
186 If ignorews is True (default is False), then differences in
187 whitespace are ignored, but there must be whitespace following
190 Return -1 on failure."""
192 if end == 0 or end > len(lines):
196 for i in range(start, end):
204 if lines[i].startswith(token):
209 def find_token_exact(lines, token, start=0, end=0):
210 return find_token(lines, token, start, end, True)
213 def find_tokens(lines, tokens, start=0, end=0, ignorews=False):
214 """ find_tokens(lines, tokens, start[[, end], ignorews]) -> int
216 Return the lowest line where one token in tokens is found, and is
217 the first element, in lines[start, end].
219 Return -1 on failure."""
220 if end == 0 or end > len(lines):
223 for i in range(start, end):
233 if lines[i].startswith(token):
238 def find_tokens_exact(lines, tokens, start=0, end=0):
239 return find_tokens(lines, tokens, start, end, True)
242 def find_re(lines, rexp, start=0, end=0):
243 """ find_re(lines, rexp, start[, end]) -> int
245 Return the lowest line where rexp, a regular expression, is found
246 in lines[start, end].
248 Return -1 on failure."""
250 if end == 0 or end > len(lines):
252 for i in range(start, end):
253 if rexp.match(lines[i]):
258 def find_token_backwards(lines, token, start):
259 """ find_token_backwards(lines, token, start) -> int
261 Return the highest line where token is found, and is the first
262 element, in lines[start, end].
264 Return -1 on failure."""
265 for i in range(start, -1, -1):
266 if lines[i].startswith(token):
271 def find_tokens_backwards(lines, tokens, start):
272 """ find_tokens_backwards(lines, token, start) -> int
274 Return the highest line where token is found, and is the first
275 element, in lines[end, start].
277 Return -1 on failure."""
278 for i in range(start, -1, -1):
281 if line.startswith(token):
286 def find_complete_lines(lines, sublines, start=0, end=0):
287 """Find first occurence of sequence `sublines` in list `lines`.
288 Return index of first line or -1 on failure.
290 Efficient search for a sub-list in a large list. Works for any values.
292 >>> find_complete_lines([1, 2, 3, 1, 1, 2], [1, 2])
295 The `start` and `end` arguments work similar to list.index()
297 >>> find_complete_lines([1, 2, 3, 1, 1 ,2], [1, 2], start=1)
299 >>> find_complete_lines([1, 2, 3, 1, 1 ,2], [1, 2], start=1, end=4)
302 The return value can be used to substitute the sub-list.
303 Take care to check before use:
306 >>> s = find_complete_lines(l, [1, 2])
308 ... l[s:s+2] = [3]; l
311 See also del_complete_lines().
315 end = end or len(lines)
319 for j, value in enumerate(sublines):
320 i = lines.index(value, start, end)
327 except ValueError: # `sublines` not found
331 def find_across_lines(lines, sub, start=0, end=0):
332 sublines = sub.splitlines()
333 if len(sublines) > 2:
334 # at least 3 lines: the middle one(s) are complete -> use index search
335 i = find_complete_lines(lines, sublines[1:-1], start+1, end-1)
339 if (lines[i-1].endswith(sublines[0]) and
340 lines[i+len(sublines)].startswith(sublines[-1])):
344 elif len(sublines) > 1:
345 # last subline must start a line
346 i = find_token(lines, sublines[-1], start, end)
349 if lines[i-1].endswith(sublines[0]):
351 else: # no line-break, may be in the middle of a line
352 if end == 0 or end > len(lines):
354 for i in range(start, end):
360 def get_value(lines, token, start=0, end=0, default="", delete=False):
361 """Find `token` in `lines` and return part of line that follows it.
363 Find the next line that looks like:
364 token followed by other stuff
366 If `delete` is True, delete the line (if found).
368 Return "followed by other stuff" with leading and trailing
371 i = find_token_exact(lines, token, start, end)
374 # TODO: establish desired behaviour, eventually change to
375 # return lines.pop(i)[len(token):].strip() # or default
376 # see test_parser_tools.py
377 l = lines[i].split(None, 1)
385 def get_quoted_value(lines, token, start=0, end=0, default="", delete=False):
386 """ get_quoted_value(lines, token, start[[, end], default]) -> string
388 Find the next line that looks like:
389 token "followed by other stuff"
390 Returns "followed by other stuff" with leading and trailing
391 whitespace and quotes removed. If there are no quotes, that is OK too.
392 So use get_value to preserve possible quotes, this one to remove them,
394 Note that we will NOT strip quotes from default!
396 val = get_value(lines, token, start, end, "", delete)
399 return val.strip('"')
402 def get_bool_value(lines, token, start=0, end=0, default=None, delete=False):
403 """ get_bool_value(lines, token, start[[, end], default]) -> string
405 Find the next line that looks like:
408 Returns True if bool_value is 1 or true and
409 False if bool_value is 0 or false
412 val = get_quoted_value(lines, token, start, end, default, delete)
414 if val == "1" or val == "true":
416 if val == "0" or val == "false":
421 def get_option_value(line, option):
422 rx = option + '\s*=\s*"([^"]+)"'
430 def set_option_value(line, option, value):
431 rx = '(' + option + '\s*=\s*")[^"]+"'
436 return re.sub(rx, '\g<1>' + value + '"', line)
439 def del_token(lines, token, start=0, end=0):
440 """ del_token(lines, token, start, end) -> int
442 Find the first line in lines where token is the first element
443 and delete that line. Returns True if we deleted a line, False
446 k = find_token_exact(lines, token, start, end)
452 def del_complete_lines(lines, sublines, start=0, end=0):
453 """Delete first occurence of `sublines` in list `lines`.
455 Efficient deletion of a sub-list in a list. Works for any values.
456 The `start` and `end` arguments work similar to list.index()
458 Returns True if a deletion was done and False if not.
460 >>> l = [1, 0, 1, 1, 1, 2]
461 >>> del_complete_lines(l, [0, 1, 1])
466 i = find_complete_lines(lines, sublines, start, end)
469 del(lines[i:i+len(sublines)])
473 def del_value(lines, token, start=0, end=0, default=None):
475 Find the next line that looks like:
476 token followed by other stuff
477 Delete that line and return "followed by other stuff"
478 with leading and trailing whitespace removed.
480 If token is not found, return `default`.
482 i = find_token_exact(lines, token, start, end)
485 return lines.pop(i)[len(token):].strip()
488 def find_beginning_of(lines, i, start_token, end_token):
491 i = find_tokens_backwards(lines, [start_token, end_token], i-1)
494 if lines[i].startswith(end_token):
503 def find_end_of(lines, i, start_token, end_token):
507 i = find_tokens(lines, [end_token, start_token], i+1)
510 if lines[i].startswith(start_token):
519 def find_nonempty_line(lines, start=0, end=0):
522 for i in range(start, end):
528 def find_end_of_inset(lines, i):
529 " Find end of inset, where lines[i] is included."
530 return find_end_of(lines, i, "\\begin_inset", "\\end_inset")
533 def find_end_of_layout(lines, i):
534 " Find end of layout, where lines[i] is included."
535 return find_end_of(lines, i, "\\begin_layout", "\\end_layout")
538 def is_in_inset(lines, i, inset, default=(-1,-1)):
540 Check if line i is in an inset of the given type.
541 If so, return starting and ending lines, otherwise `default`.
543 is_in_inset(document.body, i, "\\begin_inset Tabular")
544 returns (-1,-1) if `i` is not within a "Tabular" inset (i.e. a table).
545 If it is, then it returns the line on which the table begins and the one
547 Note that this pair will evaulate to boolean True, so (with the optional
548 default value set to False)
549 if is_in_inset(..., default=False):
550 will do what you expect.
552 start = find_token_backwards(lines, inset, i)
555 end = find_end_of_inset(lines, start)
556 if end < i: # this includes the notfound case.
561 def get_containing_inset(lines, i):
563 Finds out what kind of inset line i is within. Returns a
564 list containing (i) what follows \begin_inset on the line
565 on which the inset begins, plus the starting and ending line.
566 Returns False on any kind of error or if it isn't in an inset.
570 stins = find_token_backwards(lines, "\\begin_inset", j)
573 endins = find_end_of_inset(lines, stins)
581 inset = get_value(lines, "\\begin_inset", stins)
585 return (inset, stins, endins)
588 def get_containing_layout(lines, i):
590 Finds out what kind of layout line i is within. Returns a
591 list containing what follows \begin_layout on the line
592 on which the layout begins, plus the starting and ending line
593 and the start of the paragraph (after all params). I.e, returns:
594 (layoutname, layoutstart, layoutend, startofcontent)
595 Returns False on any kind of error.
599 stlay = find_token_backwards(lines, "\\begin_layout", j)
602 endlay = find_end_of_layout(lines, stlay)
610 lay = get_value(lines, "\\begin_layout", stlay)
614 par_params = ["\\noindent", "\\indent", "\\indent-toggle", "\\leftindent",
615 "\\start_of_appendix", "\\paragraph_spacing", "\\align",
616 "\\labelwidthstring"]
620 if lines[stpar].split(' ', 1)[0] not in par_params:
622 return (lay, stlay, endlay, stpar)
625 def count_pars_in_inset(lines, i):
627 Counts the paragraphs within this inset
629 ins = get_containing_inset(lines, i)
633 for j in range(ins[1], ins[2]):
634 m = re.match(r'\\begin_layout (.*)', lines[j])
635 if m and get_containing_inset(lines, j)[0] == ins[0]:
641 def find_end_of_sequence(lines, i):
643 Returns the end of a sequence of identical layouts.
645 lay = get_containing_layout(lines, i)
652 m = re.match(r'\\begin_layout (.*)', lines[i])
653 if m and m.group(1) != layout:
655 elif lines[i] == "\\begin_deeper":
656 j = find_end_of(lines, i, "\\begin_deeper", "\\end_deeper")
661 if m and m.group(1) == layout:
662 endlay = find_end_of_layout(lines, i)
665 if i == len(lines) - 1: