1 # This file is part of lyx2lyx
2 # -*- coding: utf-8 -*-
3 # Copyright (C) 2002-2011 Dekel Tsur <dekel@lyx.org>,
4 # José Matos <jamatos@lyx.org>, Richard Kimberly Heck <rikiheck@lyx.org>
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of the GNU General Public License
8 # as published by the Free Software Foundation; either version 2
9 # of the License, or (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 This module offers several free functions to help parse lines.
23 More documentaton is below, but here is a quick guide to what
24 they do. Optional arguments are marked by brackets.
26 find_token(lines, token[, start[, end[, ignorews]]]):
27 Returns the first line i, start <= i < end, on which
28 token is found at the beginning. Returns -1 if not
30 If ignorews is (given and) True, then differences
31 in whitespace do not count, except that there must be no
32 extra whitespace following token itself.
34 find_token_exact(lines, token[, start[, end]]]):
35 As find_token, but with ignorews set to True.
37 find_tokens(lines, tokens[, start[, end[, ignorews]]]):
38 Returns the first line i, start <= i < end, on which
39 one of the tokens in tokens is found at the beginning.
40 Returns -1 if not found.
41 If ignorews is (given and) True, then differences
42 in whitespace do not count, except that there must be no
43 extra whitespace following token itself.
45 find_tokens_exact(lines, token[, start[, end]]):
46 As find_tokens, but with ignorews True.
48 find_token_backwards(lines, token, start):
49 find_tokens_backwards(lines, tokens, start):
50 As before, but look backwards.
52 find_substring(lines, sub[, start[, end]]) -> int
53 As find_token, but sub may be anywhere in the line.
55 find_re(lines, rexp, start[, end]):
56 As find_token, but rexp is a regular expression object,
57 so it has to be passed as e.g.: re.compile(r'...').
59 get_value(lines, token[, start[, end[, default[, delete]]]]):
60 Similar to find_token, but it returns what follows the
61 token on the found line. Example:
62 get_value(document.header, "\\use_xetex", 0)
63 will find a line like:
65 and, in that case, return "true". (Note that whitespace
66 is stripped.) The final argument, default, defaults to "",
67 and is what is returned if we do not find anything. So you
68 can use that to set a default.
69 If delete is True, then delete the line if found.
71 get_quoted_value(lines, token[, start[, end[, default[, delete]]]]):
72 Similar to get_value, but it will strip quotes off the
73 value, if they are present. So use this one for cases
74 where the value is normally quoted.
76 get_option_value(line, option):
77 This assumes we have a line with something like:
79 and returns value. Returns "" if not found.
81 get_bool_value(lines, token[, start[, end[, default, delete]]]]):
82 Like get_value, but returns a boolean.
84 set_bool_value(lines, token, value[, start[, end]]):
85 Find `token` in `lines[start:end]` and set to boolean value bool(`value`).
86 Return old value. Raise ValueError if token is not in lines.
88 del_token(lines, token[, start[, end]]):
89 Like find_token, but deletes the line if it finds one.
90 Returns True if a line got deleted, otherwise False.
92 Use get_* with the optional argument "delete=True", if you want to
93 get and delete a token.
95 find_beginning_of(lines, i, start_token, end_token):
96 Here, start_token and end_token are meant to be a matching
97 pair, like "\\begin_layout" and "\\end_layout". We look for
98 the start_token that pairs with the end_token that occurs
99 on or after line i. Returns -1 if not found.
100 So, in the layout case, this would find the \\begin_layout
101 for the layout line i is in.
103 ec = find_token(document.body, "</cell", i)
104 bc = find_beginning_of(document.body, ec, \
106 Now, assuming no -1s, bc-ec wraps the cell for line i.
108 find_end_of(lines, i, start_token, end_token):
109 Like find_beginning_of, but looking for the matching
110 end_token. This might look like:
111 bc = find_token_(document.body, "<cell", i)
112 ec = find_end_of(document.body, bc, "<cell", "</cell")
113 Now, assuming no -1s, bc-ec wrap the next cell.
115 find_end_of_inset(lines, i):
116 Specialization of find_end_of for insets.
118 find_end_of_layout(lines, i):
119 Specialization of find_end_of for layouts.
121 find_end_of_sequence(lines, i):
122 Find the end of the sequence of layouts of the same kind.
123 Considers nesting. If the last paragraph in sequence is nested,
124 the position of the last \\end_deeper is returned, else
125 the position of the last \\end_layout.
127 is_in_inset(lines, i, inset, default=(-1,-1)):
128 Check if line i is in an inset of the given type.
129 If so, returns starting and ending lines. Otherwise,
132 is_in_inset(document.body, i, "\\begin_inset Tabular")
133 returns (-1,-1) unless i is within a table. If it is, then
134 it returns the line on which the table begins and the one
135 on which it ends. Note that this pair will evaulate to
137 if is_in_inset(..., default=False):
138 will do what you expect.
140 get_containing_inset(lines, i):
141 Finds out what kind of inset line i is within. Returns a
142 list containing what follows \\begin_inset on the line
143 on which the inset begins, plus the starting and ending line.
144 Returns False on any kind of error or if it isn't in an inset.
145 So get_containing_inset(document.body, i) might return:
146 ("CommandInset ref", 300, 306)
147 if i is within an InsetRef beginning on line 300 and ending
150 get_containing_layout(lines, i):
151 As get_containing_inset, but for layout. Additionally returns the
152 position of real paragraph start (after par params) as 4th value.
154 find_nonempty_line(lines, start[, end):
155 Finds the next non-empty line.
157 check_token(line, token):
158 Does line begin with token?
160 is_nonempty_line(line):
161 Does line contain something besides whitespace?
163 count_pars_in_inset(lines, i):
164 Counts the paragraphs inside an inset.
170 # Utilities for one line
171 def check_token(line, token):
172 """ check_token(line, token) -> bool
174 Return True if token is present in line and is the first element
177 Deprecated. Use line.startswith(token).
179 return line.startswith(token)
182 def is_nonempty_line(line):
183 """ is_nonempty_line(line) -> bool
185 Return False if line is either empty or it has only whitespaces,
187 return bool(line.strip())
190 # Utilities for a list of lines
191 def find_token(lines, token, start=0, end=0, ignorews=False):
192 """ find_token(lines, token, start[[, end], ignorews]) -> int
194 Return the lowest line where token is found, and is the first
195 element, in lines[start, end].
197 If ignorews is True (default is False), then differences in
198 whitespace are ignored, but there must be whitespace following
201 Use find_substring(lines, sub) to find a substring anywhere in `lines`.
203 Return -1 on failure."""
205 if end == 0 or end > len(lines):
209 for i in range(start, end):
217 if lines[i].startswith(token):
222 def find_token_exact(lines, token, start=0, end=0):
223 return find_token(lines, token, start, end, True)
226 def find_tokens(lines, tokens, start=0, end=0, ignorews=False):
227 """ find_tokens(lines, tokens, start[[, end], ignorews]) -> int
229 Return the lowest line where one token in tokens is found, and is
230 the first element, in lines[start, end].
232 Return -1 on failure."""
234 if end == 0 or end > len(lines):
237 for i in range(start, end):
247 if lines[i].startswith(token):
252 def find_tokens_exact(lines, tokens, start=0, end=0):
253 return find_tokens(lines, tokens, start, end, True)
256 def find_substring(lines, sub, start=0, end=0):
257 """ find_substring(lines, sub[, start[, end]]) -> int
259 Return the lowest line number `i` in [start, end] where
260 `sub` is a substring of line[i].
262 Return -1 on failure."""
264 if end == 0 or end > len(lines):
266 for i in range(start, end):
272 def find_re(lines, rexp, start=0, end=0):
273 """ find_re(lines, rexp[, start[, end]]) -> int
275 Return the lowest line number `i` in [start, end] where the regular
276 expression object `rexp` matches at the beginning of line[i].
277 Return -1 on failure.
279 Start your pattern with the wildcard ".*" to find a match anywhere in a
280 line. Use find_substring() to find a substring anywhere in the lines.
282 if end == 0 or end > len(lines):
284 for i in range(start, end):
285 if rexp.match(lines[i]):
290 def find_token_backwards(lines, token, start):
291 """ find_token_backwards(lines, token, start) -> int
293 Return the highest line where token is found, and is the first
294 element, in lines[start, end].
296 Return -1 on failure."""
297 for i in range(start, -1, -1):
298 if lines[i].startswith(token):
303 def find_tokens_backwards(lines, tokens, start):
304 """ find_tokens_backwards(lines, token, start) -> int
306 Return the highest line where token is found, and is the first
307 element, in lines[end, start].
309 Return -1 on failure."""
310 for i in range(start, -1, -1):
313 if line.startswith(token):
318 def find_complete_lines(lines, sublines, start=0, end=0):
319 """Find first occurence of sequence `sublines` in list `lines`.
320 Return index of first line or -1 on failure.
322 Efficient search for a sub-list in a large list. Works for any values.
324 >>> find_complete_lines([1, 2, 3, 1, 1, 2], [1, 2])
327 The `start` and `end` arguments work similar to list.index()
329 >>> find_complete_lines([1, 2, 3, 1, 1 ,2], [1, 2], start=1)
331 >>> find_complete_lines([1, 2, 3, 1, 1 ,2], [1, 2], start=1, end=4)
334 The return value can be used to substitute the sub-list.
335 Take care to check before use:
338 >>> s = find_complete_lines(l, [1, 2])
340 ... l[s:s+2] = [3]; l
343 See also del_complete_lines().
347 end = end or len(lines)
351 for j, value in enumerate(sublines):
352 i = lines.index(value, start, end)
359 except ValueError: # `sublines` not found
363 def find_across_lines(lines, sub, start=0, end=0):
364 sublines = sub.splitlines()
365 if len(sublines) > 2:
366 # at least 3 lines: the middle one(s) are complete -> use index search
367 i = find_complete_lines(lines, sublines[1:-1], start+1, end-1)
371 if (lines[i-1].endswith(sublines[0]) and
372 lines[i+len(sublines)].startswith(sublines[-1])):
376 elif len(sublines) > 1:
377 # last subline must start a line
378 i = find_token(lines, sublines[-1], start, end)
381 if lines[i-1].endswith(sublines[0]):
383 else: # no line-break, may be in the middle of a line
384 if end == 0 or end > len(lines):
386 for i in range(start, end):
392 def get_value(lines, token, start=0, end=0, default="", delete=False):
393 """Find `token` in `lines` and return part of line that follows it.
395 Find the next line that looks like:
396 token followed by other stuff
398 If `delete` is True, delete the line (if found).
400 Return "followed by other stuff" with leading and trailing
403 i = find_token_exact(lines, token, start, end)
406 # TODO: establish desired behaviour, eventually change to
407 # return lines.pop(i)[len(token):].strip() # or default
408 # see test_parser_tools.py
409 l = lines[i].split(None, 1)
417 def get_quoted_value(lines, token, start=0, end=0, default="", delete=False):
418 """ get_quoted_value(lines, token, start[[, end], default]) -> string
420 Find the next line that looks like:
421 token "followed by other stuff"
422 Returns "followed by other stuff" with leading and trailing
423 whitespace and quotes removed. If there are no quotes, that is OK too.
424 So use get_value to preserve possible quotes, this one to remove them,
426 Note that we will NOT strip quotes from default!
428 val = get_value(lines, token, start, end, "", delete)
431 return val.strip('"')
434 bool_values = {"true": True, "1": True,
435 "false": False, "0": False}
437 def get_bool_value(lines, token, start=0, end=0, default=None, delete=False):
438 """ get_bool_value(lines, token, start[[, end], default]) -> string
440 Find the next line that looks like:
443 Return True if <bool_value> is 1 or "true", False if <bool_value>
444 is 0 or "false", else `default`.
446 val = get_quoted_value(lines, token, start, end, default, delete)
447 return bool_values.get(val, default)
450 def set_bool_value(lines, token, value, start=0, end=0):
451 """Find `token` in `lines` and set to bool(`value`).
453 Return previous value. Raise `ValueError` if `token` is not in lines.
455 Cf. find_token(), get_bool_value().
457 i = find_token(lines, token, start, end)
460 oldvalue = get_bool_value(lines, token, i, i+1)
461 if oldvalue is value:
464 if get_quoted_value(lines, token, i, i+1) in ('0', '1'):
465 lines[i] = "%s %d" % (token, value)
467 lines[i] = "%s %s" % (token, str(value).lower())
472 def get_option_value(line, option):
473 rx = option + r'\s*=\s*"([^"]+)"'
481 def set_option_value(line, option, value):
482 rx = '(' + option + r'\s*=\s*")[^"]+"'
487 return re.sub(rx, r'\g<1>' + value + '"', line)
490 def del_token(lines, token, start=0, end=0):
491 """ del_token(lines, token, start, end) -> int
493 Find the first line in lines where token is the first element
494 and delete that line. Returns True if we deleted a line, False
497 k = find_token_exact(lines, token, start, end)
503 def del_complete_lines(lines, sublines, start=0, end=0):
504 """Delete first occurence of `sublines` in list `lines`.
506 Efficient deletion of a sub-list in a list. Works for any values.
507 The `start` and `end` arguments work similar to list.index()
509 Returns True if a deletion was done and False if not.
511 >>> l = [1, 0, 1, 1, 1, 2]
512 >>> del_complete_lines(l, [0, 1, 1])
517 i = find_complete_lines(lines, sublines, start, end)
520 del(lines[i:i+len(sublines)])
524 def del_value(lines, token, start=0, end=0, default=None):
526 Find the next line that looks like:
527 token followed by other stuff
528 Delete that line and return "followed by other stuff"
529 with leading and trailing whitespace removed.
531 If token is not found, return `default`.
533 i = find_token_exact(lines, token, start, end)
536 return lines.pop(i)[len(token):].strip()
539 def find_beginning_of(lines, i, start_token, end_token):
542 i = find_tokens_backwards(lines, [start_token, end_token], i-1)
545 if lines[i].startswith(end_token):
554 def find_end_of(lines, i, start_token, end_token):
558 i = find_tokens(lines, [end_token, start_token], i+1)
561 if lines[i].startswith(start_token):
570 def find_nonempty_line(lines, start=0, end=0):
573 for i in range(start, end):
579 def find_end_of_inset(lines, i):
580 " Find end of inset, where lines[i] is included."
581 return find_end_of(lines, i, "\\begin_inset", "\\end_inset")
584 def find_end_of_layout(lines, i):
585 " Find end of layout, where lines[i] is included."
586 return find_end_of(lines, i, "\\begin_layout", "\\end_layout")
589 def is_in_inset(lines, i, inset, default=(-1,-1)):
591 Check if line i is in an inset of the given type.
592 If so, return starting and ending lines, otherwise `default`.
594 is_in_inset(document.body, i, "\\begin_inset Tabular")
595 returns (-1,-1) if `i` is not within a "Tabular" inset (i.e. a table).
596 If it is, then it returns the line on which the table begins and the one
598 Note that this pair will evaulate to boolean True, so (with the optional
599 default value set to False)
600 if is_in_inset(..., default=False):
601 will do what you expect.
603 start = find_token_backwards(lines, inset, i)
606 end = find_end_of_inset(lines, start)
607 if end < i: # this includes the notfound case.
612 def get_containing_inset(lines, i):
614 Finds out what kind of inset line i is within. Returns a
615 list containing (i) what follows \\begin_inset on the line
616 on which the inset begins, plus the starting and ending line.
617 Returns False on any kind of error or if it isn't in an inset.
621 stins = find_token_backwards(lines, "\\begin_inset", j)
624 endins = find_end_of_inset(lines, stins)
632 inset = get_value(lines, "\\begin_inset", stins)
636 return (inset, stins, endins)
639 def get_containing_layout(lines, i):
641 Find out what kind of layout line `i` is within.
643 (layoutname, layoutstart, layoutend, startofcontent)
647 * end line number, and
648 * number of first paragraph line (after all params).
649 Return `False` on any kind of error.
653 stlay = find_token_backwards(lines, "\\begin_layout", j)
656 endlay = find_end_of_layout(lines, stlay)
664 layoutname = get_value(lines, "\\begin_layout", stlay)
665 if layoutname == "": # layout style missing
666 # TODO: What shall we do in this case?
668 # layoutname == "Standard" # use same fallback as the LyX parser:
669 # raise ValueError("Missing layout name on line %d"%stlay) # diagnosis
670 # return False # generic error response
671 par_params = ["\\noindent", "\\indent", "\\indent-toggle", "\\leftindent",
672 "\\start_of_appendix", "\\paragraph_spacing", "\\align",
673 "\\labelwidthstring"]
677 if lines[stpar].split(' ', 1)[0] not in par_params:
679 return (layoutname, stlay, endlay, stpar)
682 def count_pars_in_inset(lines, i):
684 Counts the paragraphs within this inset
686 ins = get_containing_inset(lines, i)
690 for j in range(ins[1], ins[2]):
691 m = re.match(r'\\begin_layout (.*)', lines[j])
692 if m and get_containing_inset(lines, j)[1] == ins[1]:
698 def find_end_of_sequence(lines, i):
700 Returns the end of a sequence of identical layouts.
702 lay = get_containing_layout(lines, i)
709 m = re.match(r'\\begin_layout (.*)', lines[i])
710 if m and m.group(1) != layout:
712 elif lines[i] == "\\begin_deeper":
713 j = find_end_of(lines, i, "\\begin_deeper", "\\end_deeper")
718 if m and m.group(1) == layout:
719 endlay = find_end_of_layout(lines, i)
722 if i == len(lines) - 1: