add wiki parser

Author: kmaehashi

atwiki/parser.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import re
+
+class AtWikiStripper(object):
+  # Comment: `// comment`
+  COMMENT = re.compile(r'^//')
+
+  # Inline annotation: `&color(#999999){text}`, `&nicovideo(url)`
+  INLINE_ANN = re.compile(r'&[a-z_]+\(([^()]*?)\)({([^{}]+?)})?'), r'\3'
+
+  # Inline links: `[[page]]`, `[[alias>URL]]`
+  INLINE_LINK = re.compile(r'\[\[(.+?)((>|>>)(.+?))?\]\]'), r'\1'
+
+  # Inline italic: `'''text'''`
+  INLINE_ITALIC = re.compile(r'\'\'\'(.+?)\'\'\''), r'\1'
+
+  # Inline bold: `''text''`
+  INLINE_BOLD = re.compile(r'\'\'(.+?)\'\''), r'\1'
+
+  # Inline del: `%%text%%`
+  INLINE_DEL = re.compile(r'%%(.+?)%%'), r'\1'
+
+  # Line annotation: `#right(){text}`, `#comment()`, `#region`
+  LINE_ANN = re.compile(r'^#[a-z_]+(\(([^()]*?)\)({([^{}]+?)})?)?\s*$'), r'\4'
+
+  # Line horizontal line: `----`
+  LINE_HR = re.compile(r'^----\s*()$'), r'\1'
+
+  # Line item list and heading: `+foo`, `-foo`, `*foo`
+  LINE_ITEMLIST = re.compile(r'^(\*+|\++|-+)(.+)$'), r'\2'
+
+  # Line quote: `>text`
+  LINE_QUOTE = re.compile(r'^>+(.+)$'), r'\1'
+
+  # Line formatted: ` text`
+  LINE_PRE = re.compile(r'^ (.+)$'), r'\1'
+
+  # Block annotation: `#exk(){{{` ... `}}}`
+  BLOCK_BEGIN_ANN = re.compile(r'^#[a-z_]+\(([^{}()]*?)\)({+)\s*$')
+  BLOCK_END_ANN = re.compile(r'^(}+)\s*$')
+
+  def __init__(self, source):
+    self._source = source
+
+  def _inline_strip(self, line, pattern, repl):
+    while True:
+      prev = line
+      line = pattern.sub(repl, line)
+      if prev == line: return line
+
+  def _line_process(self, buf, line, pattern, repl):
+    prev = line
+    line = pattern.sub(repl, line)
+    if prev == line: return False
+    buf.append(line)
+    return True
+
+  def text(self):
+    ret = []
+    lines = self._source.splitlines()
+    block_level = 0
+    for line in lines:
+      if self.COMMENT.match(line): continue
+      line = self._inline_strip(line, *self.INLINE_ANN)
+      line = self._inline_strip(line, *self.INLINE_LINK)
+      line = self._inline_strip(line, *self.INLINE_ITALIC)
+      line = self._inline_strip(line, *self.INLINE_BOLD)
+      line = self._inline_strip(line, *self.INLINE_DEL)
+      if self._line_process(ret, line, *self.LINE_ANN): continue
+      if self._line_process(ret, line, *self.LINE_HR): continue
+      if self._line_process(ret, line, *self.LINE_ITEMLIST): continue
+      if self._line_process(ret, line, *self.LINE_QUOTE): continue
+      if self._line_process(ret, line, *self.LINE_PRE): continue
+      if block_level == 0:
+        m = self.BLOCK_BEGIN_ANN.match(line)
+        if m:
+          block_level = len(m.group(2))
+          continue
+      else:
+        m = self.BLOCK_END_ANN.match(line)
+        if m and len(m.group(1)) == block_level:
+          block_level = 0
+          continue
+      ret.append(line)
+    return '\n'.join(ret)

atwiki/test/test_parser.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from unittest import TestCase
+
+from atwiki.parser import AtWikiStripper
+
+INPUT = '''
+// This is a comment.
+
+[[Link1]] / [[Link2>URL]]
+Styles: ''bold'' \'\'\'italic\'\'\' %%del%% ''bold'' 
+Special: '' A ✔︎ ( ) { } \ / ! '' 
+
+
+#right(aaa){inline} 
+#comment() 
+#region
++item 1
+++item 2
+-item 1
+--item 2
+-+item 3
+*head 1
+**head 2
+>quote 1
+>>quote 2
+ pre
+---- 
+#exk(xxx){{{
+block 1
+}}}
+#exk(){{{{{
+block 2
+}}}}}
+'''
+
+OUTPUT = '''
+
+Link1 / Link2
+Styles: bold italic del bold 
+Special:  A ✔︎ ( ) { } \ / !  
+
+
+inline
+
+
+item 1
+item 2
+item 1
+item 2
++item 3
+head 1
+head 2
+quote 1
+quote 2
+pre
+
+block 1
+block 2\
+'''
+
+class AtWikiStripperTest(TestCase):
+  def test_doc(self):
+    stripper = AtWikiStripper(INPUT)
+    self.assertEqual(OUTPUT, stripper.text())
+
+  def test_single(self):
+    stripper = AtWikiStripper('[[Test]]')
+    self.assertEqual('Test', stripper.text())