From: Oleg Broytman <phd@phdru.name>
Date: Mon, 19 May 2014 21:00:27 +0000 (+0400)
Subject: Add lexer
X-Git-Tag: v0.1~32
X-Git-Url: https://git.phdru.name/?p=phdru.name%2Fcgi-bin%2Fblog-ru%2Fsearch-tags.git;a=commitdiff_plain;h=1e45627b8f4f6074fd4d0df8379db1d8fa7f6b26

Add lexer
---

diff --git a/parse_query.py b/parse_query.py
new file mode 100644
index 0000000..4533413
--- /dev/null
+++ b/parse_query.py
@@ -0,0 +1,20 @@
+# Parse query
+
+from ply import lex, yacc
+
+literals = '()'
+
+tokens = ('NAME', 'AND_OP', 'OR_OP', 'NOT_OP')
+
+t_NAME = '[a-z][a-z0-9_]+'
+
+t_AND_OP = '&'
+
+t_OR_OP = r'\|'
+
+t_NOT_OP = '!'
+
+def t_error(t):
+    """Avoid warnings on stderr"""
+
+lexer = lex.lex()
diff --git a/test_lexer.py b/test_lexer.py
new file mode 100755
index 0000000..5fa76e8
--- /dev/null
+++ b/test_lexer.py
@@ -0,0 +1,63 @@
+#! /usr/bin/env python
+
+import unittest
+
+class TestLexer(unittest.TestCase):
+    def test_01_import(self):
+        global lexer
+        from parse_query import lexer
+
+    def test_02_tag(self):
+        global LexToken
+        from ply.lex import LexToken
+        lexer.input('xxx')
+        tokens = list(lexer)
+        self.assertEqual(len(tokens), 1)
+        lextoken = tokens[0]
+        self.assertEqual(lextoken.type, 'NAME')
+        self.assertEqual(lextoken.value, 'xxx')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 0)
+
+    def test_03_bad_tag(self):
+        from ply.lex import LexError
+        lexer.input('XXX')
+        self.assertRaises(LexError, list, lexer)
+
+    def test_04_expression(self):
+        lexer.input('!(xxx&yyy)')
+        tokens = list(lexer)
+        self.assertEqual(len(tokens), 6)
+        lextoken = tokens[0]
+        self.assertEqual(lextoken.type, 'NOT_OP')
+        self.assertEqual(lextoken.value, '!')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 0)
+        lextoken = tokens[1]
+        self.assertEqual(lextoken.type, '(')
+        self.assertEqual(lextoken.value, '(')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 1)
+        lextoken = tokens[2]
+        self.assertEqual(lextoken.type, 'NAME')
+        self.assertEqual(lextoken.value, 'xxx')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 2)
+        lextoken = tokens[3]
+        self.assertEqual(lextoken.type, 'AND_OP')
+        self.assertEqual(lextoken.value, '&')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 5)
+        lextoken = tokens[4]
+        self.assertEqual(lextoken.type, 'NAME')
+        self.assertEqual(lextoken.value, 'yyy')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 6)
+        lextoken = tokens[5]
+        self.assertEqual(lextoken.type, ')')
+        self.assertEqual(lextoken.value, ')')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 9)
+
+if __name__ == "__main__":
+    unittest.main()