From 1e45627b8f4f6074fd4d0df8379db1d8fa7f6b26 Mon Sep 17 00:00:00 2001
From: Oleg Broytman <phd@phdru.name>
Date: Tue, 20 May 2014 01:00:27 +0400
Subject: [PATCH] Add lexer

---
 parse_query.py | 20 ++++++++++++++++
 test_lexer.py  | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 83 insertions(+)
 create mode 100644 parse_query.py
 create mode 100755 test_lexer.py

diff --git a/parse_query.py b/parse_query.py
new file mode 100644
index 0000000..4533413
--- /dev/null
+++ b/parse_query.py
@@ -0,0 +1,20 @@
+# Parse query
+
+from ply import lex, yacc
+
+literals = '()'
+
+tokens = ('NAME', 'AND_OP', 'OR_OP', 'NOT_OP')
+
+t_NAME = '[a-z][a-z0-9_]+'
+
+t_AND_OP = '&'
+
+t_OR_OP = r'\|'
+
+t_NOT_OP = '!'
+
+def t_error(t):
+    """Avoid warnings on stderr"""
+
+lexer = lex.lex()
diff --git a/test_lexer.py b/test_lexer.py
new file mode 100755
index 0000000..5fa76e8
--- /dev/null
+++ b/test_lexer.py
@@ -0,0 +1,63 @@
+#! /usr/bin/env python
+
+import unittest
+
+class TestLexer(unittest.TestCase):
+    def test_01_import(self):
+        global lexer
+        from parse_query import lexer
+
+    def test_02_tag(self):
+        global LexToken
+        from ply.lex import LexToken
+        lexer.input('xxx')
+        tokens = list(lexer)
+        self.assertEqual(len(tokens), 1)
+        lextoken = tokens[0]
+        self.assertEqual(lextoken.type, 'NAME')
+        self.assertEqual(lextoken.value, 'xxx')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 0)
+
+    def test_03_bad_tag(self):
+        from ply.lex import LexError
+        lexer.input('XXX')
+        self.assertRaises(LexError, list, lexer)
+
+    def test_04_expression(self):
+        lexer.input('!(xxx&yyy)')
+        tokens = list(lexer)
+        self.assertEqual(len(tokens), 6)
+        lextoken = tokens[0]
+        self.assertEqual(lextoken.type, 'NOT_OP')
+        self.assertEqual(lextoken.value, '!')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 0)
+        lextoken = tokens[1]
+        self.assertEqual(lextoken.type, '(')
+        self.assertEqual(lextoken.value, '(')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 1)
+        lextoken = tokens[2]
+        self.assertEqual(lextoken.type, 'NAME')
+        self.assertEqual(lextoken.value, 'xxx')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 2)
+        lextoken = tokens[3]
+        self.assertEqual(lextoken.type, 'AND_OP')
+        self.assertEqual(lextoken.value, '&')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 5)
+        lextoken = tokens[4]
+        self.assertEqual(lextoken.type, 'NAME')
+        self.assertEqual(lextoken.value, 'yyy')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 6)
+        lextoken = tokens[5]
+        self.assertEqual(lextoken.type, ')')
+        self.assertEqual(lextoken.value, ')')
+        self.assertEqual(lextoken.lineno, 1)
+        self.assertEqual(lextoken.lexpos, 9)
+
+if __name__ == "__main__":
+    unittest.main()
-- 
2.39.2