+def split_ext_insert(statement):
+ """Split extended INSERT into multiple standard INSERTs"""
+ insert_tokens = []
+ values_tokens = []
+ end_tokens = []
+ expected = 'INSERT'
+ for token in statement.tokens:
+ if is_comment_or_space(token):
+ if expected == 'END':
+ end_tokens.append(token)
+ else:
+ insert_tokens.append(token)
+ continue
+ elif expected == 'INSERT':
+ if (token.ttype is T.DML) and (token.normalized == 'INSERT'):
+ insert_tokens.append(token)
+ expected = 'INTO'
+ continue
+ elif expected == 'INTO':
+ if (token.ttype is T.Keyword) and (token.normalized == 'INTO'):
+ insert_tokens.append(token)
+ expected = 'TABLE_NAME'
+ continue
+ elif expected == 'TABLE_NAME':
+ if isinstance(token, (Function, Identifier)):
+ insert_tokens.append(token)
+ expected = 'VALUES'
+ continue
+ elif expected == 'VALUES':
+ if isinstance(token, Values):
+ for subtoken in token.tokens:
+ if isinstance(subtoken, Parenthesis):
+ values_tokens.append(subtoken)
+ insert_tokens.append(Token(T.Keyword, 'VALUES'))
+ insert_tokens.append(Token(T.Whitespace, ' '))
+ expected = 'VALUES_OR_SEMICOLON'
+ continue
+ if (token.ttype is T.Keyword) and (token.normalized == 'VALUES'):
+ insert_tokens.append(token)
+ expected = 'VALUES_OR_SEMICOLON'
+ continue
+ elif expected == 'VALUES_OR_SEMICOLON':
+ if isinstance(token, Parenthesis):
+ values_tokens.append(token)
+ continue
+ elif token.ttype is T.Punctuation:
+ if token.value == ',':
+ continue
+ elif token.value == ';':
+ end_tokens.append(token)
+ expected = 'END'
+ continue
+ raise ValueError(
+ 'SQL syntax error: expected "%s", got %s "%s"' % (
+ expected, token.ttype, token.normalized))
+ new_line = Token(T.Newline, '\n')
+ new_lines = [new_line] # Insert newlines between split statements
+ for i, values in enumerate(values_tokens):
+ if i == len(values_tokens) - 1: # Last but one statement
+ # Insert newlines only between split statements but not after
+ new_lines = []
+ # The statement sets `parent` attribute of the every token to self
+ # but we don't care.
+ statement = Statement(insert_tokens + [values] +
+ end_tokens + new_lines)
+ yield statement
+
+
+def process_statement(statement, dbname='sqlite'):