New parser, bug fixes and new functionality.

.github/workflows/python-ci.yml

@@ -60,7 +60,7 @@ jobs:
       run: poetry run pytest -vv --cov=sql_metadata --cov-report=term
 
     - name: Lint with pylint
-      run: poetry run pylint sql_metadata.py
+      run: poetry run pylint sql_metadata
 
     - name: Build a distribution package
       run: poetry build -vvv

Makefile

@@ -8,7 +8,7 @@ coverage:
 	poetry run pytest -vv --cov=sql_metadata --cov-report=term
 
 lint:
-	poetry run pylint sql_metadata.py
+	poetry run pylint sql_metadata
 
 publish:
 	# run git tag -a v0.0.0 before running make publish

README.md

@@ -21,48 +21,189 @@ Supported queries syntax:
 pip install sql-metadata
 ```
 
+### Extracting raw sql-metadata tokens
+
+```python
+from sql_metadata import Parser
+
+# extract raw sql-metadata tokens
+Parser("SELECT * FROM foo").tokens
+# ['SELECT', '*', 'FROM', 'foo']
+```
+
+### Extracting columns from query
+
 ```python
->>> import sql_metadata
+from sql_metadata import Parser
+
+# get columns from query - for more examples see `tests/test_getting_columns.py`
+Parser("SELECT test, id FROM foo, bar").columns
+# ['test', 'id']
+
+Parser("INSERT /* VoteHelper::addVote xxx */  INTO `page_vote` (article_id,user_id,`time`) VALUES ('442001','27574631','20180228130846')").columns
+# ['article_id', 'user_id', 'time']
 
->>> sql_metadata.get_query_tokens("SELECT * FROM foo")
-[<DML 'SELECT' at 0x7F14FFDEB808>, <Wildcard '*' at 0x7F14FFDEB940>, <Keyword 'FROM' at 0x7F14FFDEBBB0>, <Name 'foo' at 0x7F14FFDEB9A8>]
+parser = Parser("SELECT a.* FROM product_a.users AS a JOIN product_b.users AS b ON a.ip_address = b.ip_address")
 
->>> sql_metadata.get_query_columns("SELECT test, id FROM foo, bar")
-[u'test', u'id']
+# note that aliases are auto-resolved
+parser.columns
+# ['product_a.*', 'product_a.users.ip_address', 'product_b.users.ip_address']
 
->>> sql_metadata.get_query_tables("SELECT a.* FROM product_a.users AS a JOIN product_b.users AS b ON a.ip_address = b.ip_address")
-['product_a.users', 'product_b.users']
+# note that you can also extract columns with their place in the query
+# which will return dict with lists divided into select, where, order_by, join, insert and update
+parser.columns_dict
+# {'select': ['product_a.users.*'], 'join': ['product_a.users.ip_address', 'product_b.users.ip_address']}
+```
+
+### Extracting tables from query
 
->>> sql_metadata.get_query_columns("INSERT /* VoteHelper::addVote xxx */  INTO `page_vote` (article_id,user_id,`time`) VALUES ('442001','27574631','20180228130846')")
-['article_id', 'user_id', 'time']
+```python
+from sql_metadata import Parser
 
->>> sql_metadata.get_query_columns("SELECT a.* FROM product_a.users AS a JOIN product_b.users AS b ON a.ip_address = b.ip_address")
-['a.*', 'a.ip_address', 'b.ip_address']
+# get tables from query - for more examples see `tests/test_getting_tables.py`
+Parser("SELECT a.* FROM product_a.users AS a JOIN product_b.users AS b ON a.ip_address = b.ip_address").tables
+# ['product_a.users', 'product_b.users']
 
->>> sql_metadata.get_query_tables("SELECT test, id FROM foo, bar")
-[u'foo', u'bar']
+Parser("SELECT test, id FROM foo, bar").tables
+# ['foo', 'bar']
 
->>> sql_metadata.get_query_limit_and_offset('SELECT foo_limit FROM bar_offset LIMIT 50 OFFSET 1000')
-(50, 1000)
+# you can also extract aliases of the tables as a dictionary
+parser = Parser("SELECT f.test FROM foo AS f")
 
->>> sql_metadata.get_query_limit_and_offset('SELECT foo_limit FROM bar_offset limit 2000,50')
-(50, 2000)
+# get table aliases
+parser.tables_aliases
+# {'f': 'foo'}
 
->>> sql_metadata.get_query_table_aliases("SELECT test FROM foo AS f")
-{'f': 'foo'}
+# note that aliases are auto-resolved for columns
+parser.columns
+# ["foo.test"]
 ```
 
-> See `test/test_query.py` file for more examples of a bit more complex queries.
+### Extracting values from query
+```python
+from sql_metadata import Parser
+
+parser = Parser(
+    "INSERT /* VoteHelper::addVote xxx */  INTO `page_vote` (article_id,user_id,`time`) " 
+    "VALUES ('442001','27574631','20180228130846')"
+)
+# extract values from query
+parser.values
+# ["442001", "27574631", "20180228130846"]
+
+# extract a dictionary with column-value pairs
+parser.values_dict
+#{"article_id": "442001", "user_id": "27574631", "time": "20180228130846"}
+
+# if column names are not set auto-add placeholders
+parser = Parser(
+    "INSERT IGNORE INTO `table` VALUES (9, 2.15, '123', '2017-01-01');"
+)
+parser.values
+# [9, 2.15, "123", "2017-01-01"]
+
+parser.values_dict
+#{"column_1": 9, "column_2": 2.15, "column_3": "123", "column_4": "2017-01-01"}
+```
 
-### Queries normalization
 
+### Extracting limit and offset
 ```python
->>> from sql_metadata import generalize_sql
->>> generalize_sql('SELECT /* Test */ foo FROM bar WHERE id in (1, 2, 56)')
-'SELECT foo FROM bar WHERE id in (XYZ)'
+from sql_metadata import Parser
+
+Parser('SELECT foo_limit FROM bar_offset LIMIT 50 OFFSET 1000').limit_and_offset
+# (50, 1000)
+
+Parser('SELECT foo_limit FROM bar_offset limit 2000,50').limit_and_offset
+# (50, 2000)
+```
+
+### Extracting with names
+
+```python
+from sql_metadata import Parser
+
+parser = Parser(
+    """
+WITH
+    database1.tableFromWith AS (SELECT aa.* FROM table3 as aa 
+                                left join table4 on aa.col1=table4.col2),
+    test as (select * from table3)
+SELECT
+  "xxxxx"
+FROM
+  database1.tableFromWith alias
+LEFT JOIN database2.table2 ON ("tt"."ttt"."fff" = "xx"."xxx")
+"""
+)
+
+# get names/ aliases of with statements
+parser.with_names
+# ["database1.tableFromWith", "test"]
+
+# note that names of with statements do not appear in tables
+parser.tables
+# ["table3", "table4", "database2.table2"]
+```
+
+### Extracting sub-queries
+
+```python
+from sql_metadata import Parser
+
+parser = Parser(
+"""
+SELECT COUNT(1) FROM
+(SELECT std.task_id FROM some_task_detail std WHERE std.STATUS = 1) a
+JOIN (SELECT st.task_id FROM some_task st WHERE task_type_id = 80) b
+ON a.task_id = b.task_id;
+"""
+)
+
+# get sub-queries dictionary
+parser.subqueries
+# {"a": "SELECT std.task_id FROM some_task_detail std WHERE std.STATUS = 1",
+#  "b": "SELECT st.task_id FROM some_task st WHERE task_type_id = 80"}
+
+
+# get names/ aliases of sub-queries / derived tables
+parser.subqueries_names
+# ["a", "b"]
+
+# note that you can also exclude columns coming from sub-queries
+# all columns
+parser.columns
+#["some_task_detail.task_id", "some_task_detail.STATUS", "some_task.task_id", 
+# "task_type_id", "a.task_id", "b.task_id"]
+
+# without subqueries
+parser.columns_without_subqueries
+#["some_task_detail.task_id", "some_task_detail.STATUS", "some_task.task_id", 
+# "task_type_id"]
+```
+
+See `tests` file for more examples of a bit more complex queries.
+
+### Queries normalization and comments extraction
+
+```python
+from sql_metadata import Parser
+parser = Parser('SELECT /* Test */ foo FROM bar WHERE id in (1, 2, 56)')
+
+# generalize query
+parser.generalize
+# 'SELECT foo FROM bar WHERE id in (XYZ)'
+
+# remove comments
+parser.without_comments
+# 'SELECT foo FROM bar WHERE id in (1, 2, 56)'
+
+# extract comments
+parser.comments
+# ['/* Test */']
 ```
 
-> See `test/test_normalization.py` file for more examples of a bit more complex queries.
+See `test/test_normalization.py` file for more examples of a bit more complex queries.
 
 ## Stargazers over time
 

pyproject.toml

@@ -9,7 +9,7 @@ homepage = "https://github.com/macbre/sql-metadata"
 repository = "https://github.com/macbre/sql-metadata"
 
 packages = [
-    { include="sql_metadata.py" }
+    { include="sql_metadata" }
 ]
 
 [tool.poetry.dependencies]