diff --git a/.gitignore b/.gitignore index 4c1715d..be829dc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,40 +1,175 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ *.py[cod] -# Cloud9 revision files -.c9revisions +*$py.class # C extensions *.so -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ .installed.cfg -lib -lib64 +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec # Installer logs pip-log.txt +pip-delete-this-directory.txt # Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ .coverage -.tox +.coverage.* +.cache nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ # Translations *.mo +*.pot -# Mr Developer -.mr.developer.cfg +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install different versions of packages depending on +# the platform. Pipfile.lock may vary on different platforms. For example, if you are using Mac OS +# and your colleague is using Linux, with same Pipfile, Pipfile.lock may be different. +# In that case, it may be better to ignore Pipfile.lock and let the colleague generate it themselves. +# Also, if you are developing a library, ignoring Pipfile.lock is recommended. +# See pypa/pipenv#598 for the discussion. +# Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# The poetry.lock file stores exact versions of your dependencies, so you can recreate the same +# environment across different machines. +# However, if you are developing a library, ignoring poetry.lock is recommended. +# poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm.lock +# .pdm.toml + +# PEP 582; used by e.g. pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# PyCharm files +.idea/ .project .pydevproject +.settings/ +*.iml +*.ipr +*.iws + +# VSCode +.vscode/ + +# Sublimetext +*.sublime-project +*.sublime-workspace + +# Mr Developer +.mr.developer.cfg +.mr.developer.ini + +# Cloud9 revision files +.c_revisions + +# macOS +.DS_Store +.AppleDouble +.LSOverride + +# Windows +Thumbs.db +ehthumbs.db +ehthumbs_vista.db +*.stackdump + +# MyPy +.mypy_cache/ +.dmypy.json +dmypy.json -# PyCharm -.idea +# Pyre type checker +.pyre/ \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..29cabf3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# Contributing to PgDiffPy + +We welcome contributions to PgDiffPy! Here are some ways you can contribute: + +## Reporting Issues + +If you encounter a bug or have a feature request, please open an issue on our [GitHub issue tracker](https://github.com/Dancer3809/PgDiffPy/issues). + +When reporting a bug, please include the following information: + +* A clear and descriptive title. +* The version of PgDiffPy you are using. +* The steps to reproduce the bug. +* The expected behavior and the actual behavior. +* Any relevant error messages or stack traces. + +## Submitting Pull Requests + +If you would like to contribute code to the project, please follow these steps: + +1. Fork the repository on GitHub. +2. Create a new branch for your feature or bug fix. +3. Make your changes and commit them with a clear and descriptive commit message. +4. Push your changes to your forked repository. +5. Open a pull request to the `main` branch of the original repository. + +Please ensure that your code adheres to the existing code style and that all tests pass before submitting a pull request. diff --git a/CREDITS b/CREDITS new file mode 100644 index 0000000..6936455 --- /dev/null +++ b/CREDITS @@ -0,0 +1,3 @@ +This project is a Python port of the original [apgdiff](http://apgdiff.com/) tool. All credit for the original logic and concept goes to the apgdiff team. + +This project is based on the work of [PgDiffPy](https://github.com/Gesha3809/PgDiffPy). diff --git a/README.md b/README.md index be5cc74..b59319d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,68 @@ -PgDiffPy -========= +# PgDiffPy -[![Build Status](https://travis-ci.org/Dancer3809/PgDiffPy.png)](https://travis-ci.org/[Dancer3809/PgDiffPy) +[![Build Status](https://travis-ci.org/Dancer3809/PgDiffPy.png)](https://travis-ci.org/Dancer3809/PgDiffPy) -Diff tool for Postgresql database based on apgdiff but written in Python +## Purpose +PgDiffPy is a command-line tool for comparing two PostgreSQL database schema dumps and generating a SQL script to update the old schema to match the new one. It is based on the original apgdiff tool but rewritten in Python. -[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/Dancer3809/pgdiffpy/trend.png)](https://bitdeli.com/free "Bitdeli Badge") +This tool is useful for database administrators and developers who need to manage and version-control their database schemas. It simplifies the process of deploying schema changes from a development environment to a production environment. +## Installation + +You can install PgDiffPy directly from the source repository using `pip`. It is recommended to do this within a Python virtual environment to avoid conflicts with system-wide packages. + +1. **Clone the repository:** + ```bash + git clone https://github.com/Dancer3809/PgDiffPy.git + cd PgDiffPy + ``` + +2. **Install the package:** + ```bash + pip install . + ``` + +This will install the `pgdiff` command-line tool and its dependencies. If you are using a virtual environment, the command will be available directly in your shell. + +### Alternative: Installing for the Current User + +If you prefer not to use a virtual environment, you can install the package for your user account: + +```bash +pip install --user . +``` + +This installs the package in your user's local directory (e.g., `~/.local`). To run the `pgdiff` command directly, you may need to add the local bin directory (e.g., `~/.local/bin`) to your shell's `PATH`. + +## Usage + +Once installed, you can use the `pgdiff` command-line tool to compare two database dumps: + +```bash +pgdiff [options] +``` + +### Arguments + +* `old_dump.sql`: The path to the SQL file containing the old database schema. +* `new_dump.sql`: The path to the SQL file containing the new database schema. + +### Options + +* `--add-transaction`: Adds `START TRANSACTION` and `COMMIT TRANSACTION` to the generated diff file. +* `--add-defaults`: Adds `DEFAULT ...` in case a new column has a `NOT NULL` constraint but no default value. +* `--ignore-start-with`: Ignores `START WITH` modifications on `SEQUENCE`s. +* `--ignore-function-whitespace`: Ignores multiple spaces and new lines when comparing function content. + +### Example + +```bash +pgdiff original_schema.sql new_schema.sql > migration_script.sql +``` + +This will generate a `migration_script.sql` file that you can run on the original database to apply the schema changes. + +## License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/helpers/OrderedDict.py b/helpers/OrderedDict.py deleted file mode 100644 index f3e5e35..0000000 --- a/helpers/OrderedDict.py +++ /dev/null @@ -1,249 +0,0 @@ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) \ No newline at end of file diff --git a/loaders/PgDumpLoader.py b/loaders/PgDumpLoader.py deleted file mode 100644 index db887ad..0000000 --- a/loaders/PgDumpLoader.py +++ /dev/null @@ -1,128 +0,0 @@ -import sqlparse -import re -import logging -from schema.PgDatabase import PgDatabase -from parser.CreateSchemaParser import CreateSchemaParser -from parser.CreateTableParser import CreateTableParser -from parser.AlterTableParser import AlterTableParser -from parser.CreateIndexParser import CreateIndexParser -from parser.CreateFunctionParser import CreateFunctionParser -from parser.CreateSequenceParser import CreateSequenceParser -from parser.AlterSequenceParser import AlterSequenceParser -from parser.CreateViewParser import CreateViewParser -from parser.CreateTriggerParser import CreateTriggerParser -from parser.AlterViewParser import AlterViewParser -from parser.CommentParser import CommentParser - -class PgDumpLoader(object): - - # Pattern for testing whether it is CREATE SCHEMA statement. - PATTERN_CREATE_SCHEMA = re.compile(r"^CREATE[\s]+SCHEMA[\s]+.*$", re.I | re.S) - # Pattern for parsing default schema (search_path). - PATTERN_DEFAULT_SCHEMA = re.compile(r"^SET[\s]+search_path[\s]*=[\s]*\"?([^,\s\"]+)\"?(?:,[\s]+.*)?;$", re.I | re.S) - # Pattern for testing whether it is CREATE TABLE statement. - PATTERN_CREATE_TABLE = re.compile("^CREATE[\s]+TABLE[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is CREATE VIEW statement. - PATTERN_CREATE_VIEW = re.compile("^CREATE[\s]+(?:OR[\s]+REPLACE[\s]+)?VIEW[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is ALTER TABLE statement. - PATTERN_ALTER_TABLE = re.compile("^ALTER[\s]+TABLE[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is CREATE SEQUENCE statement. - PATTERN_CREATE_SEQUENCE = re.compile("^CREATE[\s]+SEQUENCE[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is ALTER SEQUENCE statement. - PATTERN_ALTER_SEQUENCE =re.compile("^ALTER[\s]+SEQUENCE[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is CREATE INDEX statement. - PATTERN_CREATE_INDEX = re.compile("^CREATE[\s]+(?:UNIQUE[\s]+)?INDEX[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is SELECT statement. - PATTERN_SELECT = re.compile("^SELECT[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is INSERT INTO statement. - PATTERN_INSERT_INTO = re.compile("^INSERT[\s]+INTO[\\s]+.*$", re.I | re.S) - # Pattern for testing whether it is UPDATE statement. - PATTERN_UPDATE = re.compile("^UPDATE[\s].*$", re.I | re.S) - # Pattern for testing whether it is DELETE FROM statement. - PATTERN_DELETE_FROM = re.compile("^DELETE[\s]+FROM[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is CREATE TRIGGER statement. - PATTERN_CREATE_TRIGGER = re.compile("^CREATE[\s]+TRIGGER[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is CREATE FUNCTION or CREATE OR REPLACE FUNCTION statement. - PATTERN_CREATE_FUNCTION = re.compile("^CREATE[\s]+(?:OR[\s]+REPLACE[\s]+)?FUNCTION[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is ALTER VIEW statement. - PATTERN_ALTER_VIEW = re.compile("^ALTER[\s]+VIEW[\s]+.*$", re.I | re.S) - # Pattern for testing whether it is COMMENT statement. - PATTERN_COMMENT = re.compile("^COMMENT[\s]+ON[\s]+.*$", re.I | re.S) - - @staticmethod - def loadDatabaseSchema(dumpFileName, ignoreSlonyTriggers = False): - database = PgDatabase() - logging.debug('Loading %s file' % dumpFileName) - - statements = sqlparse.split(open(dumpFileName,'r')) - logging.debug('Parsed %d statements' % len(statements)) - for statement in statements: - statement = PgDumpLoader.strip_comment(statement).strip() - if PgDumpLoader.PATTERN_CREATE_SCHEMA.match(statement): - CreateSchemaParser.parse(database, statement) - continue - - match = PgDumpLoader.PATTERN_DEFAULT_SCHEMA.match(statement) - if match: - database.setDefaultSchema(match.group(1)) - continue - - if PgDumpLoader.PATTERN_CREATE_TABLE.match(statement): - CreateTableParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_ALTER_TABLE.match(statement): - AlterTableParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_CREATE_SEQUENCE.match(statement): - CreateSequenceParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_ALTER_SEQUENCE.match(statement): - AlterSequenceParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_CREATE_INDEX.match(statement): - CreateIndexParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_CREATE_VIEW.match(statement): - CreateViewParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_ALTER_VIEW.match(statement): - AlterViewParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_CREATE_TRIGGER.match(statement): - CreateTriggerParser.parse(database, statement, ignoreSlonyTriggers) - continue - - if PgDumpLoader.PATTERN_CREATE_FUNCTION.match(statement): - CreateFunctionParser.parse(database, statement) - continue - - if PgDumpLoader.PATTERN_COMMENT.match(statement): - CommentParser.parse(database, statement) - continue - - logging.info('Ignored statement: %s' % statement) - - return database - - - @staticmethod - def strip_comment(statement): - start = statement.find("--") - - while start >= 0: - end = statement.find("\n", start) - if start < end: - statement = statement[end:] - else: - statement = statement[:start] - - start = statement.find("--") - - return statement diff --git a/SearchPathHelper.py b/pgdiff/SearchPathHelper.py similarity index 100% rename from SearchPathHelper.py rename to pgdiff/SearchPathHelper.py diff --git a/PgDiff.py b/pgdiff/__init__.py similarity index 56% rename from PgDiff.py rename to pgdiff/__init__.py index 88c6070..0e4b02e 100644 --- a/PgDiff.py +++ b/pgdiff/__init__.py @@ -1,16 +1,14 @@ -import argparse -import logging -from helpers.Writer import Writer -from loaders.PgDumpLoader import PgDumpLoader -from diff.PgDiffUtils import PgDiffUtils -from SearchPathHelper import SearchPathHelper -from diff.PgDiffTables import PgDiffTables -from diff.PgDiffTriggers import PgDiffTriggers -from diff.PgDiffViews import PgDiffViews -from diff.PgDiffConstraints import PgDiffConstraints -from diff.PgDiffIndexes import PgDiffIndexes -from diff.PgDiffSequences import PgDiffSequences -from diff.PgDiffFunctions import PgDiffFunctions +from .helpers.Writer import Writer +from .loaders.PgDumpLoader import PgDumpLoader +from .diff.PgDiffUtils import PgDiffUtils +from .SearchPathHelper import SearchPathHelper +from .diff.PgDiffTables import PgDiffTables +from .diff.PgDiffTriggers import PgDiffTriggers +from .diff.PgDiffViews import PgDiffViews +from .diff.PgDiffConstraints import PgDiffConstraints +from .diff.PgDiffIndexes import PgDiffIndexes +from .diff.PgDiffSequences import PgDiffSequences +from .diff.PgDiffFunctions import PgDiffFunctions class PgDiff(object): @@ -45,37 +43,6 @@ def diff_database_schemas(writer, arguments, old_database, new_database): if arguments.addTransaction: writer.writeln("COMMIT TRANSACTION;") - # if (arguments.isOutputIgnoredStatements()) { - # if (!oldDatabase.getIgnoredStatements().isEmpty()) { - # writer.println(); - # writer.print("/* "); - # writer.println(Resources.getString( - # "OriginalDatabaseIgnoredStatements")); - - # for (final String statement : - # oldDatabase.getIgnoredStatements()) { - # writer.println(); - # writer.println(statement); - # } - - # writer.println("*/"); - # } - - # if (!newDatabase.getIgnoredStatements().isEmpty()) { - # writer.println(); - # writer.print("/* "); - # writer.println(Resources.getString("NewDatabaseIgnoredStatements")); - - # for (final String statement : - # newDatabase.getIgnoredStatements()) { - # writer.println(); - # writer.println(statement); - # } - - # writer.println("*/"); - # } - # } - @staticmethod def drop_old_schemas(writer, old_database, new_database): for oldSchemaName in old_database.schemas: @@ -91,7 +58,7 @@ def create_new_schemas(writer, old_database, new_database): @staticmethod def update_schemas(writer, arguments, old_database, new_database): # We set search path if more than one schemas or it's name is not public - set_search_path = len(new_database.schemas) > 1 or new_database.schemas.itervalues().next().name != "public" + set_search_path = len(new_database.schemas) > 1 or next(iter(new_database.schemas.values())).name != "public" for newSchemaName in new_database.schemas: if set_search_path: @@ -126,7 +93,6 @@ def update_schemas(writer, arguments, old_database, new_database): PgDiffConstraints.dropConstraints(writer, old_schema, new_schema, True, search_path_helper) PgDiffConstraints.dropConstraints(writer, old_schema, new_schema, False, search_path_helper) PgDiffIndexes.dropIndexes(writer, old_schema, new_schema, search_path_helper) - # # PgDiffTables.dropClusters(oldSchema, newSchema, search_path_helper) PgDiffTables.dropTables(writer, old_schema, new_schema, search_path_helper) PgDiffSequences.dropSequences(writer, old_schema, new_schema, search_path_helper) @@ -139,7 +105,6 @@ def update_schemas(writer, arguments, old_database, new_database): PgDiffConstraints.createConstraints(writer, old_schema, new_schema, True, search_path_helper) PgDiffConstraints.createConstraints(writer, old_schema, new_schema, False, search_path_helper) PgDiffIndexes.createIndexes(writer, old_schema, new_schema, search_path_helper) - # # PgDiffTables.createClusters(oldSchema, newSchema, search_path_helper) PgDiffTriggers.createTriggers(writer, old_schema, new_schema, search_path_helper) PgDiffViews.createViews(writer, old_schema, new_schema, search_path_helper) PgDiffViews.alterViews(writer, old_schema, new_schema, search_path_helper) @@ -148,58 +113,3 @@ def update_schemas(writer, arguments, old_database, new_database): PgDiffConstraints.alterComments(writer, old_schema, new_schema, search_path_helper) PgDiffIndexes.alterComments(writer, old_schema, new_schema, search_path_helper) PgDiffTriggers.alterComments(writer, old_schema, new_schema, search_path_helper) - - -class LogLevelAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - if values == 'DEBUG': - setattr(namespace, self.dest, logging.DEBUG) - elif values == 'INFO': - setattr(namespace, self.dest, logging.INFO) - elif values == 'WARNING': - setattr(namespace, self.dest, logging.WARNING) - elif values == 'ERROR': - setattr(namespace, self.dest, logging.ERROR) - elif values == 'CRITICAL': - setattr(namespace, self.dest, logging.CRITICAL) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(prog='PgDiffPy', usage='python PgDiff.py [options] ') - - parser.add_argument('old_dump') - parser.add_argument('new_dump') - - parser.add_argument('--add-transaction', dest='addTransaction', action='store_true', - help="Adds START TRANSACTION and COMMIT TRANSACTION to the generated diff file") - parser.add_argument('--add-defaults', dest='addDefaults', action='store_true', - help="adds DEFAULT ... in case new column has NOT NULL constraint but no default value " - "(the default value is dropped later)") - parser.add_argument('--ignore-start-with', dest='ignoreStartWith', action='store_false', - help="ignores START WITH modifications on SEQUENCEs (default is not to ignore these changes)") - parser.add_argument('--ignore-function-whitespace', dest='ignoreFunctionWhitespace', action='store_true', - help="ignores multiple spaces and new lines when comparing content of functions\n\ - \t- WARNING: this may cause functions to appear to be same in cases they are\n\ - \tnot, so use this feature only if you know what you are doing") - - parser.add_argument('--loglevel', dest='loglevel', action=LogLevelAction - , choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] - , default=logging.ERROR, help="") - - arguments = parser.parse_args() - logging.basicConfig(format=u'%(filename)s:%(lineno)d [%(levelname)s] %(message)s' - , level=arguments.loglevel) - - writer = Writer() - - try: - PgDiff.create_diff(writer, arguments) - print(writer) - except Exception as e: - if arguments.loglevel == logging.DEBUG: - import sys - import traceback - traceback.print_exception(*sys.exc_info()) - else: - print('Error: %s' % e) - exit(1) diff --git a/pgdiff/__main__.py b/pgdiff/__main__.py new file mode 100644 index 0000000..2ab237d --- /dev/null +++ b/pgdiff/__main__.py @@ -0,0 +1,70 @@ +import argparse +import logging +from . import PgDiff +from .helpers.Writer import Writer +from .loaders.PgDumpLoader import PgDumpLoader +from .diff.PgDiffUtils import PgDiffUtils +from .SearchPathHelper import SearchPathHelper +from .diff.PgDiffTables import PgDiffTables +from .diff.PgDiffTriggers import PgDiffTriggers +from .diff.PgDiffViews import PgDiffViews +from .diff.PgDiffConstraints import PgDiffConstraints +from .diff.PgDiffIndexes import PgDiffIndexes +from .diff.PgDiffSequences import PgDiffSequences +from .diff.PgDiffFunctions import PgDiffFunctions + + +class LogLevelAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + if values == 'DEBUG': + setattr(namespace, self.dest, logging.DEBUG) + elif values == 'INFO': + setattr(namespace, self.dest, logging.INFO) + elif values == 'WARNING': + setattr(namespace, self.dest, logging.WARNING) + elif values == 'ERROR': + setattr(namespace, self.dest, logging.ERROR) + elif values == 'CRITICAL': + setattr(namespace, self.dest, logging.CRITICAL) + + +def main(): + parser = argparse.ArgumentParser(prog='PgDiffPy', usage='python PgDiff.py [options] ') + + parser.add_argument('old_dump') + parser.add_argument('new_dump') + + parser.add_argument('--add-transaction', dest='addTransaction', action='store_true', + help="Adds START TRANSACTION and COMMIT TRANSACTION to the generated diff file") + parser.add_argument('--add-defaults', dest='addDefaults', action='store_true', + help=("adds DEFAULT ... in case new column has NOT NULL constraint but no default value " + "(the default value is dropped later)")) + parser.add_argument('--ignore-start-with', dest='ignoreStartWith', action='store_false', + help="ignores START WITH modifications on SEQUENCEs (default is not to ignore these changes)") + parser.add_argument('--ignore-function-whitespace', dest='ignoreFunctionWhitespace', action='store_true', + help="ignores multiple spaces and new lines when comparing content of functions\n\ \t- WARNING: this may cause functions to appear to be same in cases they are\n\ \tnot, so use this feature only if you know what you are doing") + + parser.add_argument('--loglevel', dest='loglevel', action=LogLevelAction + , choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] + , default=logging.ERROR, help="") + + arguments = parser.parse_args() + logging.basicConfig(format=u'%(filename)s:%(lineno)d [%(levelname)s] %(message)s' + , level=arguments.loglevel) + + writer = Writer() + + try: + PgDiff.create_diff(writer, arguments) + print(writer) + except Exception as e: + if arguments.loglevel == logging.DEBUG: + import sys + import traceback + traceback.print_exception(*sys.exc_info()) + else: + print('Error: %s' % e) + exit(1) + +if __name__ == "__main__": + main() diff --git a/diff/PgDiffConstraints.py b/pgdiff/diff/PgDiffConstraints.py similarity index 99% rename from diff/PgDiffConstraints.py rename to pgdiff/diff/PgDiffConstraints.py index 59a7951..5a1b146 100644 --- a/diff/PgDiffConstraints.py +++ b/pgdiff/diff/PgDiffConstraints.py @@ -1,4 +1,4 @@ -from PgDiffUtils import PgDiffUtils +from .PgDiffUtils import PgDiffUtils class PgDiffConstraints(object): diff --git a/diff/PgDiffFunctions.py b/pgdiff/diff/PgDiffFunctions.py similarity index 98% rename from diff/PgDiffFunctions.py rename to pgdiff/diff/PgDiffFunctions.py index ab214f6..8a26b35 100644 --- a/diff/PgDiffFunctions.py +++ b/pgdiff/diff/PgDiffFunctions.py @@ -1,4 +1,4 @@ -from PgDiffUtils import PgDiffUtils +from .PgDiffUtils import PgDiffUtils class PgDiffFunctions(object): diff --git a/diff/PgDiffIndexes.py b/pgdiff/diff/PgDiffIndexes.py similarity index 98% rename from diff/PgDiffIndexes.py rename to pgdiff/diff/PgDiffIndexes.py index bdf2363..bfd52ac 100644 --- a/diff/PgDiffIndexes.py +++ b/pgdiff/diff/PgDiffIndexes.py @@ -1,3 +1,5 @@ +from .PgDiffUtils import PgDiffUtils + class PgDiffIndexes(object): @staticmethod def createIndexes(writer, oldSchema, newSchema, searchPathHelper): diff --git a/diff/PgDiffSequences.py b/pgdiff/diff/PgDiffSequences.py similarity index 99% rename from diff/PgDiffSequences.py rename to pgdiff/diff/PgDiffSequences.py index 6c9234f..ca4522b 100644 --- a/diff/PgDiffSequences.py +++ b/pgdiff/diff/PgDiffSequences.py @@ -1,4 +1,4 @@ -from PgDiffUtils import PgDiffUtils +from .PgDiffUtils import PgDiffUtils class PgDiffSequences(object): diff --git a/diff/PgDiffTables.py b/pgdiff/diff/PgDiffTables.py similarity index 99% rename from diff/PgDiffTables.py rename to pgdiff/diff/PgDiffTables.py index 1d56ec8..b5d97e5 100644 --- a/diff/PgDiffTables.py +++ b/pgdiff/diff/PgDiffTables.py @@ -1,5 +1,5 @@ -from PgDiffUtils import PgDiffUtils -from schema.PgColumn import PgColumnUtils +from .PgDiffUtils import PgDiffUtils +from ..schema.PgColumn import PgColumnUtils class PgDiffTables(object): diff --git a/diff/PgDiffTriggers.py b/pgdiff/diff/PgDiffTriggers.py similarity index 99% rename from diff/PgDiffTriggers.py rename to pgdiff/diff/PgDiffTriggers.py index 7212ea4..52cf380 100644 --- a/diff/PgDiffTriggers.py +++ b/pgdiff/diff/PgDiffTriggers.py @@ -1,4 +1,4 @@ -from PgDiffUtils import PgDiffUtils +from .PgDiffUtils import PgDiffUtils class PgDiffTriggers(object): diff --git a/diff/PgDiffUtils.py b/pgdiff/diff/PgDiffUtils.py similarity index 100% rename from diff/PgDiffUtils.py rename to pgdiff/diff/PgDiffUtils.py diff --git a/diff/PgDiffViews.py b/pgdiff/diff/PgDiffViews.py similarity index 99% rename from diff/PgDiffViews.py rename to pgdiff/diff/PgDiffViews.py index fde88dc..7261067 100644 --- a/diff/PgDiffViews.py +++ b/pgdiff/diff/PgDiffViews.py @@ -1,4 +1,4 @@ -from PgDiffUtils import PgDiffUtils +from .PgDiffUtils import PgDiffUtils class PgDiffViews(object): diff --git a/diff/__init__.py b/pgdiff/diff/__init__.py similarity index 100% rename from diff/__init__.py rename to pgdiff/diff/__init__.py diff --git a/helpers/Writer.py b/pgdiff/helpers/Writer.py similarity index 100% rename from helpers/Writer.py rename to pgdiff/helpers/Writer.py diff --git a/helpers/__init__.py b/pgdiff/helpers/__init__.py similarity index 100% rename from helpers/__init__.py rename to pgdiff/helpers/__init__.py diff --git a/pgdiff/loaders/PgDumpLoader.py b/pgdiff/loaders/PgDumpLoader.py new file mode 100644 index 0000000..af06370 --- /dev/null +++ b/pgdiff/loaders/PgDumpLoader.py @@ -0,0 +1,149 @@ +import sqlparse +import re +import logging +from ..schema.PgDatabase import PgDatabase +from ..parser.CreateSchemaParser import CreateSchemaParser +from ..parser.CreateTableParser import CreateTableParser +from ..parser.AlterTableParser import AlterTableParser +from ..parser.CreateIndexParser import CreateIndexParser +from ..parser.CreateFunctionParser import CreateFunctionParser +from ..parser.CreateSequenceParser import CreateSequenceParser +from ..parser.AlterSequenceParser import AlterSequenceParser +from ..parser.CreateViewParser import CreateViewParser +from ..parser.CreateTriggerParser import CreateTriggerParser +from ..parser.AlterViewParser import AlterViewParser +from ..parser.CommentParser import CommentParser + +class PgDumpLoader(object): + + # Pattern for testing whether it is CREATE SCHEMA statement. + PATTERN_CREATE_SCHEMA = re.compile(r"^CREATE[\s]+SCHEMA[\s]+.*$", re.I | re.S) + # Pattern for parsing default schema (search_path). + PATTERN_DEFAULT_SCHEMA = re.compile(r"^SET[\s]+search_path[\s]*=[\s]*\"?([^,\s\"]+)\"?(?:,[\s]+.*)?;$", re.I | re.S) + # Pattern for testing whether it is CREATE TABLE statement. + PATTERN_CREATE_TABLE = re.compile("^CREATE[\s]+TABLE[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is CREATE VIEW statement. + PATTERN_CREATE_VIEW = re.compile("^CREATE[\s]+(?:OR[\s]+REPLACE[\s]+)?VIEW[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is ALTER TABLE statement. + PATTERN_ALTER_TABLE = re.compile("^ALTER[\s]+TABLE[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is CREATE SEQUENCE statement. + PATTERN_CREATE_SEQUENCE = re.compile("^CREATE[\s]+SEQUENCE[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is ALTER SEQUENCE statement. + PATTERN_ALTER_SEQUENCE =re.compile("^ALTER[\s]+SEQUENCE[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is CREATE INDEX statement. + PATTERN_CREATE_INDEX = re.compile("^CREATE[\s]+(?:UNIQUE[\s]+)?INDEX[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is SELECT statement. + PATTERN_SELECT = re.compile("^SELECT[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is INSERT INTO statement. + PATTERN_INSERT_INTO = re.compile("^INSERT[\s]+INTO[\\s]+.*$", re.I | re.S) + # Pattern for testing whether it is UPDATE statement. + PATTERN_UPDATE = re.compile("^UPDATE[\s].*$", re.I | re.S) + # Pattern for testing whether it is DELETE FROM statement. + PATTERN_DELETE_FROM = re.compile("^DELETE[\s]+FROM[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is CREATE TRIGGER statement. + PATTERN_CREATE_TRIGGER = re.compile("^CREATE[\s]+TRIGGER[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is CREATE FUNCTION or CREATE OR REPLACE FUNCTION statement. + PATTERN_CREATE_FUNCTION = re.compile("^CREATE[\s]+(?:OR[\s]+REPLACE[\s]+)?FUNCTION[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is ALTER VIEW statement. + PATTERN_ALTER_VIEW = re.compile("^ALTER[\s]+VIEW[\s]+.*$", re.I | re.S) + # Pattern for testing whether it is COMMENT statement. + PATTERN_COMMENT = re.compile("^COMMENT[\s]+ON[\s]+.*$", re.I | re.S) + + @staticmethod + def loadDatabaseSchema(dumpFileName, ignoreSlonyTriggers = False): + database = PgDatabase() + logging.debug('Loading %s file' % dumpFileName) + + statements = sqlparse.split(open(dumpFileName,'r')) + logging.debug('Parsed %d statements' % len(statements)) + + create_schema_statements = [] + create_table_statements = [] + alter_table_statements = [] + create_index_statements = [] + create_trigger_statements = [] + create_function_statements = [] + create_sequence_statements = [] + alter_sequence_statements = [] + create_view_statements = [] + alter_view_statements = [] + comment_statements = [] + other_statements = [] + + for statement in statements: + statement = PgDumpLoader.strip_comment(statement).strip() + if not statement: + continue + + if PgDumpLoader.PATTERN_CREATE_SCHEMA.match(statement): + create_schema_statements.append(statement) + elif PgDumpLoader.PATTERN_DEFAULT_SCHEMA.match(statement): + # Default schema needs to be set early + match = PgDumpLoader.PATTERN_DEFAULT_SCHEMA.match(statement) + database.setDefaultSchema(match.group(1)) + elif PgDumpLoader.PATTERN_CREATE_TABLE.match(statement): + create_table_statements.append(statement) + elif PgDumpLoader.PATTERN_ALTER_TABLE.match(statement): + alter_table_statements.append(statement) + elif PgDumpLoader.PATTERN_CREATE_SEQUENCE.match(statement): + create_sequence_statements.append(statement) + elif PgDumpLoader.PATTERN_ALTER_SEQUENCE.match(statement): + alter_sequence_statements.append(statement) + elif PgDumpLoader.PATTERN_CREATE_INDEX.match(statement): + create_index_statements.append(statement) + elif PgDumpLoader.PATTERN_CREATE_VIEW.match(statement): + create_view_statements.append(statement) + elif PgDumpLoader.PATTERN_ALTER_VIEW.match(statement): + alter_view_statements.append(statement) + elif PgDumpLoader.PATTERN_CREATE_TRIGGER.match(statement): + create_trigger_statements.append(statement) + elif PgDumpLoader.PATTERN_CREATE_FUNCTION.match(statement): + create_function_statements.append(statement) + elif PgDumpLoader.PATTERN_COMMENT.match(statement): + comment_statements.append(statement) + else: + other_statements.append(statement) + + for statement in create_schema_statements: + CreateSchemaParser.parse(database, statement) + for statement in create_table_statements: + CreateTableParser.parse(database, statement) + for statement in alter_table_statements: + AlterTableParser.parse(database, statement) + for statement in create_sequence_statements: + CreateSequenceParser.parse(database, statement) + for statement in alter_sequence_statements: + AlterSequenceParser.parse(database, statement) + for statement in create_index_statements: + CreateIndexParser.parse(database, statement) + for statement in create_view_statements: + CreateViewParser.parse(database, statement) + for statement in alter_view_statements: + AlterViewParser.parse(database, statement) + for statement in create_trigger_statements: + CreateTriggerParser.parse(database, statement, ignoreSlonyTriggers) + for statement in create_function_statements: + CreateFunctionParser.parse(database, statement) + for statement in comment_statements: + CommentParser.parse(database, statement) + + for statement in other_statements: + logging.info('Ignored statement: %s' % statement) + + return database + + + @staticmethod + def strip_comment(statement): + start = statement.find("--") + + while start >= 0: + end = statement.find("\n", start) + if start < end: + statement = statement[end:] + else: + statement = statement[:start] + + start = statement.find("--") + + return statement diff --git a/loaders/__init__.py b/pgdiff/loaders/__init__.py similarity index 100% rename from loaders/__init__.py rename to pgdiff/loaders/__init__.py diff --git a/parser/AlterSequenceParser.py b/pgdiff/parser/AlterSequenceParser.py similarity index 92% rename from parser/AlterSequenceParser.py rename to pgdiff/parser/AlterSequenceParser.py index 22e3483..d61cd5e 100644 --- a/parser/AlterSequenceParser.py +++ b/pgdiff/parser/AlterSequenceParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgSequence import PgSequence +from .Parser import Parser, ParserUtils +from ..schema.PgSequence import PgSequence class AlterSequenceParser(object): @staticmethod diff --git a/parser/AlterTableParser.py b/pgdiff/parser/AlterTableParser.py similarity index 90% rename from parser/AlterTableParser.py rename to pgdiff/parser/AlterTableParser.py index 63a9533..e1d8be9 100644 --- a/parser/AlterTableParser.py +++ b/pgdiff/parser/AlterTableParser.py @@ -1,6 +1,6 @@ import logging -from parser.Parser import Parser, ParserUtils -from schema.PgConstraint import PgConstraint +from .Parser import Parser, ParserUtils +from ..schema.PgConstraint import PgConstraint class AlterTableParser(object): @@ -10,6 +10,7 @@ def parse(database, statement): parser.expect("ALTER", "TABLE") parser.expect_optional("ONLY") + ifExists = parser.expect_optional("IF", "EXISTS") tableName = parser.parse_identifier() schemaName = ParserUtils.get_schema_name(tableName, database) @@ -34,6 +35,9 @@ def parse(database, statement): if sequence is not None: AlterTableParser.parseSequence(parser, sequence, tableName, database); return + + if ifExists: + return raise Exception("Cannot find object '%s' for statement '%s'." % (tableName, statement)) @@ -50,15 +54,15 @@ def parse(database, statement): parser.parse_identifier() elif (parser.expect_optional("ADD")): if (parser.expect_optional("FOREIGN", "KEY")): - print 'parseAddForeignKey(parser, table);' + print('parseAddForeignKey(parser, table);') elif (parser.expect_optional("CONSTRAINT")): AlterTableParser.parseAddConstraint(parser, table, schema) else: parser.throw_unsupported_command() elif (parser.expect_optional("ENABLE")): - print 'parseEnable(parser, outputIgnoredStatements, tableName, database);' + print('parseEnable(parser, outputIgnoredStatements, tableName, database);') elif (parser.expect_optional("DISABLE")): - print 'parseDisable(parser, outputIgnoredStatements, tableName, database);' + print('parseDisable(parser, outputIgnoredStatements, tableName, database);') else: parser.throw_unsupported_command() @@ -112,6 +116,10 @@ def parseAlterColumn(parser, table): parser.throw_unsupported_command() else: parser.throw_unsupported_command() + elif parser.expect_optional("DROP", "DEFAULT"): + column = table.columns.get(columnName) + if column: + column.defaultValue = None else: parser.throw_unsupported_command() diff --git a/parser/AlterViewParser.py b/pgdiff/parser/AlterViewParser.py similarity index 96% rename from parser/AlterViewParser.py rename to pgdiff/parser/AlterViewParser.py index fd1df95..8ff41b3 100644 --- a/parser/AlterViewParser.py +++ b/pgdiff/parser/AlterViewParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema import PgView +from .Parser import Parser, ParserUtils +from ..schema import PgView class AlterViewParser(object): @staticmethod diff --git a/parser/CommentParser.py b/pgdiff/parser/CommentParser.py similarity index 95% rename from parser/CommentParser.py rename to pgdiff/parser/CommentParser.py index fe186f2..4fb3178 100644 --- a/parser/CommentParser.py +++ b/pgdiff/parser/CommentParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgFunction import PgFunction, Argument +from .Parser import Parser, ParserUtils +from ..schema.PgFunction import PgFunction, Argument class CommentParser(object): @@ -49,7 +49,14 @@ def parseColumn(parser, database): objectName = ParserUtils.get_object_name(columnName) tableName = ParserUtils.get_second_object_name(columnName) schemaName = ParserUtils.get_third_object_name(columnName) - schema = database.getSchema(schemaName) + + if schemaName is None: + schema = database.getDefaultSchema() + else: + schema = database.getSchema(schemaName) + + if schema is None: + raise Exception("Cannot find schema '%s' for statement '%s'." % (schemaName, parser.statement)) table = schema.getTable(tableName) diff --git a/parser/CreateFunctionParser.py b/pgdiff/parser/CreateFunctionParser.py similarity index 96% rename from parser/CreateFunctionParser.py rename to pgdiff/parser/CreateFunctionParser.py index ed350e1..3097ad0 100644 --- a/parser/CreateFunctionParser.py +++ b/pgdiff/parser/CreateFunctionParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgFunction import PgFunction, Argument +from .Parser import Parser, ParserUtils +from ..schema.PgFunction import PgFunction, Argument class CreateFunctionParser(object): @staticmethod diff --git a/parser/CreateIndexParser.py b/pgdiff/parser/CreateIndexParser.py similarity index 87% rename from parser/CreateIndexParser.py rename to pgdiff/parser/CreateIndexParser.py index b4347db..32f3c93 100644 --- a/parser/CreateIndexParser.py +++ b/pgdiff/parser/CreateIndexParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgIndex import PgIndex +from .Parser import Parser, ParserUtils +from ..schema.PgIndex import PgIndex class CreateIndexParser(object): @staticmethod @@ -22,7 +22,7 @@ def parse(database, statement): schema = database.getSchema(schemaName) if (schema is None): - print 'ERROR: CreateIndexParser[Line 21]' + print('ERROR: CreateIndexParser[Line 21]') # throw new RuntimeException(MessageFormat.format( # Resources.getString("CannotFindSchema"), schemaName, # statement)); @@ -31,7 +31,7 @@ def parse(database, statement): table = schema.getTable(objectName) if (table is None): - print 'ERROR: CreateIndexParser[Line 32]' + print('ERROR: CreateIndexParser[Line 32]') # throw new RuntimeException(MessageFormat.format( # Resources.getString("CannotFindTable"), tableName, # statement)); diff --git a/parser/CreateSchemaParser.py b/pgdiff/parser/CreateSchemaParser.py similarity index 84% rename from parser/CreateSchemaParser.py rename to pgdiff/parser/CreateSchemaParser.py index 9b24514..117935e 100644 --- a/parser/CreateSchemaParser.py +++ b/pgdiff/parser/CreateSchemaParser.py @@ -1,11 +1,12 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgSchema import PgSchema +from .Parser import Parser, ParserUtils +from ..schema.PgSchema import PgSchema class CreateSchemaParser(object): @staticmethod def parse(database, statement): parser = Parser(statement) parser.expect("CREATE", "SCHEMA") + parser.expect_optional("IF", "NOT", "EXISTS") if parser.expect_optional("AUTHORIZATION"): schema = PgSchema(ParserUtils.get_object_name(parser.parse_identifier())) diff --git a/parser/CreateSequenceParser.py b/pgdiff/parser/CreateSequenceParser.py similarity index 92% rename from parser/CreateSequenceParser.py rename to pgdiff/parser/CreateSequenceParser.py index 0f98020..b6c8a1e 100644 --- a/parser/CreateSequenceParser.py +++ b/pgdiff/parser/CreateSequenceParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgSequence import PgSequence +from .Parser import Parser, ParserUtils +from ..schema.PgSequence import PgSequence class CreateSequenceParser(object): @staticmethod @@ -17,6 +17,9 @@ def parse(database, statement): schema.addSequence(sequence) + if parser.expect_optional("AS"): + parser.parse_identifier() + while not parser.expect_optional(";"): if parser.expect_optional("INCREMENT"): parser.expect_optional("BY") diff --git a/parser/CreateTableParser.py b/pgdiff/parser/CreateTableParser.py similarity index 83% rename from parser/CreateTableParser.py rename to pgdiff/parser/CreateTableParser.py index 376f4a8..452120d 100644 --- a/parser/CreateTableParser.py +++ b/pgdiff/parser/CreateTableParser.py @@ -1,7 +1,7 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgTable import PgTable -from schema.PgConstraint import PgConstraint -from schema.PgColumn import PgColumn +from .Parser import Parser, ParserUtils +from ..schema.PgTable import PgTable +from ..schema.PgConstraint import PgConstraint +from ..schema.PgColumn import PgColumn class CreateTableParser(object): @@ -13,8 +13,8 @@ def parse(database, statement): # Optional IF NOT EXISTS, irrelevant for our purposes parser.expect_optional("IF", "NOT", "EXISTS") - tableName = ParserUtils.get_object_name(parser.parse_identifier()) - table = PgTable(tableName) + tableName = parser.parse_identifier() + table = PgTable(ParserUtils.get_object_name(tableName)) # Figure it out why do we need this schemaName = ParserUtils.get_schema_name(tableName, database) schema = database.getSchema(schemaName) @@ -48,12 +48,16 @@ def parse(database, statement): elif parser.expect_optional("OIDS=false"): table.oids = "OIDS=false" else: - print 'table.setWith(parser.getExpression())' + print('table.setWith(parser.getExpression())') elif parser.expect_optional("TABLESPACE"): - print 'table.setTablespace(parser.parseString()' + print('table.setTablespace(parser.parseString()') + elif parser.expect_optional("PARTITION", "BY"): + parser.get_expression() else: parser.throw_unsupported_command() + + @staticmethod def parseConstraint(parser, table): constraint = PgConstraint(ParserUtils.get_object_name(parser.parse_identifier())); diff --git a/parser/CreateTriggerParser.py b/pgdiff/parser/CreateTriggerParser.py similarity index 97% rename from parser/CreateTriggerParser.py rename to pgdiff/parser/CreateTriggerParser.py index 44ed868..13338c1 100644 --- a/parser/CreateTriggerParser.py +++ b/pgdiff/parser/CreateTriggerParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgTrigger import PgTrigger +from .Parser import Parser, ParserUtils +from ..schema.PgTrigger import PgTrigger class CreateTriggerParser(object): @staticmethod diff --git a/parser/CreateViewParser.py b/pgdiff/parser/CreateViewParser.py similarity index 92% rename from parser/CreateViewParser.py rename to pgdiff/parser/CreateViewParser.py index 2efe01a..f0ed430 100644 --- a/parser/CreateViewParser.py +++ b/pgdiff/parser/CreateViewParser.py @@ -1,5 +1,5 @@ -from parser.Parser import Parser, ParserUtils -from schema.PgView import PgView +from .Parser import Parser, ParserUtils +from ..schema.PgView import PgView class CreateViewParser(object): @staticmethod diff --git a/parser/Parser.py b/pgdiff/parser/Parser.py similarity index 99% rename from parser/Parser.py rename to pgdiff/parser/Parser.py index 0bc2873..d7b059a 100644 --- a/parser/Parser.py +++ b/pgdiff/parser/Parser.py @@ -27,7 +27,7 @@ def expect_optional(self, *words): def parse_identifier(self): identifier = self._parse_identifier() - if self.statement[self.position] == '.': + while self.position < len(self.statement) and self.statement[self.position] == '.': self.position += 1 identifier += '.' + self._parse_identifier() diff --git a/parser/__init__.py b/pgdiff/parser/__init__.py similarity index 100% rename from parser/__init__.py rename to pgdiff/parser/__init__.py diff --git a/schema/PgColumn.py b/pgdiff/schema/PgColumn.py similarity index 88% rename from schema/PgColumn.py rename to pgdiff/schema/PgColumn.py index be1e4d8..c1018e4 100644 --- a/schema/PgColumn.py +++ b/pgdiff/schema/PgColumn.py @@ -1,11 +1,12 @@ import re -from diff.PgDiffUtils import PgDiffUtils +from ..diff.PgDiffUtils import PgDiffUtils class PgColumn(object): PATTERN_NULL = re.compile(r"^(.+)[\s]+NULL$", re.I) PATTERN_NOT_NULL = re.compile(r"^(.+)[\s]+NOT[\s]+NULL$", re.I) PATTERN_DEFAULT = re.compile(r"^(.+)[\s]+DEFAULT[\s]+(.+)$", re.I) + PATTERN_GENERATED = re.compile(r"^(.+)[\s]+GENERATED[\s]+ALWAYS[\s]+AS[\s]+(.+)[\s]+STORED$", re.I) def __init__(self, name): self.name = name @@ -14,6 +15,7 @@ def __init__(self, name): self.statistics = None self.storage = None self.comment = None + self.generated = None def parseDefinition(self, definition): string = definition @@ -36,6 +38,11 @@ def parseDefinition(self, definition): string = matcher.group(1).strip() self.defaultValue = matcher.group(2).strip() + matcher = self.PATTERN_GENERATED.match(string) + if matcher: + string = matcher.group(1).strip() + self.generated = matcher.group(2).strip() + self.type = string def getFullDefinition(self, addDefaults): diff --git a/schema/PgConstraint.py b/pgdiff/schema/PgConstraint.py similarity index 97% rename from schema/PgConstraint.py rename to pgdiff/schema/PgConstraint.py index 13da075..37cc6b5 100644 --- a/schema/PgConstraint.py +++ b/pgdiff/schema/PgConstraint.py @@ -1,5 +1,5 @@ import re -from diff.PgDiffUtils import PgDiffUtils +from ..diff.PgDiffUtils import PgDiffUtils class PgConstraint(object): PATTERN_PRIMARY_KEY = re.compile(r".*PRIMARY[\s]+KEY.*", re.I) diff --git a/schema/PgDatabase.py b/pgdiff/schema/PgDatabase.py similarity index 86% rename from schema/PgDatabase.py rename to pgdiff/schema/PgDatabase.py index 638043f..6d8d71a 100644 --- a/schema/PgDatabase.py +++ b/pgdiff/schema/PgDatabase.py @@ -1,5 +1,5 @@ import logging -from PgSchema import PgSchema +from .PgSchema import PgSchema class PgDatabase(object): @@ -20,4 +20,4 @@ def getDefaultSchema(self): # Returns schemaName object or Default schema if schemaName is None def getSchema(self, schema_name): - return self.schemas.get(schema_name, self.default_schema) \ No newline at end of file + return self.schemas.get(schema_name) \ No newline at end of file diff --git a/schema/PgFunction.py b/pgdiff/schema/PgFunction.py similarity index 96% rename from schema/PgFunction.py rename to pgdiff/schema/PgFunction.py index 0088d7f..a5ba247 100644 --- a/schema/PgFunction.py +++ b/pgdiff/schema/PgFunction.py @@ -1,7 +1,7 @@ import hashlib +import base64 -from diff.PgDiffUtils import PgDiffUtils -from helpers.OrderedDict import OrderedDict +from ..diff.PgDiffUtils import PgDiffUtils class PgFunction(object): def __init__(self): @@ -122,7 +122,7 @@ def getSignature(self): sbSQL.append(')') - return hashlib.md5(''.join(sbSQL)).digest().encode("base64") + return base64.b64encode(hashlib.md5(''.join(sbSQL).encode('utf-8')).digest()) class Argument(object): diff --git a/schema/PgIndex.py b/pgdiff/schema/PgIndex.py similarity index 96% rename from schema/PgIndex.py rename to pgdiff/schema/PgIndex.py index 204908b..03e9c20 100644 --- a/schema/PgIndex.py +++ b/pgdiff/schema/PgIndex.py @@ -1,4 +1,4 @@ -from diff.PgDiffUtils import PgDiffUtils +from ..diff.PgDiffUtils import PgDiffUtils class PgIndex(object): def __init__(self, name): diff --git a/schema/PgSchema.py b/pgdiff/schema/PgSchema.py similarity index 92% rename from schema/PgSchema.py rename to pgdiff/schema/PgSchema.py index 5aa27d4..526c709 100644 --- a/schema/PgSchema.py +++ b/pgdiff/schema/PgSchema.py @@ -1,9 +1,8 @@ -from helpers.OrderedDict import OrderedDict -from diff.PgDiffUtils import PgDiffUtils +from ..diff.PgDiffUtils import PgDiffUtils class PgSchema(object): def __init__(self, schemaName): - self.tables = OrderedDict() + self.tables = {} self.indexes = dict() self.functions = dict() self.sequences = dict() diff --git a/schema/PgSequence.py b/pgdiff/schema/PgSequence.py similarity index 97% rename from schema/PgSequence.py rename to pgdiff/schema/PgSequence.py index 5380e61..0c47810 100644 --- a/schema/PgSequence.py +++ b/pgdiff/schema/PgSequence.py @@ -1,4 +1,4 @@ -from diff.PgDiffUtils import PgDiffUtils +from ..diff.PgDiffUtils import PgDiffUtils class PgSequence(object): def __init__(self, name): diff --git a/schema/PgTable.py b/pgdiff/schema/PgTable.py similarity index 96% rename from schema/PgTable.py rename to pgdiff/schema/PgTable.py index 2ba0b55..f5c0d09 100644 --- a/schema/PgTable.py +++ b/pgdiff/schema/PgTable.py @@ -1,10 +1,9 @@ -from diff.PgDiffUtils import PgDiffUtils -from helpers.OrderedDict import OrderedDict +from ..diff.PgDiffUtils import PgDiffUtils class PgTable(object): def __init__(self, tableName): self.name=tableName - self.columns=OrderedDict() + self.columns={} self.indexes=dict() self.constraints=dict() self.triggers=dict() diff --git a/schema/PgTrigger.py b/pgdiff/schema/PgTrigger.py similarity index 98% rename from schema/PgTrigger.py rename to pgdiff/schema/PgTrigger.py index 4c148b7..79ef317 100644 --- a/schema/PgTrigger.py +++ b/pgdiff/schema/PgTrigger.py @@ -1,4 +1,4 @@ -from diff.PgDiffUtils import PgDiffUtils +from ..diff.PgDiffUtils import PgDiffUtils class PgTrigger(object): diff --git a/schema/PgView.py b/pgdiff/schema/PgView.py similarity index 66% rename from schema/PgView.py rename to pgdiff/schema/PgView.py index fd035aa..829d9ff 100644 --- a/schema/PgView.py +++ b/pgdiff/schema/PgView.py @@ -1,4 +1,4 @@ -from diff.PgDiffUtils import PgDiffUtils +from ..diff.PgDiffUtils import PgDiffUtils class PgView(object): def __init__(self, name): @@ -10,14 +10,24 @@ def __init__(self, name): self.columnComments = dict() self.query = None - def __cmp__(self, other): - if set(self.columnNames) != set(other.columnNames): - return 1 + def __eq__(self, other): + if other is None: + return False + + if self.columnNames is None and other.columnNames is not None: + return False + elif self.columnNames is not None and other.columnNames is None: + return False + elif self.columnNames is not None and other.columnNames is not None and set(self.columnNames) != set(other.columnNames): + return False if self.query.strip() != other.query.strip(): - return 1 + return False + + return True - return 0 + def __ne__(self, other): + return not self.__eq__(other) def getCreationSQL(self): sbSQL = [] @@ -41,13 +51,13 @@ def getCreationSQL(self): sbSQL.append(self.query) sbSQL.append(';') - for defaultValue in self.defaultValues: + for columnName, defaultValue in self.defaultValues.items(): sbSQL.append("\n\nALTER VIEW ") sbSQL.append(PgDiffUtils.getQuotedName(self.name)) sbSQL.append(" ALTER COLUMN ") - sbSQL.append(PgDiffUtils.getQuotedName(defaultValue.columnName)) + sbSQL.append(PgDiffUtils.getQuotedName(columnName)) sbSQL.append(" SET DEFAULT ") - sbSQL.append(defaultValue.defaultValue) + sbSQL.append(defaultValue) sbSQL.append(';') if self.comment: @@ -57,12 +67,14 @@ def getCreationSQL(self): sbSQL.append(self.comment) sbSQL.append(';') - for columnComment in self.columnComments: - if columnComment.comment: + for columnName, columnComment in self.columnComments.items(): + if columnComment: sbSQL.append("\n\nCOMMENT ON COLUMN ") - sbSQL.append(PgDiffUtils.getQuotedName(columnComment.columnName)) + sbSQL.append(PgDiffUtils.getQuotedName(self.name)) + sbSQL.append('.') + sbSQL.append(PgDiffUtils.getQuotedName(columnName)) sbSQL.append(" IS ") - sbSQL.append(columnComment.getComment()) + sbSQL.append(columnComment) sbSQL.append(';') return ''.join(sbSQL) diff --git a/schema/__init__.py b/pgdiff/schema/__init__.py similarity index 100% rename from schema/__init__.py rename to pgdiff/schema/__init__.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3531de1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "PgDiffPy" +version = "1.0.0" +authors = [ + { name="Prateek", email="for.groups+GITHUB@gmail.com" }, +] +description = "A Python-based tool for comparing PostgreSQL database schemas." +readme = "README.md" +requires-python = ">=3.6" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "sqlparse>=0.1.8", +] + +[project.urls] +"Homepage" = "https://github.com/Dancer3809/PgDiffPy" + +[project.scripts] +pgdiff = "pgdiff.__main__:main" + +[tool.setuptools.packages.find] +where = ["."] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5e9e573..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sqlparse>=0.1.8 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..fc1f76c --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup + +setup() \ No newline at end of file