diff --git a/CHANGELOG.md b/CHANGELOG.md index b1b886b..cd9e8fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The file is formatted as described on http://keepachangelog.com/. ## [Unreleased] +## [2.0.4] - 2020-01-14 + +### Fixed + +* Support KNIME 4.3 ([#9](https://github.com/3D-e-Chem/knime-python-wrapper/pull/9) + [#10](https://github.com/3D-e-Chem/knime-python-wrapper/pull/10)) + +### Removed + +* Support for KNIME versions older than 4.3 + ## [2.0.3] - 2019-07-02 ### Changed @@ -53,3 +63,11 @@ The file is formatted as described on http://keepachangelog.com/. * Abstract PythonWrapperNode classes * Test utility to run tests which call PythonKernel execute + +[Unreleased]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.4...HEAD +[2.0.4]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.3...v2.0.4 +[2.0.3]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.2...v2.0.3 +[2.0.2]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v2.0.1...v2.0.2 +[2.0.1]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v1.1.0...v2.0.1 +[1.1.0]: https://github.com/3D-e-Chem/knime-python-wrapper/compare/v1.0.0...v1.1.0 +[1.0.0]: https://github.com/3D-e-Chem/knime-python-wrapper/releases/tag/v1.0.0 diff --git a/p2/category.xml b/p2/category.xml index eaf6e08..2d3c64f 100644 --- a/p2/category.xml +++ b/p2/category.xml @@ -1,6 +1,6 @@ - + diff --git a/p2/pom.xml b/p2/pom.xml index 62341a4..f76d690 100644 --- a/p2/pom.xml +++ b/p2/pom.xml @@ -4,7 +4,7 @@ nl.esciencecenter.e3dchem.python nl.esciencecenter.e3dchem.python - 2.0.3-SNAPSHOT + 2.0.4-SNAPSHOT nl.esciencecenter.e3dchem.python.p2 eclipse-repository diff --git a/plugin/META-INF/MANIFEST.MF b/plugin/META-INF/MANIFEST.MF index 42b9877..0b5f572 100644 --- a/plugin/META-INF/MANIFEST.MF +++ b/plugin/META-INF/MANIFEST.MF @@ -2,15 +2,15 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Abstract Python wrapper KNIME node and helpers Bundle-SymbolicName: nl.esciencecenter.e3dchem.python.plugin;singleton:=true -Bundle-Version: 2.0.3.qualifier +Bundle-Version: 2.0.4.qualifier Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Bundle-Vendor: Netherlands eScience Center -Require-Bundle: org.knime.core;bundle-version="[4.0.0,5.0.0)", - org.knime.base;bundle-version="[4.0.0,5.0.0)", - org.knime.python2;bundle-version="[4.0.0,5.0.0)", - org.knime.python2.serde.csv;bundle-version="[4.0.0,5.0.0)", - org.knime.python2.serde.flatbuffers;bundle-version="[4.0.0,5.0.0)", - org.knime.python2.serde.arrow;bundle-version="[4.0.0,5.0.0)", - org.knime.python2.serde.arrow.libs;bundle-version="[4.0.0,5.0.0)" +Require-Bundle: org.knime.core;bundle-version="[4.3.0,5.0.0)", + org.knime.base;bundle-version="[4.3.0,5.0.0)", + org.knime.python2;bundle-version="[4.3.0,5.0.0)", + org.knime.python2.envconfigs;bundle-version="[4.3.0,5.0.0)", + org.knime.python2.serde.csv;bundle-version="[4.3.0,5.0.0)", + org.knime.python2.serde.flatbuffers;bundle-version="[4.3.0,5.0.0)", + org.knime.python2.serde.arrow;bundle-version="[4.3.0,5.0.0)" Bundle-ClassPath: . Export-Package: nl.esciencecenter.e3dchem.python diff --git a/plugin/pom.xml b/plugin/pom.xml index 029a033..5f2856b 100644 --- a/plugin/pom.xml +++ b/plugin/pom.xml @@ -4,7 +4,7 @@ nl.esciencecenter.e3dchem.python nl.esciencecenter.e3dchem.python - 2.0.3-SNAPSHOT + 2.0.4-SNAPSHOT nl.esciencecenter.e3dchem.python.plugin eclipse-plugin diff --git a/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeConfig.java b/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeConfig.java index 54da9ba..456c71a 100644 --- a/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeConfig.java +++ b/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeConfig.java @@ -1,26 +1,22 @@ package nl.esciencecenter.e3dchem.python; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.Locale; -import java.util.Objects; import java.util.Set; -import org.knime.python2.DefaultPythonCommand; +import org.knime.core.node.InvalidSettingsException; +import org.knime.core.node.NodeSettingsRO; +import org.knime.core.node.NodeSettingsWO; +import org.knime.core.node.workflow.FlowVariable; import org.knime.python2.PythonCommand; import org.knime.python2.PythonModuleSpec; import org.knime.python2.PythonVersion; +import org.knime.python2.config.PythonCommandFlowVariableConfig; import org.knime.python2.extensions.serializationlibrary.SentinelOption; import org.knime.python2.extensions.serializationlibrary.SerializationOptions; import org.knime.python2.generic.VariableNames; import org.knime.python2.kernel.PythonKernelOptions; import org.knime.python2.prefs.PythonPreferences; -import org.knime.core.node.InvalidSettingsException; -import org.knime.core.node.NodeSettingsRO; -import org.knime.core.node.NodeSettingsWO; -import org.knime.core.node.workflow.FlowVariable; /** * Configuration for {@link PythonWrapperNodeModel}. @@ -31,32 +27,22 @@ */ public class PythonWrapperNodeConfig { private static final String CFG_PYTHON_VERSION_OPTION = "pythonVersionOption"; - static final String CFG_PYTHON2COMMAND = "python2Command"; - static final String CFG_PYTHON3COMMAND = "python3Command"; + private static final String CFG_PYTHON2_COMMAND = "python2Command"; + + private static final String CFG_PYTHON3_COMMAND = "python3Command"; private static final String CFG_CONVERT_MISSING_TO_PYTHON = "convertMissingToPython"; private static final String CFG_CONVERT_MISSING_FROM_PYTHON = "convertMissingFromPython"; private static final String CFG_SENTINEL_OPTION = "sentinelOption"; private static final String CFG_SENTINEL_VALUE = "sentinelValue"; private static final String CFG_CHUNK_SIZE = "chunkSize"; - /** - * {@link #m_python2Command} and {@link #m_python3Command} are special in that they currently aren't configurable - * via a Python scripting node's dialog but only using flow variables. If no respective flow variables are set, - * their value is retrieved from the Python preferences. - */ - private static final String INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE = ""; - private PythonVersion m_pythonVersion = PythonPreferences.getPythonVersionPreference(); - /** - * {@code null} means to fall back to {@link PythonPreferences#getPython2CommandPreference()}. - */ - private PythonCommand m_python2Command = null; + private PythonCommandFlowVariableConfig m_python2CommandConfig = new PythonCommandFlowVariableConfig(CFG_PYTHON2_COMMAND, + PythonVersion.PYTHON2, PythonPreferences::getCondaInstallationPath); - /** - * {@code null} means to fall back to {@link PythonPreferences#getPython2CommandPreference()}. - */ - private PythonCommand m_python3Command = null; + private PythonCommandFlowVariableConfig m_python3CommandConfig = new PythonCommandFlowVariableConfig(CFG_PYTHON3_COMMAND, + PythonVersion.PYTHON3, PythonPreferences::getCondaInstallationPath); private int m_chunkSize = SerializationOptions.DEFAULT_CHUNK_SIZE; @@ -118,13 +104,20 @@ public void setPythonVersion(final PythonVersion pythonVersion) { m_pythonVersion = pythonVersion; } + /** + * @return The config of the Python 2 command. + */ + public PythonCommandFlowVariableConfig getPython2CommandConfig() { + return m_python2CommandConfig; + } + /** * @return The Python 2 command to use. May be {@code null} in which case no specific Python 2 command is configured * and one has to resort to - e.g., - the {@link PythonPreferences#getPython2CommandPreference() global * preferences}. */ public PythonCommand getPython2Command() { - return m_python2Command; + return m_python2CommandConfig.getCommand().orElse(null); } /** @@ -133,7 +126,14 @@ public PythonCommand getPython2Command() { * {@link PythonPreferences#getPython2CommandPreference() global preferences}. */ public void setPython2Command(final PythonCommand python2Command) { - m_python2Command = python2Command; + m_python2CommandConfig.setCommand(python2Command); + } + + /** + * @return The config of the Python 3 command. + */ + public PythonCommandFlowVariableConfig getPython3CommandConfig() { + return m_python3CommandConfig; } /** @@ -141,8 +141,8 @@ public void setPython2Command(final PythonCommand python2Command) { * and one has to resort to - e.g., - the {@link PythonPreferences#getPython3CommandPreference() global * preferences}. */ - public PythonCommand getPython3Command() { - return m_python3Command; + public PythonCommand getPython3Command() { + return m_python3CommandConfig.getCommand().orElse(null); } /** @@ -151,9 +151,9 @@ public PythonCommand getPython3Command() { * {@link PythonPreferences#getPython3CommandPreference() global preferences}. */ public void setPython3Command(final PythonCommand python3Command) { - m_python3Command = python3Command; + m_python3CommandConfig.setCommand(python3Command); } - + /** * * @return The configured number of rows to transfer to/from Python per chunk of an input/output table. @@ -273,13 +273,13 @@ public void saveTo(final NodeSettingsWO settings) { */ public void saveToInDialog(NodeSettingsWO settings) { settings.addString(CFG_PYTHON_VERSION_OPTION, getPythonVersion().getId()); - settings.addString(CFG_PYTHON2COMMAND, commandToString(getPython2Command())); - settings.addString(CFG_PYTHON3COMMAND, commandToString(getPython3Command())); settings.addInt(CFG_CHUNK_SIZE, getChunkSize()); settings.addBoolean(CFG_CONVERT_MISSING_TO_PYTHON, getConvertMissingToPython()); settings.addBoolean(CFG_CONVERT_MISSING_FROM_PYTHON, getConvertMissingFromPython()); settings.addString(CFG_SENTINEL_OPTION, getSentinelOption().name()); settings.addInt(CFG_SENTINEL_VALUE, getSentinelValue()); + m_python2CommandConfig.saveSettingsTo(settings); + m_python3CommandConfig.saveSettingsTo(settings); } /** @@ -300,39 +300,22 @@ public void loadFrom(final NodeSettingsRO settings) throws InvalidSettingsExcept * * @param settings * The settings to load from + * @throws InvalidSettingsException */ - public void loadFromInDialog(final NodeSettingsRO settings) { + public void loadFromInDialog(final NodeSettingsRO settings) throws InvalidSettingsException { final String pythonVersionString = settings.getString(CFG_PYTHON_VERSION_OPTION, getPythonVersion().getId()); // Backward compatibility: old saved versions may be all upper case. setPythonVersion(PythonVersion.fromId(pythonVersionString.toLowerCase(Locale.ROOT))); - final String python2CommandString = - settings.getString(CFG_PYTHON2COMMAND, commandToString(getPython2Command())); - setPython2Command(commandFromString(python2CommandString)); - final String python3CommandString = - settings.getString(CFG_PYTHON3COMMAND, commandToString(getPython3Command())); - setPython3Command(commandFromString(python3CommandString)); setChunkSize(settings.getInt(CFG_CHUNK_SIZE, getChunkSize())); setConvertMissingToPython(settings.getBoolean(CFG_CONVERT_MISSING_TO_PYTHON, getConvertMissingToPython())); setConvertMissingFromPython( settings.getBoolean(CFG_CONVERT_MISSING_FROM_PYTHON, getConvertMissingFromPython())); setSentinelOption(SentinelOption.valueOf(settings.getString(CFG_SENTINEL_OPTION, getSentinelOption().name()))); setSentinelValue(settings.getInt(CFG_SENTINEL_VALUE, getSentinelValue())); + m_python2CommandConfig.loadSettingsFrom(settings); + m_python3CommandConfig.loadSettingsFrom(settings); } - private static String commandToString(final PythonCommand command) { - return command != null // - ? command.toString() // - : INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE; - } - - private static PythonCommand commandFromString(final String commandString) { - return Objects.equals(commandString, INDICATE_FALLBACK_TO_PREFERENCES_COMMAND_VALUE) // - ? null // - // TODO: This only works for ordinary paths ("manual configuration"), not for Conda directory + environment - // name ("Conda configuration"). - : new DefaultPythonCommand(commandString); - } - /** * Set of key/value pairs which inside Python script will be a dictionary * named by {@link #getOptionsName()}. @@ -352,9 +335,8 @@ public Set getOptionsValues() { public PythonKernelOptions getKernelOptions() { final SerializationOptions serializationOptions = new SerializationOptions(getChunkSize(), getConvertMissingToPython(), getConvertMissingFromPython(), getSentinelOption(), getSentinelValue()); - PythonKernelOptions opt = new PythonKernelOptions(getPythonVersion(), getPython2Command(), getPython3Command(), - serializationOptions); - return opt.forAddedAdditionalRequiredModules(additionalRequiredModules).forExternalCustomPath(externalCustomPath); + PythonKernelOptions opt = new PythonKernelOptions(); + return opt.forSerializationOptions(serializationOptions).forAddedAdditionalRequiredModules(additionalRequiredModules).forExternalCustomPath(externalCustomPath); } /** diff --git a/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeModel.java b/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeModel.java index a256532..190ec4d 100644 --- a/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeModel.java +++ b/plugin/src/java/nl/esciencecenter/e3dchem/python/PythonWrapperNodeModel.java @@ -7,8 +7,10 @@ import java.io.InputStreamReader; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.LinkedList; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import org.knime.base.node.util.exttool.ExtToolOutputNodeModel; @@ -22,14 +24,21 @@ import org.knime.core.node.port.PortType; import org.knime.core.node.workflow.FlowVariable; import org.knime.core.node.workflow.FlowVariable.Type; +import org.knime.python2.PythonCommand; +import org.knime.python2.PythonModuleSpec; +import org.knime.python2.config.PythonFlowVariableOptions; +import org.knime.python2.kernel.PythonCancelable; +import org.knime.python2.kernel.PythonCanceledExecutionException; import org.knime.python2.kernel.PythonExecutionMonitorCancelable; +import org.knime.python2.kernel.PythonIOException; import org.knime.python2.kernel.PythonKernel; +import org.knime.python2.kernel.PythonKernelOptions; +import org.knime.python2.kernel.PythonKernelQueue; /** * Implements a {@link NodeModel} for nodes that launch external Python script. * - * @param - * Configuration + * @param Configuration */ public abstract class PythonWrapperNodeModel extends ExtToolOutputNodeModel { protected C m_config = createConfig(); @@ -48,14 +57,52 @@ protected final C getConfig() { return m_config; } + /** + * Gets the kernel specific options. + * + * @return the kernel specific options + */ + protected PythonKernelOptions getKernelOptions() { + final PythonKernelOptions options = getConfig().getKernelOptions(); + final String serializerId = new PythonFlowVariableOptions(getAvailableFlowVariables()).getSerializerId() + .orElse(null); + return options.forSerializationOptions(options.getSerializationOptions().forSerializerId(serializerId)); + } + + // Below has been copied from KNIME Python node code (https://github.com/knime/knime-python/blob/analytics-platform/4.3.0/org.knime.python2.nodes/src/org/knime/python2/nodes/PythonNodeModel.java) and + // adjusted. + protected PythonKernel getNextKernelFromQueue(final PythonCancelable cancelable) + throws PythonCanceledExecutionException, PythonIOException { + return getNextKernelFromQueue(Collections.emptySet(), Collections.emptySet(), cancelable); + } + + protected PythonKernel getNextKernelFromQueue(final Set requiredAdditionalModules, + final PythonCancelable cancelable) throws PythonCanceledExecutionException, PythonIOException { + return getNextKernelFromQueue(requiredAdditionalModules, Collections.emptySet(), cancelable); + } + + protected PythonKernel getNextKernelFromQueue(final Set requiredAdditionalModules, + final Set optionalAdditionalModules, final PythonCancelable cancelable) + throws PythonCanceledExecutionException, PythonIOException { + final PythonKernelOptions options = getKernelOptions(); + final PythonCommand command = options.getUsePython3() // + ? options.getPython3Command() // + : options.getPython2Command(); + return PythonKernelQueue.getNextKernel(command, requiredAdditionalModules, optionalAdditionalModules, options, + cancelable); + } + public BufferedDataTable[] execute(BufferedDataTable[] inData, ExecutionContext exec) throws Exception { - // Below has been copied from Knime Python node source code and + // Below has been copied from KNIME Python node source code (https://github.com/knime/knime-python/blob/analytics-platform/4.3.0/org.knime.python2.nodes/src/org/knime/python2/nodes/source/PythonSourceNodeModel.java) and // adjusted. - PythonKernel kernel = new PythonKernel(getConfig().getKernelOptions()); - try { - return executeKernel(inData, exec, kernel); - } finally { - kernel.close(); + final PythonExecutionMonitorCancelable cancelable = new PythonExecutionMonitorCancelable(exec); + try (final PythonKernel kernel = getNextKernelFromQueue(cancelable)) { + kernel.setOptions(getConfig().getKernelOptions()); + try { + return executeKernel(inData, exec, kernel); + } finally { + kernel.close(); + } } } @@ -96,11 +143,10 @@ public BufferedDataTable[] executeKernel(BufferedDataTable[] inData, ExecutionCo /** * Push new variables to the stack. * - * Only pushes new variables to the stack if they are new or changed in type - * or value. + * Only pushes new variables to the stack if they are new or changed in type or + * value. * - * @param newVariables - * The flow variables to push + * @param newVariables The flow variables to push */ protected void addNewVariables(Collection newVariables) { // Below has been copied from Knime Python node source code and diff --git a/pom.xml b/pom.xml index 583c31b..71a5adc 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 nl.esciencecenter.e3dchem.python nl.esciencecenter.e3dchem.python - 2.0.3-SNAPSHOT + 2.0.4-SNAPSHOT pom diff --git a/targetplatform/KNIME-AP-4.0.target b/targetplatform/KNIME-AP-4.0.target index d605f6f..91fe31d 100644 --- a/targetplatform/KNIME-AP-4.0.target +++ b/targetplatform/KNIME-AP-4.0.target @@ -2,6 +2,7 @@ + @@ -11,11 +12,11 @@ - + - + @@ -24,7 +25,7 @@ -Dsun.java2d.d3d=false -Dosgi.classloader.lock=classname -XX:+UnlockDiagnosticVMOptions --XX:+UnsyncloadClass +-XX:+UnsyncloadClass -XX:+UseG1GC -ea -Dorg.eclipse.swt.internal.gtk.cairoGraphics=false diff --git a/targetplatform/pom.xml b/targetplatform/pom.xml index 56e7adb..a6a5335 100644 --- a/targetplatform/pom.xml +++ b/targetplatform/pom.xml @@ -20,7 +20,7 @@ Contributors: nl.esciencecenter.e3dchem.python nl.esciencecenter.e3dchem.python - 2.0.3-SNAPSHOT + 2.0.4-SNAPSHOT diff --git a/tests/META-INF/MANIFEST.MF b/tests/META-INF/MANIFEST.MF index 6729d21..f9a5b68 100644 --- a/tests/META-INF/MANIFEST.MF +++ b/tests/META-INF/MANIFEST.MF @@ -2,13 +2,13 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Tests for Abstract Python wrapper KNIME node and helpers Bundle-SymbolicName: nl.esciencecenter.e3dchem.python.tests -Bundle-Version: 2.0.3.qualifier +Bundle-Version: 2.0.4.qualifier Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Require-Bundle: org.junit;bundle-version="4.12.0", - org.knime.core;bundle-version="[4.0.0,5.0.0)", - org.knime.base;bundle-version="[4.0.0,5.0.0)", - org.knime.testing;bundle-version="[4.0.0,5.0.0)", - org.knime.python2;bundle-version="[4.0.0,5.0.0)", +Require-Bundle: org.junit;bundle-version="[4.12.0,5.0.0)", + org.knime.core;bundle-version="[4.3.0,5.0.0)", + org.knime.base;bundle-version="[4.3.0,5.0.0)", + org.knime.testing;bundle-version="[4.3.0,5.0.0)", + org.knime.python2;bundle-version="[4.3.0,5.0.0)", nl.esciencecenter.e3dchem.python.plugin Bundle-Vendor: Netherlands eScience Center Bundle-ClassPath: ., diff --git a/tests/pom.xml b/tests/pom.xml index 0b0e022..d4ea227 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -5,7 +5,7 @@ nl.esciencecenter.e3dchem.python nl.esciencecenter.e3dchem.python - 2.0.3-SNAPSHOT + 2.0.4-SNAPSHOT nl.esciencecenter.e3dchem.python.tests @@ -111,6 +111,22 @@ ${tycho.testArgLine} ${tycho.test.jvmArgs} + + org.eclipse.tycho + target-platform-configuration + ${tycho.version} + + + + + eclipse-feature + org.knime.features.core + 0.0.0 + + + + +