Refactor tests (#251)

* added test util file

* refactored into a separate testing class

* renamed check to assert

* simplified some code

* updated test_sparse

* renamed utils to testing_utils

* moved optimize_with_hotstart and check_hist_file to test class

* updated tp109

* changed sol.fStar to a dictionary format

* add support for multiple solutions in asserts

* updated hs015

* generalized testing functions

* changed obj back to scalar instead of dict

* updated base class API

* only skip tests if non built-in optimizer

* updated isort

* Update .flake8

* added isort.cfg

* moved test_large_snopt to test_large_sparse

* switched to parameterized tests

* made sparse tests smaller

* added version check for SNOPT < 7.7.7

* updated NSGA2 test

* stopped storing opt in testing class, was causing obscure pickle error

* isort

* removed unnecessary code from ALPSO wrapper

* switched from packaging to pkg_resources

* switched to using self.id() to access the test name

* removed duplicate code

* added some comments

Author: ewu63

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-exclude =
+extend-exclude =
     # OptView and related files need to be fixed eventually
     pyoptsparse/postprocessing/OptView.py
     pyoptsparse/postprocessing/OptView_baseclass.py

.isort.cfg

@@ -0,0 +1,10 @@
+[isort]
+profile=black
+line_length=120
+force_sort_within_sections=true
+import_heading_stdlib=Standard Python modules
+import_heading_thirdparty=External modules
+import_heading_firstparty=First party modules
+import_heading_localfolder=Local modules
+skip_glob=**__init__.py,**setup.py
+known_local_folder=testing_utils

pyoptsparse/pyALPSO/pyALPSO.py

@@ -181,8 +181,6 @@ def objconfunc(x):
 
             # Create the optimization solution
             sol = self._createSolution(optTime, sol_inform, opt_f, opt_x)
-            for key in sol.objectives.keys():
-                sol.objectives[key].value = opt_f
         else:  # We are not on the root process so go into waiting loop:
             self._waitLoop()
             sol = None

test/test_hs015.py

@@ -5,14 +5,16 @@
 
 # External modules
 import numpy as np
-from numpy.testing import assert_allclose
+from parameterized import parameterized
 
 # First party modules
-from pyoptsparse import OPT, History, Optimization
-from pyoptsparse.pyOpt_error import Error
+from pyoptsparse import History, Optimization
 
+# Local modules
+from testing_utils import OptTest
 
-class TestHS15(unittest.TestCase):
+
+class TestHS15(OptTest):
 
     ## Solve test problem HS15 from the Hock & Schittkowski collection.
     #
@@ -27,6 +29,29 @@ class TestHS15(unittest.TestCase):
     #  at (-0.79212, -1.26243), with final objective = 360.4.
     ##
 
+    name = "HS015"
+    DVs = {"xvars"}
+    cons = {"con"}
+    objs = {"obj"}
+    extras = {"extra1", "extra2"}
+    fStar = [
+        306.5,
+        360.379767,
+    ]
+    xStar = [
+        {"xvars": (0.5, 2.0)},
+        {"xvars": (-0.79212322, -1.26242985)},
+    ]
+    tol = {
+        "SLSQP": 1e-8,
+        "NLPQLP": 1e-12,
+        "IPOPT": 1e-4,
+        "ParOpt": 1e-6,
+        "CONMIN": 1e-10,
+        "PSQP": 5e-12,
+    }
+    optOptions = {}
+
     def objfunc(self, xdict):
         self.nf += 1
         x = xdict["xvars"]
@@ -53,190 +78,30 @@ def sens(self, xdict, funcs):
         fail = False
         return funcsSens, fail
 
-    def optimize(self, optName, tol, optOptions={}, storeHistory=False, hotStart=None, x0=[-2, 1.0]):
-        self.nf = 0  # number of function evaluations
-        self.ng = 0  # number of gradient evaluations
+    def setup_optProb(self):
         # Optimization Object
-        optProb = Optimization("HS15 Constraint Problem", self.objfunc)
+        self.optProb = Optimization("HS15 Constraint Problem", self.objfunc)
 
         # Design Variables
         lower = [-5.0, -5.0]
         upper = [0.5, 5.0]
-        optProb.addVarGroup("xvars", 2, lower=lower, upper=upper, value=x0)
+        value = [-2, 1.0]
+        self.optProb.addVarGroup("xvars", 2, lower=lower, upper=upper, value=value)
 
         # Constraints
         lower = [1.0, 0.0]
         upper = [None, None]
-        optProb.addConGroup("con", 2, lower=lower, upper=upper)
+        self.optProb.addConGroup("con", 2, lower=lower, upper=upper)
 
         # Objective
-        optProb.addObj("obj")
-
-        # Check optimization problem:
-        print(optProb)
-
-        # Optimizer
-        try:
-            opt = OPT(optName, options=optOptions)
-        except Error:
-            raise unittest.SkipTest("Optimizer not available:", optName)
-
-        # Solution
-        if storeHistory is not None:
-            if storeHistory is True:
-                self.histFileName = "%s_hs015_Hist.hst" % (optName.lower())
-            elif isinstance(storeHistory, str):
-                self.histFileName = storeHistory
-        else:
-            self.histFileName = None
-
-        sol = opt(optProb, sens=self.sens, storeHistory=self.histFileName, hotStart=hotStart)
-
-        # Test printing solution to screen
-        print(sol)
-
-        # Check Solution
-        self.fStar1 = 306.5
-        self.fStar2 = 360.379767
-
-        self.xStar1 = (0.5, 2.0)
-        self.xStar2 = (-0.79212322, -1.26242985)
-
-        dv = sol.getDVs()
-        sol_xvars = [sol.variables["xvars"][i].value for i in range(2)]
-        assert_allclose(sol_xvars, dv["xvars"], atol=tol, rtol=tol)
-        # we check either optimum via try/except
-        try:
-            if optName == "SNOPT" and opt.version != "7.7.7":
-                assert_allclose(sol.objectives["obj"].value, self.fStar1, atol=tol, rtol=tol)
-            else:
-                assert_allclose(sol.fStar, self.fStar1, atol=tol, rtol=tol)
-            assert_allclose(dv["xvars"], self.xStar1, atol=tol, rtol=tol)
-        except AssertionError:
-            if optName == "SNOPT" and opt.version != "7.7.7":
-                assert_allclose(sol.objectives["obj"].value, self.fStar2, atol=tol, rtol=tol)
-            else:
-                assert_allclose(sol.fStar, self.fStar2, atol=tol, rtol=tol)
-            assert_allclose(dv["xvars"], self.xStar2, atol=tol, rtol=tol)
-
-    def check_hist_file(self, optimizer, tol):
-        """
-        We check the history file here along with the API
-        """
-        hist = History(self.histFileName, flag="r")
-        # Metadata checks
-        metadata = hist.getMetadata()
-        self.assertEqual(metadata["optimizer"], optimizer)
-        metadata_def_keys = [
-            "optName",
-            "optOptions",
-            "nprocs",
-            "startTime",
-            "endTime",
-            "optTime",
-            "version",
-            "optVersion",
-        ]
-        for key in metadata_def_keys:
-            self.assertIn(key, metadata)
-            # we test that SNOPT version is stored correctly
-            if optimizer == "SNOPT" and key == "optVersion":
-                self.assertNotEqual(metadata[key], None)
-        hist.getOptProb()
-
-        # Info checks
-        self.assertEqual(hist.getDVNames(), ["xvars"])
-        self.assertEqual(hist.getConNames(), ["con"])
-        self.assertEqual(hist.getObjNames(), ["obj"])
-        dvInfo = hist.getDVInfo()
-        self.assertEqual(len(dvInfo), 1)
-        self.assertEqual(dvInfo["xvars"], hist.getDVInfo(key="xvars"))
-        conInfo = hist.getConInfo()
-        self.assertEqual(len(conInfo), 1)
-        self.assertEqual(conInfo["con"], hist.getConInfo(key="con"))
-        objInfo = hist.getObjInfo()
-        self.assertEqual(len(objInfo), 1)
-        self.assertEqual(objInfo["obj"], hist.getObjInfo(key="obj"))
-        for key in ["lower", "upper", "scale"]:
-            self.assertIn(key, dvInfo["xvars"])
-            self.assertIn(key, conInfo["con"])
-        self.assertIn("scale", objInfo["obj"])
-
-        # callCounter checks
-        callCounters = hist.getCallCounters()
-        last = hist.read("last")  # 'last' key should be present
-        self.assertIn(last, callCounters)
-
-        # iterKeys checks
-        iterKeys = hist.getIterKeys()
-        for key in ["xuser", "fail", "isMajor"]:
-            self.assertIn(key, iterKeys)
-
-        # extraFuncsNames checks
-        extraFuncsNames = hist.getExtraFuncsNames()
-        for key in ["extra1", "extra2"]:
-            self.assertIn(key, extraFuncsNames)
-
-        # getValues checks
-        val = hist.getValues()
-
-        # this check is only used for optimizers that guarantee '0' and 'last' contain funcs
-        if optimizer in ["SNOPT", "PSQP"]:
-            val = hist.getValues(callCounters=["0", "last"], stack=True)
-            self.assertEqual(val["isMajor"].size, 2)
-            self.assertTrue(val["isMajor"][0])  # the first callCounter must be a major iteration
-            self.assertTrue(val["isMajor"][-1])  # the last callCounter must be a major iteration
-            # check optimum stored in history file against xstar
-            assert_allclose(val["xuser"][-1], self.xStar1, atol=tol, rtol=tol)
-
-    def optimize_with_hotstart(self, optName, tol, optOptions={}):
-        """
-        This code will perform 4 optimizations, one real opt and three restarts.
-        In this process, it will check various combinations of storeHistory and hotStart filenames.
-        It will also call `check_hist_file` after the first optimization.
-        """
-        self.optimize(optName, tol, storeHistory=True, optOptions=optOptions, x0=[-2.001, 1.001])
-        self.assertGreater(self.nf, 0)
-        self.assertGreater(self.ng, 0)
-        self.check_hist_file(optName, tol)
-
-        # re-optimize with hotstart
-        self.optimize(optName, tol, storeHistory=False, hotStart=self.histFileName, optOptions=optOptions)
-        # we should have zero actual function/gradient evaluations
-        self.assertEqual(self.nf, 0)
-        self.assertEqual(self.ng, 0)
-        # another test with hotstart, this time with storeHistory = hotStart
-        self.optimize(optName, tol, storeHistory=True, hotStart=self.histFileName, optOptions=optOptions)
-        # we should have zero actual function/gradient evaluations
-        self.assertEqual(self.nf, 0)
-        self.assertEqual(self.ng, 0)
-        # another test with hotstart, this time with a non-existing history file
-        # this will perform a cold start
-        self.optimize(optName, tol, storeHistory=True, hotStart="notexisting.hst", optOptions=optOptions)
-        self.assertGreater(self.nf, 0)
-        self.assertGreater(self.ng, 0)
-        self.check_hist_file(optName, tol)
-        # final test with hotstart, this time with a different storeHistory
-        self.optimize(
-            optName,
-            tol,
-            storeHistory="{}_new_hotstart.hst".format(optName),
-            hotStart=self.histFileName,
-            optOptions=optOptions,
-        )
-        # we should have zero actual function/gradient evaluations
-        self.assertEqual(self.nf, 0)
-        self.assertEqual(self.ng, 0)
+        self.optProb.addObj("obj")
 
     def test_snopt(self):
-        test_name = "hs015_SNOPT"
+        self.optName = "SNOPT"
+        self.setup_optProb()
         store_vars = ["step", "merit", "feasibility", "optimality", "penalty", "Hessian", "condZHZ", "slack", "lambda"]
-        optOptions = {
-            "Save major iteration variables": store_vars,
-            "Print file": "{}.out".format(test_name),
-            "Summary file": "{}_summary.out".format(test_name),
-        }
-        self.optimize_with_hotstart("SNOPT", 1e-12, optOptions=optOptions)
+        optOptions = {"Save major iteration variables": store_vars}
+        self.optimize_with_hotstart(1e-12, optOptions=optOptions)
 
         hist = History(self.histFileName, flag="r")
         data = hist.getValues(callCounters=["last"])
@@ -249,34 +114,20 @@ def test_snopt(self):
         self.assertEqual(data["feasibility"].shape, (1, 1))
         self.assertEqual(data["slack"].shape, (1, 2))
         self.assertEqual(data["lambda"].shape, (1, 2))
-
-    def test_slsqp(self):
-        optOptions = {"IFILE": "hs015_SLSQP.out"}
-        self.optimize_with_hotstart("SLSQP", 1e-8, optOptions=optOptions)
-
-    def test_nlpqlp(self):
-        optOptions = {"iFile": "hs015_NLPQLP.out"}
-        self.optimize_with_hotstart("NLPQLP", 1e-12, optOptions=optOptions)
-
-    def test_ipopt(self):
-        optOptions = {"output_file": "hs015_IPOPT.out"}
-        self.optimize_with_hotstart("IPOPT", 1e-4, optOptions=optOptions)
-
-    def test_paropt(self):
-        optOptions = {"output_file": "hs015_ParOpt.out"}
-        self.optimize_with_hotstart("ParOpt", 1e-6, optOptions=optOptions)
-
-    def test_conmin(self):
-        optOptions = {
-            "DELFUN": 1e-10,
-            "DABFUN": 1e-10,
-            "IFILE": "hs015_CONMIN.out",
-        }
-        self.optimize_with_hotstart("CONMIN", 1e-10, optOptions=optOptions)
-
-    def test_psqp(self):
-        optOptions = {"IFILE": "hs015_PSQP.out"}
-        self.optimize_with_hotstart("PSQP", 5e-12, optOptions=optOptions)
+        # dv = sol.getDVs()
+        # sol_xvars = [sol.variables["xvars"][i].value for i in range(2)]
+        # assert_allclose(sol_xvars, dv["xvars"], atol=tol, rtol=tol)
+
+    @parameterized.expand(["IPOPT", "SLSQP", "PSQP", "CONMIN", "NLPQLP", "ParOpt"])
+    def test_optimization(self, optName):
+        self.optName = optName
+        self.setup_optProb()
+        optOptions = self.optOptions.pop(optName, None)
+        sol = self.optimize(optOptions=optOptions)
+        # Check Solution
+        self.assert_solution_allclose(sol, self.tol[optName])
+        # Check informs
+        self.assert_inform_equal(sol)
 
 
 if __name__ == "__main__":