[BEAM-12907] Run DataFrame API tests with multiple pandas versions (#15587)

* Run DataFrame API tests with multiple pandas versions

* create tasks in a loop, make sure they're run in precommit

* Don't define tasks in a loop

Author: TheNeuralBit

sdks/python/test-suites/tox/py38/build.gradle

@@ -34,12 +34,43 @@ apply from: "../common.gradle"
 // TODO(BEAM-8954): Remove this once tox uses isolated builds.
 testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage
 
-(0..5).each {version ->
-  // Create a test task for each major version of pyarrow
-  toxTask "testPy38pyarrow-$version", "py38-pyarrow-$version"
-  test.dependsOn "testPy38pyarrow-$version"
-  preCommitPy38.dependsOn "testPy38pyarrow-$version"
-}
+// Create a test task for each major version of pyarrow
+toxTask "testPy38pyarrow-0", "py38-pyarrow-0"
+test.dependsOn "testPy38pyarrow-0"
+preCommitPy38.dependsOn "testPy38pyarrow-0"
+
+toxTask "testPy38pyarrow-1", "py38-pyarrow-1"
+test.dependsOn "testPy38pyarrow-1"
+preCommitPy38.dependsOn "testPy38pyarrow-1"
+
+toxTask "testPy38pyarrow-2", "py38-pyarrow-2"
+test.dependsOn "testPy38pyarrow-2"
+preCommitPy38.dependsOn "testPy38pyarrow-2"
+
+toxTask "testPy38pyarrow-3", "py38-pyarrow-3"
+test.dependsOn "testPy38pyarrow-3"
+preCommitPy38.dependsOn "testPy38pyarrow-3"
+
+toxTask "testPy38pyarrow-4", "py38-pyarrow-4"
+test.dependsOn "testPy38pyarrow-4"
+preCommitPy38.dependsOn "testPy38pyarrow-4"
+
+toxTask "testPy38pyarrow-5", "py38-pyarrow-5"
+test.dependsOn "testPy38pyarrow-5"
+preCommitPy38.dependsOn "testPy38pyarrow-5"
+
+// Create a test task for each minor version of pandas
+toxTask "testPy38pandas-11", "py38-pandas-11"
+test.dependsOn "testPy38pandas-11"
+preCommitPy38.dependsOn "testPy38pandas-11"
+
+toxTask "testPy38pandas-12", "py38-pandas-12"
+test.dependsOn "testPy38pandas-12"
+preCommitPy38.dependsOn "testPy38pandas-12"
+
+toxTask "testPy38pandas-13", "py38-pandas-13"
+test.dependsOn "testPy38pandas-13"
+preCommitPy38.dependsOn "testPy38pandas-13"
 
 toxTask "whitespacelint", "whitespacelint"
 

sdks/python/tox.ini

@@ -250,3 +250,14 @@ commands =
   # Run pytest directly rather using run_pytest.sh. It doesn't handle
   # selecting tests with -m (BEAM-12985)
   pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}
+
+[testenv:py{36,37,38}-pandas-{11,12,13}]
+deps =
+  11: pandas>=1.1.0,<1.2.0
+  12: pandas>=1.2.0,<1.3.0
+  13: pandas>=1.3.0,<1.4.0
+commands =
+  # Log pandas and numpy version for debugging
+  /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"
+  # Run all DataFrame API unit tests
+  {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/dataframe'