Fix flaky tests

README.md

@@ -1,7 +1,7 @@
 Problem Builder and Step Builder
 --------------------------------
 
-[![Build Status](https://travis-ci.org/open-craft/problem-builder.svg?branch=master)](https://travis-ci.org/open-craft/problem-builder)
+[![Circle CI](https://circleci.com/gh/open-craft/problem-builder.svg?style=svg)](https://circleci.com/gh/open-craft/problem-builder)
 
 This repository provides two XBlocks: Problem Builder and Step Builder.
 

circle.yml

@@ -0,0 +1,24 @@
+machine:
+  python:
+    version: 2.7.10
+dependencies:
+  override:
+    - "pip install -U pip wheel setuptools"
+    - "pip install -e git://github.com/edx/xblock-sdk.git@22c1b2f173919bef22f2d9d9295ec5396d02dffd#egg=xblock-sdk"
+    - "pip install -r requirements.txt"
+    - "pip install -r $VIRTUAL_ENV/src/xblock-sdk/requirements/base.txt"
+    - "pip install -r $VIRTUAL_ENV/src/xblock-sdk/requirements/test.txt"
+    - "pip uninstall -y xblock-problem-builder && python setup.py sdist && pip install dist/xblock-problem-builder-2.0.tar.gz"
+    - "pip install -r test_requirements.txt"
+    - "mkdir var"
+test:
+  override:
+    - "if [ $CIRCLE_NODE_INDEX = '0' ]; then pep8 problem_builder --max-line-length=120; fi":
+        parallel: true
+    - "if [ $CIRCLE_NODE_INDEX = '1' ]; then pylint problem_builder --disable=all --enable=function-redefined,undefined-variable,unused-variable; fi":
+        parallel: true
+    - "python run_tests.py":
+        parallel: true
+        files:
+          - "problem_builder/v1/tests/**/*.py"
+          - "problem_builder/tests/**/*.py"

problem_builder/tests/integration/test_author_changes.py

@@ -2,6 +2,7 @@
 If an author makes changes to the block after students have started using it, will bad things
 happen?
 """
+import time
 from .base_test import ProblemBuilderBaseTest, MentoringAssessmentBaseTest
 import re
 
@@ -20,6 +21,12 @@ def refresh_page(self):
         [Re]load the page with our scenario
         """
         self.pb_block_dom = self.go_to_view("student_view")
+        # At this point the ajax request that initializes the Mentoring block
+        # might be still in progres. Race conditions resulting in duplicate field data
+        # can occur if we try to reload the block at the same time.
+        # Sleep 200ms to wait for the ajax request to finish, unfortunately I wasn't
+        # able to find a better way.
+        time.sleep(0.2)
         self.reload_pb_block()
 
     def reload_pb_block(self):

problem_builder/tests/integration/test_step_builder.py

@@ -1,5 +1,3 @@
-import time
-
 from mock import patch
 from ddt import ddt, data
 from selenium.webdriver.support.ui import WebDriverWait
@@ -1332,7 +1330,8 @@ def check_viewport(self):
 
         def is_scrolled_to_top(driver):
             scroll_top = int(driver.execute_script("return $(window).scrollTop()"))
-            return abs(scroll_top - step_builder_offset) < 1
+            tolerance = 2
+            return abs(scroll_top - step_builder_offset) <= tolerance
 
         wait = WebDriverWait(self.browser, 5)
         wait.until(is_scrolled_to_top)
@@ -1342,7 +1341,7 @@ def scroll_down(self):
 
     def test_scroll_into_view(self):
         # Make window small, so that we have to scroll.
-        self.browser.set_window_size(400, 400)
+        self.browser.set_window_size(600, 400)
         step_builder, controls = self.load_assessment_scenario("step_builder_long_steps.xml", {})
         # First step
         self.check_viewport()

run_tests.py

@@ -9,7 +9,6 @@
 
 import os
 import sys
-
 import logging
 
 logging_level_overrides = {
@@ -18,7 +17,38 @@
     'workbench.runtime': logging.ERROR,
 }
 
+def patch_broken_pipe_error():
+    """Monkey Patch BaseServer.handle_error to not write a stacktrace to stderr on broken pipe.
+    This message is automatically suppressed in Django 1.8, so this monkey patch can be
+    removed once the workbench upgrades to Django >= 1.8.
+    http://stackoverflow.com/a/22618740/51397"""
+
+    import socket
+    from SocketServer import BaseServer
+    from wsgiref import handlers
+
+    handle_error = BaseServer.handle_error
+    log_exception = handlers.BaseHandler.log_exception
+
+    def is_broken_pipe_error():
+        exc_type, exc_value = sys.exc_info()[:2]
+        if issubclass(exc_type, socket.error) and exc_value.args[0] == 32:
+            return True
+        return False
+
+    def my_handle_error(self, request, client_address):
+        if not is_broken_pipe_error():
+            handle_error(self, request, client_address)
+
+    def my_log_exception(self, exc_info):
+        if not is_broken_pipe_error():
+            log_exception(self, exc_info)
+
+    BaseServer.handle_error = my_handle_error
+    handlers.BaseHandler.log_exception = my_log_exception
+
 if __name__ == "__main__":
+    patch_broken_pipe_error()
     # Use the workbench settings file:
     os.environ.setdefault("DJANGO_SETTINGS_MODULE", "workbench.settings")
     # Configure a range of ports in case the default port of 8081 is in use