From e949c273d01bbdfc7b5540d082e35b45838e0611 Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 12 Aug 2013 21:00:48 +0000
Subject: [PATCH 01/20] Add sysadmin dashboard - for seeting overview of system
 status, for deleting and loading courses, for seeing log of git imports of
 courseware. Includes script for importing course XML from github site.

---
 lms/djangoapps/dashboard/sysadmin.py          | 458 ++++++++++++++++++
 lms/envs/common.py                            |   2 +
 lms/envs/dev.py                               |   4 +
 lms/templates/sysadmin_dashboard.html         | 196 ++++++++
 lms/templates/sysadmin_dashboard_gitlogs.html |  92 ++++
 lms/urls.py                                   |   8 +
 scripts/cms_git_add_course                    | 122 +++++
 7 files changed, 882 insertions(+)
 create mode 100644 lms/djangoapps/dashboard/sysadmin.py
 create mode 100644 lms/templates/sysadmin_dashboard.html
 create mode 100644 lms/templates/sysadmin_dashboard_gitlogs.html
 create mode 100755 scripts/cms_git_add_course

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
new file mode 100644
index 000000000000..e0fe3db78ed4
--- /dev/null
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -0,0 +1,458 @@
+# MITx sysadmin dashboard
+
+import csv
+import itertools
+import json
+import logging
+import os
+import requests
+import string
+import subprocess
+import time
+import urllib
+
+from random import choice
+from StringIO import StringIO
+from datetime import datetime
+
+from django.conf import settings
+from django.contrib.auth.models import User, Group
+from django.http import HttpResponse, Http404
+from courseware.models import StudentModule
+from student.models import CourseEnrollment, CourseEnrollmentAllowed
+from student.models import UserProfile, Registration
+from external_auth.models import ExternalAuthMap
+
+from courseware.access import (has_access, get_access_group_name,
+                               course_beta_test_group_name)
+from courseware.courses import get_course_with_access, get_course_by_id
+
+from django.contrib.auth import logout, authenticate, login
+from django_future.csrf import ensure_csrf_cookie
+from django.views.decorators.cache import cache_control
+from mitxmako.shortcuts import render_to_response
+from xmodule.course_module import CourseDescriptor
+from xmodule.modulestore import Location
+from xmodule.modulestore.django import modulestore
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.store_utilities import delete_course
+
+import track.views
+
+log = logging.getLogger(__name__)
+
+
+def escape(s):
+    """escape HTML special characters in string"""
+    return str(s).replace('<','&lt;').replace('>','&gt;')
+
+def git_info_for_course(cdir):
+    gdir = settings.DATA_DIR / cdir
+    info = [gdir,'','']
+    if os.path.exists(gdir):
+        cmd = "cd %s; git log -1" % gdir
+        for k in os.popen(cmd).readlines():
+            if 'commit' in k:
+                info[0] = k.split()[1]
+            elif 'Author' in k:
+                info[2] = k.split()[1]
+            elif 'Date' in k:
+                info[1] = k.split(' ',1)[1].strip()
+    return info
+
+
+def fix_external_auth_map_passwords():
+    msg = ''
+    for eamap in ExternalAuthMap.objects.all():
+        u = eamap.user
+        pw = eamap.internal_password
+        if u is None:
+            continue
+        try:
+            testuser = authenticate(username=u.username, password=pw)
+        except Exception as err:
+            msg += "Failed in authenticating %s, error %s\n" % (u,err)
+            continue
+        if testuser is None:
+            msg += "Failed in authenticating %s; " % (u)
+            msg += "fixed password"
+            u.set_password(pw)
+            u.save()
+            continue
+    if not msg:
+        msg = "All ok!"
+    return msg
+
+@ensure_csrf_cookie
+@cache_control(no_cache=True, no_store=True, must_revalidate=True)
+def sysadmin_dashboard(request):
+    """
+    Sysadmin dashboard.
+    
+    Provides:
+        
+        1. enrollment numbers
+        2. loading new courses from github
+        3. reloading XML from files
+
+    """
+    if not request.user.is_staff:
+        raise Http404
+
+    msg = ''
+    problems = []
+    plots = []
+    datatable = {}
+    
+    def_ms = modulestore()
+    is_using_mongo = 'mongo' in str(def_ms.__class__)
+
+    if is_using_mongo:
+        courses = def_ms.get_courses()
+        courses = dict([c.id, c] for c in courses)	# no course directory
+    else:
+        courses = def_ms.courses.items()
+
+    # the instructor dashboard page is modal: grades, psychometrics, admin
+    # keep that state in request.session (defaults to grades mode)
+    dash_mode = request.POST.get('dash_mode','')
+    if dash_mode:
+        request.session['dash_mode'] = dash_mode
+    else:
+        dash_mode = request.session.get('dash_mode','Status')
+    
+    # helper functions
+
+    def return_csv(fn, datatable, fp=None):
+        if fp is None:
+            response = HttpResponse(mimetype='text/csv')
+            response['Content-Disposition'] = 'attachment; filename={0}'.format(fn)
+        else:
+            response = fp
+        writer = csv.writer(response, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
+        writer.writerow(datatable['header'])
+        for datarow in datatable['data']:
+            encoded_row = [unicode(s).encode('utf-8') for s in datarow]
+            writer.writerow(encoded_row)
+        return response
+
+    def get_staff_group(course):
+        return get_group(course, 'staff')
+
+    def get_instructor_group(course):
+        return get_group(course, 'instructor')
+
+    def get_group(course, groupname):
+        grpname = get_access_group_name(course, groupname)
+        try:
+            group = Group.objects.get(name=grpname)
+        except Group.DoesNotExist:
+            group = Group(name=grpname)     # create the group
+            group.save()
+        return group
+
+    # default datatable depends on dash_mode
+    
+    if dash_mode=='Status':
+        datatable = dict(header=['Statistic','Value'],
+                         title="Site statistics")
+        datatable['data'] = [['Total number of users', User.objects.all().count()]]
+
+    elif dash_mode=='Courses':
+        data = []
+
+        #for cdir, course in def_ms.courses.items():
+        for cdir, course in courses.items():
+            data.append([course.display_name, cdir] + git_info_for_course(cdir))
+
+        datatable = dict(header=['Course Name', 'dir', 'git commit', 'last change', 'last editor'],
+                         title="Information about all courses",
+                         data=data)
+
+    elif dash_mode=='Enrollment' or dash_mode=="Staffing":
+        data = []
+
+        #for cdir, course in def_ms.courses.items():
+        for cdir, course in courses.items():
+            datum = [course.display_name, course.id]
+            datum += [CourseEnrollment.objects.filter(course_id=course.id).count()]
+            datum += [get_group(course, 'staff').user_set.all().count()]
+            datum += [','.join([x.username for x in get_group(course, 'instructor').user_set.all()])]
+            data.append(datum)
+
+        datatable = dict(header=['Course Name', 'course_id', '# enrolled', '# staff', 'instructors'],
+                         title="Enrollment information for all courses",
+                         data=data)
+
+    # process actions from form POST
+    action = request.POST.get('action', '')
+    track.views.server_track(request, action, {}, page='sysdashboard')
+
+    if "Download list of all users (csv file)" in action:
+        datatable = dict(header=['username', 'email'],
+                         title="List of all users",
+                         data=[[u.username, u.email] for u in User.objects.all()])
+        return return_csv('users_%s.csv' % request.META['SERVER_NAME'],datatable)
+        
+    elif "Check and repair external Auth Map" in action:
+        msg += '<pre>'
+        msg += fix_external_auth_map_passwords()
+        msg += '</pre>'
+        datatable = {}
+
+    elif "Create user" in action:
+        uname = request.POST.get('student_uname','').strip()
+        name = request.POST.get('student_fullname','').strip()
+
+        def GenPasswd(length=8, chars=string.letters + string.digits):
+            return ''.join([choice(chars) for i in range(length)])
+
+        def create_user(uname, name, do_mit=False):
+            if not uname:
+                return "Must provide username"
+            if not name:
+                return "Must provide full name"
+            msg = ''
+            if do_mit:
+                if not '@' in uname:
+                    email = '%s@MIT.EDU' % uname
+                else:
+                    email = uname
+                if not email.endswith('@MIT.EDU'):
+                    msg += 'email must end in @MIT.EDU'
+                    return msg
+                mit_domain = 'ssl:MIT'
+                if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
+                    msg += "Failed - email %s already exists as external_id" % email
+                    return msg
+                make_eamap = True
+            else:
+                email = uname
+                if not '@' in email:
+                    msg += 'email address required (not username)'
+                    return msg
+            password = GenPasswd(12)
+            user = User(username=uname, email=email, is_active=True)
+            user.set_password(password)
+            try:
+                user.save()
+            except IntegrityError:
+                msg += "Oops, failed to create user %s, IntegrityError" % user
+                return msg
+                
+            r = Registration()
+            r.register(user)
+            
+            up = UserProfile(user=user)
+            up.name = name
+            up.save()
+            
+            if make_eamap:
+                credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
+                eamap = ExternalAuthMap(external_id = email,
+                                        external_email = email,
+                                        external_domain = mit_domain,
+                                        external_name = name,
+                                        internal_password = password,
+                                        external_credentials = json.dumps(credentials),
+                    )
+                eamap.user = user
+                eamap.dtsignup = datetime.now()
+                eamap.save()
+            
+            msg += "User %s created successfully!" % user
+            return msg
+            
+        msg += create_user(uname, name, do_mit=settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'])
+        datatable = {}
+
+
+    elif "Delete user" in action:
+        uname = request.POST.get('student_uname','').strip()
+
+        def delete_user(uname):
+            if not uname:
+                return "Must provide username"
+            if '@' in uname:
+                try:
+                    u = User.objects.get(email=uname)
+                except Exception, err:
+                    msg = "Cannot find user with email address %s" % uname
+                    return msg
+            else:
+                try:
+                    u = User.objects.get(username=uname)
+                except Exception, err:
+                    msg = "Cannot find user with username %s" % uname
+                    return msg
+            u.delete()
+            return "Deleted user %s" % uname
+
+        msg += delete_user(uname)
+
+    elif "Download staff and instructor list (csv file)" in action:
+        data = []
+        roles = ['instructor','staff']
+        #for cdir, course in def_ms.courses.items():
+        for cdir, course in courses.items():
+            for role in roles:
+                for u in get_group(course, role).user_set.all():
+                    datum = [course.id, role, u.username, u.email, u.profile.name]
+                    data.append(datum)
+            
+        datatable = dict(header=['course_id', 'role', 'username', 'email', 'full_name'],
+                         title="List of all course staff and instructors",
+                         data=data)
+        return return_csv('staff_%s.csv' % request.META['SERVER_NAME'],datatable)
+        
+
+    elif action=="Delete course from site":
+        
+        course_id = request.POST.get('course_id','').strip()
+        ok = False
+        if course_id in courses:
+            ok = True
+            course = courses[course_id]
+        else:
+            try:
+                course = get_course_by_id(course_id)
+                ok = True
+            except Exception as err:
+                msg += "Error - cannot get course with ID %s<br/><pre>%s</pre>" % (course_id, escape(err))
+
+        if ok and not is_using_mongo:
+            cdir = course.metadata.get('data_dir', course.location.course)
+            def_ms.courses.pop(cdir)
+
+            # now move the directory (don't actually delete it)
+            nd = cdir + '_deleted_%s' % int(time.time())
+            os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / nd)
+            os.system('chmod -x %s' % (settings.DATA_DIR / nd))
+
+            msg += "<font color='red'>Deleted %s = %s (%s)</font>" % (cdir, course.id, course.display_name)
+
+        elif ok and is_using_mongo:
+            # delete course that is stored with mongodb backend
+            loc = course.location
+            #ms = modulestore('direct')
+            cs = contentstore()
+            commit = True
+            ret = delete_course(def_ms, cs, loc, commit)
+            # don't delete user permission groups, though
+            msg += "<font color='red'>Deleted %s = %s (%s)</font>" % (loc, course.id, course.display_name)
+    
+
+    elif action=="Load new course from github":
+        
+        gitloc = request.POST.get('repo_location','').strip().replace(' ','').replace(';','')
+
+        def get_course_from_git(gitloc):
+            msg = ''
+            if (not gitloc.endswith('.git')) or ('http:' in gitloc) or ('https:' in gitloc):
+                msg += "The git repo location should end with '.git', and be for SSH access"
+                return msg
+
+            if is_using_mongo:
+                acscript = getattr(settings, 'CMS_ADD_COURSE_SCRIPT', '')
+                if not acscript or not os.path.exists(acscript):
+                    msg = "<font color='red'>Must configure CMS_ADD_COURSE_SCRIPT in settings first!</font>"
+                    return msg
+                cmd = '{0} "{1}"'.format(acscript, gitloc)
+                logging.debug('Adding course with command: {0}'.format(cmd))
+                ret = subprocess.Popen(cmd, shell=True, executable = "/bin/bash",
+                                       stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+                ret = ''.join(ret)
+                msg = "<font color='red'>Added course from {0}</font>".format(gitloc)
+                msg += "<pre>{0}</pre>".format(ret.replace('<','&lt;'))
+                return msg
+
+            cdir = gitloc.rsplit('/',1)[1][:-4]
+            gdir = settings.DATA_DIR / cdir
+            if os.path.exists(gdir):
+                msg += "The course %s already exists in the data directory! (reloading anyway)" % cdir
+                # return msg
+            else:
+                cmd = "cd %s; git clone %s" % (settings.DATA_DIR, gitloc)
+                msg += '<pre>%s</pre>' % escape(os.popen(cmd).read())
+            if not os.path.exists(gdir):
+                msg += "Failed to clone repository to %s" % gdir
+                return msg
+            def_ms.try_load_course(cdir)	# load into modulestore
+            errlog = def_ms.errored_courses.get(cdir,'')
+            if errlog:
+                msg += '<hr width="50%%"><pre>%s</pre>' % escape(errlog)
+            else:
+                course = def_ms.courses[cdir]
+                msg += "Loaded course %s (%s)<br/>Errors:" % (cdir, course.display_name)
+                errors = def_ms.get_item_errors(course.location)
+                if not errors:
+                    msg += "None"
+                else:
+                    msg += "<ul>"
+                    for (summary, err) in errors:
+                        msg += '<li><pre>%s: %s</pre></li>' % (escape(summary), escape(err))
+                    msg += "</ul>"
+                datatable['data'].append([course.display_name, cdir] + git_info_for_course(cdir))
+            return msg
+            
+        msg += get_course_from_git(gitloc)
+
+    else:	# default to showing status summary
+        msg += '<h2>Courses loaded in the modulestore</h2>'
+        msg += '<ol>'
+        #for cdir, course in def_ms.courses.items():
+        for cdir, course in courses.items():
+            msg += '<li>%s (%s)</li>' % (escape(cdir),
+                                         course.location.url())
+        msg += '</ol>'
+
+        
+    
+    #----------------------------------------
+    # context for rendering
+
+    context = {'datatable': datatable,
+               'plots': plots,
+               'msg': msg,
+               'djangopid' : os.getpid(),
+               'modeflag': {dash_mode: 'selectedmode'},
+               'mitx_version' : getattr(settings,'MITX_VERSION_STRING',''),
+               }
+
+    return render_to_response('sysadmin_dashboard.html', context)
+    
+#-----------------------------------------------------------------------------
+
+def view_git_logs(request, course_id=None):
+
+    import mongoengine	# don't import that until we need it, here
+
+    class CourseImportLog(mongoengine.Document):
+        course_id = mongoengine.StringField(max_length=128)
+        location = mongoengine.StringField(max_length=168)
+        import_log = mongoengine.StringField(max_length=20*65535)
+        git_log = mongoengine.StringField(max_length=65535)
+        repo_dir = mongoengine.StringField(max_length=128)
+        created = mongoengine.DateTimeField()
+        meta = { 'indexes': ['course_id', 'created'],
+                 'allow_inheritance': False, }
+
+    DBNAME = "xlog"
+
+    mdb = mongoengine.connect(DBNAME)
+    
+    if course_id is None:
+        cilset = CourseImportLog.objects.all().order_by('-created')
+    else:
+        log.debug('course_id=%s' % course_id)
+        cilset = CourseImportLog.objects.filter(course_id=course_id).order_by('-created')
+        log.debug('cilset length=%s' % len(cilset))
+        
+    context = {'cilset': cilset,
+               'course_id': course_id,
+               }
+
+    return render_to_response('sysadmin_dashboard_gitlogs.html', context)
+    
+    
+    
\ No newline at end of file
diff --git a/lms/envs/common.py b/lms/envs/common.py
index 0cbcbb774ab3..adb4094844ed 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -84,6 +84,8 @@
 
     'ENABLE_MASQUERADE': True,  # allow course staff to change to student view of courseware
 
+    'ENABLE_SYSADMIN_DASHBOARD': False, # sysadmin dashboard, to see what courses are loaded, to delete & load courses
+
     'DISABLE_LOGIN_BUTTON': False,  # used in systems where login is automatic, eg MIT SSL
 
     'STUB_VIDEO_FOR_TESTING': False,  # do not display video when running automated acceptance tests
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index 622ff6acf7e2..5baee2337478 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -30,6 +30,7 @@
 MITX_FEATURES['ENABLE_SERVICE_STATUS'] = True
 MITX_FEATURES['ENABLE_HINTER_INSTRUCTOR_VIEW'] = True
 MITX_FEATURES['ENABLE_INSTRUCTOR_BETA_DASHBOARD'] = False
+MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
 
 WIKI_ENABLED = True
 
@@ -256,6 +257,9 @@
 if SEGMENT_IO_LMS_KEY:
     MITX_FEATURES['SEGMENT_IO_LMS'] = True
 
+########################## GIT WORKFLOW ########################
+
+CMS_ADD_COURSE_SCRIPT = REPO_ROOT / "scripts/cms_git_add_course"
 
 ########################## USER API ########################
 EDX_API_KEY = None
diff --git a/lms/templates/sysadmin_dashboard.html b/lms/templates/sysadmin_dashboard.html
new file mode 100644
index 000000000000..ff09485690ed
--- /dev/null
+++ b/lms/templates/sysadmin_dashboard.html
@@ -0,0 +1,196 @@
+<%inherit file="/main.html" />
+<%! from django.core.urlresolvers import reverse %>
+<%namespace name='static' file='/static_content.html'/>
+
+<%block name="headextra">
+  <%static:css group='course'/>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
+</%block>
+
+<style type="text/css">
+table.stat_table {
+	font-family: verdana,arial,sans-serif;
+	font-size:11px;
+	color:#333333;
+	border-width: 1px;
+	border-color: #666666;
+	border-collapse: collapse;
+}
+table.stat_table th {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #dedede;
+}
+table.stat_table td {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #ffffff;
+}
+
+a.selectedmode { background-color: yellow; }
+
+textarea {
+  height: 200px;
+}
+</style>
+
+<script language="JavaScript" type="text/javascript">
+function goto( mode)
+{
+  document.dashform.dash_mode.value = mode;
+  document.dashform.submit() ;
+}
+</script>
+
+<section class="container">
+<div class="sysadmin-dashboard-wrapper">
+
+  <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt"
+>
+    <h1>Sysadmin Dashboard</h1>
+
+    <h2>[ <a href="#" onclick="goto('Status');" class="${modeflag.get('Status')}">Status</a> |
+          <a href="#" onclick="goto('Courses');" class="${modeflag.get('Courses')}">Courses</a> |
+          <a href="#" onclick="goto('Enrollment');" class="${modeflag.get('Enrollment')}">Enrollment</a> |
+          <a href="#" onclick="goto('Staffing');" class="${modeflag.get('Staffing')}">Staffing</a> |
+          <a href="/gitlogs">GitLogs</a>
+           ]
+    </h2>
+
+  <div style="text-align:right"><span id="djangopid">${djangopid}</span>
+  | <span id="mitxver">${mitx_version}</span></div>
+
+    <form name="dashform" method="POST">
+    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
+    <input type="hidden" name="dash_mode" value="">
+
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Status'):
+
+    <p>
+    Email or username: <input type="text" name="student_uname" size=40 >
+    <input type="submit" name="action" value="Delete user">
+    </p>
+
+    <p>
+    Full Name: <input type="text" name="student_fullname" size=40 >
+    <input type="submit" name="action" value="Create user">
+    </p>
+
+    <p>
+    <input type="submit" name="action" value="Download list of all users (csv file)">
+    </p>
+
+    <p>
+    <input type="submit" name="action" value="Check and repair external Auth Map">
+    </p>
+
+    <hr width="40%" style="align:left">
+ %endif
+
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Enrollment'):
+
+<p>Go to each individual course's Instructor dashboard to manage
+course enrollment.</p>
+
+%endif
+
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Staffing'):
+
+<h3>Manage course staff and instructors</h3><br/>
+
+    <p>
+    <input type="submit" name="action" value="Download staff and instructor list (csv file)">
+    </p>
+<!--
+    <p>You may upload a list of staff/instructors for all the courses.
+    The safest approach is to download the list, modify it, then upload.</p>
+    <p><font color="red">Careful!  The uploaded file <b>overwrites</b>
+    existing list, globally covering all courses!</font></p>
+    <p>
+    Staff and instructor list (CSV file): <input type="file" name="staff_file" >
+    <input type="submit" name="action" value="Upload staff and instructor list">
+    </p>
+-->
+
+%endif
+
+##----------------------------------------------------------------------------- 
+%if modeflag.get('Courses'):
+    <h3>Administer Courses</h3><br/>
+
+    <p>
+    Repo location (ssh): <input type="text" name="repo_location" size=60 >
+    <input type="submit" name="action" value="Load new course from github">
+    </p>
+    <p>
+    Course ID or dir: <input type="text" name="course_id" size=60 >
+    <input type="submit" name="action" value="Delete course from site">
+    </p>
+%endif
+
+##----------------------------------------------------------------------------- 
+    </form>
+
+%if msg:
+    <p></p><p>${msg}</p>
+%endif
+##----------------------------------------------------------------------------- 
+##----------------------------------------------------------------------------- 
+
+%if datatable:
+
+    <br/>
+    <br/>
+    <p>
+      <hr width="100%">
+      <h2>${datatable['title']}</h2>
+      <table class="stat_table">
+        <tr>
+	%for hname in datatable['header']:
+	  <th>${hname}</th>
+	%endfor
+	</tr>
+        %for row in datatable['data']:
+          <tr>
+	    %for value in row:
+	      <td>${value}</td>
+	    %endfor
+	  </tr>
+        %endfor
+      </table>
+    </p>
+%endif
+
+##-----------------------------------------------------------------------------  
+%if plots:
+
+    %for plot in plots:
+      <br/>
+      <h3>${plot['title']}</h3>
+      <br/>
+      <p>${plot['info']}</p>
+      <br/>
+      <div id="plot_${plot['id']}" style="width:600px;height:300px;"></div>
+      <script type="text/javascript">
+          $(function () {
+              ${plot['data']}
+              $.plot($("#plot_${plot['id']}"), ${plot['cmd']}  );
+          });
+      </script>
+      <br/>
+      <br/>
+    %endfor
+
+%endif
+
+  </section>
+</div>
+</section>
diff --git a/lms/templates/sysadmin_dashboard_gitlogs.html b/lms/templates/sysadmin_dashboard_gitlogs.html
new file mode 100644
index 000000000000..64bcc9d15560
--- /dev/null
+++ b/lms/templates/sysadmin_dashboard_gitlogs.html
@@ -0,0 +1,92 @@
+<%inherit file="/main.html" />
+<%! from django.core.urlresolvers import reverse %>
+<%namespace name='static' file='/static_content.html'/>
+
+<%block name="headextra">
+  <%static:css group='course'/>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
+  <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
+</%block>
+
+<style type="text/css">
+table.stat_table {
+	font-family: verdana,arial,sans-serif;
+	font-size:11px;
+	color:#333333;
+	border-width: 1px;
+	border-color: #666666;
+	border-collapse: collapse;
+}
+table.stat_table th {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #dedede;
+}
+table.stat_table td {
+	border-width: 1px;
+	padding: 8px;
+	border-style: solid;
+	border-color: #666666;
+	background-color: #ffffff;
+}
+
+a.selectedmode { background-color: yellow; }
+
+textarea {
+  height: 200px;
+}
+</style>
+
+<section class="container">
+<div class="sysadmin-dashboard-wrapper">
+
+  <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
+    <h1>git logs</h1>
+
+% if course_id is None:
+
+    <table class="stat_table">
+    <tr><th>Date</th><th>Course ID</th><th>git action</th></tr>
+
+    % for cil in cilset[:10]:
+
+        <tr>
+	    <td>${cil.created}</td>
+	    <td><a href="/gitlogs/${cil.course_id}">${cil.course_id}</a></td>
+	    <td>${cil.git_log}</td>
+	</tr>
+
+    % endfor
+    
+    </table>
+
+% else:
+
+    <h2>Recent git load activity for ${course_id}</h2>
+
+    <table class="stat_table">
+    <tr><th>Date</th><th>Course ID</th><th>git action</th></tr>
+
+    % for cil in cilset[:2]:
+
+        <tr>
+	    <td>${cil.created}</td>
+	    <td><a href="/gitlogs/${cil.course_id}">${cil.course_id}</a></td>
+	    <td>${cil.git_log}</td>
+	</tr>
+	
+	<tr><td colspan="3">
+	    <pre>${cil.import_log}</pre>
+	</td></tr>
+
+    % endfor
+    
+    </table>
+
+% endif
+
+  </section>
+</div>
+</section>
diff --git a/lms/urls.py b/lms/urls.py
index b32c0263d0ef..0e291507e881 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -73,6 +73,14 @@
     url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict),
 )
 
+# sysadmin dashboard, to see what courses are loaded, to delete & load courses
+if settings.MITX_FEATURES["ENABLE_SYSADMIN_DASHBOARD"]:
+    urlpatterns += (
+        url(r'^sysadmin$', 'dashboard.sysadmin.sysadmin_dashboard', name="sysadmin"),
+        url(r'^gitlogs$', 'dashboard.sysadmin.view_git_logs', name="gitlogs"),
+        url(r'^gitlogs/(?P<course_id>.+)$', 'dashboard.sysadmin.view_git_logs'),
+    )
+
 #Semi-static views (these need to be rendered and have the login bar, but don't change)
 urlpatterns += (
     url(r'^404$', 'static_template_view.views.render',
diff --git a/scripts/cms_git_add_course b/scripts/cms_git_add_course
new file mode 100755
index 000000000000..3b159a4916a1
--- /dev/null
+++ b/scripts/cms_git_add_course
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+#
+# python script to pull a git repo and import into cms / edge mongodb content database.
+#
+# usage:
+#
+#    python cms_git_add_course <git-ssh-url> [<directory>]
+#
+# argument is git ssh url, like: git@github.com:mitocw/edx4edx_lite.git
+# if the directory is given, that is used and presumed to contain the git repo
+# 
+
+import os, sys, string, re
+import datetime
+import mongoengine		# used to store import log 
+
+DIR = "/mnt/data_import"
+
+#-----------------------------------------------------------------------------
+# pull from git
+
+repo = sys.argv[1]
+m = re.match('git@[^ ]+\.git', repo)
+if not m:
+    print "Oops, not a git ssh url?"
+    print "Expecting something like git@github.com:mitocw/edx4edx_lite.git"
+    sys.exit(-1)
+
+if len(sys.argv)>2:
+    rdir = sys.argv[2]
+    rdir = os.path.basename(rdir)
+else:
+    rdir = repo.rsplit('/',1)[-1].rsplit('.git',1)[0]
+print "rdir = %s" % rdir
+
+rdirp = '%s/%s' % (DIR, rdir)
+if os.path.exists(rdirp):
+    print "directory already exists, doing a git pull instead of git clone"
+    cmd = 'cd %s/%s; git pull' % (DIR, rdir)
+else:
+    cmd = 'cd %s; git clone "%s"' % (DIR, repo)
+
+print cmd
+ret_git = os.popen(cmd).read()
+print ret_git
+
+if not os.path.exists('%s/%s' % (DIR, rdir)):
+    print "git clone failed!"
+    sys.exit(-1)
+
+# get commit id
+commit_id = os.popen('cd %s; git log -n 1 | head -1' % rdirp).read().strip().split(' ')[1]
+
+ret_git += "\nCommit ID: %s" % commit_id
+
+# get branch
+branch = ''
+for k in os.popen('cd %s; git branch' % rdirp).readlines():
+    if k[0]=='*':
+        branch = k[2:].strip()
+
+ret_git += "   \nBranch: %s" % branch
+
+#-----------------------------------------------------------------------------
+# import
+
+#cmd = "./DJANGO-ADMIN-CMS import %s %s" % (DIR, rdir)
+cmd = "./DJANGO-ADMIN-CMS import --nostatic %s %s" % (DIR, rdir)
+print cmd
+ret_import = os.popen(cmd).read()
+print ret_import
+
+course_id = "unknown"
+location = "unknown"
+
+# extract course ID from output of import-command-run and make symlink
+# this is needed in order for custom course scripts to work
+m = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)', ret_import)
+if m:
+    location = m.group(1).strip()
+    print "location = %s" % location
+    course_id = location.replace('i4x://','').replace('/course/','/').split('\n')[0].strip()
+    
+    cdir = "%s/%s" % (DIR, course_id.split('/')[1])
+    print "Studio course dir = %s" % cdir
+    
+    if os.path.exists(cdir) and not os.path.islink(cdir):
+        print "   -> exists, but is not symlink"
+        print os.popen('ls -l %s' % cdir).read()
+        print os.popen('rmdir %s' % cdir).read()
+
+    if not os.path.exists(cdir):
+        print "   -> creating symlink"
+        print os.popen('ln -s %s %s' % (rdirp, cdir)).read()
+        print os.popen('ls -l %s' % cdir).read()
+        
+#-----------------------------------------------------------------------------
+# store import-command-run output in mongo
+
+class CourseImportLog(mongoengine.Document):
+    course_id = mongoengine.StringField(max_length=128)
+    location = mongoengine.StringField(max_length=168)
+    import_log = mongoengine.StringField(max_length=20*65535)
+    git_log = mongoengine.StringField(max_length=65535)
+    repo_dir = mongoengine.StringField(max_length=128)
+    created = mongoengine.DateTimeField()
+    meta = { 'indexes': ['course_id', 'created'],
+             'allow_inheritance': False, }
+
+DBNAME = "xlog"
+
+mdb = mongoengine.connect(DBNAME)
+
+cil = CourseImportLog(course_id=course_id,
+                      location=location,
+                      repo_dir=rdir,
+                      created=datetime.datetime.now(),
+                      import_log=ret_import,
+                      git_log=ret_git)
+cil.save()
+
+print "saved CourseImportLog for %s" % cil.course_id

From 29929c62a5d407eb7df530c60fa051c8488f5b5d Mon Sep 17 00:00:00 2001
From: ichuang <ichuang@mit.edu>
Date: Mon, 12 Aug 2013 21:08:01 +0000
Subject: [PATCH 02/20] add mongoengine to requirements

---
 requirements/edx/base.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index 9179315797c6..9294ff51239b 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -30,6 +30,7 @@ glob2==0.3
 lxml==3.0.1
 mako==0.7.3
 Markdown==2.2.1
+mongoengine==0.8.3
 networkx==1.7
 nltk==2.0.4
 paramiko==1.9.0

From 9b8b89a9cc6cc33aed84b4de73b7745d0c0f5b0a Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Thu, 24 Oct 2013 13:56:56 -0400
Subject: [PATCH 03/20] This is a major revamp to make the sysadmin dashboard
 testable and more portable to different installation types and situations.

---
 .../management/commands/import.py             |  11 +-
 .../courseware/management/commands/import.py  |   1 +
 lms/djangoapps/dashboard/sysadmin.py          | 778 ++++++++++--------
 .../dashboard/tests/test_sysadmin.py          | 370 +++++++++
 lms/envs/aws.py                               |   6 +
 lms/envs/common.py                            |   1 +
 lms/envs/dev.py                               |   6 +-
 lms/envs/test.py                              |   4 +
 lms/templates/sysadmin_dashboard.html         | 125 +--
 lms/templates/sysadmin_dashboard_gitlogs.html |  56 +-
 lms/urls.py                                   |   3 +-
 requirements/edx/base.txt                     |   2 +-
 scripts/cms_git_add_course                    | 122 ---
 scripts/git_add_course.py                     | 212 +++++
 14 files changed, 1157 insertions(+), 540 deletions(-)
 create mode 120000 lms/djangoapps/courseware/management/commands/import.py
 create mode 100644 lms/djangoapps/dashboard/tests/test_sysadmin.py
 delete mode 100755 scripts/cms_git_add_course
 create mode 100755 scripts/git_add_course.py

diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py
index e0d58b32f0dd..2e16bf1258ef 100644
--- a/cms/djangoapps/contentstore/management/commands/import.py
+++ b/cms/djangoapps/contentstore/management/commands/import.py
@@ -31,9 +31,16 @@ def handle(self, *args, **options):
             course_dirs = args[1:]
         else:
             course_dirs = None
-        print("Importing.  Data_dir={data}, course_dirs={courses}".format(
+        self.stdout.write("Importing.  Data_dir={data}, course_dirs={courses}\n".format(
             data=data_dir,
             courses=course_dirs,
             dis=do_import_static))
-        import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False,
+        try:
+            mstore = modulestore('direct')
+        except KeyError:
+            self.stdout.write('Unable to load direct modulestore, trying '
+                              'default\n')
+            mstore = modulestore('default')
+
+        import_from_xml(mstore, data_dir, course_dirs, load_error_modules=False,
                         static_content_store=contentstore(), verbose=True, do_import_static=do_import_static)
diff --git a/lms/djangoapps/courseware/management/commands/import.py b/lms/djangoapps/courseware/management/commands/import.py
new file mode 120000
index 000000000000..36b7e3c6fcd7
--- /dev/null
+++ b/lms/djangoapps/courseware/management/commands/import.py
@@ -0,0 +1 @@
+../../../../../cms/djangoapps/contentstore/management/commands/import.py
\ No newline at end of file
diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index e0fe3db78ed4..37b7f1b881e0 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -1,458 +1,566 @@
-# MITx sysadmin dashboard
-
+"""
+This module creates a sysadmin dashboard for managing and viewing
+courses.
+"""
 import csv
-import itertools
 import json
 import logging
 import os
-import requests
-import string
-import subprocess
 import time
-import urllib
+import imp
+import StringIO
 
-from random import choice
-from StringIO import StringIO
 from datetime import datetime
 
 from django.conf import settings
 from django.contrib.auth.models import User, Group
-from django.http import HttpResponse, Http404
-from courseware.models import StudentModule
-from student.models import CourseEnrollment, CourseEnrollmentAllowed
+from django.utils.translation import ugettext as _
+from student.models import CourseEnrollment
 from student.models import UserProfile, Registration
 from external_auth.models import ExternalAuthMap
+from external_auth.views import generate_password
 
-from courseware.access import (has_access, get_access_group_name,
-                               course_beta_test_group_name)
-from courseware.courses import get_course_with_access, get_course_by_id
+from courseware.access import get_access_group_name
+from courseware.courses import get_course_by_id
 
-from django.contrib.auth import logout, authenticate, login
+from django.contrib.auth import authenticate
+from django.core.exceptions import PermissionDenied
 from django_future.csrf import ensure_csrf_cookie
 from django.views.decorators.cache import cache_control
+from django.db import IntegrityError
+from django.http import HttpResponse
+from django.utils.html import escape
+from django.contrib.admin.views.decorators import staff_member_required
+from django.http import Http404
+
 from mitxmako.shortcuts import render_to_response
-from xmodule.course_module import CourseDescriptor
-from xmodule.modulestore import Location
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.store_utilities import delete_course
+import mongoengine
 
 import track.views
 
 log = logging.getLogger(__name__)
 
 
-def escape(s):
-    """escape HTML special characters in string"""
-    return str(s).replace('<','&lt;').replace('>','&gt;')
+class CourseImportLog(mongoengine.Document):
+    """Mongoengine model for git log"""
+    # pylint: disable-msg=R0924
+
+    course_id = mongoengine.StringField(max_length=128)
+    location = mongoengine.StringField(max_length=168)
+    import_log = mongoengine.StringField(max_length=20 * 65535)
+    git_log = mongoengine.StringField(max_length=65535)
+    repo_dir = mongoengine.StringField(max_length=128)
+    created = mongoengine.DateTimeField()
+    meta = {'indexes': ['course_id', 'created'],
+            'allow_inheritance': False}
+
 
 def git_info_for_course(cdir):
+    """This pulls out some git info like the last commit"""
+
+    cmd = ''
     gdir = settings.DATA_DIR / cdir
-    info = [gdir,'','']
+    info = [gdir, '', '']
     if os.path.exists(gdir):
-        cmd = "cd %s; git log -1" % gdir
+        cmd = 'cd {0}; git log -1'.format(gdir)
         for k in os.popen(cmd).readlines():
             if 'commit' in k:
                 info[0] = k.split()[1]
             elif 'Author' in k:
                 info[2] = k.split()[1]
             elif 'Date' in k:
-                info[1] = k.split(' ',1)[1].strip()
+                info[1] = k.split(' ', 1)[1].strip()
     return info
 
 
+def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
+    """This downloads and runs the checks for importing a course in git"""
+
+    msg = u''
+    if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
+       gitloc.startswith('https:') or gitloc.startswith('git:')):
+        msg += \
+            _("The git repo location should end with '.git', and be a valid url")
+        return msg
+
+    if is_using_mongo:
+        acscript = getattr(settings, 'GIT_ADD_COURSE_SCRIPT', '')
+        if not acscript or not os.path.exists(acscript):
+            msg = u"<font color='red'>{0} - {1}</font>".format(
+                _('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'), acscript)
+            return msg
+
+        # import course script directly and call add_repo function
+        git_add_script = imp.load_source('git_add_script', acscript)
+        logging.debug(
+            _('Adding course using add repo from {0} and repo {1}').format(
+                acscript, gitloc))
+
+        # Grab logging output for debugging imports
+        output = StringIO.StringIO()
+
+        import_logger = logging.getLogger(
+            'xmodule.modulestore.xml_importer')
+        git_logger = logging.getLogger('git_add_script')
+        xml_logger = logging.getLogger('xmodule.modulestore.xml')
+
+        import_log_handler = logging.StreamHandler(output)
+        import_log_handler.setLevel(logging.DEBUG)
+
+        for logger in [import_logger, git_logger, xml_logger, ]:
+            logger.addHandler(import_log_handler)
+
+        git_add_script.add_repo(gitloc, None)
+
+        ret = output.getvalue()
+
+        # Remove handler hijacks
+        for logger in [import_logger, git_logger, xml_logger, ]:
+            logger.removeHandler(import_log_handler)
+        msg = u"<font color='red'>{0} {1}</font>".format(
+            _('Added course from'), gitloc)
+        msg += _("<pre>{0}</pre>").format(escape(ret))
+        return msg
+
+    cdir = (gitloc.rsplit('/', 1)[1])[:-4]
+    gdir = settings.DATA_DIR / cdir
+    if os.path.exists(gdir):
+        msg += _("The course {0} already exists in the data directory! "
+                 "(reloading anyway)").format(cdir)
+        cmd = 'cd {0}; git pull'.format(settings.DATA_DIR, gitloc)
+    else:
+        cmd = 'cd {0}; git clone {1}'.format(settings.DATA_DIR, gitloc)
+        msg += u'<pre>%s</pre>' % escape(os.popen(cmd).read())
+    if not os.path.exists(gdir):
+        msg += _('Failed to clone repository to {0}').format(gdir)
+        return msg
+    def_ms.try_load_course(os.path.abspath(gdir))  # load into modulestore
+    errlog = def_ms.errored_courses.get(cdir, '')
+    if errlog:
+        msg += u'<hr width="50%"><pre>{0}</pre>'.format(escape(errlog))
+    else:
+        course = def_ms.courses[os.path.abspath(gdir)]
+        msg += _('Loaded course {0} {1}<br/>Errors:').format(cdir,
+                                                             course.display_name)
+        errors = def_ms.get_item_errors(course.location)
+        if not errors:
+            msg += u'None'
+        else:
+            msg += u'<ul>'
+            for (summary, err) in errors:
+                msg += \
+                    u'<li><pre>{0}: {1}</pre></li>'.format(escape(summary),
+                                                           escape(err))
+            msg += u'</ul>'
+        datatable['data'].append([course.display_name, cdir]
+                                 + git_info_for_course(cdir))
+    return msg
+
+
 def fix_external_auth_map_passwords():
+    """
+    This corrects any passwords that have drifted from eamp to
+    internal django auth
+    """
+
     msg = ''
     for eamap in ExternalAuthMap.objects.all():
-        u = eamap.user
-        pw = eamap.internal_password
-        if u is None:
+        euser = eamap.user
+        epass = eamap.internal_password
+        if euser is None:
             continue
         try:
-            testuser = authenticate(username=u.username, password=pw)
-        except Exception as err:
-            msg += "Failed in authenticating %s, error %s\n" % (u,err)
+            testuser = authenticate(username=euser.username, password=epass)
+        except (TypeError, PermissionDenied), err:
+            msg += _('Failed in authenticating {0}, error {1}\n'
+                     ).format(euser, err)
             continue
         if testuser is None:
-            msg += "Failed in authenticating %s; " % (u)
-            msg += "fixed password"
-            u.set_password(pw)
-            u.save()
+            msg += _('Failed in authenticating {0}\n').format(euser)
+            msg += _('fixed password')
+            euser.set_password(epass)
+            euser.save()
             continue
     if not msg:
-        msg = "All ok!"
+        msg = _('All ok!')
     return msg
 
+
+def create_user(uname, name, password=None, do_mit=False):
+    """ Creates a user (both SSL and regular)"""
+
+    if not uname:
+        return _('Must provide username')
+    if not name:
+        return _('Must provide full name')
+
+    make_eamap = False
+
+    msg = u''
+    if do_mit:
+        if not '@' in uname:
+            email = '{0}@MIT.EDU'.format(uname)
+        else:
+            email = uname
+        if not email.endswith('@MIT.EDU'):
+            msg += u'email must end in @MIT.EDU'
+            return msg
+        mit_domain = 'ssl:MIT'
+        if ExternalAuthMap.objects.filter(external_id=email,
+                                          external_domain=mit_domain):
+            msg += _('Failed - email {0} already exists as external_id'
+                     ).format(email)
+            return msg
+        make_eamap = True
+        new_password = generate_password()
+    else:
+        if not password:
+            return _('Password must be supplied if not using certificates')
+
+        email = uname
+
+        if not '@' in email:
+            msg += _('email address required (not username)')
+            return msg
+        new_password = password
+
+    user = User(username=uname, email=email, is_active=True)
+    user.set_password(new_password)
+    try:
+        user.save()
+    except IntegrityError:
+        msg += _('Oops, failed to create user {0}, IntegrityError'
+                 ).format(user)
+        return msg
+
+    reg = Registration()
+    reg.register(user)
+
+    profile = UserProfile(user=user)
+    profile.name = name
+    profile.save()
+
+    if make_eamap:
+        credentials = \
+            '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}'.format(name, email)
+        eamap = ExternalAuthMap(
+            external_id=email,
+            external_email=email,
+            external_domain=mit_domain,
+            external_name=name,
+            internal_password=new_password,
+            external_credentials=json.dumps(credentials),
+        )
+        eamap.user = user
+        eamap.dtsignup = datetime.now()
+        eamap.save()
+
+    msg += _('User {0} created successfully!').format(user)
+    return msg
+
+
+def delete_user(uname):
+    """Deletes a user from django auth"""
+
+    if not uname:
+        return _('Must provide username')
+    if '@' in uname:
+        try:
+            user = User.objects.get(email=uname)
+        except User.DoesNotExist, err:
+            msg = _('Cannot find user with email address {0}'
+                    ).format(uname)
+            return msg
+    else:
+        try:
+            user = User.objects.get(username=uname)
+        except User.DoesNotExist, err:
+            msg = _('Cannot find user with username {0} - {1}'
+                    ).format(uname, err.msg)
+            return msg
+    user.delete()
+    return _('Deleted user {0}').format(uname)
+
+
+def return_csv(filename, datatable):
+    """Convenient function for handling the http response of a csv"""
+
+    response = HttpResponse(mimetype='text/csv')
+    response['Content-Disposition'] = 'attachment; filename={0}'.format(filename)
+
+    writer = csv.writer(response, dialect='excel', quotechar='"',
+                        quoting=csv.QUOTE_ALL)
+    writer.writerow(datatable['header'])
+    for datarow in datatable['data']:
+        encoded_row = [unicode(s).encode('utf-8') for s in datarow]
+        writer.writerow(encoded_row)
+    return response
+
+
+def get_staff_group(course):
+    """Gets staff members for course"""
+
+    return get_group(course, 'staff')
+
+
+def get_instructor_group(course):
+    """Gets instructors for course"""
+
+    return get_group(course, 'instructor')
+
+
+def get_group(course, groupname):
+    """Gets the course group"""
+
+    grpname = get_access_group_name(course, groupname)
+    try:
+        group = Group.objects.get(name=grpname)
+    except Group.DoesNotExist:
+        group = Group(name=grpname)  # create the group
+        group.save()
+    return group
+
+
+@staff_member_required
 @ensure_csrf_cookie
 @cache_control(no_cache=True, no_store=True, must_revalidate=True)
 def sysadmin_dashboard(request):
     """
     Sysadmin dashboard.
-    
     Provides:
-        
         1. enrollment numbers
         2. loading new courses from github
         3. reloading XML from files
-
     """
-    if not request.user.is_staff:
-        raise Http404
+    # pylint: disable-msg=R0915
 
-    msg = ''
-    problems = []
+    msg = u''
     plots = []
     datatable = {}
-    
+
     def_ms = modulestore()
     is_using_mongo = 'mongo' in str(def_ms.__class__)
 
     if is_using_mongo:
         courses = def_ms.get_courses()
-        courses = dict([c.id, c] for c in courses)	# no course directory
+        courses = dict([c.id, c] for c in courses)  # no course directory
     else:
         courses = def_ms.courses.items()
 
-    # the instructor dashboard page is modal: grades, psychometrics, admin
-    # keep that state in request.session (defaults to grades mode)
-    dash_mode = request.POST.get('dash_mode','')
+    # the sysadmin dashboard page is modal: status, courses, enrollment,
+    # staffing, gitlogs
+    # keep that state in request.session (defaults to status mode)
+    dash_mode = request.POST.get('dash_mode', '')
     if dash_mode:
         request.session['dash_mode'] = dash_mode
     else:
-        dash_mode = request.session.get('dash_mode','Status')
-    
-    # helper functions
-
-    def return_csv(fn, datatable, fp=None):
-        if fp is None:
-            response = HttpResponse(mimetype='text/csv')
-            response['Content-Disposition'] = 'attachment; filename={0}'.format(fn)
-        else:
-            response = fp
-        writer = csv.writer(response, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL)
-        writer.writerow(datatable['header'])
-        for datarow in datatable['data']:
-            encoded_row = [unicode(s).encode('utf-8') for s in datarow]
-            writer.writerow(encoded_row)
-        return response
-
-    def get_staff_group(course):
-        return get_group(course, 'staff')
-
-    def get_instructor_group(course):
-        return get_group(course, 'instructor')
-
-    def get_group(course, groupname):
-        grpname = get_access_group_name(course, groupname)
-        try:
-            group = Group.objects.get(name=grpname)
-        except Group.DoesNotExist:
-            group = Group(name=grpname)     # create the group
-            group.save()
-        return group
+        dash_mode = request.session.get('dash_mode', _('Status'))
 
     # default datatable depends on dash_mode
-    
-    if dash_mode=='Status':
-        datatable = dict(header=['Statistic','Value'],
-                         title="Site statistics")
-        datatable['data'] = [['Total number of users', User.objects.all().count()]]
-
-    elif dash_mode=='Courses':
+    if dash_mode == _('Status'):
+        datatable = dict(header=[_('Statistic'), _('Value')],
+                         title=_('Site statistics'))
+        datatable['data'] = [[_('Total number of users'),
+                             User.objects.all().count()]]
+    elif dash_mode == _('Courses'):
         data = []
 
-        #for cdir, course in def_ms.courses.items():
-        for cdir, course in courses.items():
-            data.append([course.display_name, cdir] + git_info_for_course(cdir))
-
-        datatable = dict(header=['Course Name', 'dir', 'git commit', 'last change', 'last editor'],
-                         title="Information about all courses",
+        if hasattr(courses, 'items'):
+            course_iter = courses.items()
+        else:
+            course_iter = courses
+        for (cdir, course) in course_iter:
+            data.append([course.display_name, cdir]
+                        + git_info_for_course(cdir))
+
+        datatable = dict(header=[_('Course Name'), _('dir'),
+                         _('git commit'), _('last change'),
+                         _('last editor')],
+                         title=_('Information about all courses'),
                          data=data)
-
-    elif dash_mode=='Enrollment' or dash_mode=="Staffing":
+    elif dash_mode == _('Staffing and Enrollment'):
         data = []
 
-        #for cdir, course in def_ms.courses.items():
-        for cdir, course in courses.items():
+        if hasattr(courses, 'items'):
+            course_iter = courses.items()
+        else:
+            course_iter = courses
+        for (cdir, course) in course_iter:
             datum = [course.display_name, course.id]
-            datum += [CourseEnrollment.objects.filter(course_id=course.id).count()]
+            datum += \
+                [CourseEnrollment.objects.filter(course_id=course.id).count()]
             datum += [get_group(course, 'staff').user_set.all().count()]
-            datum += [','.join([x.username for x in get_group(course, 'instructor').user_set.all()])]
+            datum += [','.join([x.username for x in get_group(course,
+                      'instructor').user_set.all()])]
             data.append(datum)
 
-        datatable = dict(header=['Course Name', 'course_id', '# enrolled', '# staff', 'instructors'],
-                         title="Enrollment information for all courses",
+        datatable = dict(header=[_('Course Name'), _('course_id'),
+                                 _('# enrolled'), _('# staff'), _('instructors')],
+                         title=_('Enrollment information for all courses'),
                          data=data)
 
     # process actions from form POST
     action = request.POST.get('action', '')
     track.views.server_track(request, action, {}, page='sysdashboard')
 
-    if "Download list of all users (csv file)" in action:
-        datatable = dict(header=['username', 'email'],
-                         title="List of all users",
-                         data=[[u.username, u.email] for u in User.objects.all()])
-        return return_csv('users_%s.csv' % request.META['SERVER_NAME'],datatable)
-        
-    elif "Check and repair external Auth Map" in action:
-        msg += '<pre>'
+    if _('Download list of all users (csv file)') in action:
+        datatable = dict(header=[_('username'), _('email')],
+                         title=_('List of all users'),
+                         data=[[u.username, u.email] for u in
+                               User.objects.all()])
+        return return_csv('users_{0}.csv'.format(
+            request.META['SERVER_NAME']), datatable)
+    elif _('Check and repair external Auth Map') in action:
+        msg += u'<pre>'
         msg += fix_external_auth_map_passwords()
-        msg += '</pre>'
-        datatable = {}
-
-    elif "Create user" in action:
-        uname = request.POST.get('student_uname','').strip()
-        name = request.POST.get('student_fullname','').strip()
-
-        def GenPasswd(length=8, chars=string.letters + string.digits):
-            return ''.join([choice(chars) for i in range(length)])
-
-        def create_user(uname, name, do_mit=False):
-            if not uname:
-                return "Must provide username"
-            if not name:
-                return "Must provide full name"
-            msg = ''
-            if do_mit:
-                if not '@' in uname:
-                    email = '%s@MIT.EDU' % uname
-                else:
-                    email = uname
-                if not email.endswith('@MIT.EDU'):
-                    msg += 'email must end in @MIT.EDU'
-                    return msg
-                mit_domain = 'ssl:MIT'
-                if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain):
-                    msg += "Failed - email %s already exists as external_id" % email
-                    return msg
-                make_eamap = True
-            else:
-                email = uname
-                if not '@' in email:
-                    msg += 'email address required (not username)'
-                    return msg
-            password = GenPasswd(12)
-            user = User(username=uname, email=email, is_active=True)
-            user.set_password(password)
-            try:
-                user.save()
-            except IntegrityError:
-                msg += "Oops, failed to create user %s, IntegrityError" % user
-                return msg
-                
-            r = Registration()
-            r.register(user)
-            
-            up = UserProfile(user=user)
-            up.name = name
-            up.save()
-            
-            if make_eamap:
-                credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email)
-                eamap = ExternalAuthMap(external_id = email,
-                                        external_email = email,
-                                        external_domain = mit_domain,
-                                        external_name = name,
-                                        internal_password = password,
-                                        external_credentials = json.dumps(credentials),
-                    )
-                eamap.user = user
-                eamap.dtsignup = datetime.now()
-                eamap.save()
-            
-            msg += "User %s created successfully!" % user
-            return msg
-            
-        msg += create_user(uname, name, do_mit=settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'])
+        msg += u'</pre>'
         datatable = {}
-
-
-    elif "Delete user" in action:
-        uname = request.POST.get('student_uname','').strip()
-
-        def delete_user(uname):
-            if not uname:
-                return "Must provide username"
-            if '@' in uname:
-                try:
-                    u = User.objects.get(email=uname)
-                except Exception, err:
-                    msg = "Cannot find user with email address %s" % uname
-                    return msg
-            else:
-                try:
-                    u = User.objects.get(username=uname)
-                except Exception, err:
-                    msg = "Cannot find user with username %s" % uname
-                    return msg
-            u.delete()
-            return "Deleted user %s" % uname
-
+    elif _('Create user') in action:
+        uname = request.POST.get('student_uname', '').strip()
+        name = request.POST.get('student_fullname', '').strip()
+        password = request.POST.get('student_password', '').strip()
+
+        msg += create_user(uname, name, password,
+                           do_mit=settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'])
+    elif _('Delete user') in action:
+        uname = request.POST.get('student_uname', '').strip()
         msg += delete_user(uname)
-
-    elif "Download staff and instructor list (csv file)" in action:
+    elif _('Download staff and instructor list (csv file)') in action:
         data = []
-        roles = ['instructor','staff']
-        #for cdir, course in def_ms.courses.items():
-        for cdir, course in courses.items():
+        roles = ['instructor', 'staff']
+
+        if hasattr(courses, 'items'):
+            course_iter = courses.items()
+        else:
+            course_iter = courses
+        for (cdir, course) in course_iter:
             for role in roles:
-                for u in get_group(course, role).user_set.all():
-                    datum = [course.id, role, u.username, u.email, u.profile.name]
+                for user in get_group(course, role).user_set.all():
+                    datum = [course.id, role, user.username, user.email,
+                             user.profile.name]
                     data.append(datum)
-            
-        datatable = dict(header=['course_id', 'role', 'username', 'email', 'full_name'],
-                         title="List of all course staff and instructors",
+        datatable = dict(header=[_('course_id'),
+                                 _('role'), _('username'),
+                                 _('email'), _('full_name')],
+                         title=_('List of all course staff and instructors'),
                          data=data)
-        return return_csv('staff_%s.csv' % request.META['SERVER_NAME'],datatable)
-        
-
-    elif action=="Delete course from site":
-        
-        course_id = request.POST.get('course_id','').strip()
-        ok = False
+        return return_csv('staff_{0}.csv'.format(
+            request.META['SERVER_NAME']), datatable)
+    elif action == _('Delete course from site'):
+        course_id = request.POST.get('course_id', '').strip()
+        course_found = False
         if course_id in courses:
-            ok = True
+            course_found = True
             course = courses[course_id]
         else:
             try:
                 course = get_course_by_id(course_id)
-                ok = True
-            except Exception as err:
-                msg += "Error - cannot get course with ID %s<br/><pre>%s</pre>" % (course_id, escape(err))
-
-        if ok and not is_using_mongo:
-            cdir = course.metadata.get('data_dir', course.location.course)
+                course_found = True
+            except Http404, err:
+                msg += \
+                    _('Error - cannot get course with ID {0}<br/><pre>{1}</pre>'
+                      ).format(course_id, escape(err))
+
+        if course_found and not is_using_mongo:
+            cdir = course.data_dir
             def_ms.courses.pop(cdir)
 
             # now move the directory (don't actually delete it)
-            nd = cdir + '_deleted_%s' % int(time.time())
-            os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / nd)
-            os.system('chmod -x %s' % (settings.DATA_DIR / nd))
+            new_dir = cdir + '_deleted_{0}'.format(int(time.time()))
+            os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / new_dir)
 
-            msg += "<font color='red'>Deleted %s = %s (%s)</font>" % (cdir, course.id, course.display_name)
+            msg += u"<font color='red'>Deleted {0} = {1} ({2})</font>".format(
+                cdir, course.id, course.display_name)
 
-        elif ok and is_using_mongo:
+        elif course_found and is_using_mongo:
             # delete course that is stored with mongodb backend
             loc = course.location
-            #ms = modulestore('direct')
-            cs = contentstore()
+            content_store = contentstore()
             commit = True
-            ret = delete_course(def_ms, cs, loc, commit)
+            delete_course(def_ms, content_store, loc, commit)
             # don't delete user permission groups, though
-            msg += "<font color='red'>Deleted %s = %s (%s)</font>" % (loc, course.id, course.display_name)
-    
-
-    elif action=="Load new course from github":
-        
-        gitloc = request.POST.get('repo_location','').strip().replace(' ','').replace(';','')
-
-        def get_course_from_git(gitloc):
-            msg = ''
-            if (not gitloc.endswith('.git')) or ('http:' in gitloc) or ('https:' in gitloc):
-                msg += "The git repo location should end with '.git', and be for SSH access"
-                return msg
-
-            if is_using_mongo:
-                acscript = getattr(settings, 'CMS_ADD_COURSE_SCRIPT', '')
-                if not acscript or not os.path.exists(acscript):
-                    msg = "<font color='red'>Must configure CMS_ADD_COURSE_SCRIPT in settings first!</font>"
-                    return msg
-                cmd = '{0} "{1}"'.format(acscript, gitloc)
-                logging.debug('Adding course with command: {0}'.format(cmd))
-                ret = subprocess.Popen(cmd, shell=True, executable = "/bin/bash",
-                                       stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-                ret = ''.join(ret)
-                msg = "<font color='red'>Added course from {0}</font>".format(gitloc)
-                msg += "<pre>{0}</pre>".format(ret.replace('<','&lt;'))
-                return msg
-
-            cdir = gitloc.rsplit('/',1)[1][:-4]
-            gdir = settings.DATA_DIR / cdir
-            if os.path.exists(gdir):
-                msg += "The course %s already exists in the data directory! (reloading anyway)" % cdir
-                # return msg
-            else:
-                cmd = "cd %s; git clone %s" % (settings.DATA_DIR, gitloc)
-                msg += '<pre>%s</pre>' % escape(os.popen(cmd).read())
-            if not os.path.exists(gdir):
-                msg += "Failed to clone repository to %s" % gdir
-                return msg
-            def_ms.try_load_course(cdir)	# load into modulestore
-            errlog = def_ms.errored_courses.get(cdir,'')
-            if errlog:
-                msg += '<hr width="50%%"><pre>%s</pre>' % escape(errlog)
-            else:
-                course = def_ms.courses[cdir]
-                msg += "Loaded course %s (%s)<br/>Errors:" % (cdir, course.display_name)
-                errors = def_ms.get_item_errors(course.location)
-                if not errors:
-                    msg += "None"
-                else:
-                    msg += "<ul>"
-                    for (summary, err) in errors:
-                        msg += '<li><pre>%s: %s</pre></li>' % (escape(summary), escape(err))
-                    msg += "</ul>"
-                datatable['data'].append([course.display_name, cdir] + git_info_for_course(cdir))
-            return msg
-            
-        msg += get_course_from_git(gitloc)
-
-    else:	# default to showing status summary
-        msg += '<h2>Courses loaded in the modulestore</h2>'
-        msg += '<ol>'
-        #for cdir, course in def_ms.courses.items():
-        for cdir, course in courses.items():
-            msg += '<li>%s (%s)</li>' % (escape(cdir),
-                                         course.location.url())
-        msg += '</ol>'
-
-        
-    
-    #----------------------------------------
+            msg += \
+                u"<font color='red'>{0} {1} = {2} ({3})</font>".format(
+                    _('Deleted'), loc, course.id, course.display_name)
+
+    elif action == _('Load new course from github'):
+        gitloc = request.POST.get('repo_location', '').strip().replace(
+            ' ', '').replace(';', '')
+        msg += get_course_from_git(gitloc, is_using_mongo, def_ms, datatable)
+
+    # default to showing status summary
+    else:
+        if hasattr(courses, 'items'):
+            course_iter = courses.items()
+        else:
+            course_iter = courses
+        msg += u'<h2>{0}</h2>'.format(
+            _('Courses loaded in the modulestore'))
+        msg += u'<ol>'
+        for (cdir, course) in course_iter:
+            msg += u'<li>{0} ({1})</li>'.format(
+                escape(cdir), course.location.url())
+        msg += u'</ol>'
+
+    # ----------------------------------------
     # context for rendering
 
-    context = {'datatable': datatable,
-               'plots': plots,
-               'msg': msg,
-               'djangopid' : os.getpid(),
-               'modeflag': {dash_mode: 'selectedmode'},
-               'mitx_version' : getattr(settings,'MITX_VERSION_STRING',''),
-               }
+    context = {
+        'datatable': datatable,
+        'plots': plots,
+        'msg': msg,
+        'djangopid': os.getpid(),
+        'modeflag': {dash_mode: 'selectedmode'},
+        'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+    }
 
     return render_to_response('sysadmin_dashboard.html', context)
-    
-#-----------------------------------------------------------------------------
 
-def view_git_logs(request, course_id=None):
 
-    import mongoengine	# don't import that until we need it, here
+# -----------------------------------------------------------------------------
 
-    class CourseImportLog(mongoengine.Document):
-        course_id = mongoengine.StringField(max_length=128)
-        location = mongoengine.StringField(max_length=168)
-        import_log = mongoengine.StringField(max_length=20*65535)
-        git_log = mongoengine.StringField(max_length=65535)
-        repo_dir = mongoengine.StringField(max_length=128)
-        created = mongoengine.DateTimeField()
-        meta = { 'indexes': ['course_id', 'created'],
-                 'allow_inheritance': False, }
-
-    DBNAME = "xlog"
+@staff_member_required
+def view_git_logs(request, course_id=None):
+    """Shows logs of imports that happened as a result of a git import"""
+    # pylint: disable-msg=W0613
+
+    # Set defaults even if it isn't defined in settings
+    mongo_db = {
+        'host': 'localhost',
+        'user': '',
+        'password': '',
+        'db': 'xlog',
+    }
+
+    # Allow overrides
+    if hasattr(settings, 'MONGODB_LOG'):
+        for config_item in ['host', 'user', 'password', 'db', ]:
+            mongo_db[config_item] = settings.MONGODB_LOG.get(
+                config_item, mongo_db[config_item])
+
+    mongouri = 'mongodb://{0}/{1}'.format(mongo_db['host'],
+                                          mongo_db['db'])
+
+    try:
+        mdb = mongoengine.connect(mongo_db['db'], host=mongouri,
+                                  username=mongo_db['user'],
+                                  password=mongo_db['password'])
+    except mongoengine.connection.ConnectionError, e:
+        logging.critical(_('Unable to connect to mongodb to save log, please check '
+                           'MONGODB_LOG settings. error: {0}').format(str(e)))
 
-    mdb = mongoengine.connect(DBNAME)
-    
     if course_id is None:
         cilset = CourseImportLog.objects.all().order_by('-created')
     else:
-        log.debug('course_id=%s' % course_id)
-        cilset = CourseImportLog.objects.filter(course_id=course_id).order_by('-created')
-        log.debug('cilset length=%s' % len(cilset))
-        
-    context = {'cilset': cilset,
-               'course_id': course_id,
-               }
-
-    return render_to_response('sysadmin_dashboard_gitlogs.html', context)
-    
-    
-    
\ No newline at end of file
+        log.debug('course_id={0}'.format(course_id))
+        cilset = CourseImportLog.objects.filter(
+            course_id=course_id).order_by('-created')
+        log.debug('cilset length={0}'.format(len(cilset)))
+    mdb.disconnect()
+    context = {'cilset': cilset, 'course_id': course_id}
+
+    return render_to_response('sysadmin_dashboard_gitlogs.html',
+                              context)
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
new file mode 100644
index 000000000000..8f8e1fd1dda0
--- /dev/null
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -0,0 +1,370 @@
+"""
+Provide tests for sysadmin dashboard feature in sysadmin.py
+"""
+
+import unittest
+import os
+import shutil
+
+from django.test.client import Client
+from django.test.utils import override_settings
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.urlresolvers import reverse
+from django.utils.translation import ugettext as _
+
+from dashboard.sysadmin import create_user
+from external_auth.models import ExternalAuthMap
+from django.contrib.auth.hashers import check_password
+from xmodule.modulestore.django import modulestore
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+from django.utils.html import escape
+from dashboard.sysadmin import CourseImportLog
+import mongoengine
+
+TEST_MONGODB_LOG = {
+    'host': 'localhost',
+    'user': '',
+    'password': '',
+    'db': 'test_xlog',
+}
+
+
+class SysadminBaseTestCase(ModuleStoreTestCase):
+    """ Base class with common methods used in XML and Mongo tests"""
+
+    def setUp(self):
+        super(SysadminBaseTestCase, self).setUp()
+        self.user = User.objects.create_user('test_user', 'test_user+sysadmin@edx.org', 'foo')
+        self.client = Client()
+
+    def _setstaff_login(self):
+        """ Makes the test user staff and logs them in"""
+
+        self.user.is_staff = True
+        self.user.save()
+        self.client.login(username=self.user.username, password='foo')
+
+    def _add_edx4edx(self):
+        """Adds the edx4edx sample course"""
+
+        return self.client.post(reverse('sysadmin'), {
+            'dash_mode': _('Courses'),
+            'repo_location': 'https://github.com/mitocw/edx4edx_lite.git',
+            'action': _('Load new course from github'), })
+
+    def _rm_edx4edx(self):
+        """Deletes the sample course from the XML store"""
+        # pylint: disable-msg=E1103
+
+        def_ms = modulestore()
+        try:
+            # using XML stor
+            course = def_ms.courses.get('{0}/edx4edx_lite'.format(os.path.abspath(settings.DATA_DIR)), None)
+        except AttributeError:
+            # Using mongo store
+            course = def_ms.get_course('MITx/edx4edx/edx4edx')
+
+        # Delete git loaded course
+        return self.client.post(reverse('sysadmin'),
+                                {'dash_mode': _('Courses'),
+                                 'course_id': course.id,
+                                 'action': _('Delete course from site'), })
+
+
+@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set")
+class TestSysadmin(SysadminBaseTestCase):
+    """
+    Check that landing page is the status page
+    """
+
+    def test_staff_access(self):
+        # pylint: disable-msg=E1103
+
+        response = self.client.get(reverse('sysadmin'))
+        self.assertEqual('/sysadmin', response.context['next'])
+
+        response = self.client.get(reverse('gitlogs'))
+        self.assertEqual('/gitlogs', response.context['next'])
+
+        logged_in = self.client.login(username=self.user.username,
+                                      password='foo')
+        self.assertTrue(logged_in)
+
+        response = self.client.get(reverse('sysadmin'))
+        self.assertEqual('/sysadmin', response.context['next'])
+
+        response = self.client.get(reverse('gitlogs'))
+        self.assertEqual('/gitlogs', response.context['next'])
+
+        self.user.is_staff = True
+        self.user.save()
+
+        self.client.logout()
+        self.client.login(username=self.user.username, password='foo')
+
+        response = self.client.get(reverse('sysadmin'))
+        self.assertFalse(hasattr(response.context, 'next'))
+
+        response = self.client.get(reverse('gitlogs'))
+        self.assertFalse(hasattr(response.context, 'next'))
+
+    def test_user_mod(self):
+        """Create and delete a user"""
+
+        self._setstaff_login()
+
+        self.client.login(username=self.user.username, password='foo')
+
+        # Create user
+        self.client.post(reverse('sysadmin'),
+                         {'dash_mode': _('Status'),
+                          'action': _('Create user'),
+                          'student_uname': 'test_cuser+sysadmin@edx.org',
+                          'student_fullname': 'test cuser',
+                          'student_password': 'foozor', })
+
+        self.assertIsNotNone(
+            User.objects.get(username='test_cuser+sysadmin@edx.org',
+                             email='test_cuser+sysadmin@edx.org'))
+
+        # login as new user to confirm
+        self.assertTrue(self.client.login(username='test_cuser+sysadmin@edx.org',
+                                          password='foozor'))
+
+        self.client.logout()
+        self.client.login(username=self.user.username, password='foo')
+
+        # Delete user
+        self.client.post(reverse('sysadmin'),
+                         {'dash_mode': _('Status'),
+                          'action': _('Delete user'),
+                          'student_uname': 'test_cuser+sysadmin@edx.org',
+                          'student_fullname': 'test cuser', })
+
+        self.assertEqual(0, len(User.objects.filter(
+            username='test_cuser+sysadmin@edx.org',
+            email='test_cuser+sysadmin@edx.org')))
+
+        self.assertEqual(1, len(User.objects.all()))
+
+    def test_user_csv(self):
+        """Download and validate user CSV"""
+
+        self._setstaff_login()
+
+        response = self.client.post(reverse('sysadmin'), {
+            'dash_mode': _('Status'),
+            'action': _('Download list of all users (csv file)'),
+        })
+
+        self.assertIn('attachment', response['Content-Disposition'])
+        self.assertEqual('text/csv', response['Content-Type'])
+        self.assertIn('test_user', response.content)
+        self.assertTrue(2, len(response.content.splitlines()))
+
+    def test_authmap_repair(self):
+        """Run authmap check and repair"""
+
+        self._setstaff_login()
+
+        create_user('test0', 'test test', do_mit=True)
+        # Will raise exception, so no assert needed
+        eamap = ExternalAuthMap.objects.get(external_name='test test')
+        mitu = User.objects.get(username='test0')
+
+        self.assertTrue(check_password(eamap.internal_password, mitu.password))
+
+        mitu.set_password('not autogenerated')
+        mitu.save()
+
+        self.assertFalse(check_password(eamap.internal_password, mitu.password))
+
+        response = self.client.post(reverse('sysadmin'), {
+            'dash_mode': _('Status'),
+            'action': _('Check and repair external Auth Map'), })
+
+        self.assertIn('{0} test0'.format(_('Failed in authenticating')), response.content)
+        self.assertIn(_('fixed password'), response.content)
+
+        self.assertTrue(self.client.login(username='test0', password=eamap.internal_password))
+
+        # Check for all OK
+        self._setstaff_login()
+        response = self.client.post(reverse('sysadmin'),
+                                    {'dash_mode': _('Status'),
+                                     'action': _('Check and repair external Auth Map'), })
+        self.assertIn(_('All ok!'), response.content)
+
+    def test_xml_course_add_delete(self):
+        """add and delete course from xml module store"""
+
+        self._setstaff_login()
+
+        # Try bad git repo
+        response = self.client.post(reverse('sysadmin'), {
+            'dash_mode': _('Courses'),
+            'repo_location': 'github.com/mitocw/edx4edx_lite',
+            'action': _('Load new course from github'), })
+        self.assertIn(_("The git repo location should end with '.git', and be a valid url"), response.content.decode('utf-8'))
+
+        # Create git loaded course
+        response = self._add_edx4edx()
+
+        def_ms = modulestore()
+        self.assertIn('xml', str(def_ms.__class__))
+        course = def_ms.courses.get('{0}/edx4edx_lite'.format(
+            os.path.abspath(settings.DATA_DIR)), None)
+        self.assertIsNotNone(course)
+
+        response = self._rm_edx4edx()
+        course = def_ms.courses.get('{0}/edx4edx_lite'.format(
+            os.path.abspath(settings.DATA_DIR)), None)
+        self.assertIsNone(course)
+
+    def test_git_pull(self):
+        """Make sure we can pull"""
+
+        self._setstaff_login()
+
+        response = self._add_edx4edx()
+        response = self._add_edx4edx()
+        self.assertIn(_("The course {0} already exists in the data directory! "
+                        "(reloading anyway)").format('edx4edx_lite'),
+                      response.content.decode('utf-8'))
+        self._rm_edx4edx()
+
+    def test_staff_csv(self):
+        """Download and validate staff CSV"""
+
+        self._setstaff_login()
+        self._add_edx4edx()
+
+        response = self.client.post(reverse('sysadmin'), {
+            'dash_mode': _('Staffing and Enrollment'),
+            'action': _('Download staff and instructor list (csv file)'),
+        })
+
+        self.assertIn('attachment', response['Content-Disposition'])
+        self.assertEqual('text/csv', response['Content-Type'])
+        columns = [_('course_id'), _('role'), _('username'), _('email'), _('full_name'), ]
+        self.assertIn(','.join('"' + c + '"' for c in columns), response.content)
+
+        self._rm_edx4edx()
+
+    def test_enrollment_page(self):
+        """
+        Adds a course and makes sure that it shows up on the staffing and
+        enrollment page
+        """
+
+        self._setstaff_login()
+        self._add_edx4edx()
+        response = self.client.post(reverse('sysadmin'), {'dash_mode': _('Staffing and Enrollment')})
+        print(response.content)
+        self.assertIn('edx4edx', response.content)
+        self._rm_edx4edx()
+
+
+@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
+@override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
+@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set")
+class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
+    """
+    Check that importing into the mongo module store works
+    """
+
+    @classmethod
+    def tearDownClass(cls):
+        super(TestSysAdminMongoCourseImport, cls).tearDownClass()
+        # Delete git repos and mongo objects
+        try:
+            shutil.rmtree(getattr(settings, 'GIT_REPO_DIR'))
+        except OSError:
+            pass
+
+        try:
+            mongoengine.connect(TEST_MONGODB_LOG['db'])
+            CourseImportLog.objects.all().delete()
+        except mongoengine.connection.ConnectionError:
+            pass
+
+    def _setstaff_login(self):
+        """ Makes the test user staff and logs them in"""
+
+        self.user.is_staff = True
+        self.user.save()
+
+        self.client.login(username=self.user.username, password='foo')
+
+    def test_missing_repo_dir(self):
+        """Ensure that we handle a missing repo dir"""
+
+        self._setstaff_login()
+
+        if os.path.isdir(getattr(settings, 'GIT_REPO_DIR')):
+            shutil.rmtree(getattr(settings, 'GIT_REPO_DIR'))
+
+        # Create git loaded course
+        response = self._add_edx4edx()
+        self.assertIn(escape(_("Path {0} doesn't exist, please create it, or configure a "
+                               "different path with GIT_REPO_DIR").format(settings.GIT_REPO_DIR)),
+                      response.content.decode('UTF-8'))
+
+    def test_mongo_course_add_delete(self):
+        """same as TestSysadmin.test_xml_course_add_delete, but use mongo store"""
+
+        self._setstaff_login()
+        try:
+            os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
+        except OSError:
+            pass
+
+        def_ms = modulestore()
+        self.assertIn('mongo', str(def_ms.__class__))
+
+        self._add_edx4edx()
+        course = def_ms.get_course('MITx/edx4edx/edx4edx')
+        self.assertIsNotNone(course)
+
+        self._rm_edx4edx()
+        course = def_ms.get_course('MITx/edx4edx/edx4edx')
+        self.assertIsNone(course)
+
+    def test_gitlogs(self):
+        """Create a log entry and make sure it exists"""
+
+        self._setstaff_login()
+        try:
+            os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
+        except OSError:
+            pass
+
+        self._add_edx4edx()
+        response = self.client.get(reverse('gitlogs'))
+        print(response.content)
+        print(CourseImportLog.objects.all())
+        # Check that our earlier import has a log with a link to details
+        self.assertIn('/gitlogs/MITx/edx4edx/edx4edx', response.content)
+
+        response = self.client.get(
+            reverse('gitlogs_detail', kwargs={'course_id': 'MITx/edx4edx/edx4edx'}))
+
+        self.assertIn('======&gt; IMPORTING course to location', response.content)
+
+        self._rm_edx4edx()
+
+    @override_settings(GIT_ADD_COURSE_SCRIPT='')
+    def test_no_script_set(self):
+        """ Test if settings are right on mongo store import"""
+
+        self._setstaff_login()
+
+        response = self.client.post(reverse('sysadmin'), {
+            'dash_mode': _('Courses'),
+            'repo_location': 'https://github.com/mitocw/edx4edx_lite.git',
+            'action': _('Load new course from github'), })
+        self.assertIn(_('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'),
+                      response.content)
diff --git a/lms/envs/aws.py b/lms/envs/aws.py
index 436563913a18..4c63ecda4a86 100644
--- a/lms/envs/aws.py
+++ b/lms/envs/aws.py
@@ -199,6 +199,11 @@
 FEEDBACK_SUBMISSION_EMAIL = ENV_TOKENS.get("FEEDBACK_SUBMISSION_EMAIL")
 MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS)
 
+# git repo loading  environment
+GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', None)
+GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', None)
+GIT_ADD_COURSE_SCRIPT = ENV_TOKENS.get('GIT_ADD_COURSE_SCRIPT', None)
+
 for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items():
     oldvalue = CODE_JAIL.get(name)
     if isinstance(oldvalue, dict):
@@ -242,6 +247,7 @@
 MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE)
 CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE)
 DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG',DOC_STORE_CONFIG)
+MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG')
 
 OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE',
                                                OPEN_ENDED_GRADING_INTERFACE)
diff --git a/lms/envs/common.py b/lms/envs/common.py
index 03cdfe74d0e7..ff1d953d503c 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -484,6 +484,7 @@
 # These are listed, eg at https://github.com/MITx/mitx/admin/hooks
 
 ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
+GIT_ADD_COURSE_SCRIPT = REPO_ROOT / "scripts/git_add_course.py"
 
 #################################### AWS #######################################
 # S3BotoStorage insists on a timeout for uploaded assets. We should make it
diff --git a/lms/envs/dev.py b/lms/envs/dev.py
index 9d3a4f8a955e..bfbafd4b4fa1 100644
--- a/lms/envs/dev.py
+++ b/lms/envs/dev.py
@@ -31,8 +31,8 @@
 MITX_FEATURES['ENABLE_SERVICE_STATUS'] = True
 MITX_FEATURES['ENABLE_INSTRUCTOR_EMAIL'] = True
 MITX_FEATURES['ENABLE_HINTER_INSTRUCTOR_VIEW'] = True
-MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
 MITX_FEATURES['ENABLE_INSTRUCTOR_BETA_DASHBOARD'] = True
+MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
 MITX_FEATURES['MULTIPLE_ENROLLMENT_ROLES'] = True
 MITX_FEATURES['ENABLE_SHOPPING_CART'] = True
 MITX_FEATURES['AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'] = True
@@ -264,10 +264,6 @@
 if SEGMENT_IO_LMS_KEY:
     MITX_FEATURES['SEGMENT_IO_LMS'] = True
 
-########################## GIT WORKFLOW ########################
-
-CMS_ADD_COURSE_SCRIPT = REPO_ROOT / "scripts/cms_git_add_course"
-
 ###################### Payment ##############################3
 
 CC_PROCESSOR['CyberSource']['SHARED_SECRET'] = os.environ.get('CYBERSOURCE_SHARED_SECRET', '')
diff --git a/lms/envs/test.py b/lms/envs/test.py
index 42a4d8443658..7291c408419f 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -197,6 +197,10 @@
 CC_PROCESSOR['CyberSource']['SERIAL_NUMBER'] = "0123456789012345678901"
 CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = "/shoppingcart/payment_fake"
 
+########################### SYSADMIN DASHBOARD ################################
+MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
+GIT_REPO_DIR = TEST_ROOT / "course_repos"
+
 ################################# CELERY ######################################
 
 CELERY_ALWAYS_EAGER = True
diff --git a/lms/templates/sysadmin_dashboard.html b/lms/templates/sysadmin_dashboard.html
index ff09485690ed..28cbfb05a823 100644
--- a/lms/templates/sysadmin_dashboard.html
+++ b/lms/templates/sysadmin_dashboard.html
@@ -1,9 +1,10 @@
 <%inherit file="/main.html" />
 <%! from django.core.urlresolvers import reverse %>
+<%! from django.utils.translation import ugettext as _ %>
 <%namespace name='static' file='/static_content.html'/>
 
 <%block name="headextra">
-  <%static:css group='course'/>
+  <%static:css group='style-course'/>
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
 </%block>
@@ -50,97 +51,107 @@
 <section class="container">
 <div class="sysadmin-dashboard-wrapper">
 
-  <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt"
->
-    <h1>Sysadmin Dashboard</h1>
+  <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
+    <h1>${_('Sysadmin Dashboard')}</h1>
 
-    <h2>[ <a href="#" onclick="goto('Status');" class="${modeflag.get('Status')}">Status</a> |
-          <a href="#" onclick="goto('Courses');" class="${modeflag.get('Courses')}">Courses</a> |
-          <a href="#" onclick="goto('Enrollment');" class="${modeflag.get('Enrollment')}">Enrollment</a> |
-          <a href="#" onclick="goto('Staffing');" class="${modeflag.get('Staffing')}">Staffing</a> |
-          <a href="/gitlogs">GitLogs</a>
+    <h2>[ <a href="#" onclick="goto('Status');" class="${modeflag.get('Status')}">${_('Status')}</a> |
+          <a href="#" onclick="goto('Courses');" class="${modeflag.get('Courses')}">${_('Courses')}</a> |
+          <a href="#" onclick="goto('Staffing and Enrollment');" class="${modeflag.get('Staffing and Enrollment')}">${_('Staffing and Enrollment')}</a> |
+          <a href="/gitlogs">${_('GitLogs')}</a>
            ]
     </h2>
 
-  <div style="text-align:right"><span id="djangopid">${djangopid}</span>
-  | <span id="mitxver">${mitx_version}</span></div>
-
     <form name="dashform" method="POST">
     <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
     <input type="hidden" name="dash_mode" value="">
 
 ##----------------------------------------------------------------------------- 
 %if modeflag.get('Status'):
-
-    <p>
-    Email or username: <input type="text" name="student_uname" size=40 >
-    <input type="submit" name="action" value="Delete user">
-    </p>
-
+	<h3>${_('User Management')}</h3>
+	<ul class="list-input">
+	  <li class="field text" style="padding-bottom: 1.2em">
+		<label for="student_uname">${_('Email or username')}</label>
+		<input type="text" name="student_uname" size=40 />
+	  </li>
+
+	  <li class="field text">
+		<label for="student_fullname">${_('Full Name')}</label>
+		<input type="text" name="student_fullname" size=40 />
+	  </li>
+	  <li class="field text">
+		<label for="student_password">${_('Password')}</label>
+		<input type="password" name="student_password" size=40 />
+	  </li>
+	</ul>
+
+	<div class="form-actions">
+	  <p>
+		<input type="submit" name="action" value="${_('Delete user')}" />
+		<input type="submit" name="action" value="${_('Create user')}">
+	  </p>
+    </div>
+
+	<hr />
     <p>
-    Full Name: <input type="text" name="student_fullname" size=40 >
-    <input type="submit" name="action" value="Create user">
+    <input type="submit" name="action" value="${_('Download list of all users (csv file)')}" style="width: 300px;">
     </p>
 
     <p>
-    <input type="submit" name="action" value="Download list of all users (csv file)">
-    </p>
-
-    <p>
-    <input type="submit" name="action" value="Check and repair external Auth Map">
+    <input type="submit" name="action" value="${_('Check and repair external Auth Map')}" style="width: 300px;">
     </p>
 
     <hr width="40%" style="align:left">
  %endif
 
 ##----------------------------------------------------------------------------- 
-%if modeflag.get('Enrollment'):
+%if modeflag.get('Staffing and Enrollment'):
 
-<p>Go to each individual course's Instructor dashboard to manage
-course enrollment.</p>
+<p>${_("Go to each individual course's Instructor dashboard to manage course enrollment.")}</p>
+<hr />
 
-%endif
-
-##----------------------------------------------------------------------------- 
-%if modeflag.get('Staffing'):
-
-<h3>Manage course staff and instructors</h3><br/>
+<h3>${_('Manage course staff and instructors')}</h3><br/>
 
     <p>
-    <input type="submit" name="action" value="Download staff and instructor list (csv file)">
+    <input type="submit" name="action" value="${_('Download staff and instructor list (csv file)')}">
     </p>
-<!--
-    <p>You may upload a list of staff/instructors for all the courses.
-    The safest approach is to download the list, modify it, then upload.</p>
-    <p><font color="red">Careful!  The uploaded file <b>overwrites</b>
-    existing list, globally covering all courses!</font></p>
-    <p>
-    Staff and instructor list (CSV file): <input type="file" name="staff_file" >
-    <input type="submit" name="action" value="Upload staff and instructor list">
-    </p>
--->
 
 %endif
 
 ##----------------------------------------------------------------------------- 
 %if modeflag.get('Courses'):
-    <h3>Administer Courses</h3><br/>
-
-    <p>
-    Repo location (ssh): <input type="text" name="repo_location" size=60 >
-    <input type="submit" name="action" value="Load new course from github">
-    </p>
-    <p>
-    Course ID or dir: <input type="text" name="course_id" size=60 >
-    <input type="submit" name="action" value="Delete course from site">
-    </p>
+    <h3>${_('Administer Courses')}</h3><br/>
+
+    <ul class="list-input">
+	  <li class="field text">
+		<label for="repo_location">
+		  ${_('Repo location')}:
+		</label>
+		<input type="text" name="repo_location" style="width:60%" />
+	  </li>
+	</ul>
+	<div class="form-actions">
+	  <input type="submit" name="action" value="${_('Load new course from github')}">
+	</div>
+	<hr />
+    <ul class="list-input">
+	  <li class="field text">
+		<label for="course_id">
+		  ${_('Course ID or dir')}:
+		</label>
+		<input type="text" name="course_id" style="width:60%" />
+	  </li>
+	</ul>
+	<div class="form-actions">
+      <input type="submit" name="action" value="${_('Delete course from site')}">
+	</div>
+	<hr style="width:40%" />
 %endif
 
 ##----------------------------------------------------------------------------- 
     </form>
 
 %if msg:
-    <p></p><p>${msg}</p>
+    <p>${msg}</p>
 %endif
 ##----------------------------------------------------------------------------- 
 ##----------------------------------------------------------------------------- 
@@ -192,5 +203,7 @@ <h3>${plot['title']}</h3>
 %endif
 
   </section>
+  <div style="text-align:right; float: right"><span id="djangopid">${_('Django PID')}: ${djangopid}</span>
+  | <span id="mitxver">${_('Platform Version')}: ${mitx_version}</span></div>
 </div>
 </section>
diff --git a/lms/templates/sysadmin_dashboard_gitlogs.html b/lms/templates/sysadmin_dashboard_gitlogs.html
index 64bcc9d15560..be44bf060e75 100644
--- a/lms/templates/sysadmin_dashboard_gitlogs.html
+++ b/lms/templates/sysadmin_dashboard_gitlogs.html
@@ -1,13 +1,20 @@
 <%inherit file="/main.html" />
 <%! from django.core.urlresolvers import reverse %>
+<%! from django.utils.translation import ugettext as _ %>
 <%namespace name='static' file='/static_content.html'/>
 
 <%block name="headextra">
-  <%static:css group='course'/>
+  <%static:css group='style-course'/>
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
 </%block>
-
+<script language="JavaScript" type="text/javascript">
+function goto( mode)
+{
+  document.dashform.dash_mode.value = mode;
+  document.dashform.submit() ;
+}
+</script>
 <style type="text/css">
 table.stat_table {
 	font-family: verdana,arial,sans-serif;
@@ -43,31 +50,44 @@
 <div class="sysadmin-dashboard-wrapper">
 
   <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
-    <h1>git logs</h1>
+    <h1>${_('Sysadmin Dashboard')}</h1>
 
-% if course_id is None:
+    <h2>[ <a href="#" onclick="goto('Status');">${_('Status')}</a> |
+          <a href="#" onclick="goto('Courses');">${_('Courses')}</a> |
+         <a href="#" onclick="goto('Staffing and Enrollment');">${_('Staffing and Enrollment')}</a> |
+           <a href="${reverse('gitlogs')}" class="selectedmode">${_('GitLogs')}</a>
+           ]
+    </h2>
 
-    <table class="stat_table">
-    <tr><th>Date</th><th>Course ID</th><th>git action</th></tr>
+    <form name="dashform" method="POST" action="${reverse('sysadmin')}">
+    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
+    <input type="hidden" name="dash_mode" value="">
 
-    % for cil in cilset[:10]:
+    <h3>${_('Git Logs')}</h3>
 
-        <tr>
-	    <td>${cil.created}</td>
-	    <td><a href="/gitlogs/${cil.course_id}">${cil.course_id}</a></td>
-	    <td>${cil.git_log}</td>
-	</tr>
+%if course_id is None:
 
-    % endfor
-    
+    <table class="stat_table">
+	  <thead>
+		<tr><th>${_('Date')}</th><th>${_('Course ID')}</th><th>${_('Git Action')}</th></tr>
+	  </thead>
+	  <tbody>
+		%for cil in cilset[:10]:
+        <tr>
+	      <td>${cil.created}</td>
+	      <td><a href="/gitlogs/${cil.course_id}">${cil.course_id}</a></td>
+	      <td>${cil.git_log}</td>
+		</tr>
+		%endfor
+	  </tbody>
     </table>
 
-% else:
+%else:
 
-    <h2>Recent git load activity for ${course_id}</h2>
+    <h2>${_('Recent git load activity for')} ${course_id}</h2>
 
     <table class="stat_table">
-    <tr><th>Date</th><th>Course ID</th><th>git action</th></tr>
+    <tr><th>${_('Date')}</th><th>${_('Course ID')}</th><th>${_('git action')}</th></tr>
 
     % for cil in cilset[:2]:
 
@@ -78,7 +98,7 @@ <h2>Recent git load activity for ${course_id}</h2>
 	</tr>
 	
 	<tr><td colspan="3">
-	    <pre>${cil.import_log}</pre>
+	    <pre>${cil.import_log | h}</pre>
 	</td></tr>
 
     % endfor
diff --git a/lms/urls.py b/lms/urls.py
index 678bf7871fc4..8c844d7ca0c3 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -88,7 +88,8 @@
     urlpatterns += (
         url(r'^sysadmin$', 'dashboard.sysadmin.sysadmin_dashboard', name="sysadmin"),
         url(r'^gitlogs$', 'dashboard.sysadmin.view_git_logs', name="gitlogs"),
-        url(r'^gitlogs/(?P<course_id>.+)$', 'dashboard.sysadmin.view_git_logs'),
+        url(r'^gitlogs/(?P<course_id>.+)$', 'dashboard.sysadmin.view_git_logs',
+            name="gitlogs_detail"),
     )
 
 #Semi-static views (these need to be rendered and have the login bar, but don't change)
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index 6c23b2b3add6..2994f8881922 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -40,7 +40,7 @@ lazy==1.1
 lxml==3.0.1
 mako==0.7.3
 Markdown==2.2.1
-mongoengine==0.8.3
+mongoengine==0.7.10
 networkx==1.7
 nltk==2.0.4
 oauthlib==0.5.1
diff --git a/scripts/cms_git_add_course b/scripts/cms_git_add_course
deleted file mode 100755
index 3b159a4916a1..000000000000
--- a/scripts/cms_git_add_course
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/usr/bin/env python
-#
-# python script to pull a git repo and import into cms / edge mongodb content database.
-#
-# usage:
-#
-#    python cms_git_add_course <git-ssh-url> [<directory>]
-#
-# argument is git ssh url, like: git@github.com:mitocw/edx4edx_lite.git
-# if the directory is given, that is used and presumed to contain the git repo
-# 
-
-import os, sys, string, re
-import datetime
-import mongoengine		# used to store import log 
-
-DIR = "/mnt/data_import"
-
-#-----------------------------------------------------------------------------
-# pull from git
-
-repo = sys.argv[1]
-m = re.match('git@[^ ]+\.git', repo)
-if not m:
-    print "Oops, not a git ssh url?"
-    print "Expecting something like git@github.com:mitocw/edx4edx_lite.git"
-    sys.exit(-1)
-
-if len(sys.argv)>2:
-    rdir = sys.argv[2]
-    rdir = os.path.basename(rdir)
-else:
-    rdir = repo.rsplit('/',1)[-1].rsplit('.git',1)[0]
-print "rdir = %s" % rdir
-
-rdirp = '%s/%s' % (DIR, rdir)
-if os.path.exists(rdirp):
-    print "directory already exists, doing a git pull instead of git clone"
-    cmd = 'cd %s/%s; git pull' % (DIR, rdir)
-else:
-    cmd = 'cd %s; git clone "%s"' % (DIR, repo)
-
-print cmd
-ret_git = os.popen(cmd).read()
-print ret_git
-
-if not os.path.exists('%s/%s' % (DIR, rdir)):
-    print "git clone failed!"
-    sys.exit(-1)
-
-# get commit id
-commit_id = os.popen('cd %s; git log -n 1 | head -1' % rdirp).read().strip().split(' ')[1]
-
-ret_git += "\nCommit ID: %s" % commit_id
-
-# get branch
-branch = ''
-for k in os.popen('cd %s; git branch' % rdirp).readlines():
-    if k[0]=='*':
-        branch = k[2:].strip()
-
-ret_git += "   \nBranch: %s" % branch
-
-#-----------------------------------------------------------------------------
-# import
-
-#cmd = "./DJANGO-ADMIN-CMS import %s %s" % (DIR, rdir)
-cmd = "./DJANGO-ADMIN-CMS import --nostatic %s %s" % (DIR, rdir)
-print cmd
-ret_import = os.popen(cmd).read()
-print ret_import
-
-course_id = "unknown"
-location = "unknown"
-
-# extract course ID from output of import-command-run and make symlink
-# this is needed in order for custom course scripts to work
-m = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)', ret_import)
-if m:
-    location = m.group(1).strip()
-    print "location = %s" % location
-    course_id = location.replace('i4x://','').replace('/course/','/').split('\n')[0].strip()
-    
-    cdir = "%s/%s" % (DIR, course_id.split('/')[1])
-    print "Studio course dir = %s" % cdir
-    
-    if os.path.exists(cdir) and not os.path.islink(cdir):
-        print "   -> exists, but is not symlink"
-        print os.popen('ls -l %s' % cdir).read()
-        print os.popen('rmdir %s' % cdir).read()
-
-    if not os.path.exists(cdir):
-        print "   -> creating symlink"
-        print os.popen('ln -s %s %s' % (rdirp, cdir)).read()
-        print os.popen('ls -l %s' % cdir).read()
-        
-#-----------------------------------------------------------------------------
-# store import-command-run output in mongo
-
-class CourseImportLog(mongoengine.Document):
-    course_id = mongoengine.StringField(max_length=128)
-    location = mongoengine.StringField(max_length=168)
-    import_log = mongoengine.StringField(max_length=20*65535)
-    git_log = mongoengine.StringField(max_length=65535)
-    repo_dir = mongoengine.StringField(max_length=128)
-    created = mongoengine.DateTimeField()
-    meta = { 'indexes': ['course_id', 'created'],
-             'allow_inheritance': False, }
-
-DBNAME = "xlog"
-
-mdb = mongoengine.connect(DBNAME)
-
-cil = CourseImportLog(course_id=course_id,
-                      location=location,
-                      repo_dir=rdir,
-                      created=datetime.datetime.now(),
-                      import_log=ret_import,
-                      git_log=ret_git)
-cil.save()
-
-print "saved CourseImportLog for %s" % cil.course_id
diff --git a/scripts/git_add_course.py b/scripts/git_add_course.py
new file mode 100755
index 000000000000..6cd53bad0df9
--- /dev/null
+++ b/scripts/git_add_course.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# pylint: disable-msg=C0111
+# python script to pull a git repo and import into cms / edge mongodb content database.
+#
+# usage:
+#
+#    python git_add_course.py <git-ssh-url> [<directory>]
+#
+# argument is git ssh url, like: git@github.com:mitocw/edx4edx_lite.git
+# if the directory is given, that is used and presumed to contain the git repo
+#
+
+import os
+import sys
+import re
+import datetime
+import mongoengine  # used to store import log
+import StringIO
+import logging
+
+from django.utils.translation import ugettext as _
+
+from django.conf import settings
+from django.core import management
+from django.core.management.base import CommandError
+from xmodule.modulestore.django import modulestore
+
+log = logging.getLogger(__name__)
+
+GIT_REPO_DIR = getattr(settings,
+                       'GIT_REPO_DIR', '/opt/edx/course_repos')
+GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
+
+
+class CourseImportLog(mongoengine.Document):
+    """Mongoengine model for git log"""
+    # pylint: disable-msg=R0924
+
+    course_id = mongoengine.StringField(max_length=128)
+    location = mongoengine.StringField(max_length=168)
+    import_log = mongoengine.StringField(max_length=20 * 65535)
+    git_log = mongoengine.StringField(max_length=65535)
+    repo_dir = mongoengine.StringField(max_length=128)
+    created = mongoengine.DateTimeField()
+    meta = {'indexes': ['course_id', 'created'],
+            'allow_inheritance': False}
+
+
+def add_repo(repo, rdir_in):
+    """This will add a git repo into the mongo modulestore"""
+    # pylint: disable-msg=R0915
+
+    # Set defaults even if it isn't defined in settings
+    mongo_db = {
+        'host': 'localhost',
+        'user': '',
+        'password': '',
+        'db': 'xlog',
+    }
+
+    # Allow overrides
+    if hasattr(settings, 'MONGODB_LOG'):
+        for config_item in ['host', 'user', 'password', 'db', ]:
+            mongo_db[config_item] = settings.MONGODB_LOG.get(
+                config_item, mongo_db[config_item])
+
+    if not os.path.isdir(GIT_REPO_DIR):
+        log.critical(_("Path {0} doesn't exist, please create it, or configure a "
+                       "different path with GIT_REPO_DIR").format(GIT_REPO_DIR))
+        return -1
+
+    # -----------------------------------------------------------------------------
+    # pull from git
+    if not (repo.endswith('.git') or repo.startswith('http:') or
+       repo.startswith('https:') or repo.startswith('git:')):
+
+        log.error(_('Oops, not a git ssh url?'))
+        log.error(_('Expecting something like git@github.com:mitocw/edx4edx_lite.git'))
+        return -1
+
+    if rdir_in:
+        rdir = rdir_in
+        rdir = os.path.basename(rdir)
+    else:
+        rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
+
+    log.debug('rdir = {0}'.format(rdir))
+
+    rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
+    if os.path.exists(rdirp):
+        log.info(_('directory already exists, doing a git pull instead of git clone'))
+        cmd = 'cd {0}/{1}; git pull'.format(GIT_REPO_DIR, rdir)
+    else:
+        cmd = 'cd {0}; git clone "{1}"'.format(GIT_REPO_DIR, repo)
+
+    log.debug(cmd)
+    ret_git = os.popen(cmd).read()
+    log.debug(ret_git)
+
+    if not os.path.exists('{0}/{1}'.format(GIT_REPO_DIR, rdir)):
+        log.error(_('git clone failed!'))
+        return -1
+
+    # get commit id
+    commit_id = os.popen('cd {0}; git log -n 1 | head -1'.format(rdirp)).read().strip().split(' ')[1]
+
+    ret_git += _('\nCommit ID: {0}').format(commit_id)
+
+    # get branch
+    branch = ''
+    for k in os.popen('cd {0}; git branch'.format(rdirp)).readlines():
+        if k[0] == '*':
+            branch = k[2:].strip()
+
+    ret_git += '   \nBranch: {0}'.format(branch)
+
+    # Get XML logging logger and capture debug to parse results
+    output = StringIO.StringIO()
+    import_logger = logging.getLogger('xmodule.modulestore.xml_importer')
+    import_log_handler = logging.StreamHandler(output)
+    import_log_handler.setLevel(logging.DEBUG)
+    import_logger.addHandler(import_log_handler)
+    try:
+        management.call_command('import', GIT_REPO_DIR, rdir,
+                                nostatic=not GIT_IMPORT_STATIC)
+    except CommandError, ex:
+        log.critical(_('Unable to run import command.'))
+        log.critical(_('Error was {0}').format(str(ex)))
+        return -1
+
+    ret_import = output.getvalue()
+
+    course_id = 'unknown'
+    location = 'unknown'
+
+    # extract course ID from output of import-command-run and make symlink
+    # this is needed in order for custom course scripts to work
+    match = re.search('(?ms)===> IMPORTING course to location ([^ \n]+)',
+                      ret_import)
+    if match:
+        location = match.group(1).strip()
+        log.debug('location = {0}'.format(location))
+        course_id = location.replace('i4x://', '').replace(
+            '/course/', '/').split('\n')[0].strip()
+
+        cdir = '{0}/{1}'.format(GIT_REPO_DIR, course_id.split('/')[1])
+        log.debug(_('Studio course dir = {0}').format(cdir))
+
+        if os.path.exists(cdir) and not os.path.islink(cdir):
+            log.debug(_('   -> exists, but is not symlink'))
+            log.debug(os.popen('ls -l {0}'.format(cdir)).read())
+            log.debug(os.popen('rmdir {0}'.format(cdir)).read())
+
+        if not os.path.exists(cdir):
+            log.debug(_('   -> creating symlink'))
+            log.debug(os.popen('ln -s {0} {1}'.format(rdirp,
+                      cdir)).read())
+            log.debug(os.popen('ls -l {0}'.format(cdir)).read())
+
+    # -----------------------------------------------------------------------------
+    # store import-command-run output in mongo
+    mongouri = 'mongodb://{0}/{1}'.format(mongo_db['host'],
+                                          mongo_db['db'])
+    try:
+        mdb = mongoengine.connect(mongo_db['db'], host=mongouri,
+                                  username=mongo_db['user'],
+                                  password=mongo_db['password'])
+    except mongoengine.connection.ConnectionError, ex:
+        log.critical(_('Unable to connect to mongodb to save log, please '
+                       'check MONGODB_LOG settings'))
+        log.critical(_('Error was: {0}').format(str(ex)))
+        return -1
+    logging.critical(mongoengine.connection.get_db())
+    cil = CourseImportLog(
+        course_id=course_id,
+        location=location,
+        repo_dir=rdir,
+        created=datetime.datetime.now(),
+        import_log=ret_import,
+        git_log=ret_git,
+    )
+    cil.save()
+
+    log.debug(_('saved CourseImportLog for {0}').format(cil.course_id))
+    mdb.disconnect()
+    return 0
+
+if __name__ == '__main__':
+    # pylint: disable-msg=C0103
+
+    if len(sys.argv) < 2:
+        print(_('This script requires at least one argument, the git URL'))
+        sys.exit(1)
+
+    if len(sys.argv) > 3:
+        print(_('This script requires no more than two arguments.'))
+        sys.exit(1)
+
+    rdir_arg = None
+
+    # check that we are using mongo modulestore
+    if not 'mongo' in str(modulestore().__class__):
+        print _('This script requires a mongo module store')
+        sys.exit(1)
+
+    if len(sys.argv) > 2:
+        rdir_arg = sys.argv[2]
+
+    if add_repo(sys.argv[1], rdir_arg) != 0:
+        sys.exit(1)

From 0c0b28cbc67029431bb9d15e209c9e39cbbb2c43 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Fri, 25 Oct 2013 17:53:11 -0400
Subject: [PATCH 04/20] After testing with mongo cluster with aws.py settings,
 there are a few fixes

---
 lms/djangoapps/dashboard/sysadmin.py |  9 ++++-----
 lms/envs/aws.py                      |  4 ++--
 scripts/git_add_course.py            | 10 +++++-----
 3 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 37b7f1b881e0..1e959ede3aae 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -541,13 +541,12 @@ def view_git_logs(request, course_id=None):
             mongo_db[config_item] = settings.MONGODB_LOG.get(
                 config_item, mongo_db[config_item])
 
-    mongouri = 'mongodb://{0}/{1}'.format(mongo_db['host'],
-                                          mongo_db['db'])
+    mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
+        mongo_db['user'], mongo_db['password'],
+        mongo_db['host'], mongo_db['db'])
 
     try:
-        mdb = mongoengine.connect(mongo_db['db'], host=mongouri,
-                                  username=mongo_db['user'],
-                                  password=mongo_db['password'])
+        mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
     except mongoengine.connection.ConnectionError, e:
         logging.critical(_('Unable to connect to mongodb to save log, please check '
                            'MONGODB_LOG settings. error: {0}').format(str(e)))
diff --git a/lms/envs/aws.py b/lms/envs/aws.py
index 4c63ecda4a86..b83b5a7c3a2f 100644
--- a/lms/envs/aws.py
+++ b/lms/envs/aws.py
@@ -201,8 +201,8 @@
 
 # git repo loading  environment
 GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', None)
-GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', None)
-GIT_ADD_COURSE_SCRIPT = ENV_TOKENS.get('GIT_ADD_COURSE_SCRIPT', None)
+GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True)
+GIT_ADD_COURSE_SCRIPT = ENV_TOKENS.get('GIT_ADD_COURSE_SCRIPT', GIT_ADD_COURSE_SCRIPT)
 
 for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items():
     oldvalue = CODE_JAIL.get(name)
diff --git a/scripts/git_add_course.py b/scripts/git_add_course.py
index 6cd53bad0df9..a1188c80ac65 100755
--- a/scripts/git_add_course.py
+++ b/scripts/git_add_course.py
@@ -161,12 +161,12 @@ def add_repo(repo, rdir_in):
 
     # -----------------------------------------------------------------------------
     # store import-command-run output in mongo
-    mongouri = 'mongodb://{0}/{1}'.format(mongo_db['host'],
-                                          mongo_db['db'])
+    mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
+        mongo_db['user'], mongo_db['password'],
+        mongo_db['host'], mongo_db['db'])
+
     try:
-        mdb = mongoengine.connect(mongo_db['db'], host=mongouri,
-                                  username=mongo_db['user'],
-                                  password=mongo_db['password'])
+        mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
     except mongoengine.connection.ConnectionError, ex:
         log.critical(_('Unable to connect to mongodb to save log, please '
                        'check MONGODB_LOG settings'))

From fc3baf7d6fdc4f6385632865fa4e7e319aa6ed82 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Mon, 28 Oct 2013 09:57:27 -0400
Subject: [PATCH 05/20] Erroneous logging message removed

---
 scripts/git_add_course.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/git_add_course.py b/scripts/git_add_course.py
index a1188c80ac65..4307b0388217 100755
--- a/scripts/git_add_course.py
+++ b/scripts/git_add_course.py
@@ -172,7 +172,6 @@ def add_repo(repo, rdir_in):
                        'check MONGODB_LOG settings'))
         log.critical(_('Error was: {0}').format(str(ex)))
         return -1
-    logging.critical(mongoengine.connection.get_db())
     cil = CourseImportLog(
         course_id=course_id,
         location=location,

From 7deed40171856a67b8b04fcc073672809f348700 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Mon, 28 Oct 2013 13:16:33 -0400
Subject: [PATCH 06/20] More production/dev fixes

---
 lms/djangoapps/dashboard/sysadmin.py | 12 +++++++++---
 scripts/git_add_course.py            |  5 ++++-
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 1e959ede3aae..6b9795a4e6f5 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -111,6 +111,8 @@ def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
         import_log_handler.setLevel(logging.DEBUG)
 
         for logger in [import_logger, git_logger, xml_logger, ]:
+            logger.old_level = logger.level
+            logger.setLevel(logging.DEBUG)
             logger.addHandler(import_log_handler)
 
         git_add_script.add_repo(gitloc, None)
@@ -119,6 +121,7 @@ def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
 
         # Remove handler hijacks
         for logger in [import_logger, git_logger, xml_logger, ]:
+            logger.setLevel(logger.old_level)
             logger.removeHandler(import_log_handler)
         msg = u"<font color='red'>{0} {1}</font>".format(
             _('Added course from'), gitloc)
@@ -546,10 +549,13 @@ def view_git_logs(request, course_id=None):
         mongo_db['host'], mongo_db['db'])
 
     try:
-        mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
-    except mongoengine.connection.ConnectionError, e:
+        if mongo_db['user'] and mongo_db['password']:
+            mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
+        else:
+            mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
+    except mongoengine.connection.ConnectionError, ex:
         logging.critical(_('Unable to connect to mongodb to save log, please check '
-                           'MONGODB_LOG settings. error: {0}').format(str(e)))
+                           'MONGODB_LOG settings. error: {0}').format(str(ex)))
 
     if course_id is None:
         cilset = CourseImportLog.objects.all().order_by('-created')
diff --git a/scripts/git_add_course.py b/scripts/git_add_course.py
index 4307b0388217..469e2233f11a 100755
--- a/scripts/git_add_course.py
+++ b/scripts/git_add_course.py
@@ -166,7 +166,10 @@ def add_repo(repo, rdir_in):
         mongo_db['host'], mongo_db['db'])
 
     try:
-        mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
+        if mongo_db['user'] and mongo_db['password']:
+            mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
+        else:
+            mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
     except mongoengine.connection.ConnectionError, ex:
         log.critical(_('Unable to connect to mongodb to save log, please '
                        'check MONGODB_LOG settings'))

From 096d0ac6b521492262244d0df0fe86e0bb0a281c Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Mon, 28 Oct 2013 14:09:25 -0400
Subject: [PATCH 07/20] Added addtional import logger for debugging course
 imports

---
 lms/djangoapps/dashboard/sysadmin.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 6b9795a4e6f5..5475ab8d41a3 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -106,11 +106,12 @@ def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
             'xmodule.modulestore.xml_importer')
         git_logger = logging.getLogger('git_add_script')
         xml_logger = logging.getLogger('xmodule.modulestore.xml')
+        xml_seq_logger = logging.getLogger('xmodule.seq_module')
 
         import_log_handler = logging.StreamHandler(output)
         import_log_handler.setLevel(logging.DEBUG)
 
-        for logger in [import_logger, git_logger, xml_logger, ]:
+        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
             logger.old_level = logger.level
             logger.setLevel(logging.DEBUG)
             logger.addHandler(import_log_handler)
@@ -120,7 +121,7 @@ def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
         ret = output.getvalue()
 
         # Remove handler hijacks
-        for logger in [import_logger, git_logger, xml_logger, ]:
+        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
             logger.setLevel(logger.old_level)
             logger.removeHandler(import_log_handler)
         msg = u"<font color='red'>{0} {1}</font>".format(

From 57fe6679b94cab4c757b026c7b6f4028cbb89f87 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Wed, 30 Oct 2013 12:41:18 -0400
Subject: [PATCH 08/20] Added additional loggers for storing in mongo instead
 of just showing on manual import

---
 scripts/git_add_course.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/scripts/git_add_course.py b/scripts/git_add_course.py
index 469e2233f11a..245913436421 100755
--- a/scripts/git_add_course.py
+++ b/scripts/git_add_course.py
@@ -119,9 +119,18 @@ def add_repo(repo, rdir_in):
     # Get XML logging logger and capture debug to parse results
     output = StringIO.StringIO()
     import_logger = logging.getLogger('xmodule.modulestore.xml_importer')
+    git_logger = logging.getLogger('git_add_script')
+    xml_logger = logging.getLogger('xmodule.modulestore.xml')
+    xml_seq_logger = logging.getLogger('xmodule.seq_module')
+
     import_log_handler = logging.StreamHandler(output)
     import_log_handler.setLevel(logging.DEBUG)
-    import_logger.addHandler(import_log_handler)
+
+    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+        logger.old_level = logger.level
+        logger.setLevel(logging.DEBUG)
+        logger.addHandler(import_log_handler)
+
     try:
         management.call_command('import', GIT_REPO_DIR, rdir,
                                 nostatic=not GIT_IMPORT_STATIC)
@@ -132,6 +141,11 @@ def add_repo(repo, rdir_in):
 
     ret_import = output.getvalue()
 
+    # Remove handler hijacks
+    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+        logger.setLevel(logger.old_level)
+        logger.removeHandler(import_log_handler)
+
     course_id = 'unknown'
     location = 'unknown'
 

From 2c2296922165c57e334c831539306154c0f1e7d2 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Tue, 5 Nov 2013 17:40:49 -0500
Subject: [PATCH 09/20] This changes it to allow course staff to see their git
 import logs

---
 lms/djangoapps/dashboard/sysadmin.py          | 14 +++++-
 .../dashboard/tests/test_sysadmin.py          | 49 ++++++++++++++++---
 2 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 5475ab8d41a3..ae9e7025d286 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -31,12 +31,14 @@
 from django.http import HttpResponse
 from django.utils.html import escape
 from django.contrib.admin.views.decorators import staff_member_required
+from django.contrib.auth.decorators import login_required
 from django.http import Http404
 
 from mitxmako.shortcuts import render_to_response
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.store_utilities import delete_course
+from courseware.access import has_access
 import mongoengine
 
 import track.views
@@ -526,7 +528,7 @@ def sysadmin_dashboard(request):
 
 # -----------------------------------------------------------------------------
 
-@staff_member_required
+@login_required
 def view_git_logs(request, course_id=None):
     """Shows logs of imports that happened as a result of a git import"""
     # pylint: disable-msg=W0613
@@ -559,8 +561,18 @@ def view_git_logs(request, course_id=None):
                            'MONGODB_LOG settings. error: {0}').format(str(ex)))
 
     if course_id is None:
+
+        # Require staff if not going to specific course
+        if not request.user.is_staff:
+            raise Http404
         cilset = CourseImportLog.objects.all().order_by('-created')
     else:
+        course = get_course_by_id(course_id)
+        # Allow only course team, instructors, and staff
+        if not (request.user.is_staff or
+                has_access(request.user, course, 'instructor') or
+                has_access(request.user, course, 'staff')):
+            raise Http404
         log.debug('course_id={0}'.format(course_id))
         cilset = CourseImportLog.objects.filter(
             course_id=course_id).order_by('-created')
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 8f8e1fd1dda0..c345b3b3c7f3 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -17,10 +17,12 @@
 from dashboard.sysadmin import create_user
 from external_auth.models import ExternalAuthMap
 from django.contrib.auth.hashers import check_password
+from django.contrib.auth.models import Group
 from xmodule.modulestore.django import modulestore
 from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from django.utils.html import escape
+from courseware.access import get_access_group_name, has_access
 from dashboard.sysadmin import CourseImportLog
 import mongoengine
 
@@ -86,9 +88,6 @@ def test_staff_access(self):
         response = self.client.get(reverse('sysadmin'))
         self.assertEqual('/sysadmin', response.context['next'])
 
-        response = self.client.get(reverse('gitlogs'))
-        self.assertEqual('/gitlogs', response.context['next'])
-
         logged_in = self.client.login(username=self.user.username,
                                       password='foo')
         self.assertTrue(logged_in)
@@ -97,7 +96,7 @@ def test_staff_access(self):
         self.assertEqual('/sysadmin', response.context['next'])
 
         response = self.client.get(reverse('gitlogs'))
-        self.assertEqual('/gitlogs', response.context['next'])
+        self.assertEqual(response.status_code, 404)
 
         self.user.is_staff = True
         self.user.save()
@@ -344,8 +343,7 @@ def test_gitlogs(self):
 
         self._add_edx4edx()
         response = self.client.get(reverse('gitlogs'))
-        print(response.content)
-        print(CourseImportLog.objects.all())
+
         # Check that our earlier import has a log with a link to details
         self.assertIn('/gitlogs/MITx/edx4edx/edx4edx', response.content)
 
@@ -356,6 +354,45 @@ def test_gitlogs(self):
 
         self._rm_edx4edx()
 
+    def test_gitlog_courseteam_access(self):
+        """Ensure course team users are allowed to access only their own course"""
+
+        try:
+            os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
+        except OSError:
+            pass
+
+        self._setstaff_login()
+        self._add_edx4edx()
+        self.user.is_staff = False
+        self.user.save()
+        logged_in = self.client.login(username=self.user.username,
+                                      password='foo')
+        response = self.client.get(reverse('gitlogs'))
+
+        # Make sure our non privileged user doesn't have access to all logs
+        self.assertEqual(response.status_code, 404)
+
+        # Add user as staff in course team
+        def_ms = modulestore()
+        course = def_ms.get_course('MITx/edx4edx/edx4edx')
+        
+        staff_groupname = get_access_group_name(course, 'staff')
+        group, _ = Group.objects.get_or_create(name=staff_groupname)
+        self.user.groups.add(group)
+        
+        self.assertTrue(has_access(self.user, course, 'staff'))
+        logged_in = self.client.login(username=self.user.username,
+                                      password='foo')
+        self.assertTrue(logged_in)
+
+        response = self.client.get(
+            reverse('gitlogs_detail', kwargs={'course_id': 'MITx/edx4edx/edx4edx'}))
+        print(response.content)
+        self.assertIn('======&gt; IMPORTING course to location', response.content)
+
+        self._rm_edx4edx()
+
     @override_settings(GIT_ADD_COURSE_SCRIPT='')
     def test_no_script_set(self):
         """ Test if settings are right on mongo store import"""

From 705f623fd1e35cd0ee95a955f91a27596b3abe67 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Thu, 14 Nov 2013 13:04:30 -0500
Subject: [PATCH 10/20] =?UTF-8?q?Code=20review=20based=20fixes:=20nitpicks?=
 =?UTF-8?q?=20style=20L70-71:=20subprocess=20at=20top=20of=20sysadmin=20L7?=
 =?UTF-8?q?7:=20git=20log=20=E2=80=94format=20L351:=20modulestore=20type?=
 =?UTF-8?q?=20check=20-=20use=20isinstance=20and=20if=20not=20xml=20use=20?=
 =?UTF-8?q?mongo=20so=20that=20mongo=20is=20used=20in=20mixed=20modulestor?=
 =?UTF-8?q?e=20L91:=20split=20mongo=20import=20and=20xml=20import=20L258:?=
 =?UTF-8?q?=20Use=20setting=20for=20eamap,=20do=5Fmit,=20etc?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 lms/djangoapps/dashboard/sysadmin.py          | 147 ++++++++++--------
 .../dashboard/tests/test_sysadmin.py          |   4 +-
 2 files changed, 84 insertions(+), 67 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index ae9e7025d286..a3b788b3d061 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -6,6 +6,7 @@
 import json
 import logging
 import os
+import subprocess
 import time
 import imp
 import StringIO
@@ -15,8 +16,7 @@
 from django.conf import settings
 from django.contrib.auth.models import User, Group
 from django.utils.translation import ugettext as _
-from student.models import CourseEnrollment
-from student.models import UserProfile, Registration
+from student.models import CourseEnrollment, UserProfile, Registration
 from external_auth.models import ExternalAuthMap
 from external_auth.views import generate_password
 
@@ -36,7 +36,9 @@
 
 from mitxmako.shortcuts import render_to_response
 from xmodule.modulestore.django import modulestore
+
 from xmodule.contentstore.django import contentstore
+from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.store_utilities import delete_course
 from courseware.access import has_access
 import mongoengine
@@ -65,71 +67,72 @@ def git_info_for_course(cdir):
 
     cmd = ''
     gdir = settings.DATA_DIR / cdir
-    info = [gdir, '', '']
-    if os.path.exists(gdir):
-        cmd = 'cd {0}; git log -1'.format(gdir)
-        for k in os.popen(cmd).readlines():
-            if 'commit' in k:
-                info[0] = k.split()[1]
-            elif 'Author' in k:
-                info[2] = k.split()[1]
-            elif 'Date' in k:
-                info[1] = k.split(' ', 1)[1].strip()
-    return info
+    info = ['', '', '']
+    if not os.path.exists(gdir):
+        return info
 
+    cmd = ['git', 'log', '-1',
+           '--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}',]
+    try:
+        output_json = json.loads(subprocess.check_output(cmd, cwd=gdir))
+        info = [ output_json['commit'],
+                 output_json['date'], 
+                 output_json['author'], ]
+    except (ValueError, subprocess.CalledProcessError):
+        pass
 
-def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
-    """This downloads and runs the checks for importing a course in git"""
+    return info
 
-    msg = u''
-    if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
-       gitloc.startswith('https:') or gitloc.startswith('git:')):
-        msg += \
-            _("The git repo location should end with '.git', and be a valid url")
+def import_mongo_course(gitloc, def_ms):
+    """
+    Imports course using management command and captures logging output
+    at debug level for display in template
+    """
+
+    acscript = getattr(settings, 'GIT_ADD_COURSE_SCRIPT', '')
+    if not acscript or not os.path.exists(acscript):
+        msg = u"<font color='red'>{0} - {1}</font>".format(
+            _('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'), acscript)
         return msg
 
-    if is_using_mongo:
-        acscript = getattr(settings, 'GIT_ADD_COURSE_SCRIPT', '')
-        if not acscript or not os.path.exists(acscript):
-            msg = u"<font color='red'>{0} - {1}</font>".format(
-                _('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'), acscript)
-            return msg
+    # import course script directly and call add_repo function
+    git_add_script = imp.load_source('git_add_script', acscript)
+    logging.debug(
+        _('Adding course using add repo from {0} and repo {1}').format(
+            acscript, gitloc))
 
-        # import course script directly and call add_repo function
-        git_add_script = imp.load_source('git_add_script', acscript)
-        logging.debug(
-            _('Adding course using add repo from {0} and repo {1}').format(
-                acscript, gitloc))
+    # Grab logging output for debugging imports
+    output = StringIO.StringIO()
 
-        # Grab logging output for debugging imports
-        output = StringIO.StringIO()
+    import_logger = logging.getLogger(
+        'xmodule.modulestore.xml_importer')
+    git_logger = logging.getLogger('git_add_script')
+    xml_logger = logging.getLogger('xmodule.modulestore.xml')
+    xml_seq_logger = logging.getLogger('xmodule.seq_module')
 
-        import_logger = logging.getLogger(
-            'xmodule.modulestore.xml_importer')
-        git_logger = logging.getLogger('git_add_script')
-        xml_logger = logging.getLogger('xmodule.modulestore.xml')
-        xml_seq_logger = logging.getLogger('xmodule.seq_module')
+    import_log_handler = logging.StreamHandler(output)
+    import_log_handler.setLevel(logging.DEBUG)
 
-        import_log_handler = logging.StreamHandler(output)
-        import_log_handler.setLevel(logging.DEBUG)
+    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+        logger.old_level = logger.level
+        logger.setLevel(logging.DEBUG)
+        logger.addHandler(import_log_handler)
 
-        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
-            logger.old_level = logger.level
-            logger.setLevel(logging.DEBUG)
-            logger.addHandler(import_log_handler)
+    git_add_script.add_repo(gitloc, None)
 
-        git_add_script.add_repo(gitloc, None)
+    ret = output.getvalue()
 
-        ret = output.getvalue()
+    # Remove handler hijacks
+    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+        logger.setLevel(logger.old_level)
+        logger.removeHandler(import_log_handler)
+    msg = u"<font color='red'>{0} {1}</font>".format(
+        _('Added course from'), gitloc)
+    msg += _("<pre>{0}</pre>").format(escape(ret))
+    return msg
 
-        # Remove handler hijacks
-        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
-            logger.setLevel(logger.old_level)
-            logger.removeHandler(import_log_handler)
-        msg = u"<font color='red'>{0} {1}</font>".format(
-            _('Added course from'), gitloc)
-        msg += _("<pre>{0}</pre>").format(escape(ret))
-        return msg
+def import_xml_course(gitloc, def_ms, datatable):
+    """Imports a git course into the XMLModuleStore"""
 
     cdir = (gitloc.rsplit('/', 1)[1])[:-4]
     gdir = settings.DATA_DIR / cdir
@@ -165,6 +168,19 @@ def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
                                  + git_info_for_course(cdir))
     return msg
 
+def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
+    """This downloads and runs the checks for importing a course in git"""
+
+    msg = u''
+    if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
+            gitloc.startswith('https:') or gitloc.startswith('git:')):
+        msg += _("The git repo location should end with '.git', and be a valid url")
+        return msg
+
+    if is_using_mongo:
+        return import_mongo_course(gitloc, def_ms)
+
+    return import_xml_course(gitloc, def_ms, datatable)
 
 def fix_external_auth_map_passwords():
     """
@@ -195,7 +211,7 @@ def fix_external_auth_map_passwords():
     return msg
 
 
-def create_user(uname, name, password=None, do_mit=False):
+def create_user(uname, name, password=None):
     """ Creates a user (both SSL and regular)"""
 
     if not uname:
@@ -203,10 +219,8 @@ def create_user(uname, name, password=None, do_mit=False):
     if not name:
         return _('Must provide full name')
 
-    make_eamap = False
-
     msg = u''
-    if do_mit:
+    if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
         if not '@' in uname:
             email = '{0}@MIT.EDU'.format(uname)
         else:
@@ -220,7 +234,6 @@ def create_user(uname, name, password=None, do_mit=False):
             msg += _('Failed - email {0} already exists as external_id'
                      ).format(email)
             return msg
-        make_eamap = True
         new_password = generate_password()
     else:
         if not password:
@@ -249,9 +262,8 @@ def create_user(uname, name, password=None, do_mit=False):
     profile.name = name
     profile.save()
 
-    if make_eamap:
-        credentials = \
-            '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}'.format(name, email)
+    if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
+        credentials = '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}'.format(name, email)
         eamap = ExternalAuthMap(
             external_id=email,
             external_email=email,
@@ -348,7 +360,11 @@ def sysadmin_dashboard(request):
     datatable = {}
 
     def_ms = modulestore()
-    is_using_mongo = 'mongo' in str(def_ms.__class__)
+
+    # Prefer mongo if using mixed or mongo store
+    is_using_mongo = True
+    if isinstance(def_ms, XMLModuleStore):
+        is_using_mongo = False
 
     if is_using_mongo:
         courses = def_ms.get_courses()
@@ -379,6 +395,8 @@ def sysadmin_dashboard(request):
         else:
             course_iter = courses
         for (cdir, course) in course_iter:
+            if '/' in cdir:
+                cdir = cdir.rsplit('/', 1)[1]
             data.append([course.display_name, cdir]
                         + git_info_for_course(cdir))
 
@@ -429,8 +447,7 @@ def sysadmin_dashboard(request):
         name = request.POST.get('student_fullname', '').strip()
         password = request.POST.get('student_password', '').strip()
 
-        msg += create_user(uname, name, password,
-                           do_mit=settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'])
+        msg += create_user(uname, name, password)
     elif _('Delete user') in action:
         uname = request.POST.get('student_uname', '').strip()
         msg += delete_user(uname)
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index c345b3b3c7f3..941f20c9509d 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -63,7 +63,7 @@ def _rm_edx4edx(self):
 
         def_ms = modulestore()
         try:
-            # using XML stor
+            # using XML store
             course = def_ms.courses.get('{0}/edx4edx_lite'.format(os.path.abspath(settings.DATA_DIR)), None)
         except AttributeError:
             # Using mongo store
@@ -79,7 +79,7 @@ def _rm_edx4edx(self):
 @unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set")
 class TestSysadmin(SysadminBaseTestCase):
     """
-    Check that landing page is the status page
+    Test sysadmin dashboard features using XMLModuleStore
     """
 
     def test_staff_access(self):

From aed34d73fbef54355b82fb5eb91df0865709fe0d Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Thu, 14 Nov 2013 18:42:06 -0500
Subject: [PATCH 11/20] Refactor of git_add_course.py from shell script to
 management command

---
 .../dashboard/management/__init__.py          |  0
 .../dashboard/management/commands/__init__.py |  0
 .../management/commands}/git_add_course.py    | 65 ++++++++-----------
 lms/djangoapps/dashboard/sysadmin.py          | 37 ++++-------
 .../dashboard/tests/test_sysadmin.py          | 17 ++---
 lms/envs/aws.py                               |  1 -
 lms/envs/common.py                            |  2 +-
 7 files changed, 47 insertions(+), 75 deletions(-)
 create mode 100644 lms/djangoapps/dashboard/management/__init__.py
 create mode 100644 lms/djangoapps/dashboard/management/commands/__init__.py
 rename {scripts => lms/djangoapps/dashboard/management/commands}/git_add_course.py (84%)
 mode change 100755 => 100644

diff --git a/lms/djangoapps/dashboard/management/__init__.py b/lms/djangoapps/dashboard/management/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/lms/djangoapps/dashboard/management/commands/__init__.py b/lms/djangoapps/dashboard/management/commands/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/scripts/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py
old mode 100755
new mode 100644
similarity index 84%
rename from scripts/git_add_course.py
rename to lms/djangoapps/dashboard/management/commands/git_add_course.py
index 245913436421..63b49f9faaee
--- a/scripts/git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py
@@ -1,17 +1,6 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# pylint: disable-msg=C0111
-# python script to pull a git repo and import into cms / edge mongodb content database.
-#
-# usage:
-#
-#    python git_add_course.py <git-ssh-url> [<directory>]
-#
-# argument is git ssh url, like: git@github.com:mitocw/edx4edx_lite.git
-# if the directory is given, that is used and presumed to contain the git repo
-#
-
+"""
+Script for importing courseware from git/xml into a mongo modulestore
+"""
 import os
 import sys
 import re
@@ -21,19 +10,18 @@
 import logging
 
 from django.utils.translation import ugettext as _
-
 from django.conf import settings
 from django.core import management
-from django.core.management.base import CommandError
+from django.core.management.base import BaseCommand, CommandError, make_option
+
 from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.xml import XMLModuleStore
 
 log = logging.getLogger(__name__)
 
-GIT_REPO_DIR = getattr(settings,
-                       'GIT_REPO_DIR', '/opt/edx/course_repos')
+GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos')
 GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
 
-
 class CourseImportLog(mongoengine.Document):
     """Mongoengine model for git log"""
     # pylint: disable-msg=R0924
@@ -47,7 +35,6 @@ class CourseImportLog(mongoengine.Document):
     meta = {'indexes': ['course_id', 'created'],
             'allow_inheritance': False}
 
-
 def add_repo(repo, rdir_in):
     """This will add a git repo into the mongo modulestore"""
     # pylint: disable-msg=R0915
@@ -203,26 +190,30 @@ def add_repo(repo, rdir_in):
     mdb.disconnect()
     return 0
 
-if __name__ == '__main__':
-    # pylint: disable-msg=C0103
+class Command(BaseCommand):
+    """
+    Pull a git repo and import into the mongo based content database.
+    """
+
+    help = _('Import the specified git repository into the modulestore and directory')
+
+    def handle(self, *args, **options):
+        """Check inputs and run the command"""
+        # pylint: disable-msg=C0103
 
-    if len(sys.argv) < 2:
-        print(_('This script requires at least one argument, the git URL'))
-        sys.exit(1)
+        if isinstance(modulestore, XMLModuleStore):
+            raise CommandError(_('This script requires a mongo module store'))
 
-    if len(sys.argv) > 3:
-        print(_('This script requires no more than two arguments.'))
-        sys.exit(1)
+        if len(args) < 1:
+            raise CommandError(_('This script requires at least one argument, the git URL'))
 
-    rdir_arg = None
+        if len(args) > 2:
+            raise CommandError(_('This script requires no more than two arguments.'))
 
-    # check that we are using mongo modulestore
-    if not 'mongo' in str(modulestore().__class__):
-        print _('This script requires a mongo module store')
-        sys.exit(1)
+        rdir_arg = None
 
-    if len(sys.argv) > 2:
-        rdir_arg = sys.argv[2]
+        if len(args) > 1:
+            rdir_arg = args[1]
 
-    if add_repo(sys.argv[1], rdir_arg) != 0:
-        sys.exit(1)
+        if add_repo(args[0], rdir_arg) != 0:
+            raise CommandError(_('Repo was not added, check log output for details'))
diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index a3b788b3d061..d85486242fb8 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -20,11 +20,10 @@
 from external_auth.models import ExternalAuthMap
 from external_auth.views import generate_password
 
-from courseware.access import get_access_group_name
-from courseware.courses import get_course_by_id
-
 from django.contrib.auth import authenticate
 from django.core.exceptions import PermissionDenied
+from django.core import management
+from django.core.management.base import CommandError
 from django_future.csrf import ensure_csrf_cookie
 from django.views.decorators.cache import cache_control
 from django.db import IntegrityError
@@ -36,12 +35,14 @@
 
 from mitxmako.shortcuts import render_to_response
 from xmodule.modulestore.django import modulestore
-
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.store_utilities import delete_course
-from courseware.access import has_access
+from courseware.access import has_access, get_access_group_name
+from courseware.courses import get_course_by_id
+
 import mongoengine
+import dashboard.management.commands.git_add_course as git_add_course
 
 import track.views
 
@@ -89,24 +90,15 @@ def import_mongo_course(gitloc, def_ms):
     at debug level for display in template
     """
 
-    acscript = getattr(settings, 'GIT_ADD_COURSE_SCRIPT', '')
-    if not acscript or not os.path.exists(acscript):
-        msg = u"<font color='red'>{0} - {1}</font>".format(
-            _('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'), acscript)
-        return msg
+    msg = u''
 
-    # import course script directly and call add_repo function
-    git_add_script = imp.load_source('git_add_script', acscript)
-    logging.debug(
-        _('Adding course using add repo from {0} and repo {1}').format(
-            acscript, gitloc))
+    logging.debug(_('Adding course using git repo {0}').format(gitloc))
 
     # Grab logging output for debugging imports
     output = StringIO.StringIO()
 
-    import_logger = logging.getLogger(
-        'xmodule.modulestore.xml_importer')
-    git_logger = logging.getLogger('git_add_script')
+    import_logger = logging.getLogger('xmodule.modulestore.xml_importer')
+    git_logger = logging.getLogger('dashboard.management.commands.git_add_course')
     xml_logger = logging.getLogger('xmodule.modulestore.xml')
     xml_seq_logger = logging.getLogger('xmodule.seq_module')
 
@@ -118,15 +110,15 @@ def import_mongo_course(gitloc, def_ms):
         logger.setLevel(logging.DEBUG)
         logger.addHandler(import_log_handler)
 
-    git_add_script.add_repo(gitloc, None)
 
+    git_add_course.add_repo(gitloc, None)
     ret = output.getvalue()
 
     # Remove handler hijacks
     for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
         logger.setLevel(logger.old_level)
         logger.removeHandler(import_log_handler)
-    msg = u"<font color='red'>{0} {1}</font>".format(
+    msg = u"<h4 style='color:blue'>{0} {1}</h4>".format(
         _('Added course from'), gitloc)
     msg += _("<pre>{0}</pre>").format(escape(ret))
     return msg
@@ -134,6 +126,7 @@ def import_mongo_course(gitloc, def_ms):
 def import_xml_course(gitloc, def_ms, datatable):
     """Imports a git course into the XMLModuleStore"""
 
+    msg = u''
     cdir = (gitloc.rsplit('/', 1)[1])[:-4]
     gdir = settings.DATA_DIR / cdir
     if os.path.exists(gdir):
@@ -171,11 +164,9 @@ def import_xml_course(gitloc, def_ms, datatable):
 def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
     """This downloads and runs the checks for importing a course in git"""
 
-    msg = u''
     if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
             gitloc.startswith('https:') or gitloc.startswith('git:')):
-        msg += _("The git repo location should end with '.git', and be a valid url")
-        return msg
+        return _("The git repo location should end with '.git', and be a valid url")
 
     if is_using_mongo:
         return import_mongo_course(gitloc, def_ms)
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 941f20c9509d..77551f9eb37d 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -33,6 +33,8 @@
     'db': 'test_xlog',
 }
 
+MITX_FEATURES_WITH_SSL_AUTH = settings.MITX_FEATURES.copy()
+MITX_FEATURES_WITH_SSL_AUTH['AUTH_USE_MIT_CERTIFICATES'] = True
 
 class SysadminBaseTestCase(ModuleStoreTestCase):
     """ Base class with common methods used in XML and Mongo tests"""
@@ -164,12 +166,13 @@ def test_user_csv(self):
         self.assertIn('test_user', response.content)
         self.assertTrue(2, len(response.content.splitlines()))
 
+    @override_settings(MITX_FEATURES=MITX_FEATURES_WITH_SSL_AUTH)
     def test_authmap_repair(self):
         """Run authmap check and repair"""
 
         self._setstaff_login()
 
-        create_user('test0', 'test test', do_mit=True)
+        create_user('test0', 'test test')
         # Will raise exception, so no assert needed
         eamap = ExternalAuthMap.objects.get(external_name='test test')
         mitu = User.objects.get(username='test0')
@@ -393,15 +396,3 @@ def test_gitlog_courseteam_access(self):
 
         self._rm_edx4edx()
 
-    @override_settings(GIT_ADD_COURSE_SCRIPT='')
-    def test_no_script_set(self):
-        """ Test if settings are right on mongo store import"""
-
-        self._setstaff_login()
-
-        response = self.client.post(reverse('sysadmin'), {
-            'dash_mode': _('Courses'),
-            'repo_location': 'https://github.com/mitocw/edx4edx_lite.git',
-            'action': _('Load new course from github'), })
-        self.assertIn(_('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'),
-                      response.content)
diff --git a/lms/envs/aws.py b/lms/envs/aws.py
index b83b5a7c3a2f..61a464b6b051 100644
--- a/lms/envs/aws.py
+++ b/lms/envs/aws.py
@@ -202,7 +202,6 @@
 # git repo loading  environment
 GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', None)
 GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True)
-GIT_ADD_COURSE_SCRIPT = ENV_TOKENS.get('GIT_ADD_COURSE_SCRIPT', GIT_ADD_COURSE_SCRIPT)
 
 for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items():
     oldvalue = CODE_JAIL.get(name)
diff --git a/lms/envs/common.py b/lms/envs/common.py
index ff1d953d503c..1c01d7b7f45f 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -484,7 +484,6 @@
 # These are listed, eg at https://github.com/MITx/mitx/admin/hooks
 
 ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178']
-GIT_ADD_COURSE_SCRIPT = REPO_ROOT / "scripts/git_add_course.py"
 
 #################################### AWS #######################################
 # S3BotoStorage insists on a timeout for uploaded assets. We should make it
@@ -918,6 +917,7 @@
     'eventtracking.django',
     'util',
     'certificates',
+    'dashboard',
     'instructor',
     'instructor_task',
     'open_ended_grading',

From 9bc44c0ef314207ed2af7d39abf43739c18401f9 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Fri, 15 Nov 2013 17:03:43 -0500
Subject: [PATCH 12/20] Refactor of sysadmin dashboard to use class views, and
 url includes

---
 .../management/commands/git_add_course.py     |   17 +-
 lms/djangoapps/dashboard/models.py            |   15 +-
 lms/djangoapps/dashboard/sysadmin.py          | 1018 +++++++++--------
 lms/djangoapps/dashboard/sysadmin_urls.py     |   12 +
 .../dashboard/tests/test_sysadmin.py          |   75 +-
 lms/templates/sysadmin_dashboard.html         |  125 +-
 lms/templates/sysadmin_dashboard_gitlogs.html |   72 +-
 lms/urls.py                                   |    5 +-
 8 files changed, 712 insertions(+), 627 deletions(-)
 create mode 100644 lms/djangoapps/dashboard/sysadmin_urls.py

diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py
index 63b49f9faaee..5cc137863d3a 100644
--- a/lms/djangoapps/dashboard/management/commands/git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py
@@ -5,7 +5,6 @@
 import sys
 import re
 import datetime
-import mongoengine  # used to store import log
 import StringIO
 import logging
 
@@ -17,24 +16,14 @@
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml import XMLModuleStore
 
+import mongoengine
+from dashboard.models import CourseImportLog
+
 log = logging.getLogger(__name__)
 
 GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos')
 GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
 
-class CourseImportLog(mongoengine.Document):
-    """Mongoengine model for git log"""
-    # pylint: disable-msg=R0924
-
-    course_id = mongoengine.StringField(max_length=128)
-    location = mongoengine.StringField(max_length=168)
-    import_log = mongoengine.StringField(max_length=20 * 65535)
-    git_log = mongoengine.StringField(max_length=65535)
-    repo_dir = mongoengine.StringField(max_length=128)
-    created = mongoengine.DateTimeField()
-    meta = {'indexes': ['course_id', 'created'],
-            'allow_inheritance': False}
-
 def add_repo(repo, rdir_in):
     """This will add a git repo into the mongo modulestore"""
     # pylint: disable-msg=R0915
diff --git a/lms/djangoapps/dashboard/models.py b/lms/djangoapps/dashboard/models.py
index 6b2021999398..ddc2eb9aeb2e 100644
--- a/lms/djangoapps/dashboard/models.py
+++ b/lms/djangoapps/dashboard/models.py
@@ -1 +1,14 @@
-# Create your models here.
+import mongoengine
+
+class CourseImportLog(mongoengine.Document):
+    """Mongoengine model for git log"""
+
+    course_id = mongoengine.StringField(max_length=128)
+    location = mongoengine.StringField(max_length=168)
+    import_log = mongoengine.StringField(max_length=20 * 65535)
+    git_log = mongoengine.StringField(max_length=65535)
+    repo_dir = mongoengine.StringField(max_length=128)
+    created = mongoengine.DateTimeField()
+    meta = {'indexes': ['course_id', 'created'],
+            'allow_inheritance': False}
+
diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index d85486242fb8..b991f3f66e16 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -26,12 +26,14 @@
 from django.core.management.base import CommandError
 from django_future.csrf import ensure_csrf_cookie
 from django.views.decorators.cache import cache_control
+from django.utils.decorators import method_decorator
 from django.db import IntegrityError
 from django.http import HttpResponse
 from django.utils.html import escape
 from django.contrib.admin.views.decorators import staff_member_required
 from django.contrib.auth.decorators import login_required
 from django.http import Http404
+from django.views.generic.base import TemplateView
 
 from mitxmako.shortcuts import render_to_response
 from xmodule.modulestore.django import modulestore
@@ -42,363 +44,503 @@
 from courseware.courses import get_course_by_id
 
 import mongoengine
+from dashboard.models import CourseImportLog
 import dashboard.management.commands.git_add_course as git_add_course
 
 import track.views
 
 log = logging.getLogger(__name__)
 
+class SysadminDashboardView(TemplateView):
+    """Base class for sysadmin dashboard views with common methods"""
+
+    template_name = 'sysadmin_dashboard.html'
+
+    def __init__(self, **kwargs):
+        self.def_ms = modulestore()
+        self.is_using_mongo = True
+        if isinstance(self.def_ms, XMLModuleStore):
+            self.is_using_mongo = False
+        self.msg = u''
+        return super(SysadminDashboardView, self).__init__(**kwargs)
+
+    @method_decorator(ensure_csrf_cookie)
+    @method_decorator(login_required)
+    @method_decorator(cache_control(no_cache=True, no_store=True, must_revalidate=True))
+    def dispatch(self, *args, **kwargs):
+        return super(SysadminDashboardView, self).dispatch(*args, **kwargs)
+
+    def get_courses(self):
+        # Prefer mongo if using mixed or mongo store
+        if self.is_using_mongo:
+            courses = self.def_ms.get_courses()
+            courses = dict([c.id, c] for c in courses)  # no course directory
+        else:
+            courses = self.def_ms.courses.items()
+        return courses
 
-class CourseImportLog(mongoengine.Document):
-    """Mongoengine model for git log"""
-    # pylint: disable-msg=R0924
+    def return_csv(self, filename, datatable):
+        """Convenient function for handling the http response of a csv"""
 
-    course_id = mongoengine.StringField(max_length=128)
-    location = mongoengine.StringField(max_length=168)
-    import_log = mongoengine.StringField(max_length=20 * 65535)
-    git_log = mongoengine.StringField(max_length=65535)
-    repo_dir = mongoengine.StringField(max_length=128)
-    created = mongoengine.DateTimeField()
-    meta = {'indexes': ['course_id', 'created'],
-            'allow_inheritance': False}
+        response = HttpResponse(mimetype='text/csv')
+        response['Content-Disposition'] = 'attachment; filename={0}'.format(filename)
 
+        writer = csv.writer(response, dialect='excel', quotechar='"',
+                            quoting=csv.QUOTE_ALL)
+        writer.writerow(datatable['header'])
+        for datarow in datatable['data']:
+            encoded_row = [unicode(s).encode('utf-8') for s in datarow]
+            writer.writerow(encoded_row)
+        return response
 
-def git_info_for_course(cdir):
-    """This pulls out some git info like the last commit"""
 
-    cmd = ''
-    gdir = settings.DATA_DIR / cdir
-    info = ['', '', '']
-    if not os.path.exists(gdir):
-        return info
+class Users(SysadminDashboardView):
+    """
+    The status view provides Web based user management, a listing of
+    courses loaded, and user statistics
+    """
+
+    def fix_external_auth_map_passwords(self):
+        """
+        This corrects any passwords that have drifted from eamap to
+        internal django auth.  Needs to be removed when fixed in external_auth
+        """
+
+        msg = ''
+        for eamap in ExternalAuthMap.objects.all():
+            euser = eamap.user
+            epass = eamap.internal_password
+            if euser is None:
+                continue
+            try:
+                testuser = authenticate(username=euser.username, password=epass)
+            except (TypeError, PermissionDenied), err:
+                msg += _('Failed in authenticating {0}, error {1}\n'
+                         ).format(euser, err)
+                continue
+            if testuser is None:
+                msg += _('Failed in authenticating {0}\n').format(euser)
+                msg += _('fixed password')
+                euser.set_password(epass)
+                euser.save()
+                continue
+        if not msg:
+            msg = _('All ok!')
+        return msg
 
-    cmd = ['git', 'log', '-1',
-           '--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}',]
-    try:
-        output_json = json.loads(subprocess.check_output(cmd, cwd=gdir))
-        info = [ output_json['commit'],
-                 output_json['date'], 
-                 output_json['author'], ]
-    except (ValueError, subprocess.CalledProcessError):
-        pass
+    def create_user(self, uname, name, password=None):
+        """ Creates a user (both SSL and regular)"""
+
+        if not uname:
+            return _('Must provide username')
+        if not name:
+            return _('Must provide full name')
+
+        msg = u''
+        if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
+            if not '@' in uname:
+                email = '{0}@MIT.EDU'.format(uname)
+            else:
+                email = uname
+            if not email.endswith('@MIT.EDU'):
+                msg += u'email must end in @MIT.EDU'
+                return msg
+            mit_domain = 'ssl:MIT'
+            if ExternalAuthMap.objects.filter(external_id=email,
+                                              external_domain=mit_domain):
+                msg += _('Failed - email {0} already exists as external_id'
+                         ).format(email)
+                return msg
+            new_password = generate_password()
+        else:
+            if not password:
+                return _('Password must be supplied if not using certificates')
 
-    return info
+            email = uname
 
-def import_mongo_course(gitloc, def_ms):
-    """
-    Imports course using management command and captures logging output
-    at debug level for display in template
-    """
+            if not '@' in email:
+                msg += _('email address required (not username)')
+                return msg
+            new_password = password
+
+        user = User(username=uname, email=email, is_active=True)
+        user.set_password(new_password)
+        try:
+            user.save()
+        except IntegrityError:
+            msg += _('Oops, failed to create user {0}, IntegrityError'
+                     ).format(user)
+            return msg
 
-    msg = u''
-
-    logging.debug(_('Adding course using git repo {0}').format(gitloc))
-
-    # Grab logging output for debugging imports
-    output = StringIO.StringIO()
-
-    import_logger = logging.getLogger('xmodule.modulestore.xml_importer')
-    git_logger = logging.getLogger('dashboard.management.commands.git_add_course')
-    xml_logger = logging.getLogger('xmodule.modulestore.xml')
-    xml_seq_logger = logging.getLogger('xmodule.seq_module')
-
-    import_log_handler = logging.StreamHandler(output)
-    import_log_handler.setLevel(logging.DEBUG)
-
-    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
-        logger.old_level = logger.level
-        logger.setLevel(logging.DEBUG)
-        logger.addHandler(import_log_handler)
-
-
-    git_add_course.add_repo(gitloc, None)
-    ret = output.getvalue()
-
-    # Remove handler hijacks
-    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
-        logger.setLevel(logger.old_level)
-        logger.removeHandler(import_log_handler)
-    msg = u"<h4 style='color:blue'>{0} {1}</h4>".format(
-        _('Added course from'), gitloc)
-    msg += _("<pre>{0}</pre>").format(escape(ret))
-    return msg
-
-def import_xml_course(gitloc, def_ms, datatable):
-    """Imports a git course into the XMLModuleStore"""
-
-    msg = u''
-    cdir = (gitloc.rsplit('/', 1)[1])[:-4]
-    gdir = settings.DATA_DIR / cdir
-    if os.path.exists(gdir):
-        msg += _("The course {0} already exists in the data directory! "
-                 "(reloading anyway)").format(cdir)
-        cmd = 'cd {0}; git pull'.format(settings.DATA_DIR, gitloc)
-    else:
-        cmd = 'cd {0}; git clone {1}'.format(settings.DATA_DIR, gitloc)
-        msg += u'<pre>%s</pre>' % escape(os.popen(cmd).read())
-    if not os.path.exists(gdir):
-        msg += _('Failed to clone repository to {0}').format(gdir)
+        reg = Registration()
+        reg.register(user)
+
+        profile = UserProfile(user=user)
+        profile.name = name
+        profile.save()
+
+        if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
+            credentials = '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}'.format(name, email)
+            eamap = ExternalAuthMap(
+                external_id=email,
+                external_email=email,
+                external_domain=mit_domain,
+                external_name=name,
+                internal_password=new_password,
+                external_credentials=json.dumps(credentials),
+            )
+            eamap.user = user
+            eamap.dtsignup = datetime.now()
+            eamap.save()
+
+        msg += _('User {0} created successfully!').format(user)
         return msg
-    def_ms.try_load_course(os.path.abspath(gdir))  # load into modulestore
-    errlog = def_ms.errored_courses.get(cdir, '')
-    if errlog:
-        msg += u'<hr width="50%"><pre>{0}</pre>'.format(escape(errlog))
-    else:
-        course = def_ms.courses[os.path.abspath(gdir)]
-        msg += _('Loaded course {0} {1}<br/>Errors:').format(cdir,
-                                                             course.display_name)
-        errors = def_ms.get_item_errors(course.location)
-        if not errors:
-            msg += u'None'
+
+    def delete_user(self, uname):
+        """Deletes a user from django auth"""
+
+        if not uname:
+            return _('Must provide username')
+        if '@' in uname:
+            try:
+                user = User.objects.get(email=uname)
+            except User.DoesNotExist, err:
+                msg = _('Cannot find user with email address {0}'
+                        ).format(uname)
+                return msg
         else:
-            msg += u'<ul>'
-            for (summary, err) in errors:
-                msg += \
-                    u'<li><pre>{0}: {1}</pre></li>'.format(escape(summary),
-                                                           escape(err))
-            msg += u'</ul>'
-        datatable['data'].append([course.display_name, cdir]
-                                 + git_info_for_course(cdir))
-    return msg
+            try:
+                user = User.objects.get(username=uname)
+            except User.DoesNotExist, err:
+                msg = _('Cannot find user with username {0} - {1}'
+                        ).format(uname, str(err))
+                return msg
+        user.delete()
+        return _('Deleted user {0}').format(uname)
 
-def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable):
-    """This downloads and runs the checks for importing a course in git"""
+    def make_common_context(self):
+        """Returns the datatable used for this view"""
+
+        self.datatable = {}
+        courses = self.get_courses()
+
+        self.datatable = dict(header=[_('Statistic'), _('Value')],
+                         title=_('Site statistics'))
+        self.datatable['data'] = [[_('Total number of users'),
+                              User.objects.all().count()]]
 
-    if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
-            gitloc.startswith('https:') or gitloc.startswith('git:')):
-        return _("The git repo location should end with '.git', and be a valid url")
+        if hasattr(courses, 'items'):
+            course_iter = courses.items()
+        else:
+            course_iter = courses
 
-    if is_using_mongo:
-        return import_mongo_course(gitloc, def_ms)
+        self.msg += u'<h2>{0}</h2>'.format(
+            _('Courses loaded in the modulestore'))
+        self.msg += u'<ol>'
+        for (cdir, course) in course_iter:
+            self.msg += u'<li>{0} ({1})</li>'.format(
+                escape(cdir), course.location.url())
+        self.msg += u'</ol>'
 
-    return import_xml_course(gitloc, def_ms, datatable)
+    def get(self, request):
 
-def fix_external_auth_map_passwords():
+        if not request.user.is_staff:
+            raise Http404
+        self.make_common_context()
+
+        context = {
+            'datatable': self.datatable,
+            'msg': self.msg,
+            'djangopid': os.getpid(),
+            'modeflag': {'users': 'active-section'},
+            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+        }
+        return render_to_response(self.template_name, context)
+
+    def post(self, request):
+        """Handle various actions available on page"""
+
+        if not request.user.is_staff:
+            raise Http404
+
+        self.make_common_context()
+
+        action = request.POST.get('action', '')
+        track.views.server_track(request, action, {}, page='user_sysdashboard')
+
+        if action == 'download_users':
+            self.datatable = dict(header=[_('username'), _('email')],
+                             title=_('List of all users'),
+                             data=[[u.username, u.email] for u in
+                                   User.objects.all()])
+            return self.return_csv('users_{0}.csv'.format(
+                request.META['SERVER_NAME']), self.datatable)
+        elif action == 'repair_eamap':
+            self.msg = u'<h4>{0}</h4><pre>{1}</pre>{2}'.format(
+                _('Repair Results'), self.fix_external_auth_map_passwords(), self.msg)
+            self.datatable = {}
+        elif action == 'create_user':
+            uname = request.POST.get('student_uname', '').strip()
+            name = request.POST.get('student_fullname', '').strip()
+            password = request.POST.get('student_password', '').strip()
+            self.msg = u'<h4>{0}</h4><p>{1}</p><hr />{2}'.format(
+                _('Create User Results'), self.create_user(uname, name, password), self.msg)
+        elif action == 'del_user':
+            uname = request.POST.get('student_uname', '').strip()
+            self.msg = u'<h4>{0}</h4><p>{1}</p><hr />{2}'.format(
+                _('Delete User Results'), self.delete_user(uname), self.msg)
+
+        context = {
+            'datatable': self.datatable,
+            'msg': self.msg,
+            'djangopid': os.getpid(),
+            'modeflag': {'users': 'active-section'},
+            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+        }
+        return render_to_response(self.template_name, context)
+
+class Courses(SysadminDashboardView):
     """
-    This corrects any passwords that have drifted from eamp to
-    internal django auth
+    This manages adding/updating courses from git, deleting courses, and
+    provides course listing information.
     """
 
-    msg = ''
-    for eamap in ExternalAuthMap.objects.all():
-        euser = eamap.user
-        epass = eamap.internal_password
-        if euser is None:
-            continue
+    def git_info_for_course(self, cdir):
+        """This pulls out some git info like the last commit"""
+
+        cmd = ''
+        gdir = settings.DATA_DIR / cdir
+        info = ['', '', '']
+        if not os.path.exists(gdir):
+            return info
+
+        cmd = ['git', 'log', '-1',
+               '--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}',]
         try:
-            testuser = authenticate(username=euser.username, password=epass)
-        except (TypeError, PermissionDenied), err:
-            msg += _('Failed in authenticating {0}, error {1}\n'
-                     ).format(euser, err)
-            continue
-        if testuser is None:
-            msg += _('Failed in authenticating {0}\n').format(euser)
-            msg += _('fixed password')
-            euser.set_password(epass)
-            euser.save()
-            continue
-    if not msg:
-        msg = _('All ok!')
-    return msg
-
-
-def create_user(uname, name, password=None):
-    """ Creates a user (both SSL and regular)"""
-
-    if not uname:
-        return _('Must provide username')
-    if not name:
-        return _('Must provide full name')
-
-    msg = u''
-    if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
-        if not '@' in uname:
-            email = '{0}@MIT.EDU'.format(uname)
-        else:
-            email = uname
-        if not email.endswith('@MIT.EDU'):
-            msg += u'email must end in @MIT.EDU'
-            return msg
-        mit_domain = 'ssl:MIT'
-        if ExternalAuthMap.objects.filter(external_id=email,
-                                          external_domain=mit_domain):
-            msg += _('Failed - email {0} already exists as external_id'
-                     ).format(email)
-            return msg
-        new_password = generate_password()
-    else:
-        if not password:
-            return _('Password must be supplied if not using certificates')
+            output_json = json.loads(subprocess.check_output(cmd, cwd=gdir))
+            info = [ output_json['commit'],
+                     output_json['date'], 
+                     output_json['author'], ]
+        except (ValueError, subprocess.CalledProcessError):
+            pass
 
-        email = uname
+        return info
 
-        if not '@' in email:
-            msg += _('email address required (not username)')
-            return msg
-        new_password = password
-
-    user = User(username=uname, email=email, is_active=True)
-    user.set_password(new_password)
-    try:
-        user.save()
-    except IntegrityError:
-        msg += _('Oops, failed to create user {0}, IntegrityError'
-                 ).format(user)
-        return msg
+    def get_course_from_git(self, gitloc, datatable):
+        """This downloads and runs the checks for importing a course in git"""
 
-    reg = Registration()
-    reg.register(user)
-
-    profile = UserProfile(user=user)
-    profile.name = name
-    profile.save()
-
-    if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
-        credentials = '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}'.format(name, email)
-        eamap = ExternalAuthMap(
-            external_id=email,
-            external_email=email,
-            external_domain=mit_domain,
-            external_name=name,
-            internal_password=new_password,
-            external_credentials=json.dumps(credentials),
-        )
-        eamap.user = user
-        eamap.dtsignup = datetime.now()
-        eamap.save()
-
-    msg += _('User {0} created successfully!').format(user)
-    return msg
-
-
-def delete_user(uname):
-    """Deletes a user from django auth"""
-
-    if not uname:
-        return _('Must provide username')
-    if '@' in uname:
-        try:
-            user = User.objects.get(email=uname)
-        except User.DoesNotExist, err:
-            msg = _('Cannot find user with email address {0}'
-                    ).format(uname)
-            return msg
-    else:
-        try:
-            user = User.objects.get(username=uname)
-        except User.DoesNotExist, err:
-            msg = _('Cannot find user with username {0} - {1}'
-                    ).format(uname, err.msg)
-            return msg
-    user.delete()
-    return _('Deleted user {0}').format(uname)
+        if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
+                gitloc.startswith('https:') or gitloc.startswith('git:')):
+            return _("The git repo location should end with '.git', and be a valid url")
 
+        if self.is_using_mongo:
+            return self.import_mongo_course(gitloc)
 
-def return_csv(filename, datatable):
-    """Convenient function for handling the http response of a csv"""
+        return self.import_xml_course(gitloc, datatable)
 
-    response = HttpResponse(mimetype='text/csv')
-    response['Content-Disposition'] = 'attachment; filename={0}'.format(filename)
+    def import_mongo_course(self, gitloc):
+        """
+        Imports course using management command and captures logging output
+        at debug level for display in template
+        """
 
-    writer = csv.writer(response, dialect='excel', quotechar='"',
-                        quoting=csv.QUOTE_ALL)
-    writer.writerow(datatable['header'])
-    for datarow in datatable['data']:
-        encoded_row = [unicode(s).encode('utf-8') for s in datarow]
-        writer.writerow(encoded_row)
-    return response
+        msg = u''
 
+        logging.debug(_('Adding course using git repo {0}').format(gitloc))
 
-def get_staff_group(course):
-    """Gets staff members for course"""
+        # Grab logging output for debugging imports
+        output = StringIO.StringIO()
 
-    return get_group(course, 'staff')
+        import_logger = logging.getLogger('xmodule.modulestore.xml_importer')
+        git_logger = logging.getLogger('dashboard.management.commands.git_add_course')
+        xml_logger = logging.getLogger('xmodule.modulestore.xml')
+        xml_seq_logger = logging.getLogger('xmodule.seq_module')
 
+        import_log_handler = logging.StreamHandler(output)
+        import_log_handler.setLevel(logging.DEBUG)
 
-def get_instructor_group(course):
-    """Gets instructors for course"""
+        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+            logger.old_level = logger.level
+            logger.setLevel(logging.DEBUG)
+            logger.addHandler(import_log_handler)
 
-    return get_group(course, 'instructor')
 
+        git_add_course.add_repo(gitloc, None)
+        ret = output.getvalue()
 
-def get_group(course, groupname):
-    """Gets the course group"""
+        # Remove handler hijacks
+        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+            logger.setLevel(logger.old_level)
+            logger.removeHandler(import_log_handler)
+        msg = u"<h4 style='color:blue'>{0} {1}</h4>".format(
+            _('Added course from'), gitloc)
+        msg += _("<pre>{0}</pre>").format(escape(ret))
+        return msg
 
-    grpname = get_access_group_name(course, groupname)
-    try:
-        group = Group.objects.get(name=grpname)
-    except Group.DoesNotExist:
-        group = Group(name=grpname)  # create the group
-        group.save()
-    return group
+    def import_xml_course(self, gitloc, datatable):
+        """Imports a git course into the XMLModuleStore"""
 
+        msg = u''
+        cdir = (gitloc.rsplit('/', 1)[1])[:-4]
+        gdir = settings.DATA_DIR / cdir
+        if os.path.exists(gdir):
+            msg += _("The course {0} already exists in the data directory! "
+                     "(reloading anyway)").format(cdir)
+            cmd = 'cd {0}; git pull'.format(settings.DATA_DIR, gitloc)
+        else:
+            cmd = 'cd {0}; git clone {1}'.format(settings.DATA_DIR, gitloc)
+            msg += u'<pre>%s</pre>' % escape(os.popen(cmd).read())
+        if not os.path.exists(gdir):
+            msg += _('Failed to clone repository to {0}').format(gdir)
+            return msg
+        self.def_ms.try_load_course(os.path.abspath(gdir))  # load into modulestore
+        errlog = self.def_ms.errored_courses.get(cdir, '')
+        if errlog:
+            msg += u'<hr width="50%"><pre>{0}</pre>'.format(escape(errlog))
+        else:
+            course = self.def_ms.courses[os.path.abspath(gdir)]
+            msg += _('Loaded course {0} {1}<br/>Errors:').format(cdir,
+                                                                 course.display_name)
+            errors = self.def_ms.get_item_errors(course.location)
+            if not errors:
+                msg += u'None'
+            else:
+                msg += u'<ul>'
+                for (summary, err) in errors:
+                    msg += \
+                        u'<li><pre>{0}: {1}</pre></li>'.format(escape(summary),
+                                                               escape(err))
+                msg += u'</ul>'
+            datatable['data'].append([course.display_name, cdir]
+                                     + self.git_info_for_course(cdir))
+        return msg
 
-@staff_member_required
-@ensure_csrf_cookie
-@cache_control(no_cache=True, no_store=True, must_revalidate=True)
-def sysadmin_dashboard(request):
-    """
-    Sysadmin dashboard.
-    Provides:
-        1. enrollment numbers
-        2. loading new courses from github
-        3. reloading XML from files
-    """
-    # pylint: disable-msg=R0915
-
-    msg = u''
-    plots = []
-    datatable = {}
-
-    def_ms = modulestore()
-
-    # Prefer mongo if using mixed or mongo store
-    is_using_mongo = True
-    if isinstance(def_ms, XMLModuleStore):
-        is_using_mongo = False
-
-    if is_using_mongo:
-        courses = def_ms.get_courses()
-        courses = dict([c.id, c] for c in courses)  # no course directory
-    else:
-        courses = def_ms.courses.items()
-
-    # the sysadmin dashboard page is modal: status, courses, enrollment,
-    # staffing, gitlogs
-    # keep that state in request.session (defaults to status mode)
-    dash_mode = request.POST.get('dash_mode', '')
-    if dash_mode:
-        request.session['dash_mode'] = dash_mode
-    else:
-        dash_mode = request.session.get('dash_mode', _('Status'))
-
-    # default datatable depends on dash_mode
-    if dash_mode == _('Status'):
-        datatable = dict(header=[_('Statistic'), _('Value')],
-                         title=_('Site statistics'))
-        datatable['data'] = [[_('Total number of users'),
-                             User.objects.all().count()]]
-    elif dash_mode == _('Courses'):
+    def make_datatable(self):
+        """Creates course information datatable"""
+        
         data = []
-
+        courses = self.get_courses()
         if hasattr(courses, 'items'):
             course_iter = courses.items()
         else:
             course_iter = courses
         for (cdir, course) in course_iter:
+            gdir = cdir
             if '/' in cdir:
-                cdir = cdir.rsplit('/', 1)[1]
+                gdir = cdir.rsplit('/', 1)[1]
             data.append([course.display_name, cdir]
-                        + git_info_for_course(cdir))
+                        + self.git_info_for_course(gdir))
 
-        datatable = dict(header=[_('Course Name'), _('dir'),
-                         _('git commit'), _('last change'),
-                         _('last editor')],
+        return  dict(header=[_('Course Name'), _('Directory/ID'),
+                         _('Git Commit'), _('Last Change'),
+                         _('Last Editor')],
                          title=_('Information about all courses'),
                          data=data)
-    elif dash_mode == _('Staffing and Enrollment'):
+
+    def get(self, request):
+        """Displays forms and course information"""
+
+        if not request.user.is_staff:
+            raise Http404
+
+        context = {
+            'datatable': self.make_datatable(),
+            'msg': self.msg,
+            'djangopid': os.getpid(),
+            'modeflag': {'courses': 'active-section'},
+            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+        }
+        return render_to_response(self.template_name, context)
+
+    def post(self, request):
+        """Handle all actions from courses view"""
+
+        if not request.user.is_staff:
+            raise Http404
+
+        action = request.POST.get('action', '')
+        track.views.server_track(request, action, {}, page='courses_sysdashboard')
+
+        courses = self.get_courses()
+        if action == _('add_course'):
+            gitloc = request.POST.get('repo_location', '').strip().replace(
+                ' ', '').replace(';', '')
+            datatable = self.make_datatable()
+            self.msg += self.get_course_from_git(gitloc, datatable)
+
+        elif action == 'del_course':
+            course_id = request.POST.get('course_id', '').strip()
+            course_found = False
+            if course_id in courses:
+                course_found = True
+                course = courses[course_id]
+            else:
+                try:
+                    course = get_course_by_id(course_id)
+                    course_found = True
+                except Exception, err:
+                    self.msg += _('Error - cannot get course with ID {0}<br/><pre>{1}</pre>').format(
+                        course_id, escape(str(err)))
+
+            if course_found and not self.is_using_mongo:
+                cdir = course.data_dir
+                self.def_ms.courses.pop(cdir)
+
+                # now move the directory (don't actually delete it)
+                new_dir = cdir + '_deleted_{0}'.format(int(time.time()))
+                os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / new_dir)
+
+                self.msg += u"<font color='red'>Deleted {0} = {1} ({2})</font>".format(
+                    cdir, course.id, course.display_name)
+
+            elif course_found and self.is_using_mongo:
+                # delete course that is stored with mongodb backend
+                loc = course.location
+                content_store = contentstore()
+                commit = True
+                delete_course(self.def_ms, content_store, loc, commit)
+                # don't delete user permission groups, though
+                self.msg += \
+                    u"<font color='red'>{0} {1} = {2} ({3})</font>".format(
+                        _('Deleted'), loc, course.id, course.display_name)
+            datatable = self.make_datatable()
+
+        context = {
+            'datatable': datatable,
+            'msg': self.msg,
+            'djangopid': os.getpid(),
+            'modeflag': {'courses': 'active-section'},
+            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+        }
+        return render_to_response(self.template_name, context)
+
+
+class Staffing(SysadminDashboardView):
+    """
+    The status view provides a view of staffing and enrollment in
+    courses that include an option to download the data as a csv.
+    """
+
+    def get_group(self, course, groupname):
+        """Gets the course group"""
+
+        grpname = get_access_group_name(course, groupname)
+        try:
+            group = Group.objects.get(name=grpname)
+        except Group.DoesNotExist:
+            group = Group(name=grpname)  # create the group
+            group.save()
+        return group
+
+    def get(self, request):
+        """Displays course Enrollment and staffing course statistics"""
+
+        if not request.user.is_staff:
+            raise Http404
         data = []
 
+        courses = self.get_courses()
+
         if hasattr(courses, 'items'):
             course_iter = courses.items()
         else:
@@ -407,8 +549,8 @@ def sysadmin_dashboard(request):
             datum = [course.display_name, course.id]
             datum += \
                 [CourseEnrollment.objects.filter(course_id=course.id).count()]
-            datum += [get_group(course, 'staff').user_set.all().count()]
-            datum += [','.join([x.username for x in get_group(course,
+            datum += [self.get_group(course, 'staff').user_set.all().count()]
+            datum += [','.join([x.username for x in self.get_group(course,
                       'instructor').user_set.all()])]
             data.append(datum)
 
@@ -416,177 +558,111 @@ def sysadmin_dashboard(request):
                                  _('# enrolled'), _('# staff'), _('instructors')],
                          title=_('Enrollment information for all courses'),
                          data=data)
+        context = {
+            'datatable': datatable,
+            'msg': self.msg,
+            'djangopid': os.getpid(),
+            'modeflag': {'staffing': 'active-section'},
+            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+        }
+        return render_to_response(self.template_name, context)
+
+    def post(self, request):
+        """Handle all actions from staffing and enrollment view"""
+
+        action = request.POST.get('action', '')
+        track.views.server_track(request, action, {}, page='staffing_sysdashboard')
+
+        if action == 'get_staff_csv':
+            data = []
+            roles = ['instructor', 'staff']
+
+            courses = self.get_courses()
+            if hasattr(courses, 'items'):
+                course_iter = courses.items()
+            else:
+                course_iter = courses
+            for (cdir, course) in course_iter:
+                for role in roles:
+                    for user in self.get_group(course, role).user_set.all():
+                        datum = [course.id, role, user.username, user.email,
+                                 user.profile.name]
+                        data.append(datum)
+            datatable = dict(header=[_('course_id'),
+                                     _('role'), _('username'),
+                                     _('email'), _('full_name')],
+                             title=_('List of all course staff and instructors'),
+                             data=data)
+            return self.return_csv('staff_{0}.csv'.format(
+                request.META['SERVER_NAME']), datatable)
+
+        return self.get(request)
+
+
+class GitLogs(TemplateView):
+    """
+    This provides a view into the import of courses from git repositories.
+    It is convenient for allowing course teams to see what may be wrong with
+    their xml
+    """
 
-    # process actions from form POST
-    action = request.POST.get('action', '')
-    track.views.server_track(request, action, {}, page='sysdashboard')
-
-    if _('Download list of all users (csv file)') in action:
-        datatable = dict(header=[_('username'), _('email')],
-                         title=_('List of all users'),
-                         data=[[u.username, u.email] for u in
-                               User.objects.all()])
-        return return_csv('users_{0}.csv'.format(
-            request.META['SERVER_NAME']), datatable)
-    elif _('Check and repair external Auth Map') in action:
-        msg += u'<pre>'
-        msg += fix_external_auth_map_passwords()
-        msg += u'</pre>'
-        datatable = {}
-    elif _('Create user') in action:
-        uname = request.POST.get('student_uname', '').strip()
-        name = request.POST.get('student_fullname', '').strip()
-        password = request.POST.get('student_password', '').strip()
-
-        msg += create_user(uname, name, password)
-    elif _('Delete user') in action:
-        uname = request.POST.get('student_uname', '').strip()
-        msg += delete_user(uname)
-    elif _('Download staff and instructor list (csv file)') in action:
-        data = []
-        roles = ['instructor', 'staff']
+    template_name = 'sysadmin_dashboard_gitlogs.html'
 
-        if hasattr(courses, 'items'):
-            course_iter = courses.items()
-        else:
-            course_iter = courses
-        for (cdir, course) in course_iter:
-            for role in roles:
-                for user in get_group(course, role).user_set.all():
-                    datum = [course.id, role, user.username, user.email,
-                             user.profile.name]
-                    data.append(datum)
-        datatable = dict(header=[_('course_id'),
-                                 _('role'), _('username'),
-                                 _('email'), _('full_name')],
-                         title=_('List of all course staff and instructors'),
-                         data=data)
-        return return_csv('staff_{0}.csv'.format(
-            request.META['SERVER_NAME']), datatable)
-    elif action == _('Delete course from site'):
-        course_id = request.POST.get('course_id', '').strip()
-        course_found = False
-        if course_id in courses:
-            course_found = True
-            course = courses[course_id]
+    @method_decorator(login_required)
+    def get(self, request, course_id=None):
+        """Shows logs of imports that happened as a result of a git import"""
+
+        # Set defaults even if it isn't defined in settings
+        mongo_db = {
+            'host': 'localhost',
+            'user': '',
+            'password': '',
+            'db': 'xlog',
+        }
+
+        # Allow overrides
+        if hasattr(settings, 'MONGODB_LOG'):
+            for config_item in ['host', 'user', 'password', 'db', ]:
+                mongo_db[config_item] = settings.MONGODB_LOG.get(
+                    config_item, mongo_db[config_item])
+
+        mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
+            mongo_db['user'], mongo_db['password'],
+            mongo_db['host'], mongo_db['db'])
+
+        error_msg = ''
+
+        try:
+            if mongo_db['user'] and mongo_db['password']:
+                mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
+            else:
+                mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
+        except mongoengine.connection.ConnectionError, ex:
+            logging.critical(_('Unable to connect to mongodb to save log, please check '
+                               'MONGODB_LOG settings. error: {0}').format(str(ex)))
+
+        if course_id is None:
+            # Require staff if not going to specific course
+            if not request.user.is_staff:
+                raise Http404
+            cilset = CourseImportLog.objects.all().order_by('-created')
         else:
             try:
                 course = get_course_by_id(course_id)
-                course_found = True
-            except Http404, err:
-                msg += \
-                    _('Error - cannot get course with ID {0}<br/><pre>{1}</pre>'
-                      ).format(course_id, escape(err))
-
-        if course_found and not is_using_mongo:
-            cdir = course.data_dir
-            def_ms.courses.pop(cdir)
-
-            # now move the directory (don't actually delete it)
-            new_dir = cdir + '_deleted_{0}'.format(int(time.time()))
-            os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / new_dir)
-
-            msg += u"<font color='red'>Deleted {0} = {1} ({2})</font>".format(
-                cdir, course.id, course.display_name)
-
-        elif course_found and is_using_mongo:
-            # delete course that is stored with mongodb backend
-            loc = course.location
-            content_store = contentstore()
-            commit = True
-            delete_course(def_ms, content_store, loc, commit)
-            # don't delete user permission groups, though
-            msg += \
-                u"<font color='red'>{0} {1} = {2} ({3})</font>".format(
-                    _('Deleted'), loc, course.id, course.display_name)
-
-    elif action == _('Load new course from github'):
-        gitloc = request.POST.get('repo_location', '').strip().replace(
-            ' ', '').replace(';', '')
-        msg += get_course_from_git(gitloc, is_using_mongo, def_ms, datatable)
-
-    # default to showing status summary
-    else:
-        if hasattr(courses, 'items'):
-            course_iter = courses.items()
-        else:
-            course_iter = courses
-        msg += u'<h2>{0}</h2>'.format(
-            _('Courses loaded in the modulestore'))
-        msg += u'<ol>'
-        for (cdir, course) in course_iter:
-            msg += u'<li>{0} ({1})</li>'.format(
-                escape(cdir), course.location.url())
-        msg += u'</ol>'
-
-    # ----------------------------------------
-    # context for rendering
-
-    context = {
-        'datatable': datatable,
-        'plots': plots,
-        'msg': msg,
-        'djangopid': os.getpid(),
-        'modeflag': {dash_mode: 'selectedmode'},
-        'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
-    }
-
-    return render_to_response('sysadmin_dashboard.html', context)
-
-
-# -----------------------------------------------------------------------------
-
-@login_required
-def view_git_logs(request, course_id=None):
-    """Shows logs of imports that happened as a result of a git import"""
-    # pylint: disable-msg=W0613
-
-    # Set defaults even if it isn't defined in settings
-    mongo_db = {
-        'host': 'localhost',
-        'user': '',
-        'password': '',
-        'db': 'xlog',
-    }
-
-    # Allow overrides
-    if hasattr(settings, 'MONGODB_LOG'):
-        for config_item in ['host', 'user', 'password', 'db', ]:
-            mongo_db[config_item] = settings.MONGODB_LOG.get(
-                config_item, mongo_db[config_item])
-
-    mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
-        mongo_db['user'], mongo_db['password'],
-        mongo_db['host'], mongo_db['db'])
-
-    try:
-        if mongo_db['user'] and mongo_db['password']:
-            mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
-        else:
-            mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
-    except mongoengine.connection.ConnectionError, ex:
-        logging.critical(_('Unable to connect to mongodb to save log, please check '
-                           'MONGODB_LOG settings. error: {0}').format(str(ex)))
-
-    if course_id is None:
-
-        # Require staff if not going to specific course
-        if not request.user.is_staff:
-            raise Http404
-        cilset = CourseImportLog.objects.all().order_by('-created')
-    else:
-        course = get_course_by_id(course_id)
-        # Allow only course team, instructors, and staff
-        if not (request.user.is_staff or
-                has_access(request.user, course, 'instructor') or
-                has_access(request.user, course, 'staff')):
-            raise Http404
-        log.debug('course_id={0}'.format(course_id))
-        cilset = CourseImportLog.objects.filter(
-            course_id=course_id).order_by('-created')
-        log.debug('cilset length={0}'.format(len(cilset)))
-    mdb.disconnect()
-    context = {'cilset': cilset, 'course_id': course_id}
-
-    return render_to_response('sysadmin_dashboard_gitlogs.html',
-                              context)
+            except Exception:
+                    cilset = None
+                    error_msg = _('Cannot find course {0}').format(course_id)
+
+            # Allow only course team, instructors, and staff
+            if not (request.user.is_staff or
+                    has_access(request.user, course, 'instructor') or
+                    has_access(request.user, course, 'staff')):
+                raise Http404
+            log.debug('course_id={0}'.format(course_id))
+            cilset = CourseImportLog.objects.filter(
+                course_id=course_id).order_by('-created')
+            log.debug('cilset length={0}'.format(len(cilset)))
+        mdb.disconnect()
+        context = {'cilset': cilset, 'course_id': course_id, 'error_msg': error_msg}
+
+        return render_to_response(self.template_name, context)
diff --git a/lms/djangoapps/dashboard/sysadmin_urls.py b/lms/djangoapps/dashboard/sysadmin_urls.py
new file mode 100644
index 000000000000..0e04b94c1d90
--- /dev/null
+++ b/lms/djangoapps/dashboard/sysadmin_urls.py
@@ -0,0 +1,12 @@
+from django.conf.urls import include, patterns, url
+from django.views.generic import TemplateView
+from dashboard import sysadmin
+
+urlpatterns = patterns(
+    '',
+    url(r'^$', sysadmin.Users.as_view(), name="sysadmin"),
+    url(r'^courses/?$', sysadmin.Courses.as_view(), name="sysadmin_courses"),
+    url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"),    
+    url(r'^gitlogs/?$', sysadmin.GitLogs.as_view(), name="gitlogs"),
+    url(r'^gitlogs/(?P<course_id>.+)$', sysadmin.GitLogs.as_view(), name="gitlogs_detail"),
+)
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 77551f9eb37d..17747757febf 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -14,7 +14,7 @@
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
 
-from dashboard.sysadmin import create_user
+from dashboard.sysadmin import Users
 from external_auth.models import ExternalAuthMap
 from django.contrib.auth.hashers import check_password
 from django.contrib.auth.models import Group
@@ -23,7 +23,8 @@
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from django.utils.html import escape
 from courseware.access import get_access_group_name, has_access
-from dashboard.sysadmin import CourseImportLog
+from dashboard.models import CourseImportLog
+from xmodule.modulestore.xml import XMLModuleStore
 import mongoengine
 
 TEST_MONGODB_LOG = {
@@ -54,10 +55,9 @@ def _setstaff_login(self):
     def _add_edx4edx(self):
         """Adds the edx4edx sample course"""
 
-        return self.client.post(reverse('sysadmin'), {
-            'dash_mode': _('Courses'),
+        return self.client.post(reverse('sysadmin_courses'), {
             'repo_location': 'https://github.com/mitocw/edx4edx_lite.git',
-            'action': _('Load new course from github'), })
+            'action': 'add_course', })
 
     def _rm_edx4edx(self):
         """Deletes the sample course from the XML store"""
@@ -72,10 +72,9 @@ def _rm_edx4edx(self):
             course = def_ms.get_course('MITx/edx4edx/edx4edx')
 
         # Delete git loaded course
-        return self.client.post(reverse('sysadmin'),
-                                {'dash_mode': _('Courses'),
-                                 'course_id': course.id,
-                                 'action': _('Delete course from site'), })
+        return self.client.post(reverse('sysadmin_courses'), 
+                                 { 'course_id': course.id,
+                                 'action': 'del_course', })
 
 
 @unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set")
@@ -87,15 +86,18 @@ class TestSysadmin(SysadminBaseTestCase):
     def test_staff_access(self):
         # pylint: disable-msg=E1103
 
-        response = self.client.get(reverse('sysadmin'))
-        self.assertEqual('/sysadmin', response.context['next'])
+        test_views = ['sysadmin', 'sysadmin_courses', 'sysadmin_staffing',] 
+        for view in test_views:
+            response = self.client.get(reverse(view))
+            self.assertEqual(response.status_code, 302)
 
         logged_in = self.client.login(username=self.user.username,
                                       password='foo')
         self.assertTrue(logged_in)
 
-        response = self.client.get(reverse('sysadmin'))
-        self.assertEqual('/sysadmin', response.context['next'])
+        for view in test_views:
+            response = self.client.get(reverse(view))
+            self.assertEqual(response.status_code, 404)
 
         response = self.client.get(reverse('gitlogs'))
         self.assertEqual(response.status_code, 404)
@@ -106,11 +108,12 @@ def test_staff_access(self):
         self.client.logout()
         self.client.login(username=self.user.username, password='foo')
 
-        response = self.client.get(reverse('sysadmin'))
-        self.assertFalse(hasattr(response.context, 'next'))
+        for view in test_views:
+            response = self.client.get(reverse(view))
+            self.assertTrue(response.status_code, 200)
 
         response = self.client.get(reverse('gitlogs'))
-        self.assertFalse(hasattr(response.context, 'next'))
+        self.assertTrue(response.status_code, 200)
 
     def test_user_mod(self):
         """Create and delete a user"""
@@ -121,8 +124,7 @@ def test_user_mod(self):
 
         # Create user
         self.client.post(reverse('sysadmin'),
-                         {'dash_mode': _('Status'),
-                          'action': _('Create user'),
+                         {'action': 'create_user',
                           'student_uname': 'test_cuser+sysadmin@edx.org',
                           'student_fullname': 'test cuser',
                           'student_password': 'foozor', })
@@ -140,10 +142,9 @@ def test_user_mod(self):
 
         # Delete user
         self.client.post(reverse('sysadmin'),
-                         {'dash_mode': _('Status'),
-                          'action': _('Delete user'),
-                          'student_uname': 'test_cuser+sysadmin@edx.org',
-                          'student_fullname': 'test cuser', })
+                         {'action': 'del_user',
+                         'student_uname': 'test_cuser+sysadmin@edx.org',
+                         'student_fullname': 'test cuser', })
 
         self.assertEqual(0, len(User.objects.filter(
             username='test_cuser+sysadmin@edx.org',
@@ -156,10 +157,7 @@ def test_user_csv(self):
 
         self._setstaff_login()
 
-        response = self.client.post(reverse('sysadmin'), {
-            'dash_mode': _('Status'),
-            'action': _('Download list of all users (csv file)'),
-        })
+        response = self.client.post(reverse('sysadmin'), {'action': 'download_users',})
 
         self.assertIn('attachment', response['Content-Disposition'])
         self.assertEqual('text/csv', response['Content-Type'])
@@ -172,7 +170,7 @@ def test_authmap_repair(self):
 
         self._setstaff_login()
 
-        create_user('test0', 'test test')
+        Users().create_user('test0', 'test test')
         # Will raise exception, so no assert needed
         eamap = ExternalAuthMap.objects.get(external_name='test test')
         mitu = User.objects.get(username='test0')
@@ -184,9 +182,7 @@ def test_authmap_repair(self):
 
         self.assertFalse(check_password(eamap.internal_password, mitu.password))
 
-        response = self.client.post(reverse('sysadmin'), {
-            'dash_mode': _('Status'),
-            'action': _('Check and repair external Auth Map'), })
+        response = self.client.post(reverse('sysadmin'), {'action': 'repair_eamap', })
 
         self.assertIn('{0} test0'.format(_('Failed in authenticating')), response.content)
         self.assertIn(_('fixed password'), response.content)
@@ -195,9 +191,7 @@ def test_authmap_repair(self):
 
         # Check for all OK
         self._setstaff_login()
-        response = self.client.post(reverse('sysadmin'),
-                                    {'dash_mode': _('Status'),
-                                     'action': _('Check and repair external Auth Map'), })
+        response = self.client.post(reverse('sysadmin'),{ 'action': 'repair_eamap', })
         self.assertIn(_('All ok!'), response.content)
 
     def test_xml_course_add_delete(self):
@@ -206,10 +200,9 @@ def test_xml_course_add_delete(self):
         self._setstaff_login()
 
         # Try bad git repo
-        response = self.client.post(reverse('sysadmin'), {
-            'dash_mode': _('Courses'),
+        response = self.client.post(reverse('sysadmin_courses'), {
             'repo_location': 'github.com/mitocw/edx4edx_lite',
-            'action': _('Load new course from github'), })
+            'action': 'add_course', })
         self.assertIn(_("The git repo location should end with '.git', and be a valid url"), response.content.decode('utf-8'))
 
         # Create git loaded course
@@ -244,10 +237,7 @@ def test_staff_csv(self):
         self._setstaff_login()
         self._add_edx4edx()
 
-        response = self.client.post(reverse('sysadmin'), {
-            'dash_mode': _('Staffing and Enrollment'),
-            'action': _('Download staff and instructor list (csv file)'),
-        })
+        response = self.client.post(reverse('sysadmin_staffing'), {'action': 'get_staff_csv', })
 
         self.assertIn('attachment', response['Content-Disposition'])
         self.assertEqual('text/csv', response['Content-Type'])
@@ -264,8 +254,7 @@ def test_enrollment_page(self):
 
         self._setstaff_login()
         self._add_edx4edx()
-        response = self.client.post(reverse('sysadmin'), {'dash_mode': _('Staffing and Enrollment')})
-        print(response.content)
+        response = self.client.get(reverse('sysadmin_staffing'))
         self.assertIn('edx4edx', response.content)
         self._rm_edx4edx()
 
@@ -325,7 +314,7 @@ def test_mongo_course_add_delete(self):
             pass
 
         def_ms = modulestore()
-        self.assertIn('mongo', str(def_ms.__class__))
+        self.assertFalse(isinstance(def_ms, XMLModuleStore))
 
         self._add_edx4edx()
         course = def_ms.get_course('MITx/edx4edx/edx4edx')
diff --git a/lms/templates/sysadmin_dashboard.html b/lms/templates/sysadmin_dashboard.html
index 28cbfb05a823..21dcf4c869dc 100644
--- a/lms/templates/sysadmin_dashboard.html
+++ b/lms/templates/sysadmin_dashboard.html
@@ -10,6 +10,12 @@
 </%block>
 
 <style type="text/css">
+a.active-section {
+	color: #551A8B;
+}
+.sysadmin-dashboard-content h2 a {
+	margin-right: 1.2em;
+}
 table.stat_table {
 	font-family: verdana,arial,sans-serif;
 	font-size:11px;
@@ -40,34 +46,25 @@
 }
 </style>
 
-<script language="JavaScript" type="text/javascript">
-function goto( mode)
-{
-  document.dashform.dash_mode.value = mode;
-  document.dashform.submit() ;
-}
-</script>
-
 <section class="container">
 <div class="sysadmin-dashboard-wrapper">
 
   <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
     <h1>${_('Sysadmin Dashboard')}</h1>
-
-    <h2>[ <a href="#" onclick="goto('Status');" class="${modeflag.get('Status')}">${_('Status')}</a> |
-          <a href="#" onclick="goto('Courses');" class="${modeflag.get('Courses')}">${_('Courses')}</a> |
-          <a href="#" onclick="goto('Staffing and Enrollment');" class="${modeflag.get('Staffing and Enrollment')}">${_('Staffing and Enrollment')}</a> |
-          <a href="/gitlogs">${_('GitLogs')}</a>
-           ]
+	<hr />
+    <h2 class="instructor-nav">
+	  <a href="${reverse('sysadmin')}" class="${modeflag.get('users')}">${_('Users')}</a>
+      <a href="${reverse('sysadmin_courses')}" class="${modeflag.get('courses')}">${_('Courses')}</a>
+      <a href="${reverse('sysadmin_staffing')}" class="${modeflag.get('staffing')}">${_('Staffing and Enrollment')}</a> 
+      <a href="${reverse('gitlogs')}">${_('Git Logs')}</a>
     </h2>
+	<hr />
+%if modeflag.get('users'):
+	<h3>${_('User Management')}</h3>
 
-    <form name="dashform" method="POST">
-    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
-    <input type="hidden" name="dash_mode" value="">
+    <form name="action" method="POST">
+    <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
 
-##----------------------------------------------------------------------------- 
-%if modeflag.get('Status'):
-	<h3>${_('User Management')}</h3>
 	<ul class="list-input">
 	  <li class="field text" style="padding-bottom: 1.2em">
 		<label for="student_uname">${_('Email or username')}</label>
@@ -86,75 +83,76 @@ <h3>${_('User Management')}</h3>
 
 	<div class="form-actions">
 	  <p>
-		<input type="submit" name="action" value="${_('Delete user')}" />
-		<input type="submit" name="action" value="${_('Create user')}">
+		<button type="submit" name="action" value="del_user">${_('Delete user')}</button>
+		<button type="submit" name="action" value="create_user">${_('Create user')}</button>
 	  </p>
     </div>
 
 	<hr />
     <p>
-    <input type="submit" name="action" value="${_('Download list of all users (csv file)')}" style="width: 300px;">
+      <button type="submit" name="action" value="download_users" style="width: 350px;">
+		${_('Download list of all users (csv file)')}
+	  </button>
     </p>
 
     <p>
-    <input type="submit" name="action" value="${_('Check and repair external Auth Map')}" style="width: 300px;">
+    <button type="submit" name="action" value="repair_eamap" style="width: 350px;">
+	  ${_('Check and repair external authentication map')}
+	</button>
     </p>
 
     <hr width="40%" style="align:left">
+	</form>
  %endif
 
-##----------------------------------------------------------------------------- 
-%if modeflag.get('Staffing and Enrollment'):
+%if modeflag.get('staffing'):
 
 <p>${_("Go to each individual course's Instructor dashboard to manage course enrollment.")}</p>
 <hr />
 
 <h3>${_('Manage course staff and instructors')}</h3><br/>
-
-    <p>
-    <input type="submit" name="action" value="${_('Download staff and instructor list (csv file)')}">
-    </p>
+<form name="action" method="POST">
+  <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
+  <button type="submit" name="action" value="get_staff_csv">${_('Download staff and instructor list (csv file)')}</button>
+</form>
 
 %endif
 
-##----------------------------------------------------------------------------- 
-%if modeflag.get('Courses'):
-    <h3>${_('Administer Courses')}</h3><br/>
-
-    <ul class="list-input">
-	  <li class="field text">
-		<label for="repo_location">
-		  ${_('Repo location')}:
-		</label>
-		<input type="text" name="repo_location" style="width:60%" />
-	  </li>
-	</ul>
-	<div class="form-actions">
-	  <input type="submit" name="action" value="${_('Load new course from github')}">
-	</div>
-	<hr />
-    <ul class="list-input">
-	  <li class="field text">
-		<label for="course_id">
-		  ${_('Course ID or dir')}:
-		</label>
-		<input type="text" name="course_id" style="width:60%" />
-	  </li>
-	</ul>
-	<div class="form-actions">
-      <input type="submit" name="action" value="${_('Delete course from site')}">
-	</div>
-	<hr style="width:40%" />
+%if modeflag.get('courses'):
+<h3>${_('Administer Courses')}</h3><br/>
+
+<form name="action" method="POST">
+  <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }" />
+  <ul class="list-input">
+	<li class="field text">
+	  <label for="repo_location">
+		${_('Repo location')}:
+	  </label>
+	  <input type="text" name="repo_location" style="width:60%" />
+	</li>
+  </ul>
+  <div class="form-actions">
+	<button type="submit" name="action" value="add_course">${_('Load new course from github')}</button>
+  </div>
+  <hr />
+  <ul class="list-input">
+	<li class="field text">
+	  <label for="course_id">
+		${_('Course ID or dir')}:
+	  </label>
+	  <input type="text" name="course_id" style="width:60%" />
+	</li>
+  </ul>
+  <div class="form-actions">
+	<button type="submit" name="action" value="del_course">${_('Delete course from site')}</button>
+  </div>
+</form>
+<hr style="width:40%" />
 %endif
 
-##----------------------------------------------------------------------------- 
-    </form>
-
 %if msg:
     <p>${msg}</p>
 %endif
-##----------------------------------------------------------------------------- 
-##----------------------------------------------------------------------------- 
 
 %if datatable:
 
@@ -180,7 +178,6 @@ <h2>${datatable['title']}</h2>
     </p>
 %endif
 
-##-----------------------------------------------------------------------------  
 %if plots:
 
     %for plot in plots:
diff --git a/lms/templates/sysadmin_dashboard_gitlogs.html b/lms/templates/sysadmin_dashboard_gitlogs.html
index be44bf060e75..7a63fb4d1ff3 100644
--- a/lms/templates/sysadmin_dashboard_gitlogs.html
+++ b/lms/templates/sysadmin_dashboard_gitlogs.html
@@ -8,14 +8,13 @@
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.js')}"></script>
   <script type="text/javascript" src="${static.url('js/vendor/flot/jquery.flot.axislabels.js')}"></script>
 </%block>
-<script language="JavaScript" type="text/javascript">
-function goto( mode)
-{
-  document.dashform.dash_mode.value = mode;
-  document.dashform.submit() ;
-}
-</script>
 <style type="text/css">
+a.active-section {
+	color: #551A8B;
+}
+.sysadmin-dashboard-content h2 a {
+	margin-right: 1.2em;
+}
 table.stat_table {
 	font-family: verdana,arial,sans-serif;
 	font-size:11px;
@@ -46,18 +45,22 @@
 }
 </style>
 
+<section class="container">
+<div class="sysadmin-dashboard-wrapper">
+
 <section class="container">
 <div class="sysadmin-dashboard-wrapper">
 
   <section class="sysadmin-dashboard-content" style="margin-left:10pt;margin-top:10pt;margin-right:10pt;margin-bottom:20pt">
     <h1>${_('Sysadmin Dashboard')}</h1>
-
-    <h2>[ <a href="#" onclick="goto('Status');">${_('Status')}</a> |
-          <a href="#" onclick="goto('Courses');">${_('Courses')}</a> |
-         <a href="#" onclick="goto('Staffing and Enrollment');">${_('Staffing and Enrollment')}</a> |
-           <a href="${reverse('gitlogs')}" class="selectedmode">${_('GitLogs')}</a>
-           ]
+	<hr />
+    <h2 class="instructor-nav">
+	  <a href="${reverse('sysadmin')}">${_('Users')}</a>
+      <a href="${reverse('sysadmin_courses')}">${_('Courses')}</a>
+      <a href="${reverse('sysadmin_staffing')}">${_('Staffing and Enrollment')}</a> 
+      <a href="${reverse('gitlogs')}" class="active-section">${_('Git Logs')}</a>
     </h2>
+	<hr />
 
     <form name="dashform" method="POST" action="${reverse('sysadmin')}">
     <input type="hidden" name="csrfmiddlewaretoken" value="${ csrf_token }">
@@ -66,16 +69,19 @@ <h2>[ <a href="#" onclick="goto('Status');">${_('Status')}</a> |
     <h3>${_('Git Logs')}</h3>
 
 %if course_id is None:
-
     <table class="stat_table">
 	  <thead>
-		<tr><th>${_('Date')}</th><th>${_('Course ID')}</th><th>${_('Git Action')}</th></tr>
+		<tr>
+		  <th>${_('Date')}</th>
+		  <th>${_('Course ID')}</th>
+		  <th>${_('Git Action')}</th>
+		</tr>
 	  </thead>
 	  <tbody>
 		%for cil in cilset[:10]:
         <tr>
 	      <td>${cil.created}</td>
-	      <td><a href="/gitlogs/${cil.course_id}">${cil.course_id}</a></td>
+	      <td><a href="${reverse('gitlogs')}/${cil.course_id}">${cil.course_id}</a></td>
 	      <td>${cil.git_log}</td>
 		</tr>
 		%endfor
@@ -87,22 +93,28 @@ <h3>${_('Git Logs')}</h3>
     <h2>${_('Recent git load activity for')} ${course_id}</h2>
 
     <table class="stat_table">
-    <tr><th>${_('Date')}</th><th>${_('Course ID')}</th><th>${_('git action')}</th></tr>
-
-    % for cil in cilset[:2]:
-
+	  <thead>
+		<tr>
+		  <th>${_('Date')}</th>
+		  <th>${_('Course ID')}</th>
+		  <th>${_('git action')}</th>
+		</tr>
+	  </thead>
+	  <tbody>
+		% for cil in cilset[:2]:
         <tr>
-	    <td>${cil.created}</td>
-	    <td><a href="/gitlogs/${cil.course_id}">${cil.course_id}</a></td>
-	    <td>${cil.git_log}</td>
-	</tr>
+	      <td>${cil.created}</td>
+	      <td><a href="${reverse('gitlogs')}/${cil.course_id}">${cil.course_id}</a></td>
+	      <td>${cil.git_log}</td>
+		</tr>
 	
-	<tr><td colspan="3">
-	    <pre>${cil.import_log | h}</pre>
-	</td></tr>
-
-    % endfor
-    
+		<tr>
+		  <td colspan="3">
+			<pre>${cil.import_log | h}</pre>
+		  </td>
+		</tr>
+		% endfor
+      </tbody>
     </table>
 
 % endif
diff --git a/lms/urls.py b/lms/urls.py
index 8c844d7ca0c3..8ccd7fc91e37 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -86,10 +86,7 @@
 # sysadmin dashboard, to see what courses are loaded, to delete & load courses
 if settings.MITX_FEATURES["ENABLE_SYSADMIN_DASHBOARD"]:
     urlpatterns += (
-        url(r'^sysadmin$', 'dashboard.sysadmin.sysadmin_dashboard', name="sysadmin"),
-        url(r'^gitlogs$', 'dashboard.sysadmin.view_git_logs', name="gitlogs"),
-        url(r'^gitlogs/(?P<course_id>.+)$', 'dashboard.sysadmin.view_git_logs',
-            name="gitlogs_detail"),
+        url(r'^sysadmin/', include('dashboard.sysadmin_urls')),
     )
 
 #Semi-static views (these need to be rendered and have the login bar, but don't change)

From f2fdc7d1f52156ac72cf020455cc3b91125a011a Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Tue, 19 Nov 2013 10:25:59 -0500
Subject: [PATCH 13/20] Class based view refactoring Switched SSL domain to be
 settings instead of hard coded Modified csv return to use memory efficient
 generator and django qs iterator

Fixup
---
 .../management/commands/git_add_course.py     |  37 ++--
 lms/djangoapps/dashboard/models.py            |   5 +-
 lms/djangoapps/dashboard/sysadmin.py          | 170 +++++++++++-------
 lms/djangoapps/dashboard/sysadmin_urls.py     |  12 +-
 .../dashboard/tests/test_sysadmin.py          | 106 +++++++----
 lms/envs/aws.py                               |   5 +
 lms/envs/common.py                            |   2 +-
 7 files changed, 213 insertions(+), 124 deletions(-)

diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py
index 5cc137863d3a..ee19a6665fae 100644
--- a/lms/djangoapps/dashboard/management/commands/git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py
@@ -1,8 +1,8 @@
 """
 Script for importing courseware from git/xml into a mongo modulestore
 """
+
 import os
-import sys
 import re
 import datetime
 import StringIO
@@ -11,7 +11,7 @@
 from django.utils.translation import ugettext as _
 from django.conf import settings
 from django.core import management
-from django.core.management.base import BaseCommand, CommandError, make_option
+from django.core.management.base import BaseCommand, CommandError
 
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml import XMLModuleStore
@@ -24,9 +24,10 @@
 GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos')
 GIT_IMPORT_STATIC = getattr(settings, 'GIT_IMPORT_STATIC', True)
 
+
 def add_repo(repo, rdir_in):
     """This will add a git repo into the mongo modulestore"""
-    # pylint: disable-msg=R0915
+    # pylint: disable=R0915
 
     # Set defaults even if it isn't defined in settings
     mongo_db = {
@@ -43,17 +44,18 @@ def add_repo(repo, rdir_in):
                 config_item, mongo_db[config_item])
 
     if not os.path.isdir(GIT_REPO_DIR):
-        log.critical(_("Path {0} doesn't exist, please create it, or configure a "
-                       "different path with GIT_REPO_DIR").format(GIT_REPO_DIR))
+        log.critical(_("Path {0} doesn't exist, please create it, "
+                       "or configure a different path with "
+                       "GIT_REPO_DIR").format(GIT_REPO_DIR))
         return -1
 
-    # -----------------------------------------------------------------------------
     # pull from git
     if not (repo.endswith('.git') or repo.startswith('http:') or
        repo.startswith('https:') or repo.startswith('git:')):
 
         log.error(_('Oops, not a git ssh url?'))
-        log.error(_('Expecting something like git@github.com:mitocw/edx4edx_lite.git'))
+        log.error(_('Expecting something like '
+                    'git@github.com:mitocw/edx4edx_lite.git'))
         return -1
 
     if rdir_in:
@@ -66,7 +68,8 @@ def add_repo(repo, rdir_in):
 
     rdirp = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
     if os.path.exists(rdirp):
-        log.info(_('directory already exists, doing a git pull instead of git clone'))
+        log.info(_('directory already exists, doing a git pull instead '
+                   'of git clone'))
         cmd = 'cd {0}/{1}; git pull'.format(GIT_REPO_DIR, rdir)
     else:
         cmd = 'cd {0}; git clone "{1}"'.format(GIT_REPO_DIR, repo)
@@ -80,7 +83,8 @@ def add_repo(repo, rdir_in):
         return -1
 
     # get commit id
-    commit_id = os.popen('cd {0}; git log -n 1 | head -1'.format(rdirp)).read().strip().split(' ')[1]
+    commit_id = os.popen('cd {0}; git log -n 1 | head -1'.format(
+        rdirp)).read().strip().split(' ')[1]
 
     ret_git += _('\nCommit ID: {0}').format(commit_id)
 
@@ -149,7 +153,6 @@ def add_repo(repo, rdir_in):
                       cdir)).read())
             log.debug(os.popen('ls -l {0}'.format(cdir)).read())
 
-    # -----------------------------------------------------------------------------
     # store import-command-run output in mongo
     mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
         mongo_db['user'], mongo_db['password'],
@@ -179,25 +182,28 @@ def add_repo(repo, rdir_in):
     mdb.disconnect()
     return 0
 
+
 class Command(BaseCommand):
     """
     Pull a git repo and import into the mongo based content database.
     """
 
-    help = _('Import the specified git repository into the modulestore and directory')
+    help = _('Import the specified git repository into the '
+             'modulestore and directory')
 
     def handle(self, *args, **options):
         """Check inputs and run the command"""
-        # pylint: disable-msg=C0103
 
         if isinstance(modulestore, XMLModuleStore):
             raise CommandError(_('This script requires a mongo module store'))
 
         if len(args) < 1:
-            raise CommandError(_('This script requires at least one argument, the git URL'))
+            raise CommandError(_('This script requires at least one argument, '
+                                 'the git URL'))
 
         if len(args) > 2:
-            raise CommandError(_('This script requires no more than two arguments.'))
+            raise CommandError(_('This script requires no more than two '
+                                 'arguments.'))
 
         rdir_arg = None
 
@@ -205,4 +211,5 @@ def handle(self, *args, **options):
             rdir_arg = args[1]
 
         if add_repo(args[0], rdir_arg) != 0:
-            raise CommandError(_('Repo was not added, check log output for details'))
+            raise CommandError(_('Repo was not added, check log output '
+                                 'for details'))
diff --git a/lms/djangoapps/dashboard/models.py b/lms/djangoapps/dashboard/models.py
index ddc2eb9aeb2e..096288f6d4e8 100644
--- a/lms/djangoapps/dashboard/models.py
+++ b/lms/djangoapps/dashboard/models.py
@@ -1,7 +1,11 @@
+"""Models for dashboard application"""
+
 import mongoengine
 
+
 class CourseImportLog(mongoengine.Document):
     """Mongoengine model for git log"""
+    # pylint: disable=R0924
 
     course_id = mongoengine.StringField(max_length=128)
     location = mongoengine.StringField(max_length=168)
@@ -11,4 +15,3 @@ class CourseImportLog(mongoengine.Document):
     created = mongoengine.DateTimeField()
     meta = {'indexes': ['course_id', 'created'],
             'allow_inheritance': False}
-
diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index b991f3f66e16..1aedbea81da3 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -8,7 +8,6 @@
 import os
 import subprocess
 import time
-import imp
 import StringIO
 
 from datetime import datetime
@@ -22,15 +21,13 @@
 
 from django.contrib.auth import authenticate
 from django.core.exceptions import PermissionDenied
-from django.core import management
-from django.core.management.base import CommandError
 from django_future.csrf import ensure_csrf_cookie
 from django.views.decorators.cache import cache_control
 from django.utils.decorators import method_decorator
+from django.views.decorators.http import condition
 from django.db import IntegrityError
 from django.http import HttpResponse
 from django.utils.html import escape
-from django.contrib.admin.views.decorators import staff_member_required
 from django.contrib.auth.decorators import login_required
 from django.http import Http404
 from django.views.generic.base import TemplateView
@@ -51,26 +48,34 @@
 
 log = logging.getLogger(__name__)
 
+
 class SysadminDashboardView(TemplateView):
     """Base class for sysadmin dashboard views with common methods"""
 
     template_name = 'sysadmin_dashboard.html'
 
     def __init__(self, **kwargs):
+        """Determine modulestore type"""
+
         self.def_ms = modulestore()
         self.is_using_mongo = True
         if isinstance(self.def_ms, XMLModuleStore):
             self.is_using_mongo = False
         self.msg = u''
-        return super(SysadminDashboardView, self).__init__(**kwargs)
+        self.datatable = []
+        super(SysadminDashboardView, self).__init__(**kwargs)
 
     @method_decorator(ensure_csrf_cookie)
     @method_decorator(login_required)
-    @method_decorator(cache_control(no_cache=True, no_store=True, must_revalidate=True))
+    @method_decorator(cache_control(no_cache=True, no_store=True,
+                                    must_revalidate=True))
+    @method_decorator(condition(etag_func=None))
     def dispatch(self, *args, **kwargs):
         return super(SysadminDashboardView, self).dispatch(*args, **kwargs)
 
     def get_courses(self):
+        """ Get an iterable list of courses regardless of module store type."""
+
         # Prefer mongo if using mixed or mongo store
         if self.is_using_mongo:
             courses = self.def_ms.get_courses()
@@ -79,18 +84,36 @@ def get_courses(self):
             courses = self.def_ms.courses.items()
         return courses
 
-    def return_csv(self, filename, datatable):
-        """Convenient function for handling the http response of a csv"""
-
-        response = HttpResponse(mimetype='text/csv')
-        response['Content-Disposition'] = 'attachment; filename={0}'.format(filename)
+    def return_csv(self, filename, header, data):
+        """
+        Convenient function for handling the http response of a csv.
+        data should be iterable and is used to stream object over http
+        """
 
-        writer = csv.writer(response, dialect='excel', quotechar='"',
+        csv_file = StringIO.StringIO()
+        writer = csv.writer(csv_file, dialect='excel', quotechar='"',
                             quoting=csv.QUOTE_ALL)
-        writer.writerow(datatable['header'])
-        for datarow in datatable['data']:
-            encoded_row = [unicode(s).encode('utf-8') for s in datarow]
-            writer.writerow(encoded_row)
+
+        writer.writerow(header)
+        
+        # Setup streaming of the data
+        def read_and_flush():
+            """Read and clear buffer for optimization"""
+            csv_file.seek(0)
+            csv_data = csv_file.read()
+            csv_file.seek(0)
+            csv_file.truncate()
+            return csv_data
+
+        def csv_data():
+            """Generator for handling potentially large CSVs"""
+            for row in data:
+                writer.writerow(row)
+            csv_data = read_and_flush()
+            yield csv_data
+        response = HttpResponse(csv_data(), mimetype='text/csv')
+        response['Content-Disposition'] = 'attachment; filename={0}'.format(
+            filename)
         return response
 
 
@@ -136,14 +159,16 @@ def create_user(self, uname, name, password=None):
         if not name:
             return _('Must provide full name')
 
+        email_domain = getattr(settings, 'SSL_AUTH_EMAIL_DOMAIN', 'MIT.EDU')
+
         msg = u''
         if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
             if not '@' in uname:
-                email = '{0}@MIT.EDU'.format(uname)
+                email = '{0}@{1}'.format(uname, email_domain)
             else:
                 email = uname
-            if not email.endswith('@MIT.EDU'):
-                msg += u'email must end in @MIT.EDU'
+            if not email.endswith('@{0}'.format(email_domain)):
+                msg += u'{0} @{1}'.format(_('email must end in'), email_domain)
                 return msg
             mit_domain = 'ssl:MIT'
             if ExternalAuthMap.objects.filter(external_id=email,
@@ -180,7 +205,9 @@ def create_user(self, uname, name, password=None):
         profile.save()
 
         if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
-            credentials = '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}'.format(name, email)
+            credential_string = getattr(settings, 'SSL_AUTH_DN_FORMAT_STRING',
+                                        '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}')
+            credentials = credential_string.format(name, email)
             eamap = ExternalAuthMap(
                 external_id=email,
                 external_email=email,
@@ -205,8 +232,7 @@ def delete_user(self, uname):
             try:
                 user = User.objects.get(email=uname)
             except User.DoesNotExist, err:
-                msg = _('Cannot find user with email address {0}'
-                        ).format(uname)
+                msg = _('Cannot find user with email address {0}').format(uname)
                 return msg
         else:
             try:
@@ -225,9 +251,9 @@ def make_common_context(self):
         courses = self.get_courses()
 
         self.datatable = dict(header=[_('Statistic'), _('Value')],
-                         title=_('Site statistics'))
+                              title=_('Site statistics'))
         self.datatable['data'] = [[_('Total number of users'),
-                              User.objects.all().count()]]
+                                   User.objects.all().count()]]
 
         if hasattr(courses, 'items'):
             course_iter = courses.items()
@@ -269,22 +295,24 @@ def post(self, request):
         track.views.server_track(request, action, {}, page='user_sysdashboard')
 
         if action == 'download_users':
-            self.datatable = dict(header=[_('username'), _('email')],
-                             title=_('List of all users'),
-                             data=[[u.username, u.email] for u in
-                                   User.objects.all()])
+            header = [_('username'), _('email'), ]
+            data = [[u.username, u.email] for u in
+                    User.objects.all().iterator()]
             return self.return_csv('users_{0}.csv'.format(
-                request.META['SERVER_NAME']), self.datatable)
+                request.META['SERVER_NAME']), header, data)
         elif action == 'repair_eamap':
             self.msg = u'<h4>{0}</h4><pre>{1}</pre>{2}'.format(
-                _('Repair Results'), self.fix_external_auth_map_passwords(), self.msg)
+                _('Repair Results'),
+                self.fix_external_auth_map_passwords(),
+                self.msg)
             self.datatable = {}
         elif action == 'create_user':
             uname = request.POST.get('student_uname', '').strip()
             name = request.POST.get('student_fullname', '').strip()
             password = request.POST.get('student_password', '').strip()
             self.msg = u'<h4>{0}</h4><p>{1}</p><hr />{2}'.format(
-                _('Create User Results'), self.create_user(uname, name, password), self.msg)
+                _('Create User Results'),
+                self.create_user(uname, name, password), self.msg)
         elif action == 'del_user':
             uname = request.POST.get('student_uname', '').strip()
             self.msg = u'<h4>{0}</h4><p>{1}</p><hr />{2}'.format(
@@ -299,6 +327,7 @@ def post(self, request):
         }
         return render_to_response(self.template_name, context)
 
+
 class Courses(SysadminDashboardView):
     """
     This manages adding/updating courses from git, deleting courses, and
@@ -315,12 +344,12 @@ def git_info_for_course(self, cdir):
             return info
 
         cmd = ['git', 'log', '-1',
-               '--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}',]
+               '--format=format:{ "commit": "%H", "author": "%an %ae", "date": "%ad"}', ]
         try:
             output_json = json.loads(subprocess.check_output(cmd, cwd=gdir))
-            info = [ output_json['commit'],
-                     output_json['date'], 
-                     output_json['author'], ]
+            info = [output_json['commit'],
+                    output_json['date'],
+                    output_json['author'], ]
         except (ValueError, subprocess.CalledProcessError):
             pass
 
@@ -331,7 +360,8 @@ def get_course_from_git(self, gitloc, datatable):
 
         if not (gitloc.endswith('.git') or gitloc.startswith('http:') or
                 gitloc.startswith('https:') or gitloc.startswith('git:')):
-            return _("The git repo location should end with '.git', and be a valid url")
+            return _("The git repo location should end with '.git', "
+                     "and be a valid url")
 
         if self.is_using_mongo:
             return self.import_mongo_course(gitloc)
@@ -364,7 +394,6 @@ def import_mongo_course(self, gitloc):
             logger.setLevel(logging.DEBUG)
             logger.addHandler(import_log_handler)
 
-
         git_add_course.add_repo(gitloc, None)
         ret = output.getvalue()
 
@@ -393,14 +422,14 @@ def import_xml_course(self, gitloc, datatable):
         if not os.path.exists(gdir):
             msg += _('Failed to clone repository to {0}').format(gdir)
             return msg
-        self.def_ms.try_load_course(os.path.abspath(gdir))  # load into modulestore
+        self.def_ms.try_load_course(os.path.abspath(gdir))
         errlog = self.def_ms.errored_courses.get(cdir, '')
         if errlog:
             msg += u'<hr width="50%"><pre>{0}</pre>'.format(escape(errlog))
         else:
             course = self.def_ms.courses[os.path.abspath(gdir)]
-            msg += _('Loaded course {0} {1}<br/>Errors:').format(cdir,
-                                                                 course.display_name)
+            msg += _('Loaded course {0} {1}<br/>Errors:').format(
+                cdir, course.display_name)
             errors = self.def_ms.get_item_errors(course.location)
             if not errors:
                 msg += u'None'
@@ -417,7 +446,7 @@ def import_xml_course(self, gitloc, datatable):
 
     def make_datatable(self):
         """Creates course information datatable"""
-        
+
         data = []
         courses = self.get_courses()
         if hasattr(courses, 'items'):
@@ -431,11 +460,11 @@ def make_datatable(self):
             data.append([course.display_name, cdir]
                         + self.git_info_for_course(gdir))
 
-        return  dict(header=[_('Course Name'), _('Directory/ID'),
-                         _('Git Commit'), _('Last Change'),
-                         _('Last Editor')],
-                         title=_('Information about all courses'),
-                         data=data)
+        return dict(header=[_('Course Name'), _('Directory/ID'),
+                            _('Git Commit'), _('Last Change'),
+                            _('Last Editor')],
+                    title=_('Information about all courses'),
+                    data=data)
 
     def get(self, request):
         """Displays forms and course information"""
@@ -459,7 +488,8 @@ def post(self, request):
             raise Http404
 
         action = request.POST.get('action', '')
-        track.views.server_track(request, action, {}, page='courses_sysdashboard')
+        track.views.server_track(request, action, {},
+                                 page='courses_sysdashboard')
 
         courses = self.get_courses()
         if action == _('add_course'):
@@ -479,8 +509,10 @@ def post(self, request):
                     course = get_course_by_id(course_id)
                     course_found = True
                 except Exception, err:
-                    self.msg += _('Error - cannot get course with ID {0}<br/><pre>{1}</pre>').format(
-                        course_id, escape(str(err)))
+                    self.msg += _('Error - cannot get course with ID '
+                                  '{0}<br/><pre>{1}</pre>').format(
+                                      course_id, escape(str(err))
+                                  )
 
             if course_found and not self.is_using_mongo:
                 cdir = course.data_dir
@@ -490,8 +522,9 @@ def post(self, request):
                 new_dir = cdir + '_deleted_{0}'.format(int(time.time()))
                 os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / new_dir)
 
-                self.msg += u"<font color='red'>Deleted {0} = {1} ({2})</font>".format(
-                    cdir, course.id, course.display_name)
+                self.msg += (u"<font color='red'>Deleted "
+                             u"{0} = {1} ({2})</font>".format(
+                                 cdir, course.id, course.display_name))
 
             elif course_found and self.is_using_mongo:
                 # delete course that is stored with mongodb backend
@@ -547,15 +580,16 @@ def get(self, request):
             course_iter = courses
         for (cdir, course) in course_iter:
             datum = [course.display_name, course.id]
-            datum += \
-                [CourseEnrollment.objects.filter(course_id=course.id).count()]
+            datum += [CourseEnrollment.objects.filter(
+                course_id=course.id).count()]
             datum += [self.get_group(course, 'staff').user_set.all().count()]
             datum += [','.join([x.username for x in self.get_group(course,
                       'instructor').user_set.all()])]
             data.append(datum)
 
         datatable = dict(header=[_('Course Name'), _('course_id'),
-                                 _('# enrolled'), _('# staff'), _('instructors')],
+                                 _('# enrolled'), _('# staff'),
+                                 _('instructors')],
                          title=_('Enrollment information for all courses'),
                          data=data)
         context = {
@@ -571,7 +605,8 @@ def post(self, request):
         """Handle all actions from staffing and enrollment view"""
 
         action = request.POST.get('action', '')
-        track.views.server_track(request, action, {}, page='staffing_sysdashboard')
+        track.views.server_track(request, action, {},
+                                 page='staffing_sysdashboard')
 
         if action == 'get_staff_csv':
             data = []
@@ -588,13 +623,11 @@ def post(self, request):
                         datum = [course.id, role, user.username, user.email,
                                  user.profile.name]
                         data.append(datum)
-            datatable = dict(header=[_('course_id'),
-                                     _('role'), _('username'),
-                                     _('email'), _('full_name')],
-                             title=_('List of all course staff and instructors'),
-                             data=data)
+            header = [_('course_id'),
+                      _('role'), _('username'),
+                      _('email'), _('full_name'), ]
             return self.return_csv('staff_{0}.csv'.format(
-                request.META['SERVER_NAME']), datatable)
+                request.META['SERVER_NAME']), header, data)
 
         return self.get(request)
 
@@ -612,7 +645,7 @@ class GitLogs(TemplateView):
     def get(self, request, course_id=None):
         """Shows logs of imports that happened as a result of a git import"""
 
-        # Set defaults even if it isn't defined in settings
+        # Set mongodb defaults even if it isn't defined in settings
         mongo_db = {
             'host': 'localhost',
             'user': '',
@@ -638,8 +671,9 @@ def get(self, request, course_id=None):
             else:
                 mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
         except mongoengine.connection.ConnectionError, ex:
-            logging.critical(_('Unable to connect to mongodb to save log, please check '
-                               'MONGODB_LOG settings. error: {0}').format(str(ex)))
+            logging.critical(_('Unable to connect to mongodb to save log, '
+                               'please check MONGODB_LOG settings. '
+                               'error: {0}').format(str(ex)))
 
         if course_id is None:
             # Require staff if not going to specific course
@@ -650,8 +684,8 @@ def get(self, request, course_id=None):
             try:
                 course = get_course_by_id(course_id)
             except Exception:
-                    cilset = None
-                    error_msg = _('Cannot find course {0}').format(course_id)
+                cilset = None
+                error_msg = _('Cannot find course {0}').format(course_id)
 
             # Allow only course team, instructors, and staff
             if not (request.user.is_staff or
@@ -663,6 +697,8 @@ def get(self, request, course_id=None):
                 course_id=course_id).order_by('-created')
             log.debug('cilset length={0}'.format(len(cilset)))
         mdb.disconnect()
-        context = {'cilset': cilset, 'course_id': course_id, 'error_msg': error_msg}
+        context = {'cilset': cilset,
+                   'course_id': course_id,
+                   'error_msg': error_msg}
 
         return render_to_response(self.template_name, context)
diff --git a/lms/djangoapps/dashboard/sysadmin_urls.py b/lms/djangoapps/dashboard/sysadmin_urls.py
index 0e04b94c1d90..24fbe178ee50 100644
--- a/lms/djangoapps/dashboard/sysadmin_urls.py
+++ b/lms/djangoapps/dashboard/sysadmin_urls.py
@@ -1,12 +1,16 @@
-from django.conf.urls import include, patterns, url
-from django.views.generic import TemplateView
+"""
+Urls for sysadmin dashboard feature
+"""
+
+from django.conf.urls import patterns, url
 from dashboard import sysadmin
 
 urlpatterns = patterns(
     '',
     url(r'^$', sysadmin.Users.as_view(), name="sysadmin"),
     url(r'^courses/?$', sysadmin.Courses.as_view(), name="sysadmin_courses"),
-    url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"),    
+    url(r'^staffing/?$', sysadmin.Staffing.as_view(), name="sysadmin_staffing"),
     url(r'^gitlogs/?$', sysadmin.GitLogs.as_view(), name="gitlogs"),
-    url(r'^gitlogs/(?P<course_id>.+)$', sysadmin.GitLogs.as_view(), name="gitlogs_detail"),
+    url(r'^gitlogs/(?P<course_id>.+)$', sysadmin.GitLogs.as_view(),
+        name="gitlogs_detail"),
 )
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 17747757febf..96d2f3e65f97 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -37,16 +37,21 @@
 MITX_FEATURES_WITH_SSL_AUTH = settings.MITX_FEATURES.copy()
 MITX_FEATURES_WITH_SSL_AUTH['AUTH_USE_MIT_CERTIFICATES'] = True
 
+
 class SysadminBaseTestCase(ModuleStoreTestCase):
     """ Base class with common methods used in XML and Mongo tests"""
 
     def setUp(self):
+        """Setup test case by adding primary user."""
+
         super(SysadminBaseTestCase, self).setUp()
-        self.user = User.objects.create_user('test_user', 'test_user+sysadmin@edx.org', 'foo')
+        self.user = User.objects.create_user('test_user',
+                                             'test_user+sysadmin@edx.org',
+                                             'foo')
         self.client = Client()
 
     def _setstaff_login(self):
-        """ Makes the test user staff and logs them in"""
+        """Makes the test user staff and logs them in"""
 
         self.user.is_staff = True
         self.user.save()
@@ -61,32 +66,33 @@ def _add_edx4edx(self):
 
     def _rm_edx4edx(self):
         """Deletes the sample course from the XML store"""
-        # pylint: disable-msg=E1103
 
         def_ms = modulestore()
         try:
             # using XML store
-            course = def_ms.courses.get('{0}/edx4edx_lite'.format(os.path.abspath(settings.DATA_DIR)), None)
+            course = def_ms.courses.get('{0}/edx4edx_lite'.format(
+                os.path.abspath(settings.DATA_DIR)), None)
         except AttributeError:
             # Using mongo store
             course = def_ms.get_course('MITx/edx4edx/edx4edx')
 
         # Delete git loaded course
-        return self.client.post(reverse('sysadmin_courses'), 
-                                 { 'course_id': course.id,
+        return self.client.post(reverse('sysadmin_courses'),
+                                {'course_id': course.id,
                                  'action': 'del_course', })
 
 
-@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set")
+@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
+                     "ENABLE_SYSADMIN_DASHBOARD not set")
 class TestSysadmin(SysadminBaseTestCase):
     """
     Test sysadmin dashboard features using XMLModuleStore
     """
 
     def test_staff_access(self):
-        # pylint: disable-msg=E1103
+        """Test access controls."""
 
-        test_views = ['sysadmin', 'sysadmin_courses', 'sysadmin_staffing',] 
+        test_views = ['sysadmin', 'sysadmin_courses', 'sysadmin_staffing', ]
         for view in test_views:
             response = self.client.get(reverse(view))
             self.assertEqual(response.status_code, 302)
@@ -134,8 +140,8 @@ def test_user_mod(self):
                              email='test_cuser+sysadmin@edx.org'))
 
         # login as new user to confirm
-        self.assertTrue(self.client.login(username='test_cuser+sysadmin@edx.org',
-                                          password='foozor'))
+        self.assertTrue(self.client.login(
+            username='test_cuser+sysadmin@edx.org', password='foozor'))
 
         self.client.logout()
         self.client.login(username=self.user.username, password='foo')
@@ -143,8 +149,8 @@ def test_user_mod(self):
         # Delete user
         self.client.post(reverse('sysadmin'),
                          {'action': 'del_user',
-                         'student_uname': 'test_cuser+sysadmin@edx.org',
-                         'student_fullname': 'test cuser', })
+                          'student_uname': 'test_cuser+sysadmin@edx.org',
+                          'student_fullname': 'test cuser', })
 
         self.assertEqual(0, len(User.objects.filter(
             username='test_cuser+sysadmin@edx.org',
@@ -155,14 +161,25 @@ def test_user_mod(self):
     def test_user_csv(self):
         """Download and validate user CSV"""
 
+        num_test_users = 0
         self._setstaff_login()
 
-        response = self.client.post(reverse('sysadmin'), {'action': 'download_users',})
+        # Stuff full of users to test streaming
+        for user_num in xrange(num_test_users):
+            Users().create_user('testingman_with_long_name{}'.format(user_num),
+                                'test test')
+
+        response = self.client.post(reverse('sysadmin'),
+                                    {'action': 'download_users', })
 
         self.assertIn('attachment', response['Content-Disposition'])
         self.assertEqual('text/csv', response['Content-Type'])
         self.assertIn('test_user', response.content)
-        self.assertTrue(2, len(response.content.splitlines()))
+        self.assertTrue(num_test_users + 2, len(response.content.splitlines()))
+
+        # Clean up
+        User.objects.filter(
+            username__startswith='testingman_with_long_name').delete()
 
     @override_settings(MITX_FEATURES=MITX_FEATURES_WITH_SSL_AUTH)
     def test_authmap_repair(self):
@@ -182,16 +199,20 @@ def test_authmap_repair(self):
 
         self.assertFalse(check_password(eamap.internal_password, mitu.password))
 
-        response = self.client.post(reverse('sysadmin'), {'action': 'repair_eamap', })
+        response = self.client.post(reverse('sysadmin'),
+                                    {'action': 'repair_eamap', })
 
-        self.assertIn('{0} test0'.format(_('Failed in authenticating')), response.content)
+        self.assertIn('{0} test0'.format(_('Failed in authenticating')),
+                      response.content)
         self.assertIn(_('fixed password'), response.content)
 
-        self.assertTrue(self.client.login(username='test0', password=eamap.internal_password))
+        self.assertTrue(self.client.login(username='test0',
+                                          password=eamap.internal_password))
 
         # Check for all OK
         self._setstaff_login()
-        response = self.client.post(reverse('sysadmin'),{ 'action': 'repair_eamap', })
+        response = self.client.post(reverse('sysadmin'),
+                                    {'action': 'repair_eamap', })
         self.assertIn(_('All ok!'), response.content)
 
     def test_xml_course_add_delete(self):
@@ -203,7 +224,8 @@ def test_xml_course_add_delete(self):
         response = self.client.post(reverse('sysadmin_courses'), {
             'repo_location': 'github.com/mitocw/edx4edx_lite',
             'action': 'add_course', })
-        self.assertIn(_("The git repo location should end with '.git', and be a valid url"), response.content.decode('utf-8'))
+        self.assertIn(_("The git repo location should end with '.git', "
+                        "and be a valid url"), response.content.decode('utf-8'))
 
         # Create git loaded course
         response = self._add_edx4edx()
@@ -237,12 +259,15 @@ def test_staff_csv(self):
         self._setstaff_login()
         self._add_edx4edx()
 
-        response = self.client.post(reverse('sysadmin_staffing'), {'action': 'get_staff_csv', })
+        response = self.client.post(reverse('sysadmin_staffing'),
+                                    {'action': 'get_staff_csv', })
 
         self.assertIn('attachment', response['Content-Disposition'])
         self.assertEqual('text/csv', response['Content-Type'])
-        columns = [_('course_id'), _('role'), _('username'), _('email'), _('full_name'), ]
-        self.assertIn(','.join('"' + c + '"' for c in columns), response.content)
+        columns = [_('course_id'), _('role'), _('username'),
+                   _('email'), _('full_name'), ]
+        self.assertIn(','.join('"' + c + '"' for c in columns),
+                      response.content)
 
         self._rm_edx4edx()
 
@@ -261,7 +286,8 @@ def test_enrollment_page(self):
 
 @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
 @override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
-@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set")
+@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
+                     "ENABLE_SYSADMIN_DASHBOARD not set")
 class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
     """
     Check that importing into the mongo module store works
@@ -300,12 +326,16 @@ def test_missing_repo_dir(self):
 
         # Create git loaded course
         response = self._add_edx4edx()
-        self.assertIn(escape(_("Path {0} doesn't exist, please create it, or configure a "
-                               "different path with GIT_REPO_DIR").format(settings.GIT_REPO_DIR)),
+        self.assertIn(escape(_("Path {0} doesn't exist, please create it, or "
+                               "configure a different path with "
+                               "GIT_REPO_DIR").format(settings.GIT_REPO_DIR)),
                       response.content.decode('UTF-8'))
 
     def test_mongo_course_add_delete(self):
-        """same as TestSysadmin.test_xml_course_add_delete, but use mongo store"""
+        """
+        This is the same as TestSysadmin.test_xml_course_add_delete,
+        but it uses a mongo store
+        """
 
         self._setstaff_login()
         try:
@@ -340,14 +370,18 @@ def test_gitlogs(self):
         self.assertIn('/gitlogs/MITx/edx4edx/edx4edx', response.content)
 
         response = self.client.get(
-            reverse('gitlogs_detail', kwargs={'course_id': 'MITx/edx4edx/edx4edx'}))
+            reverse('gitlogs_detail', kwargs={
+                'course_id': 'MITx/edx4edx/edx4edx'}))
 
-        self.assertIn('======&gt; IMPORTING course to location', response.content)
+        self.assertIn('======&gt; IMPORTING course to location',
+                      response.content)
 
         self._rm_edx4edx()
 
     def test_gitlog_courseteam_access(self):
-        """Ensure course team users are allowed to access only their own course"""
+        """
+        Ensure course team users are allowed to access only their own course.
+        """
 
         try:
             os.mkdir(getattr(settings, 'GIT_REPO_DIR'))
@@ -368,20 +402,20 @@ def test_gitlog_courseteam_access(self):
         # Add user as staff in course team
         def_ms = modulestore()
         course = def_ms.get_course('MITx/edx4edx/edx4edx')
-        
+
         staff_groupname = get_access_group_name(course, 'staff')
         group, _ = Group.objects.get_or_create(name=staff_groupname)
         self.user.groups.add(group)
-        
+
         self.assertTrue(has_access(self.user, course, 'staff'))
         logged_in = self.client.login(username=self.user.username,
                                       password='foo')
         self.assertTrue(logged_in)
 
         response = self.client.get(
-            reverse('gitlogs_detail', kwargs={'course_id': 'MITx/edx4edx/edx4edx'}))
-        print(response.content)
-        self.assertIn('======&gt; IMPORTING course to location', response.content)
+            reverse('gitlogs_detail', kwargs={
+                'course_id': 'MITx/edx4edx/edx4edx'}))
+        self.assertIn('======&gt; IMPORTING course to location',
+                      response.content)
 
         self._rm_edx4edx()
-
diff --git a/lms/envs/aws.py b/lms/envs/aws.py
index 61a464b6b051..096f520b2be1 100644
--- a/lms/envs/aws.py
+++ b/lms/envs/aws.py
@@ -217,6 +217,11 @@
 if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS:
     TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS")
 
+# SSL external authentication settings
+SSL_AUTH_EMAIL_DOMAIN = ENV_TOKENS.get("SSL_AUTH_EMAIL_DOMAIN", "MIT.EDU")
+SSL_AUTH_DN_FORMAT_STRING = ENV_TOKENS.get("SSL_AUTH_DN_FORMAT_STRING",
+                                           "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}")
+
 ############################## SECURE AUTH ITEMS ###############
 # Secret things: passwords, access keys, etc.
 
diff --git a/lms/envs/common.py b/lms/envs/common.py
index 1c01d7b7f45f..8247b3d48296 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -89,7 +89,7 @@
 
     'ENABLE_MASQUERADE': True,  # allow course staff to change to student view of courseware
 
-    'ENABLE_SYSADMIN_DASHBOARD': False, # sysadmin dashboard, to see what courses are loaded, to delete & load courses
+    'ENABLE_SYSADMIN_DASHBOARD': False,  # sysadmin dashboard, to see what courses are loaded, to delete & load courses
 
     'DISABLE_LOGIN_BUTTON': False,  # used in systems where login is automatic, eg MIT SSL
 

From 4bdbbbabf32753277fe0f1ae93a36ec20fcac36d Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Tue, 19 Nov 2013 14:21:18 -0500
Subject: [PATCH 14/20] Refactor to comply with changes added in PR #1575

---
 lms/djangoapps/dashboard/sysadmin.py          | 32 ++++++-------------
 .../dashboard/tests/test_sysadmin.py          | 11 +++----
 2 files changed, 13 insertions(+), 30 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 1aedbea81da3..6c99d367d558 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -9,7 +9,6 @@
 import subprocess
 import time
 import StringIO
-
 from datetime import datetime
 
 from django.conf import settings
@@ -18,7 +17,6 @@
 from student.models import CourseEnrollment, UserProfile, Registration
 from external_auth.models import ExternalAuthMap
 from external_auth.views import generate_password
-
 from django.contrib.auth import authenticate
 from django.core.exceptions import PermissionDenied
 from django_future.csrf import ensure_csrf_cookie
@@ -31,13 +29,12 @@
 from django.contrib.auth.decorators import login_required
 from django.http import Http404
 from django.views.generic.base import TemplateView
-
 from mitxmako.shortcuts import render_to_response
 from xmodule.modulestore.django import modulestore
 from xmodule.contentstore.django import contentstore
 from xmodule.modulestore.xml import XMLModuleStore
 from xmodule.modulestore.store_utilities import delete_course
-from courseware.access import has_access, get_access_group_name
+from courseware.roles import CourseStaffRole, CourseInstructorRole
 from courseware.courses import get_course_by_id
 
 import mongoengine
@@ -95,7 +92,7 @@ def return_csv(self, filename, header, data):
                             quoting=csv.QUOTE_ALL)
 
         writer.writerow(header)
-        
+
         # Setup streaming of the data
         def read_and_flush():
             """Read and clear buffer for optimization"""
@@ -554,17 +551,6 @@ class Staffing(SysadminDashboardView):
     courses that include an option to download the data as a csv.
     """
 
-    def get_group(self, course, groupname):
-        """Gets the course group"""
-
-        grpname = get_access_group_name(course, groupname)
-        try:
-            group = Group.objects.get(name=grpname)
-        except Group.DoesNotExist:
-            group = Group(name=grpname)  # create the group
-            group.save()
-        return group
-
     def get(self, request):
         """Displays course Enrollment and staffing course statistics"""
 
@@ -582,9 +568,9 @@ def get(self, request):
             datum = [course.display_name, course.id]
             datum += [CourseEnrollment.objects.filter(
                 course_id=course.id).count()]
-            datum += [self.get_group(course, 'staff').user_set.all().count()]
-            datum += [','.join([x.username for x in self.get_group(course,
-                      'instructor').user_set.all()])]
+            datum += [CourseStaffRole(course.location).users_with_role().count()]
+            datum += [','.join([x.username for x in CourseInstructorRole(
+                course.location).users_with_role()])]
             data.append(datum)
 
         datatable = dict(header=[_('Course Name'), _('course_id'),
@@ -610,7 +596,7 @@ def post(self, request):
 
         if action == 'get_staff_csv':
             data = []
-            roles = ['instructor', 'staff']
+            roles = [CourseInstructorRole, CourseStaffRole, ]
 
             courses = self.get_courses()
             if hasattr(courses, 'items'):
@@ -619,7 +605,7 @@ def post(self, request):
                 course_iter = courses
             for (cdir, course) in course_iter:
                 for role in roles:
-                    for user in self.get_group(course, role).user_set.all():
+                    for user in role(course.location).users_with_role():
                         datum = [course.id, role, user.username, user.email,
                                  user.profile.name]
                         data.append(datum)
@@ -689,8 +675,8 @@ def get(self, request, course_id=None):
 
             # Allow only course team, instructors, and staff
             if not (request.user.is_staff or
-                    has_access(request.user, course, 'instructor') or
-                    has_access(request.user, course, 'staff')):
+                    CourseInstructorRole(course.location).has_user(request.user) or
+                    CourseStaffRole(course.location).has_user(request.user)):
                 raise Http404
             log.debug('course_id={0}'.format(course_id))
             cilset = CourseImportLog.objects.filter(
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 96d2f3e65f97..43631ab79cd9 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -8,12 +8,10 @@
 
 from django.test.client import Client
 from django.test.utils import override_settings
-
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext as _
-
 from dashboard.sysadmin import Users
 from external_auth.models import ExternalAuthMap
 from django.contrib.auth.hashers import check_password
@@ -22,9 +20,10 @@
 from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from django.utils.html import escape
-from courseware.access import get_access_group_name, has_access
+from courseware.roles import CourseStaffRole
 from dashboard.models import CourseImportLog
 from xmodule.modulestore.xml import XMLModuleStore
+
 import mongoengine
 
 TEST_MONGODB_LOG = {
@@ -403,11 +402,9 @@ def test_gitlog_courseteam_access(self):
         def_ms = modulestore()
         course = def_ms.get_course('MITx/edx4edx/edx4edx')
 
-        staff_groupname = get_access_group_name(course, 'staff')
-        group, _ = Group.objects.get_or_create(name=staff_groupname)
-        self.user.groups.add(group)
+        CourseStaffRole(course.location).add_users(self.user)
 
-        self.assertTrue(has_access(self.user, course, 'staff'))
+        self.assertTrue(CourseStaffRole(course.location).has_user(self.user))
         logged_in = self.client.login(username=self.user.username,
                                       password='foo')
         self.assertTrue(logged_in)

From b8f2949dd9ba2503832bfc58a37c73d6a3dc0815 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Tue, 19 Nov 2013 15:38:09 -0500
Subject: [PATCH 15/20] Small UI glitch and settings check fixes

---
 lms/djangoapps/dashboard/sysadmin.py          | 5 +++--
 lms/templates/sysadmin_dashboard_gitlogs.html | 3 ---
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 6c99d367d558..e7e2046c091e 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -642,8 +642,9 @@ def get(self, request, course_id=None):
         # Allow overrides
         if hasattr(settings, 'MONGODB_LOG'):
             for config_item in ['host', 'user', 'password', 'db', ]:
-                mongo_db[config_item] = settings.MONGODB_LOG.get(
-                    config_item, mongo_db[config_item])
+                if hasattr(settings.MONGODB_LOG, config_item):
+                    mongo_db[config_item] = settings.MONGODB_LOG.get(
+                        config_item, mongo_db[config_item])
 
         mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
             mongo_db['user'], mongo_db['password'],
diff --git a/lms/templates/sysadmin_dashboard_gitlogs.html b/lms/templates/sysadmin_dashboard_gitlogs.html
index 7a63fb4d1ff3..13a174393972 100644
--- a/lms/templates/sysadmin_dashboard_gitlogs.html
+++ b/lms/templates/sysadmin_dashboard_gitlogs.html
@@ -45,9 +45,6 @@
 }
 </style>
 
-<section class="container">
-<div class="sysadmin-dashboard-wrapper">
-
 <section class="container">
 <div class="sysadmin-dashboard-wrapper">
 

From d2d3e55517f75fb10aa4fc95d33cf4e2d0f4a267 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Wed, 20 Nov 2013 17:06:21 -0500
Subject: [PATCH 16/20] Several minor fixes/improvements, and the addition of a
 flag that by default disables course import for XMLModuleStores.

---
 .../management/commands/git_add_course.py     |  15 ++-
 lms/djangoapps/dashboard/sysadmin.py          | 109 ++++++++----------
 lms/djangoapps/dashboard/sysadmin_urls.py     |   1 +
 .../dashboard/tests/test_sysadmin.py          |  32 +++--
 lms/envs/aws.py                               |   2 +-
 5 files changed, 84 insertions(+), 75 deletions(-)

diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py
index ee19a6665fae..9e6737bafe37 100644
--- a/lms/djangoapps/dashboard/management/commands/git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py
@@ -8,17 +8,16 @@
 import StringIO
 import logging
 
-from django.utils.translation import ugettext as _
 from django.conf import settings
 from django.core import management
 from django.core.management.base import BaseCommand, CommandError
+from django.utils.translation import ugettext as _
+import mongoengine
 
+from dashboard.models import CourseImportLog
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.xml import XMLModuleStore
 
-import mongoengine
-from dashboard.models import CourseImportLog
-
 log = logging.getLogger(__name__)
 
 GIT_REPO_DIR = getattr(settings, 'GIT_REPO_DIR', '/opt/edx/course_repos')
@@ -40,8 +39,9 @@ def add_repo(repo, rdir_in):
     # Allow overrides
     if hasattr(settings, 'MONGODB_LOG'):
         for config_item in ['host', 'user', 'password', 'db', ]:
-            mongo_db[config_item] = settings.MONGODB_LOG.get(
-                config_item, mongo_db[config_item])
+            if hasattr(settings.MONGODB_LOG, config_item):
+                mongo_db[config_item] = settings.MONGODB_LOG.get(
+                    config_item, mongo_db[config_item])
 
     if not os.path.isdir(GIT_REPO_DIR):
         log.critical(_("Path {0} doesn't exist, please create it, "
@@ -118,6 +118,9 @@ def add_repo(repo, rdir_in):
         log.critical(_('Unable to run import command.'))
         log.critical(_('Error was {0}').format(str(ex)))
         return -1
+    except NotImplementedError, ex:
+        log.critical(_('The underlying module store does not support import.'))
+        return -1
 
     ret_import = output.getvalue()
 
diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index e7e2046c091e..d301e4fdf3a4 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -12,36 +12,36 @@
 from datetime import datetime
 
 from django.conf import settings
-from django.contrib.auth.models import User, Group
-from django.utils.translation import ugettext as _
-from student.models import CourseEnrollment, UserProfile, Registration
-from external_auth.models import ExternalAuthMap
-from external_auth.views import generate_password
 from django.contrib.auth import authenticate
+from django.contrib.auth.decorators import login_required
+from django.contrib.auth.models import User
 from django.core.exceptions import PermissionDenied
-from django_future.csrf import ensure_csrf_cookie
-from django.views.decorators.cache import cache_control
-from django.utils.decorators import method_decorator
-from django.views.decorators.http import condition
 from django.db import IntegrityError
-from django.http import HttpResponse
+from django.http import HttpResponse, Http404
+from django.utils.decorators import method_decorator
 from django.utils.html import escape
-from django.contrib.auth.decorators import login_required
-from django.http import Http404
+from django.utils.translation import ugettext as _
+from django.views.decorators.cache import cache_control
 from django.views.generic.base import TemplateView
+from django.views.decorators.http import condition
+from django_future.csrf import ensure_csrf_cookie
 from mitxmako.shortcuts import render_to_response
-from xmodule.modulestore.django import modulestore
-from xmodule.contentstore.django import contentstore
-from xmodule.modulestore.xml import XMLModuleStore
-from xmodule.modulestore.store_utilities import delete_course
-from courseware.roles import CourseStaffRole, CourseInstructorRole
-from courseware.courses import get_course_by_id
-
 import mongoengine
-from dashboard.models import CourseImportLog
-import dashboard.management.commands.git_add_course as git_add_course
 
+from courseware.courses import get_course_by_id
+from courseware.roles import CourseStaffRole, CourseInstructorRole
+import dashboard.management.commands.git_add_course as git_add_course
+from dashboard.models import CourseImportLog
+from external_auth.models import ExternalAuthMap
+from external_auth.views import generate_password
+from student.models import CourseEnrollment, UserProfile, Registration
 import track.views
+from xmodule.contentstore.django import contentstore
+from xmodule.modulestore import MONGO_MODULESTORE_TYPE
+from xmodule.modulestore.django import modulestore
+from xmodule.modulestore.store_utilities import delete_course
+from xmodule.modulestore.xml import XMLModuleStore
+
 
 log = logging.getLogger(__name__)
 
@@ -52,7 +52,10 @@ class SysadminDashboardView(TemplateView):
     template_name = 'sysadmin_dashboard.html'
 
     def __init__(self, **kwargs):
-        """Determine modulestore type"""
+        """
+        Initialize base sysadmin dashboard class with modulestore,
+        modulestore_type and return msg
+        """
 
         self.def_ms = modulestore()
         self.is_using_mongo = True
@@ -73,12 +76,9 @@ def dispatch(self, *args, **kwargs):
     def get_courses(self):
         """ Get an iterable list of courses regardless of module store type."""
 
-        # Prefer mongo if using mixed or mongo store
-        if self.is_using_mongo:
-            courses = self.def_ms.get_courses()
-            courses = dict([c.id, c] for c in courses)  # no course directory
-        else:
-            courses = self.def_ms.courses.items()
+        courses = self.def_ms.get_courses()
+        courses = dict([c.id, c] for c in courses)  # no course directory
+
         return courses
 
     def return_csv(self, filename, header, data):
@@ -170,8 +170,8 @@ def create_user(self, uname, name, password=None):
             mit_domain = 'ssl:MIT'
             if ExternalAuthMap.objects.filter(external_id=email,
                                               external_domain=mit_domain):
-                msg += _('Failed - email {0} already exists as external_id'
-                         ).format(email)
+                msg += _('Failed - email {0} already exists as '
+                         'external_id').format(email)
                 return msg
             new_password = generate_password()
         else:
@@ -190,8 +190,8 @@ def create_user(self, uname, name, password=None):
         try:
             user.save()
         except IntegrityError:
-            msg += _('Oops, failed to create user {0}, IntegrityError'
-                     ).format(user)
+            msg += _('Oops, failed to create user {0}, '
+                     'IntegrityError').format(user)
             return msg
 
         reg = Registration()
@@ -252,15 +252,10 @@ def make_common_context(self):
         self.datatable['data'] = [[_('Total number of users'),
                                    User.objects.all().count()]]
 
-        if hasattr(courses, 'items'):
-            course_iter = courses.items()
-        else:
-            course_iter = courses
-
         self.msg += u'<h2>{0}</h2>'.format(
             _('Courses loaded in the modulestore'))
         self.msg += u'<ol>'
-        for (cdir, course) in course_iter:
+        for (cdir, course) in courses.items():
             self.msg += u'<li>{0} ({1})</li>'.format(
                 escape(cdir), course.location.url())
         self.msg += u'</ol>'
@@ -293,8 +288,8 @@ def post(self, request):
 
         if action == 'download_users':
             header = [_('username'), _('email'), ]
-            data = [[u.username, u.email] for u in
-                    User.objects.all().iterator()]
+            data = ([u.username, u.email] for u in
+                    (User.objects.all().iterator()))
             return self.return_csv('users_{0}.csv'.format(
                 request.META['SERVER_NAME']), header, data)
         elif action == 'repair_eamap':
@@ -407,6 +402,12 @@ def import_xml_course(self, gitloc, datatable):
         """Imports a git course into the XMLModuleStore"""
 
         msg = u''
+        if not getattr(settings, 'GIT_IMPORT_WITH_XMLMODULESTORE', False):
+            return _('Refusing to import. GIT_IMPORT_WITH_XMLMODULESTORE is '
+                     'not turned on, and it is generally not safe to import '
+                     'into an XMLModuleStore with multithreaded. We '
+                     'recommend you enable the MongoDB based module store '
+                     'instead, unless this is a development environment.')
         cdir = (gitloc.rsplit('/', 1)[1])[:-4]
         gdir = settings.DATA_DIR / cdir
         if os.path.exists(gdir):
@@ -446,11 +447,8 @@ def make_datatable(self):
 
         data = []
         courses = self.get_courses()
-        if hasattr(courses, 'items'):
-            course_iter = courses.items()
-        else:
-            course_iter = courses
-        for (cdir, course) in course_iter:
+
+        for (cdir, course) in courses.items():
             gdir = cdir
             if '/' in cdir:
                 gdir = cdir.rsplit('/', 1)[1]
@@ -490,8 +488,7 @@ def post(self, request):
 
         courses = self.get_courses()
         if action == _('add_course'):
-            gitloc = request.POST.get('repo_location', '').strip().replace(
-                ' ', '').replace(';', '')
+            gitloc = request.POST.get('repo_location', '').strip().replace(' ', '').replace(';', '')
             datatable = self.make_datatable()
             self.msg += self.get_course_from_git(gitloc, datatable)
 
@@ -511,7 +508,8 @@ def post(self, request):
                                       course_id, escape(str(err))
                                   )
 
-            if course_found and not self.is_using_mongo:
+            is_mongo_course = (modulestore().get_modulestore_type(course_id) == MONGO_MODULESTORE_TYPE)
+            if course_found and not is_mongo_course:
                 cdir = course.data_dir
                 self.def_ms.courses.pop(cdir)
 
@@ -523,7 +521,7 @@ def post(self, request):
                              u"{0} = {1} ({2})</font>".format(
                                  cdir, course.id, course.display_name))
 
-            elif course_found and self.is_using_mongo:
+            elif course_found and is_mongo_course:
                 # delete course that is stored with mongodb backend
                 loc = course.location
                 content_store = contentstore()
@@ -560,11 +558,7 @@ def get(self, request):
 
         courses = self.get_courses()
 
-        if hasattr(courses, 'items'):
-            course_iter = courses.items()
-        else:
-            course_iter = courses
-        for (cdir, course) in course_iter:
+        for (cdir, course) in courses.items():
             datum = [course.display_name, course.id]
             datum += [CourseEnrollment.objects.filter(
                 course_id=course.id).count()]
@@ -599,11 +593,8 @@ def post(self, request):
             roles = [CourseInstructorRole, CourseStaffRole, ]
 
             courses = self.get_courses()
-            if hasattr(courses, 'items'):
-                course_iter = courses.items()
-            else:
-                course_iter = courses
-            for (cdir, course) in course_iter:
+
+            for (cdir, course) in courses.items():
                 for role in roles:
                     for user in role(course.location).users_with_role():
                         datum = [course.id, role, user.username, user.email,
diff --git a/lms/djangoapps/dashboard/sysadmin_urls.py b/lms/djangoapps/dashboard/sysadmin_urls.py
index 24fbe178ee50..489cf877cf0e 100644
--- a/lms/djangoapps/dashboard/sysadmin_urls.py
+++ b/lms/djangoapps/dashboard/sysadmin_urls.py
@@ -3,6 +3,7 @@
 """
 
 from django.conf.urls import patterns, url
+
 from dashboard import sysadmin
 
 urlpatterns = patterns(
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 43631ab79cd9..899befffe03b 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -6,25 +6,25 @@
 import os
 import shutil
 
-from django.test.client import Client
-from django.test.utils import override_settings
 from django.conf import settings
+from django.contrib.auth.hashers import check_password
 from django.contrib.auth.models import User
 from django.core.urlresolvers import reverse
+from django.test.client import Client
+from django.test.utils import override_settings
+from django.utils.html import escape
 from django.utils.translation import ugettext as _
+import mongoengine
+
+from courseware.roles import CourseStaffRole
+from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
+from dashboard.models import CourseImportLog
 from dashboard.sysadmin import Users
 from external_auth.models import ExternalAuthMap
-from django.contrib.auth.hashers import check_password
-from django.contrib.auth.models import Group
 from xmodule.modulestore.django import modulestore
-from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
-from django.utils.html import escape
-from courseware.roles import CourseStaffRole
-from dashboard.models import CourseImportLog
 from xmodule.modulestore.xml import XMLModuleStore
 
-import mongoengine
 
 TEST_MONGODB_LOG = {
     'host': 'localhost',
@@ -83,6 +83,7 @@ def _rm_edx4edx(self):
 
 @unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
                      "ENABLE_SYSADMIN_DASHBOARD not set")
+@override_settings(GIT_IMPORT_WITH_XMLMODULESTORE=True)
 class TestSysadmin(SysadminBaseTestCase):
     """
     Test sysadmin dashboard features using XMLModuleStore
@@ -240,6 +241,19 @@ def test_xml_course_add_delete(self):
             os.path.abspath(settings.DATA_DIR)), None)
         self.assertIsNone(course)
 
+    @override_settings(GIT_IMPORT_WITH_XMLMODULESTORE=False)
+    def test_xml_safety_flag(self):
+        """Make sure the settings flag to disable xml imports is working"""
+
+        self._setstaff_login()
+        response = self._add_edx4edx()
+        self.assertIn('GIT_IMPORT_WITH_XMLMODULESTORE', response.content)
+
+        def_ms = modulestore()
+        course = def_ms.courses.get('{0}/edx4edx_lite'.format(
+            os.path.abspath(settings.DATA_DIR)), None)
+        self.assertIsNone(course)
+
     def test_git_pull(self):
         """Make sure we can pull"""
 
diff --git a/lms/envs/aws.py b/lms/envs/aws.py
index a315474d6e0d..e93bd40016f3 100644
--- a/lms/envs/aws.py
+++ b/lms/envs/aws.py
@@ -216,7 +216,7 @@
 MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS)
 
 # git repo loading  environment
-GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', None)
+GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', '/edx/var/edxapp/course_repos')
 GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True)
 
 for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items():

From 5e062b22b7cb67a1d5c4133a437bf5dc9ccf8c6d Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Wed, 4 Dec 2013 14:36:41 -0500
Subject: [PATCH 17/20] sysadmin dashboard - Removal of popen, more streamlined
 git commands, and other optimizations

---
 .../management/commands/git_add_course.py     | 67 ++++++++++---------
 lms/djangoapps/dashboard/sysadmin.py          | 42 +++++++-----
 lms/djangoapps/dashboard/sysadmin_urls.py     |  1 +
 .../dashboard/tests/test_sysadmin.py          |  6 +-
 4 files changed, 66 insertions(+), 50 deletions(-)

diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py
index ee19a6665fae..30cec2169043 100644
--- a/lms/djangoapps/dashboard/management/commands/git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py
@@ -6,6 +6,7 @@
 import re
 import datetime
 import StringIO
+import subprocess
 import logging
 
 from django.utils.translation import ugettext as _
@@ -70,12 +71,15 @@ def add_repo(repo, rdir_in):
     if os.path.exists(rdirp):
         log.info(_('directory already exists, doing a git pull instead '
                    'of git clone'))
-        cmd = 'cd {0}/{1}; git pull'.format(GIT_REPO_DIR, rdir)
+        cmd = ['git', 'pull', ]
+        cwd = '{0}/{1}'.format(GIT_REPO_DIR, rdir)
     else:
-        cmd = 'cd {0}; git clone "{1}"'.format(GIT_REPO_DIR, repo)
+        cmd = ['git', 'clone', repo, ]
+        cwd = GIT_REPO_DIR
 
     log.debug(cmd)
-    ret_git = os.popen(cmd).read()
+    cwd = os.path.abspath(cwd)
+    ret_git = subprocess.check_output(cmd, cwd=cwd)
     log.debug(ret_git)
 
     if not os.path.exists('{0}/{1}'.format(GIT_REPO_DIR, rdir)):
@@ -83,46 +87,43 @@ def add_repo(repo, rdir_in):
         return -1
 
     # get commit id
-    commit_id = os.popen('cd {0}; git log -n 1 | head -1'.format(
-        rdirp)).read().strip().split(' ')[1]
+    cmd = ['git', 'log', '-1', '--format=%H', ]
+    commit_id = subprocess.check_output(cmd, cwd=rdirp)
 
     ret_git += _('\nCommit ID: {0}').format(commit_id)
 
     # get branch
-    branch = ''
-    for k in os.popen('cd {0}; git branch'.format(rdirp)).readlines():
-        if k[0] == '*':
-            branch = k[2:].strip()
-
+    cmd = ['git', 'rev-parse', '--abbrev-ref', 'HEAD', ]
+    branch = subprocess.check_output(cmd, cwd=rdirp)
     ret_git += '   \nBranch: {0}'.format(branch)
 
     # Get XML logging logger and capture debug to parse results
     output = StringIO.StringIO()
-    import_logger = logging.getLogger('xmodule.modulestore.xml_importer')
-    git_logger = logging.getLogger('git_add_script')
-    xml_logger = logging.getLogger('xmodule.modulestore.xml')
-    xml_seq_logger = logging.getLogger('xmodule.seq_module')
-
     import_log_handler = logging.StreamHandler(output)
     import_log_handler.setLevel(logging.DEBUG)
 
-    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+    logger_names = ['xmodule.modulestore.xml_importer', 'git_add_course',
+                    'xmodule.modulestore.xml', 'xmodule.seq_module', ]
+    loggers = []
+
+    for logger_name in logger_names:
+        logger = logging.getLogger(logger_name)
         logger.old_level = logger.level
         logger.setLevel(logging.DEBUG)
         logger.addHandler(import_log_handler)
+        loggers.append(logger)
 
     try:
         management.call_command('import', GIT_REPO_DIR, rdir,
                                 nostatic=not GIT_IMPORT_STATIC)
-    except CommandError, ex:
-        log.critical(_('Unable to run import command.'))
-        log.critical(_('Error was {0}').format(str(ex)))
+    except CommandError:
+        log.exception(_('Unable to run import command.'))
         return -1
 
     ret_import = output.getvalue()
 
     # Remove handler hijacks
-    for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+    for logger in loggers:
         logger.setLevel(logger.old_level)
         logger.removeHandler(import_log_handler)
 
@@ -144,14 +145,21 @@ def add_repo(repo, rdir_in):
 
         if os.path.exists(cdir) and not os.path.islink(cdir):
             log.debug(_('   -> exists, but is not symlink'))
-            log.debug(os.popen('ls -l {0}'.format(cdir)).read())
-            log.debug(os.popen('rmdir {0}'.format(cdir)).read())
+            log.debug(subprocess.check_output(['ls', 'l', ],
+                                              cwd=os.path.abspath(cdir)))
+            try:
+                os.rmdir(os.path.abspath(cdir))
+            except OSError:
+                log.exception(_('Failed to remove course directory'))
 
         if not os.path.exists(cdir):
-            log.debug(_('   -> creating symlink'))
-            log.debug(os.popen('ln -s {0} {1}'.format(rdirp,
-                      cdir)).read())
-            log.debug(os.popen('ls -l {0}'.format(cdir)).read())
+            log.debug(_('   -> creating symlink between {0} and {1}').format(rdirp, cdir))
+            try:
+                os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir))
+            except OSError:
+                log.exception(_('Unable to create course symlink'))
+            log.debug(subprocess.check_output(['ls', '-l', ],
+                                              cwd=os.path.abspath(cdir)))
 
     # store import-command-run output in mongo
     mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
@@ -163,10 +171,9 @@ def add_repo(repo, rdir_in):
             mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
         else:
             mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
-    except mongoengine.connection.ConnectionError, ex:
-        log.critical(_('Unable to connect to mongodb to save log, please '
-                       'check MONGODB_LOG settings'))
-        log.critical(_('Error was: {0}').format(str(ex)))
+    except mongoengine.connection.ConnectionError:
+        log.exception(_('Unable to connect to mongodb to save log, please '
+                        'check MONGODB_LOG settings'))
         return -1
     cil = CourseImportLog(
         course_id=course_id,
diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 6c99d367d558..77def064201e 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -12,7 +12,7 @@
 from datetime import datetime
 
 from django.conf import settings
-from django.contrib.auth.models import User, Group
+from django.contrib.auth.models import User
 from django.utils.translation import ugettext as _
 from student.models import CourseEnrollment, UserProfile, Registration
 from external_auth.models import ExternalAuthMap
@@ -73,10 +73,9 @@ def dispatch(self, *args, **kwargs):
     def get_courses(self):
         """ Get an iterable list of courses regardless of module store type."""
 
-        # Prefer mongo if using mixed or mongo store
         if self.is_using_mongo:
             courses = self.def_ms.get_courses()
-            courses = dict([c.id, c] for c in courses)  # no course directory
+            courses = {c.id: c for c in courses}  # no course directory
         else:
             courses = self.def_ms.courses.items()
         return courses
@@ -377,27 +376,29 @@ def import_mongo_course(self, gitloc):
 
         # Grab logging output for debugging imports
         output = StringIO.StringIO()
-
-        import_logger = logging.getLogger('xmodule.modulestore.xml_importer')
-        git_logger = logging.getLogger('dashboard.management.commands.git_add_course')
-        xml_logger = logging.getLogger('xmodule.modulestore.xml')
-        xml_seq_logger = logging.getLogger('xmodule.seq_module')
-
         import_log_handler = logging.StreamHandler(output)
         import_log_handler.setLevel(logging.DEBUG)
 
-        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+        logger_names = ['xmodule.modulestore.xml_importer',
+                        'dashboard.management.commands.git_add_course',
+                        'xmodule.modulestore.xml', 'xmodule.seq_module', ]
+        loggers = []
+
+        for logger_name in logger_names:
+            logger = logging.getLogger(logger_name)
             logger.old_level = logger.level
             logger.setLevel(logging.DEBUG)
             logger.addHandler(import_log_handler)
+            loggers.append(logger)
 
         git_add_course.add_repo(gitloc, None)
         ret = output.getvalue()
 
         # Remove handler hijacks
-        for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]:
+        for logger in loggers:
             logger.setLevel(logger.old_level)
             logger.removeHandler(import_log_handler)
+
         msg = u"<h4 style='color:blue'>{0} {1}</h4>".format(
             _('Added course from'), gitloc)
         msg += _("<pre>{0}</pre>").format(escape(ret))
@@ -412,10 +413,14 @@ def import_xml_course(self, gitloc, datatable):
         if os.path.exists(gdir):
             msg += _("The course {0} already exists in the data directory! "
                      "(reloading anyway)").format(cdir)
-            cmd = 'cd {0}; git pull'.format(settings.DATA_DIR, gitloc)
+            cmd = ['git', 'pull', ]
+            cwd = gdir
         else:
-            cmd = 'cd {0}; git clone {1}'.format(settings.DATA_DIR, gitloc)
-            msg += u'<pre>%s</pre>' % escape(os.popen(cmd).read())
+            cmd = ['git', 'clone', gitloc, ]
+            cwd = settings.DATA_DIR
+        cmd_output = escape(subprocess.check_output(cmd, cwd=os.path.abspath(cwd)))
+        msg += u'<pre>{0}</pre>'.format(cmd_output)
+
         if not os.path.exists(gdir):
             msg += _('Failed to clone repository to {0}').format(gdir)
             return msg
@@ -489,7 +494,7 @@ def post(self, request):
                                  page='courses_sysdashboard')
 
         courses = self.get_courses()
-        if action == _('add_course'):
+        if action == 'add_course':
             gitloc = request.POST.get('repo_location', '').strip().replace(
                 ' ', '').replace(';', '')
             datatable = self.make_datatable()
@@ -628,9 +633,14 @@ class GitLogs(TemplateView):
     template_name = 'sysadmin_dashboard_gitlogs.html'
 
     @method_decorator(login_required)
-    def get(self, request, course_id=None):
+    def get(self, request, *args, **kwargs):
         """Shows logs of imports that happened as a result of a git import"""
 
+        if 'course_id' in kwargs:
+            course_id = kwargs['course_id']
+        else:
+            course_id = None
+
         # Set mongodb defaults even if it isn't defined in settings
         mongo_db = {
             'host': 'localhost',
diff --git a/lms/djangoapps/dashboard/sysadmin_urls.py b/lms/djangoapps/dashboard/sysadmin_urls.py
index 24fbe178ee50..f412676c39a9 100644
--- a/lms/djangoapps/dashboard/sysadmin_urls.py
+++ b/lms/djangoapps/dashboard/sysadmin_urls.py
@@ -1,6 +1,7 @@
 """
 Urls for sysadmin dashboard feature
 """
+# pylint: disable=E1120
 
 from django.conf.urls import patterns, url
 from dashboard import sysadmin
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 43631ab79cd9..c519427b1f79 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -15,12 +15,11 @@
 from dashboard.sysadmin import Users
 from external_auth.models import ExternalAuthMap
 from django.contrib.auth.hashers import check_password
-from django.contrib.auth.models import Group
 from xmodule.modulestore.django import modulestore
 from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE
 from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from django.utils.html import escape
-from courseware.roles import CourseStaffRole
+from courseware.roles import CourseStaffRole, GlobalStaff
 from dashboard.models import CourseImportLog
 from xmodule.modulestore.xml import XMLModuleStore
 
@@ -52,8 +51,7 @@ def setUp(self):
     def _setstaff_login(self):
         """Makes the test user staff and logs them in"""
 
-        self.user.is_staff = True
-        self.user.save()
+        GlobalStaff().add_users(self.user)
         self.client.login(username=self.user.username, password='foo')
 
     def _add_edx4edx(self):

From bb922c23579b3a1c1cbb7d9a7d6b2ed964f59601 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Thu, 5 Dec 2013 15:32:19 -0500
Subject: [PATCH 18/20] MITX_ and mitxmako removal from master refactor

---
 lms/djangoapps/dashboard/sysadmin.py            | 16 ++++++++--------
 lms/djangoapps/dashboard/tests/test_sysadmin.py | 10 +++++-----
 lms/envs/test.py                                |  2 +-
 lms/urls.py                                     |  2 +-
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index c66913920a10..042ac4649c00 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -25,7 +25,7 @@
 from django.views.generic.base import TemplateView
 from django.views.decorators.http import condition
 from django_future.csrf import ensure_csrf_cookie
-from mitxmako.shortcuts import render_to_response
+from edxmako.shortcuts import render_to_response
 import mongoengine
 
 from courseware.courses import get_course_by_id
@@ -159,7 +159,7 @@ def create_user(self, uname, name, password=None):
         email_domain = getattr(settings, 'SSL_AUTH_EMAIL_DOMAIN', 'MIT.EDU')
 
         msg = u''
-        if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
+        if settings.FEATURES['AUTH_USE_MIT_CERTIFICATES']:
             if not '@' in uname:
                 email = '{0}@{1}'.format(uname, email_domain)
             else:
@@ -201,7 +201,7 @@ def create_user(self, uname, name, password=None):
         profile.name = name
         profile.save()
 
-        if settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']:
+        if settings.FEATURES['AUTH_USE_MIT_CERTIFICATES']:
             credential_string = getattr(settings, 'SSL_AUTH_DN_FORMAT_STRING',
                                         '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}')
             credentials = credential_string.format(name, email)
@@ -271,7 +271,7 @@ def get(self, request):
             'msg': self.msg,
             'djangopid': os.getpid(),
             'modeflag': {'users': 'active-section'},
-            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+            'mitx_version': getattr(settings, 'VERSION_STRING', ''),
         }
         return render_to_response(self.template_name, context)
 
@@ -315,7 +315,7 @@ def post(self, request):
             'msg': self.msg,
             'djangopid': os.getpid(),
             'modeflag': {'users': 'active-section'},
-            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+            'mitx_version': getattr(settings, 'VERSION_STRING', ''),
         }
         return render_to_response(self.template_name, context)
 
@@ -478,7 +478,7 @@ def get(self, request):
             'msg': self.msg,
             'djangopid': os.getpid(),
             'modeflag': {'courses': 'active-section'},
-            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+            'mitx_version': getattr(settings, 'VERSION_STRING', ''),
         }
         return render_to_response(self.template_name, context)
 
@@ -544,7 +544,7 @@ def post(self, request):
             'msg': self.msg,
             'djangopid': os.getpid(),
             'modeflag': {'courses': 'active-section'},
-            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+            'mitx_version': getattr(settings, 'VERSION_STRING', ''),
         }
         return render_to_response(self.template_name, context)
 
@@ -583,7 +583,7 @@ def get(self, request):
             'msg': self.msg,
             'djangopid': os.getpid(),
             'modeflag': {'staffing': 'active-section'},
-            'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''),
+            'mitx_version': getattr(settings, 'VERSION_STRING', ''),
         }
         return render_to_response(self.template_name, context)
 
diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py
index 95bc5bbb997b..171e0f730dae 100644
--- a/lms/djangoapps/dashboard/tests/test_sysadmin.py
+++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py
@@ -33,8 +33,8 @@
     'db': 'test_xlog',
 }
 
-MITX_FEATURES_WITH_SSL_AUTH = settings.MITX_FEATURES.copy()
-MITX_FEATURES_WITH_SSL_AUTH['AUTH_USE_MIT_CERTIFICATES'] = True
+FEATURES_WITH_SSL_AUTH = settings.FEATURES.copy()
+FEATURES_WITH_SSL_AUTH['AUTH_USE_MIT_CERTIFICATES'] = True
 
 
 class SysadminBaseTestCase(ModuleStoreTestCase):
@@ -80,7 +80,7 @@ def _rm_edx4edx(self):
                                  'action': 'del_course', })
 
 
-@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
+@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
                      "ENABLE_SYSADMIN_DASHBOARD not set")
 @override_settings(GIT_IMPORT_WITH_XMLMODULESTORE=True)
 class TestSysadmin(SysadminBaseTestCase):
@@ -180,7 +180,7 @@ def test_user_csv(self):
         User.objects.filter(
             username__startswith='testingman_with_long_name').delete()
 
-    @override_settings(MITX_FEATURES=MITX_FEATURES_WITH_SSL_AUTH)
+    @override_settings(FEATURES=FEATURES_WITH_SSL_AUTH)
     def test_authmap_repair(self):
         """Run authmap check and repair"""
 
@@ -298,7 +298,7 @@ def test_enrollment_page(self):
 
 @override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE)
 @override_settings(MONGODB_LOG=TEST_MONGODB_LOG)
-@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
+@unittest.skipUnless(settings.FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'),
                      "ENABLE_SYSADMIN_DASHBOARD not set")
 class TestSysAdminMongoCourseImport(SysadminBaseTestCase):
     """
diff --git a/lms/envs/test.py b/lms/envs/test.py
index 182fbdf0f977..1728c5d18d91 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -204,7 +204,7 @@
 CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = "/shoppingcart/payment_fake"
 
 ########################### SYSADMIN DASHBOARD ################################
-MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
+FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True
 GIT_REPO_DIR = TEST_ROOT / "course_repos"
 
 ################################# CELERY ######################################
diff --git a/lms/urls.py b/lms/urls.py
index ba11055c10d8..ca154c21c541 100644
--- a/lms/urls.py
+++ b/lms/urls.py
@@ -81,7 +81,7 @@
 )
 
 # sysadmin dashboard, to see what courses are loaded, to delete & load courses
-if settings.MITX_FEATURES["ENABLE_SYSADMIN_DASHBOARD"]:
+if settings.FEATURES["ENABLE_SYSADMIN_DASHBOARD"]:
     urlpatterns += (
         url(r'^sysadmin/', include('dashboard.sysadmin_urls')),
     )

From b11767621d61a0ed4ac9fd745cc6e244fe5c5740 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Wed, 11 Dec 2013 11:39:06 -0500
Subject: [PATCH 19/20] Bug fixes and refactors to sysadmin dashboard Corrected
 exceptions to use exception logging Corrected bad default value in aws.py for
 MONGODB_LOG Simplified mongouri format string

---
 .../management/commands/git_add_course.py        | 13 +++++--------
 lms/djangoapps/dashboard/sysadmin.py             | 16 ++++++----------
 lms/envs/aws.py                                  |  2 +-
 3 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py
index f588920e303d..e5b0dd35d2a9 100644
--- a/lms/djangoapps/dashboard/management/commands/git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py
@@ -40,9 +40,8 @@ def add_repo(repo, rdir_in):
     # Allow overrides
     if hasattr(settings, 'MONGODB_LOG'):
         for config_item in ['host', 'user', 'password', 'db', ]:
-            if hasattr(settings.MONGODB_LOG, config_item):
-                mongo_db[config_item] = settings.MONGODB_LOG.get(
-                    config_item, mongo_db[config_item])
+            mongo_db[config_item] = settings.MONGODB_LOG.get(
+                config_item, mongo_db[config_item])
 
     if not os.path.isdir(GIT_REPO_DIR):
         log.critical(_("Path {0} doesn't exist, please create it, "
@@ -119,8 +118,8 @@ def add_repo(repo, rdir_in):
     except CommandError:
         log.exception(_('Unable to run import command.'))
         return -1
-    except NotImplementedError, ex:
-        log.critical(_('The underlying module store does not support import.'))
+    except NotImplementedError:
+        log.exception(_('The underlying module store does not support import.'))
         return -1
 
     ret_import = output.getvalue()
@@ -165,9 +164,7 @@ def add_repo(repo, rdir_in):
                                               cwd=os.path.abspath(cdir)))
 
     # store import-command-run output in mongo
-    mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
-        mongo_db['user'], mongo_db['password'],
-        mongo_db['host'], mongo_db['db'])
+    mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
 
     try:
         if mongo_db['user'] and mongo_db['password']:
diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py
index 042ac4649c00..d46b9f894dc8 100644
--- a/lms/djangoapps/dashboard/sysadmin.py
+++ b/lms/djangoapps/dashboard/sysadmin.py
@@ -644,13 +644,10 @@ def get(self, request, *args, **kwargs):
         # Allow overrides
         if hasattr(settings, 'MONGODB_LOG'):
             for config_item in ['host', 'user', 'password', 'db', ]:
-                if hasattr(settings.MONGODB_LOG, config_item):
-                    mongo_db[config_item] = settings.MONGODB_LOG.get(
-                        config_item, mongo_db[config_item])
+                mongo_db[config_item] = settings.MONGODB_LOG.get(
+                    config_item, mongo_db[config_item])
 
-        mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format(
-            mongo_db['user'], mongo_db['password'],
-            mongo_db['host'], mongo_db['db'])
+        mongouri = 'mongodb://{user}:{password}@{host}/{db}'.format(**mongo_db)
 
         error_msg = ''
 
@@ -659,10 +656,9 @@ def get(self, request, *args, **kwargs):
                 mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
             else:
                 mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host'])
-        except mongoengine.connection.ConnectionError, ex:
-            logging.critical(_('Unable to connect to mongodb to save log, '
-                               'please check MONGODB_LOG settings. '
-                               'error: {0}').format(str(ex)))
+        except mongoengine.connection.ConnectionError:
+            logging.exception(_('Unable to connect to mongodb to save log, '
+                               'please check MONGODB_LOG settings.'))
 
         if course_id is None:
             # Require staff if not going to specific course
diff --git a/lms/envs/aws.py b/lms/envs/aws.py
index 0c777d1cda9a..4489403c3138 100644
--- a/lms/envs/aws.py
+++ b/lms/envs/aws.py
@@ -293,7 +293,7 @@
 MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE)
 CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE)
 DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG',DOC_STORE_CONFIG)
-MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG')
+MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {})
 
 OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE',
                                                OPEN_ENDED_GRADING_INTERFACE)

From d910d4adec4bdaf7852b89e9d4f2c09e188033c1 Mon Sep 17 00:00:00 2001
From: Carson Gee <x@carsongee.com>
Date: Wed, 11 Dec 2013 12:31:30 -0500
Subject: [PATCH 20/20] Missing dash in ls argument

---
 lms/djangoapps/dashboard/management/commands/git_add_course.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lms/djangoapps/dashboard/management/commands/git_add_course.py b/lms/djangoapps/dashboard/management/commands/git_add_course.py
index e5b0dd35d2a9..7da8228922b1 100644
--- a/lms/djangoapps/dashboard/management/commands/git_add_course.py
+++ b/lms/djangoapps/dashboard/management/commands/git_add_course.py
@@ -147,7 +147,7 @@ def add_repo(repo, rdir_in):
 
         if os.path.exists(cdir) and not os.path.islink(cdir):
             log.debug(_('   -> exists, but is not symlink'))
-            log.debug(subprocess.check_output(['ls', 'l', ],
+            log.debug(subprocess.check_output(['ls', '-l', ],
                                               cwd=os.path.abspath(cdir)))
             try:
                 os.rmdir(os.path.abspath(cdir))