diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index e0d58b32f0dd..2e16bf1258ef 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -31,9 +31,16 @@ def handle(self, *args, **options): course_dirs = args[1:] else: course_dirs = None - print("Importing. Data_dir={data}, course_dirs={courses}".format( + self.stdout.write("Importing. Data_dir={data}, course_dirs={courses}\n".format( data=data_dir, courses=course_dirs, dis=do_import_static)) - import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False, + try: + mstore = modulestore('direct') + except KeyError: + self.stdout.write('Unable to load direct modulestore, trying ' + 'default\n') + mstore = modulestore('default') + + import_from_xml(mstore, data_dir, course_dirs, load_error_modules=False, static_content_store=contentstore(), verbose=True, do_import_static=do_import_static) diff --git a/lms/djangoapps/courseware/management/commands/import.py b/lms/djangoapps/courseware/management/commands/import.py new file mode 120000 index 000000000000..36b7e3c6fcd7 --- /dev/null +++ b/lms/djangoapps/courseware/management/commands/import.py @@ -0,0 +1 @@ +../../../../../cms/djangoapps/contentstore/management/commands/import.py \ No newline at end of file diff --git a/lms/djangoapps/dashboard/sysadmin.py b/lms/djangoapps/dashboard/sysadmin.py index e0fe3db78ed4..5475ab8d41a3 100644 --- a/lms/djangoapps/dashboard/sysadmin.py +++ b/lms/djangoapps/dashboard/sysadmin.py @@ -1,458 +1,572 @@ -# MITx sysadmin dashboard - +""" +This module creates a sysadmin dashboard for managing and viewing +courses. +""" import csv -import itertools import json import logging import os -import requests -import string -import subprocess import time -import urllib +import imp +import StringIO -from random import choice -from StringIO import StringIO from datetime import datetime from django.conf import settings from django.contrib.auth.models import User, Group -from django.http import HttpResponse, Http404 -from courseware.models import StudentModule -from student.models import CourseEnrollment, CourseEnrollmentAllowed +from django.utils.translation import ugettext as _ +from student.models import CourseEnrollment from student.models import UserProfile, Registration from external_auth.models import ExternalAuthMap +from external_auth.views import generate_password -from courseware.access import (has_access, get_access_group_name, - course_beta_test_group_name) -from courseware.courses import get_course_with_access, get_course_by_id +from courseware.access import get_access_group_name +from courseware.courses import get_course_by_id -from django.contrib.auth import logout, authenticate, login +from django.contrib.auth import authenticate +from django.core.exceptions import PermissionDenied from django_future.csrf import ensure_csrf_cookie from django.views.decorators.cache import cache_control +from django.db import IntegrityError +from django.http import HttpResponse +from django.utils.html import escape +from django.contrib.admin.views.decorators import staff_member_required +from django.http import Http404 + from mitxmako.shortcuts import render_to_response -from xmodule.course_module import CourseDescriptor -from xmodule.modulestore import Location from xmodule.modulestore.django import modulestore from xmodule.contentstore.django import contentstore from xmodule.modulestore.store_utilities import delete_course +import mongoengine import track.views log = logging.getLogger(__name__) -def escape(s): - """escape HTML special characters in string""" - return str(s).replace('<','<').replace('>','>') +class CourseImportLog(mongoengine.Document): + """Mongoengine model for git log""" + # pylint: disable-msg=R0924 + + course_id = mongoengine.StringField(max_length=128) + location = mongoengine.StringField(max_length=168) + import_log = mongoengine.StringField(max_length=20 * 65535) + git_log = mongoengine.StringField(max_length=65535) + repo_dir = mongoengine.StringField(max_length=128) + created = mongoengine.DateTimeField() + meta = {'indexes': ['course_id', 'created'], + 'allow_inheritance': False} + def git_info_for_course(cdir): + """This pulls out some git info like the last commit""" + + cmd = '' gdir = settings.DATA_DIR / cdir - info = [gdir,'',''] + info = [gdir, '', ''] if os.path.exists(gdir): - cmd = "cd %s; git log -1" % gdir + cmd = 'cd {0}; git log -1'.format(gdir) for k in os.popen(cmd).readlines(): if 'commit' in k: info[0] = k.split()[1] elif 'Author' in k: info[2] = k.split()[1] elif 'Date' in k: - info[1] = k.split(' ',1)[1].strip() + info[1] = k.split(' ', 1)[1].strip() return info +def get_course_from_git(gitloc, is_using_mongo, def_ms, datatable): + """This downloads and runs the checks for importing a course in git""" + + msg = u'' + if not (gitloc.endswith('.git') or gitloc.startswith('http:') or + gitloc.startswith('https:') or gitloc.startswith('git:')): + msg += \ + _("The git repo location should end with '.git', and be a valid url") + return msg + + if is_using_mongo: + acscript = getattr(settings, 'GIT_ADD_COURSE_SCRIPT', '') + if not acscript or not os.path.exists(acscript): + msg = u"{0} - {1}".format( + _('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'), acscript) + return msg + + # import course script directly and call add_repo function + git_add_script = imp.load_source('git_add_script', acscript) + logging.debug( + _('Adding course using add repo from {0} and repo {1}').format( + acscript, gitloc)) + + # Grab logging output for debugging imports + output = StringIO.StringIO() + + import_logger = logging.getLogger( + 'xmodule.modulestore.xml_importer') + git_logger = logging.getLogger('git_add_script') + xml_logger = logging.getLogger('xmodule.modulestore.xml') + xml_seq_logger = logging.getLogger('xmodule.seq_module') + + import_log_handler = logging.StreamHandler(output) + import_log_handler.setLevel(logging.DEBUG) + + for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]: + logger.old_level = logger.level + logger.setLevel(logging.DEBUG) + logger.addHandler(import_log_handler) + + git_add_script.add_repo(gitloc, None) + + ret = output.getvalue() + + # Remove handler hijacks + for logger in [import_logger, git_logger, xml_logger, xml_seq_logger, ]: + logger.setLevel(logger.old_level) + logger.removeHandler(import_log_handler) + msg = u"{0} {1}".format( + _('Added course from'), gitloc) + msg += _("
{0}
").format(escape(ret)) + return msg + + cdir = (gitloc.rsplit('/', 1)[1])[:-4] + gdir = settings.DATA_DIR / cdir + if os.path.exists(gdir): + msg += _("The course {0} already exists in the data directory! " + "(reloading anyway)").format(cdir) + cmd = 'cd {0}; git pull'.format(settings.DATA_DIR, gitloc) + else: + cmd = 'cd {0}; git clone {1}'.format(settings.DATA_DIR, gitloc) + msg += u'
%s
' % escape(os.popen(cmd).read()) + if not os.path.exists(gdir): + msg += _('Failed to clone repository to {0}').format(gdir) + return msg + def_ms.try_load_course(os.path.abspath(gdir)) # load into modulestore + errlog = def_ms.errored_courses.get(cdir, '') + if errlog: + msg += u'
{0}
'.format(escape(errlog)) + else: + course = def_ms.courses[os.path.abspath(gdir)] + msg += _('Loaded course {0} {1}
Errors:').format(cdir, + course.display_name) + errors = def_ms.get_item_errors(course.location) + if not errors: + msg += u'None' + else: + msg += u'' + datatable['data'].append([course.display_name, cdir] + + git_info_for_course(cdir)) + return msg + + def fix_external_auth_map_passwords(): + """ + This corrects any passwords that have drifted from eamp to + internal django auth + """ + msg = '' for eamap in ExternalAuthMap.objects.all(): - u = eamap.user - pw = eamap.internal_password - if u is None: + euser = eamap.user + epass = eamap.internal_password + if euser is None: continue try: - testuser = authenticate(username=u.username, password=pw) - except Exception as err: - msg += "Failed in authenticating %s, error %s\n" % (u,err) + testuser = authenticate(username=euser.username, password=epass) + except (TypeError, PermissionDenied), err: + msg += _('Failed in authenticating {0}, error {1}\n' + ).format(euser, err) continue if testuser is None: - msg += "Failed in authenticating %s; " % (u) - msg += "fixed password" - u.set_password(pw) - u.save() + msg += _('Failed in authenticating {0}\n').format(euser) + msg += _('fixed password') + euser.set_password(epass) + euser.save() continue if not msg: - msg = "All ok!" + msg = _('All ok!') + return msg + + +def create_user(uname, name, password=None, do_mit=False): + """ Creates a user (both SSL and regular)""" + + if not uname: + return _('Must provide username') + if not name: + return _('Must provide full name') + + make_eamap = False + + msg = u'' + if do_mit: + if not '@' in uname: + email = '{0}@MIT.EDU'.format(uname) + else: + email = uname + if not email.endswith('@MIT.EDU'): + msg += u'email must end in @MIT.EDU' + return msg + mit_domain = 'ssl:MIT' + if ExternalAuthMap.objects.filter(external_id=email, + external_domain=mit_domain): + msg += _('Failed - email {0} already exists as external_id' + ).format(email) + return msg + make_eamap = True + new_password = generate_password() + else: + if not password: + return _('Password must be supplied if not using certificates') + + email = uname + + if not '@' in email: + msg += _('email address required (not username)') + return msg + new_password = password + + user = User(username=uname, email=email, is_active=True) + user.set_password(new_password) + try: + user.save() + except IntegrityError: + msg += _('Oops, failed to create user {0}, IntegrityError' + ).format(user) + return msg + + reg = Registration() + reg.register(user) + + profile = UserProfile(user=user) + profile.name = name + profile.save() + + if make_eamap: + credentials = \ + '/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}'.format(name, email) + eamap = ExternalAuthMap( + external_id=email, + external_email=email, + external_domain=mit_domain, + external_name=name, + internal_password=new_password, + external_credentials=json.dumps(credentials), + ) + eamap.user = user + eamap.dtsignup = datetime.now() + eamap.save() + + msg += _('User {0} created successfully!').format(user) return msg + +def delete_user(uname): + """Deletes a user from django auth""" + + if not uname: + return _('Must provide username') + if '@' in uname: + try: + user = User.objects.get(email=uname) + except User.DoesNotExist, err: + msg = _('Cannot find user with email address {0}' + ).format(uname) + return msg + else: + try: + user = User.objects.get(username=uname) + except User.DoesNotExist, err: + msg = _('Cannot find user with username {0} - {1}' + ).format(uname, err.msg) + return msg + user.delete() + return _('Deleted user {0}').format(uname) + + +def return_csv(filename, datatable): + """Convenient function for handling the http response of a csv""" + + response = HttpResponse(mimetype='text/csv') + response['Content-Disposition'] = 'attachment; filename={0}'.format(filename) + + writer = csv.writer(response, dialect='excel', quotechar='"', + quoting=csv.QUOTE_ALL) + writer.writerow(datatable['header']) + for datarow in datatable['data']: + encoded_row = [unicode(s).encode('utf-8') for s in datarow] + writer.writerow(encoded_row) + return response + + +def get_staff_group(course): + """Gets staff members for course""" + + return get_group(course, 'staff') + + +def get_instructor_group(course): + """Gets instructors for course""" + + return get_group(course, 'instructor') + + +def get_group(course, groupname): + """Gets the course group""" + + grpname = get_access_group_name(course, groupname) + try: + group = Group.objects.get(name=grpname) + except Group.DoesNotExist: + group = Group(name=grpname) # create the group + group.save() + return group + + +@staff_member_required @ensure_csrf_cookie @cache_control(no_cache=True, no_store=True, must_revalidate=True) def sysadmin_dashboard(request): """ Sysadmin dashboard. - Provides: - 1. enrollment numbers 2. loading new courses from github 3. reloading XML from files - """ - if not request.user.is_staff: - raise Http404 + # pylint: disable-msg=R0915 - msg = '' - problems = [] + msg = u'' plots = [] datatable = {} - + def_ms = modulestore() is_using_mongo = 'mongo' in str(def_ms.__class__) if is_using_mongo: courses = def_ms.get_courses() - courses = dict([c.id, c] for c in courses) # no course directory + courses = dict([c.id, c] for c in courses) # no course directory else: courses = def_ms.courses.items() - # the instructor dashboard page is modal: grades, psychometrics, admin - # keep that state in request.session (defaults to grades mode) - dash_mode = request.POST.get('dash_mode','') + # the sysadmin dashboard page is modal: status, courses, enrollment, + # staffing, gitlogs + # keep that state in request.session (defaults to status mode) + dash_mode = request.POST.get('dash_mode', '') if dash_mode: request.session['dash_mode'] = dash_mode else: - dash_mode = request.session.get('dash_mode','Status') - - # helper functions - - def return_csv(fn, datatable, fp=None): - if fp is None: - response = HttpResponse(mimetype='text/csv') - response['Content-Disposition'] = 'attachment; filename={0}'.format(fn) - else: - response = fp - writer = csv.writer(response, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL) - writer.writerow(datatable['header']) - for datarow in datatable['data']: - encoded_row = [unicode(s).encode('utf-8') for s in datarow] - writer.writerow(encoded_row) - return response - - def get_staff_group(course): - return get_group(course, 'staff') - - def get_instructor_group(course): - return get_group(course, 'instructor') - - def get_group(course, groupname): - grpname = get_access_group_name(course, groupname) - try: - group = Group.objects.get(name=grpname) - except Group.DoesNotExist: - group = Group(name=grpname) # create the group - group.save() - return group + dash_mode = request.session.get('dash_mode', _('Status')) # default datatable depends on dash_mode - - if dash_mode=='Status': - datatable = dict(header=['Statistic','Value'], - title="Site statistics") - datatable['data'] = [['Total number of users', User.objects.all().count()]] - - elif dash_mode=='Courses': + if dash_mode == _('Status'): + datatable = dict(header=[_('Statistic'), _('Value')], + title=_('Site statistics')) + datatable['data'] = [[_('Total number of users'), + User.objects.all().count()]] + elif dash_mode == _('Courses'): data = [] - #for cdir, course in def_ms.courses.items(): - for cdir, course in courses.items(): - data.append([course.display_name, cdir] + git_info_for_course(cdir)) - - datatable = dict(header=['Course Name', 'dir', 'git commit', 'last change', 'last editor'], - title="Information about all courses", + if hasattr(courses, 'items'): + course_iter = courses.items() + else: + course_iter = courses + for (cdir, course) in course_iter: + data.append([course.display_name, cdir] + + git_info_for_course(cdir)) + + datatable = dict(header=[_('Course Name'), _('dir'), + _('git commit'), _('last change'), + _('last editor')], + title=_('Information about all courses'), data=data) - - elif dash_mode=='Enrollment' or dash_mode=="Staffing": + elif dash_mode == _('Staffing and Enrollment'): data = [] - #for cdir, course in def_ms.courses.items(): - for cdir, course in courses.items(): + if hasattr(courses, 'items'): + course_iter = courses.items() + else: + course_iter = courses + for (cdir, course) in course_iter: datum = [course.display_name, course.id] - datum += [CourseEnrollment.objects.filter(course_id=course.id).count()] + datum += \ + [CourseEnrollment.objects.filter(course_id=course.id).count()] datum += [get_group(course, 'staff').user_set.all().count()] - datum += [','.join([x.username for x in get_group(course, 'instructor').user_set.all()])] + datum += [','.join([x.username for x in get_group(course, + 'instructor').user_set.all()])] data.append(datum) - datatable = dict(header=['Course Name', 'course_id', '# enrolled', '# staff', 'instructors'], - title="Enrollment information for all courses", + datatable = dict(header=[_('Course Name'), _('course_id'), + _('# enrolled'), _('# staff'), _('instructors')], + title=_('Enrollment information for all courses'), data=data) # process actions from form POST action = request.POST.get('action', '') track.views.server_track(request, action, {}, page='sysdashboard') - if "Download list of all users (csv file)" in action: - datatable = dict(header=['username', 'email'], - title="List of all users", - data=[[u.username, u.email] for u in User.objects.all()]) - return return_csv('users_%s.csv' % request.META['SERVER_NAME'],datatable) - - elif "Check and repair external Auth Map" in action: - msg += '
'
+    if _('Download list of all users (csv file)') in action:
+        datatable = dict(header=[_('username'), _('email')],
+                         title=_('List of all users'),
+                         data=[[u.username, u.email] for u in
+                               User.objects.all()])
+        return return_csv('users_{0}.csv'.format(
+            request.META['SERVER_NAME']), datatable)
+    elif _('Check and repair external Auth Map') in action:
+        msg += u'
'
         msg += fix_external_auth_map_passwords()
-        msg += '
' - datatable = {} - - elif "Create user" in action: - uname = request.POST.get('student_uname','').strip() - name = request.POST.get('student_fullname','').strip() - - def GenPasswd(length=8, chars=string.letters + string.digits): - return ''.join([choice(chars) for i in range(length)]) - - def create_user(uname, name, do_mit=False): - if not uname: - return "Must provide username" - if not name: - return "Must provide full name" - msg = '' - if do_mit: - if not '@' in uname: - email = '%s@MIT.EDU' % uname - else: - email = uname - if not email.endswith('@MIT.EDU'): - msg += 'email must end in @MIT.EDU' - return msg - mit_domain = 'ssl:MIT' - if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain): - msg += "Failed - email %s already exists as external_id" % email - return msg - make_eamap = True - else: - email = uname - if not '@' in email: - msg += 'email address required (not username)' - return msg - password = GenPasswd(12) - user = User(username=uname, email=email, is_active=True) - user.set_password(password) - try: - user.save() - except IntegrityError: - msg += "Oops, failed to create user %s, IntegrityError" % user - return msg - - r = Registration() - r.register(user) - - up = UserProfile(user=user) - up.name = name - up.save() - - if make_eamap: - credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email) - eamap = ExternalAuthMap(external_id = email, - external_email = email, - external_domain = mit_domain, - external_name = name, - internal_password = password, - external_credentials = json.dumps(credentials), - ) - eamap.user = user - eamap.dtsignup = datetime.now() - eamap.save() - - msg += "User %s created successfully!" % user - return msg - - msg += create_user(uname, name, do_mit=settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']) + msg += u'
' datatable = {} - - - elif "Delete user" in action: - uname = request.POST.get('student_uname','').strip() - - def delete_user(uname): - if not uname: - return "Must provide username" - if '@' in uname: - try: - u = User.objects.get(email=uname) - except Exception, err: - msg = "Cannot find user with email address %s" % uname - return msg - else: - try: - u = User.objects.get(username=uname) - except Exception, err: - msg = "Cannot find user with username %s" % uname - return msg - u.delete() - return "Deleted user %s" % uname - + elif _('Create user') in action: + uname = request.POST.get('student_uname', '').strip() + name = request.POST.get('student_fullname', '').strip() + password = request.POST.get('student_password', '').strip() + + msg += create_user(uname, name, password, + do_mit=settings.MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES']) + elif _('Delete user') in action: + uname = request.POST.get('student_uname', '').strip() msg += delete_user(uname) - - elif "Download staff and instructor list (csv file)" in action: + elif _('Download staff and instructor list (csv file)') in action: data = [] - roles = ['instructor','staff'] - #for cdir, course in def_ms.courses.items(): - for cdir, course in courses.items(): + roles = ['instructor', 'staff'] + + if hasattr(courses, 'items'): + course_iter = courses.items() + else: + course_iter = courses + for (cdir, course) in course_iter: for role in roles: - for u in get_group(course, role).user_set.all(): - datum = [course.id, role, u.username, u.email, u.profile.name] + for user in get_group(course, role).user_set.all(): + datum = [course.id, role, user.username, user.email, + user.profile.name] data.append(datum) - - datatable = dict(header=['course_id', 'role', 'username', 'email', 'full_name'], - title="List of all course staff and instructors", + datatable = dict(header=[_('course_id'), + _('role'), _('username'), + _('email'), _('full_name')], + title=_('List of all course staff and instructors'), data=data) - return return_csv('staff_%s.csv' % request.META['SERVER_NAME'],datatable) - - - elif action=="Delete course from site": - - course_id = request.POST.get('course_id','').strip() - ok = False + return return_csv('staff_{0}.csv'.format( + request.META['SERVER_NAME']), datatable) + elif action == _('Delete course from site'): + course_id = request.POST.get('course_id', '').strip() + course_found = False if course_id in courses: - ok = True + course_found = True course = courses[course_id] else: try: course = get_course_by_id(course_id) - ok = True - except Exception as err: - msg += "Error - cannot get course with ID %s
%s
" % (course_id, escape(err)) - - if ok and not is_using_mongo: - cdir = course.metadata.get('data_dir', course.location.course) + course_found = True + except Http404, err: + msg += \ + _('Error - cannot get course with ID {0}
{1}
' + ).format(course_id, escape(err)) + + if course_found and not is_using_mongo: + cdir = course.data_dir def_ms.courses.pop(cdir) # now move the directory (don't actually delete it) - nd = cdir + '_deleted_%s' % int(time.time()) - os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / nd) - os.system('chmod -x %s' % (settings.DATA_DIR / nd)) + new_dir = cdir + '_deleted_{0}'.format(int(time.time())) + os.rename(settings.DATA_DIR / cdir, settings.DATA_DIR / new_dir) - msg += "Deleted %s = %s (%s)" % (cdir, course.id, course.display_name) + msg += u"Deleted {0} = {1} ({2})".format( + cdir, course.id, course.display_name) - elif ok and is_using_mongo: + elif course_found and is_using_mongo: # delete course that is stored with mongodb backend loc = course.location - #ms = modulestore('direct') - cs = contentstore() + content_store = contentstore() commit = True - ret = delete_course(def_ms, cs, loc, commit) + delete_course(def_ms, content_store, loc, commit) # don't delete user permission groups, though - msg += "Deleted %s = %s (%s)" % (loc, course.id, course.display_name) - - - elif action=="Load new course from github": - - gitloc = request.POST.get('repo_location','').strip().replace(' ','').replace(';','') - - def get_course_from_git(gitloc): - msg = '' - if (not gitloc.endswith('.git')) or ('http:' in gitloc) or ('https:' in gitloc): - msg += "The git repo location should end with '.git', and be for SSH access" - return msg - - if is_using_mongo: - acscript = getattr(settings, 'CMS_ADD_COURSE_SCRIPT', '') - if not acscript or not os.path.exists(acscript): - msg = "Must configure CMS_ADD_COURSE_SCRIPT in settings first!" - return msg - cmd = '{0} "{1}"'.format(acscript, gitloc) - logging.debug('Adding course with command: {0}'.format(cmd)) - ret = subprocess.Popen(cmd, shell=True, executable = "/bin/bash", - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - ret = ''.join(ret) - msg = "Added course from {0}".format(gitloc) - msg += "
{0}
".format(ret.replace('<','<')) - return msg - - cdir = gitloc.rsplit('/',1)[1][:-4] - gdir = settings.DATA_DIR / cdir - if os.path.exists(gdir): - msg += "The course %s already exists in the data directory! (reloading anyway)" % cdir - # return msg - else: - cmd = "cd %s; git clone %s" % (settings.DATA_DIR, gitloc) - msg += '
%s
' % escape(os.popen(cmd).read()) - if not os.path.exists(gdir): - msg += "Failed to clone repository to %s" % gdir - return msg - def_ms.try_load_course(cdir) # load into modulestore - errlog = def_ms.errored_courses.get(cdir,'') - if errlog: - msg += '
%s
' % escape(errlog) - else: - course = def_ms.courses[cdir] - msg += "Loaded course %s (%s)
Errors:" % (cdir, course.display_name) - errors = def_ms.get_item_errors(course.location) - if not errors: - msg += "None" - else: - msg += "" - datatable['data'].append([course.display_name, cdir] + git_info_for_course(cdir)) - return msg - - msg += get_course_from_git(gitloc) - - else: # default to showing status summary - msg += '

Courses loaded in the modulestore

' - msg += '
    ' - #for cdir, course in def_ms.courses.items(): - for cdir, course in courses.items(): - msg += '
  1. %s (%s)
  2. ' % (escape(cdir), - course.location.url()) - msg += '
' - - - - #---------------------------------------- + msg += \ + u"{0} {1} = {2} ({3})".format( + _('Deleted'), loc, course.id, course.display_name) + + elif action == _('Load new course from github'): + gitloc = request.POST.get('repo_location', '').strip().replace( + ' ', '').replace(';', '') + msg += get_course_from_git(gitloc, is_using_mongo, def_ms, datatable) + + # default to showing status summary + else: + if hasattr(courses, 'items'): + course_iter = courses.items() + else: + course_iter = courses + msg += u'

{0}

'.format( + _('Courses loaded in the modulestore')) + msg += u'
    ' + for (cdir, course) in course_iter: + msg += u'
  1. {0} ({1})
  2. '.format( + escape(cdir), course.location.url()) + msg += u'
' + + # ---------------------------------------- # context for rendering - context = {'datatable': datatable, - 'plots': plots, - 'msg': msg, - 'djangopid' : os.getpid(), - 'modeflag': {dash_mode: 'selectedmode'}, - 'mitx_version' : getattr(settings,'MITX_VERSION_STRING',''), - } + context = { + 'datatable': datatable, + 'plots': plots, + 'msg': msg, + 'djangopid': os.getpid(), + 'modeflag': {dash_mode: 'selectedmode'}, + 'mitx_version': getattr(settings, 'MITX_VERSION_STRING', ''), + } return render_to_response('sysadmin_dashboard.html', context) - -#----------------------------------------------------------------------------- - -def view_git_logs(request, course_id=None): - import mongoengine # don't import that until we need it, here - class CourseImportLog(mongoengine.Document): - course_id = mongoengine.StringField(max_length=128) - location = mongoengine.StringField(max_length=168) - import_log = mongoengine.StringField(max_length=20*65535) - git_log = mongoengine.StringField(max_length=65535) - repo_dir = mongoengine.StringField(max_length=128) - created = mongoengine.DateTimeField() - meta = { 'indexes': ['course_id', 'created'], - 'allow_inheritance': False, } +# ----------------------------------------------------------------------------- - DBNAME = "xlog" +@staff_member_required +def view_git_logs(request, course_id=None): + """Shows logs of imports that happened as a result of a git import""" + # pylint: disable-msg=W0613 + + # Set defaults even if it isn't defined in settings + mongo_db = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'xlog', + } + + # Allow overrides + if hasattr(settings, 'MONGODB_LOG'): + for config_item in ['host', 'user', 'password', 'db', ]: + mongo_db[config_item] = settings.MONGODB_LOG.get( + config_item, mongo_db[config_item]) + + mongouri = 'mongodb://{0}:{1}@{2}/{3}'.format( + mongo_db['user'], mongo_db['password'], + mongo_db['host'], mongo_db['db']) + + try: + if mongo_db['user'] and mongo_db['password']: + mdb = mongoengine.connect(mongo_db['db'], host=mongouri) + else: + mdb = mongoengine.connect(mongo_db['db'], host=mongo_db['host']) + except mongoengine.connection.ConnectionError, ex: + logging.critical(_('Unable to connect to mongodb to save log, please check ' + 'MONGODB_LOG settings. error: {0}').format(str(ex))) - mdb = mongoengine.connect(DBNAME) - if course_id is None: cilset = CourseImportLog.objects.all().order_by('-created') else: - log.debug('course_id=%s' % course_id) - cilset = CourseImportLog.objects.filter(course_id=course_id).order_by('-created') - log.debug('cilset length=%s' % len(cilset)) - - context = {'cilset': cilset, - 'course_id': course_id, - } - - return render_to_response('sysadmin_dashboard_gitlogs.html', context) - - - \ No newline at end of file + log.debug('course_id={0}'.format(course_id)) + cilset = CourseImportLog.objects.filter( + course_id=course_id).order_by('-created') + log.debug('cilset length={0}'.format(len(cilset))) + mdb.disconnect() + context = {'cilset': cilset, 'course_id': course_id} + + return render_to_response('sysadmin_dashboard_gitlogs.html', + context) diff --git a/lms/djangoapps/dashboard/tests/test_sysadmin.py b/lms/djangoapps/dashboard/tests/test_sysadmin.py new file mode 100644 index 000000000000..8f8e1fd1dda0 --- /dev/null +++ b/lms/djangoapps/dashboard/tests/test_sysadmin.py @@ -0,0 +1,370 @@ +""" +Provide tests for sysadmin dashboard feature in sysadmin.py +""" + +import unittest +import os +import shutil + +from django.test.client import Client +from django.test.utils import override_settings + +from django.conf import settings +from django.contrib.auth.models import User +from django.core.urlresolvers import reverse +from django.utils.translation import ugettext as _ + +from dashboard.sysadmin import create_user +from external_auth.models import ExternalAuthMap +from django.contrib.auth.hashers import check_password +from xmodule.modulestore.django import modulestore +from courseware.tests.tests import TEST_DATA_MONGO_MODULESTORE +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from django.utils.html import escape +from dashboard.sysadmin import CourseImportLog +import mongoengine + +TEST_MONGODB_LOG = { + 'host': 'localhost', + 'user': '', + 'password': '', + 'db': 'test_xlog', +} + + +class SysadminBaseTestCase(ModuleStoreTestCase): + """ Base class with common methods used in XML and Mongo tests""" + + def setUp(self): + super(SysadminBaseTestCase, self).setUp() + self.user = User.objects.create_user('test_user', 'test_user+sysadmin@edx.org', 'foo') + self.client = Client() + + def _setstaff_login(self): + """ Makes the test user staff and logs them in""" + + self.user.is_staff = True + self.user.save() + self.client.login(username=self.user.username, password='foo') + + def _add_edx4edx(self): + """Adds the edx4edx sample course""" + + return self.client.post(reverse('sysadmin'), { + 'dash_mode': _('Courses'), + 'repo_location': 'https://github.com/mitocw/edx4edx_lite.git', + 'action': _('Load new course from github'), }) + + def _rm_edx4edx(self): + """Deletes the sample course from the XML store""" + # pylint: disable-msg=E1103 + + def_ms = modulestore() + try: + # using XML stor + course = def_ms.courses.get('{0}/edx4edx_lite'.format(os.path.abspath(settings.DATA_DIR)), None) + except AttributeError: + # Using mongo store + course = def_ms.get_course('MITx/edx4edx/edx4edx') + + # Delete git loaded course + return self.client.post(reverse('sysadmin'), + {'dash_mode': _('Courses'), + 'course_id': course.id, + 'action': _('Delete course from site'), }) + + +@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set") +class TestSysadmin(SysadminBaseTestCase): + """ + Check that landing page is the status page + """ + + def test_staff_access(self): + # pylint: disable-msg=E1103 + + response = self.client.get(reverse('sysadmin')) + self.assertEqual('/sysadmin', response.context['next']) + + response = self.client.get(reverse('gitlogs')) + self.assertEqual('/gitlogs', response.context['next']) + + logged_in = self.client.login(username=self.user.username, + password='foo') + self.assertTrue(logged_in) + + response = self.client.get(reverse('sysadmin')) + self.assertEqual('/sysadmin', response.context['next']) + + response = self.client.get(reverse('gitlogs')) + self.assertEqual('/gitlogs', response.context['next']) + + self.user.is_staff = True + self.user.save() + + self.client.logout() + self.client.login(username=self.user.username, password='foo') + + response = self.client.get(reverse('sysadmin')) + self.assertFalse(hasattr(response.context, 'next')) + + response = self.client.get(reverse('gitlogs')) + self.assertFalse(hasattr(response.context, 'next')) + + def test_user_mod(self): + """Create and delete a user""" + + self._setstaff_login() + + self.client.login(username=self.user.username, password='foo') + + # Create user + self.client.post(reverse('sysadmin'), + {'dash_mode': _('Status'), + 'action': _('Create user'), + 'student_uname': 'test_cuser+sysadmin@edx.org', + 'student_fullname': 'test cuser', + 'student_password': 'foozor', }) + + self.assertIsNotNone( + User.objects.get(username='test_cuser+sysadmin@edx.org', + email='test_cuser+sysadmin@edx.org')) + + # login as new user to confirm + self.assertTrue(self.client.login(username='test_cuser+sysadmin@edx.org', + password='foozor')) + + self.client.logout() + self.client.login(username=self.user.username, password='foo') + + # Delete user + self.client.post(reverse('sysadmin'), + {'dash_mode': _('Status'), + 'action': _('Delete user'), + 'student_uname': 'test_cuser+sysadmin@edx.org', + 'student_fullname': 'test cuser', }) + + self.assertEqual(0, len(User.objects.filter( + username='test_cuser+sysadmin@edx.org', + email='test_cuser+sysadmin@edx.org'))) + + self.assertEqual(1, len(User.objects.all())) + + def test_user_csv(self): + """Download and validate user CSV""" + + self._setstaff_login() + + response = self.client.post(reverse('sysadmin'), { + 'dash_mode': _('Status'), + 'action': _('Download list of all users (csv file)'), + }) + + self.assertIn('attachment', response['Content-Disposition']) + self.assertEqual('text/csv', response['Content-Type']) + self.assertIn('test_user', response.content) + self.assertTrue(2, len(response.content.splitlines())) + + def test_authmap_repair(self): + """Run authmap check and repair""" + + self._setstaff_login() + + create_user('test0', 'test test', do_mit=True) + # Will raise exception, so no assert needed + eamap = ExternalAuthMap.objects.get(external_name='test test') + mitu = User.objects.get(username='test0') + + self.assertTrue(check_password(eamap.internal_password, mitu.password)) + + mitu.set_password('not autogenerated') + mitu.save() + + self.assertFalse(check_password(eamap.internal_password, mitu.password)) + + response = self.client.post(reverse('sysadmin'), { + 'dash_mode': _('Status'), + 'action': _('Check and repair external Auth Map'), }) + + self.assertIn('{0} test0'.format(_('Failed in authenticating')), response.content) + self.assertIn(_('fixed password'), response.content) + + self.assertTrue(self.client.login(username='test0', password=eamap.internal_password)) + + # Check for all OK + self._setstaff_login() + response = self.client.post(reverse('sysadmin'), + {'dash_mode': _('Status'), + 'action': _('Check and repair external Auth Map'), }) + self.assertIn(_('All ok!'), response.content) + + def test_xml_course_add_delete(self): + """add and delete course from xml module store""" + + self._setstaff_login() + + # Try bad git repo + response = self.client.post(reverse('sysadmin'), { + 'dash_mode': _('Courses'), + 'repo_location': 'github.com/mitocw/edx4edx_lite', + 'action': _('Load new course from github'), }) + self.assertIn(_("The git repo location should end with '.git', and be a valid url"), response.content.decode('utf-8')) + + # Create git loaded course + response = self._add_edx4edx() + + def_ms = modulestore() + self.assertIn('xml', str(def_ms.__class__)) + course = def_ms.courses.get('{0}/edx4edx_lite'.format( + os.path.abspath(settings.DATA_DIR)), None) + self.assertIsNotNone(course) + + response = self._rm_edx4edx() + course = def_ms.courses.get('{0}/edx4edx_lite'.format( + os.path.abspath(settings.DATA_DIR)), None) + self.assertIsNone(course) + + def test_git_pull(self): + """Make sure we can pull""" + + self._setstaff_login() + + response = self._add_edx4edx() + response = self._add_edx4edx() + self.assertIn(_("The course {0} already exists in the data directory! " + "(reloading anyway)").format('edx4edx_lite'), + response.content.decode('utf-8')) + self._rm_edx4edx() + + def test_staff_csv(self): + """Download and validate staff CSV""" + + self._setstaff_login() + self._add_edx4edx() + + response = self.client.post(reverse('sysadmin'), { + 'dash_mode': _('Staffing and Enrollment'), + 'action': _('Download staff and instructor list (csv file)'), + }) + + self.assertIn('attachment', response['Content-Disposition']) + self.assertEqual('text/csv', response['Content-Type']) + columns = [_('course_id'), _('role'), _('username'), _('email'), _('full_name'), ] + self.assertIn(','.join('"' + c + '"' for c in columns), response.content) + + self._rm_edx4edx() + + def test_enrollment_page(self): + """ + Adds a course and makes sure that it shows up on the staffing and + enrollment page + """ + + self._setstaff_login() + self._add_edx4edx() + response = self.client.post(reverse('sysadmin'), {'dash_mode': _('Staffing and Enrollment')}) + print(response.content) + self.assertIn('edx4edx', response.content) + self._rm_edx4edx() + + +@override_settings(MODULESTORE=TEST_DATA_MONGO_MODULESTORE) +@override_settings(MONGODB_LOG=TEST_MONGODB_LOG) +@unittest.skipUnless(settings.MITX_FEATURES.get('ENABLE_SYSADMIN_DASHBOARD'), "ENABLE_SYSADMIN_DASHBOARD not set") +class TestSysAdminMongoCourseImport(SysadminBaseTestCase): + """ + Check that importing into the mongo module store works + """ + + @classmethod + def tearDownClass(cls): + super(TestSysAdminMongoCourseImport, cls).tearDownClass() + # Delete git repos and mongo objects + try: + shutil.rmtree(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + + try: + mongoengine.connect(TEST_MONGODB_LOG['db']) + CourseImportLog.objects.all().delete() + except mongoengine.connection.ConnectionError: + pass + + def _setstaff_login(self): + """ Makes the test user staff and logs them in""" + + self.user.is_staff = True + self.user.save() + + self.client.login(username=self.user.username, password='foo') + + def test_missing_repo_dir(self): + """Ensure that we handle a missing repo dir""" + + self._setstaff_login() + + if os.path.isdir(getattr(settings, 'GIT_REPO_DIR')): + shutil.rmtree(getattr(settings, 'GIT_REPO_DIR')) + + # Create git loaded course + response = self._add_edx4edx() + self.assertIn(escape(_("Path {0} doesn't exist, please create it, or configure a " + "different path with GIT_REPO_DIR").format(settings.GIT_REPO_DIR)), + response.content.decode('UTF-8')) + + def test_mongo_course_add_delete(self): + """same as TestSysadmin.test_xml_course_add_delete, but use mongo store""" + + self._setstaff_login() + try: + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + + def_ms = modulestore() + self.assertIn('mongo', str(def_ms.__class__)) + + self._add_edx4edx() + course = def_ms.get_course('MITx/edx4edx/edx4edx') + self.assertIsNotNone(course) + + self._rm_edx4edx() + course = def_ms.get_course('MITx/edx4edx/edx4edx') + self.assertIsNone(course) + + def test_gitlogs(self): + """Create a log entry and make sure it exists""" + + self._setstaff_login() + try: + os.mkdir(getattr(settings, 'GIT_REPO_DIR')) + except OSError: + pass + + self._add_edx4edx() + response = self.client.get(reverse('gitlogs')) + print(response.content) + print(CourseImportLog.objects.all()) + # Check that our earlier import has a log with a link to details + self.assertIn('/gitlogs/MITx/edx4edx/edx4edx', response.content) + + response = self.client.get( + reverse('gitlogs_detail', kwargs={'course_id': 'MITx/edx4edx/edx4edx'})) + + self.assertIn('======> IMPORTING course to location', response.content) + + self._rm_edx4edx() + + @override_settings(GIT_ADD_COURSE_SCRIPT='') + def test_no_script_set(self): + """ Test if settings are right on mongo store import""" + + self._setstaff_login() + + response = self.client.post(reverse('sysadmin'), { + 'dash_mode': _('Courses'), + 'repo_location': 'https://github.com/mitocw/edx4edx_lite.git', + 'action': _('Load new course from github'), }) + self.assertIn(_('Must configure GIT_ADD_COURSE_SCRIPT in settings first!'), + response.content) diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 436563913a18..b83b5a7c3a2f 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -199,6 +199,11 @@ FEEDBACK_SUBMISSION_EMAIL = ENV_TOKENS.get("FEEDBACK_SUBMISSION_EMAIL") MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS) +# git repo loading environment +GIT_REPO_DIR = ENV_TOKENS.get('GIT_REPO_DIR', None) +GIT_IMPORT_STATIC = ENV_TOKENS.get('GIT_IMPORT_STATIC', True) +GIT_ADD_COURSE_SCRIPT = ENV_TOKENS.get('GIT_ADD_COURSE_SCRIPT', GIT_ADD_COURSE_SCRIPT) + for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): oldvalue = CODE_JAIL.get(name) if isinstance(oldvalue, dict): @@ -242,6 +247,7 @@ MODULESTORE = AUTH_TOKENS.get('MODULESTORE', MODULESTORE) CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG',DOC_STORE_CONFIG) +MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG') OPEN_ENDED_GRADING_INTERFACE = AUTH_TOKENS.get('OPEN_ENDED_GRADING_INTERFACE', OPEN_ENDED_GRADING_INTERFACE) diff --git a/lms/envs/common.py b/lms/envs/common.py index 03cdfe74d0e7..ff1d953d503c 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -484,6 +484,7 @@ # These are listed, eg at https://github.com/MITx/mitx/admin/hooks ALLOWED_GITRELOAD_IPS = ['207.97.227.253', '50.57.128.197', '108.171.174.178'] +GIT_ADD_COURSE_SCRIPT = REPO_ROOT / "scripts/git_add_course.py" #################################### AWS ####################################### # S3BotoStorage insists on a timeout for uploaded assets. We should make it diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 9d3a4f8a955e..bfbafd4b4fa1 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -31,8 +31,8 @@ MITX_FEATURES['ENABLE_SERVICE_STATUS'] = True MITX_FEATURES['ENABLE_INSTRUCTOR_EMAIL'] = True MITX_FEATURES['ENABLE_HINTER_INSTRUCTOR_VIEW'] = True -MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True MITX_FEATURES['ENABLE_INSTRUCTOR_BETA_DASHBOARD'] = True +MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True MITX_FEATURES['MULTIPLE_ENROLLMENT_ROLES'] = True MITX_FEATURES['ENABLE_SHOPPING_CART'] = True MITX_FEATURES['AUTOMATIC_VERIFY_STUDENT_IDENTITY_FOR_TESTING'] = True @@ -264,10 +264,6 @@ if SEGMENT_IO_LMS_KEY: MITX_FEATURES['SEGMENT_IO_LMS'] = True -########################## GIT WORKFLOW ######################## - -CMS_ADD_COURSE_SCRIPT = REPO_ROOT / "scripts/cms_git_add_course" - ###################### Payment ##############################3 CC_PROCESSOR['CyberSource']['SHARED_SECRET'] = os.environ.get('CYBERSOURCE_SHARED_SECRET', '') diff --git a/lms/envs/test.py b/lms/envs/test.py index 42a4d8443658..7291c408419f 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -197,6 +197,10 @@ CC_PROCESSOR['CyberSource']['SERIAL_NUMBER'] = "0123456789012345678901" CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = "/shoppingcart/payment_fake" +########################### SYSADMIN DASHBOARD ################################ +MITX_FEATURES['ENABLE_SYSADMIN_DASHBOARD'] = True +GIT_REPO_DIR = TEST_ROOT / "course_repos" + ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True diff --git a/lms/templates/sysadmin_dashboard.html b/lms/templates/sysadmin_dashboard.html index ff09485690ed..28cbfb05a823 100644 --- a/lms/templates/sysadmin_dashboard.html +++ b/lms/templates/sysadmin_dashboard.html @@ -1,9 +1,10 @@ <%inherit file="/main.html" /> <%! from django.core.urlresolvers import reverse %> +<%! from django.utils.translation import ugettext as _ %> <%namespace name='static' file='/static_content.html'/> <%block name="headextra"> - <%static:css group='course'/> + <%static:css group='style-course'/> @@ -50,97 +51,107 @@
-
-

Sysadmin Dashboard

+
+

${_('Sysadmin Dashboard')}

-

[ Status | - Courses | - Enrollment | - Staffing | - GitLogs +

[ ${_('Status')} | + ${_('Courses')} | + ${_('Staffing and Enrollment')} | + ${_('GitLogs')} ]

-
${djangopid} - | ${mitx_version}
-
##----------------------------------------------------------------------------- %if modeflag.get('Status'): - -

- Email or username: - -

- +

${_('User Management')}

+
    +
  • + + +
  • + +
  • + + +
  • +
  • + + +
  • +
+ +
+

+ + +

+
+ +

- Full Name: - +

- -

- -

- +


%endif ##----------------------------------------------------------------------------- -%if modeflag.get('Enrollment'): +%if modeflag.get('Staffing and Enrollment'): -

Go to each individual course's Instructor dashboard to manage -course enrollment.

+

${_("Go to each individual course's Instructor dashboard to manage course enrollment.")}

+
-%endif - -##----------------------------------------------------------------------------- -%if modeflag.get('Staffing'): - -

Manage course staff and instructors


+

${_('Manage course staff and instructors')}


- +

- %endif ##----------------------------------------------------------------------------- %if modeflag.get('Courses'): -

Administer Courses


- -

- Repo location (ssh): - -

-

- Course ID or dir: - -

+

${_('Administer Courses')}


+ +
    +
  • + + +
  • +
+
+ +
+
+
    +
  • + + +
  • +
+
+ +
+
%endif ##-----------------------------------------------------------------------------
%if msg: -

${msg}

+

${msg}

%endif ##----------------------------------------------------------------------------- ##----------------------------------------------------------------------------- @@ -192,5 +203,7 @@

${plot['title']}

%endif
+
${_('Django PID')}: ${djangopid} + | ${_('Platform Version')}: ${mitx_version}
diff --git a/lms/templates/sysadmin_dashboard_gitlogs.html b/lms/templates/sysadmin_dashboard_gitlogs.html index 64bcc9d15560..be44bf060e75 100644 --- a/lms/templates/sysadmin_dashboard_gitlogs.html +++ b/lms/templates/sysadmin_dashboard_gitlogs.html @@ -1,13 +1,20 @@ <%inherit file="/main.html" /> <%! from django.core.urlresolvers import reverse %> +<%! from django.utils.translation import ugettext as _ %> <%namespace name='static' file='/static_content.html'/> <%block name="headextra"> - <%static:css group='course'/> + <%static:css group='style-course'/> - +