From 0fd1dceb40b9a19fadad334e87356e8040dba1c4 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Fri, 12 Aug 2016 05:17:48 +0200 Subject: [PATCH 1/8] incorporates changed needed for imathics kernel --- mathics/builtin/files.py | 3 +-- mathics/core/evaluation.py | 45 ++++++++++++++++++++++++-------------- mathics/main.py | 9 ++++---- mathics/settings.py | 4 ++++ 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/mathics/builtin/files.py b/mathics/builtin/files.py index 63418f52fa..32df1e4d7f 100644 --- a/mathics/builtin/files.py +++ b/mathics/builtin/files.py @@ -795,8 +795,7 @@ def apply(self, channel, expr, evaluation): expr = expr.get_sequence() expr = Expression('Row', Expression('List', *expr)) - evaluation.format = 'text' - text = evaluation.format_output(from_python(expr)) + text = evaluation.format_output(from_python(expr), 'text') stream.write(six.text_type(text) + '\n') return Symbol('Null') diff --git a/mathics/core/evaluation.py b/mathics/core/evaluation.py index 818f0f41e7..d877689564 100644 --- a/mathics/core/evaluation.py +++ b/mathics/core/evaluation.py @@ -144,9 +144,16 @@ def get_data(self): } +class Callbacks(object): + def __init__(self, out=None, clear_output=None, display_data=None): + self.out = out + self.clear_output = clear_output + self.display_data = display_data + + class Evaluation(object): def __init__(self, definitions=None, - out_callback=None, format='text', catch_interrupt=True): + callbacks=None, format='text', catch_interrupt=True): from mathics.core.definitions import Definitions if definitions is None: @@ -156,7 +163,7 @@ def __init__(self, definitions=None, self.timeout = False self.stopped = False self.out = [] - self.out_callback = out_callback + self.callbacks = callbacks if callbacks else Callbacks() self.listeners = {} self.options = None self.predetermined_out = None @@ -221,7 +228,7 @@ def evaluate(): self.definitions.add_rule('Out', Rule( Expression('Out', line_no), stored_result)) if result != Symbol('Null'): - return self.format_output(result) + return self.format_output(result, self.format) else: return None try: @@ -261,7 +268,7 @@ def evaluate(): if exc_result is not None: self.recursion_depth = 0 if exc_result != Symbol('Null'): - result = self.format_output(exc_result) + result = self.format_output(exc_result, self.format) result = Result(self.out, result, line_no) self.out = [] @@ -288,23 +295,27 @@ def get_stored_result(self, result): # Prevent too large results from being stored, as this can exceed the # DB's max_allowed_packet size - data = pickle.dumps(result) - if len(data) > 10000: - return Symbol('Null') + if settings.MAX_STORED_SIZE is not None: + data = pickle.dumps(result) + if len(data) > settings.MAX_STORED_SIZE: + return Symbol('Null') return result def stop(self): self.stopped = True - def format_output(self, expr): + def format_output(self, expr, format): + if isinstance(format, dict): + return dict((k, self.format_output(expr, f)) for k, f in format.items()) + from mathics.core.expression import Expression, BoxError - if self.format == 'text': + if format == 'text': result = expr.format(self, 'System`OutputForm') - elif self.format == 'xml': + elif format == 'xml': result = Expression( 'StandardForm', expr).format(self, 'System`MathMLForm') - elif self.format == 'tex': + elif format == 'tex': result = Expression('StandardForm', expr).format( self, 'System`TeXForm') else: @@ -368,20 +379,20 @@ def message(self, symbol, tag, *args): text = String("Message %s::%s not found." % (symbol_shortname, tag)) text = self.format_output(Expression( - 'StringForm', text, *(from_python(arg) for arg in args))) + 'StringForm', text, *(from_python(arg) for arg in args)), 'text') self.out.append(Message(symbol_shortname, tag, text)) - if self.out_callback: - self.out_callback(self.out[-1]) + if self.callbacks.out: + self.callbacks.out(self.out[-1]) def print_out(self, text): from mathics.core.expression import from_python - text = self.format_output(from_python(text)) + text = self.format_output(from_python(text), 'text') self.out.append(Print(text)) - if self.out_callback: - self.out_callback(self.out[-1]) + if self.callbacks.out: + self.callbacks.out(self.out[-1]) if settings.DEBUG_PRINT: print('OUT: ' + text) diff --git a/mathics/main.py b/mathics/main.py index 42070a7d93..f5f20acb35 100644 --- a/mathics/main.py +++ b/mathics/main.py @@ -13,7 +13,7 @@ from mathics.core.definitions import Definitions from mathics.core.expression import strip_context -from mathics.core.evaluation import Evaluation +from mathics.core.evaluation import Evaluation, Callbacks from mathics.core.parser import LineFeeder, FileLineFeeder from mathics import version_string, license_string, __version__ from mathics import settings @@ -238,7 +238,7 @@ def main(): if args.execute: for expr in args.execute: print(shell.get_in_prompt() + expr) - evaluation = Evaluation(shell.definitions, out_callback=shell.out_callback) + evaluation = Evaluation(shell.definitions, callbacks=Callbacks(out=shell.out_callback)) result = evaluation.parse_evaluate(expr, timeout=settings.TIMEOUT) shell.print_result(result) @@ -249,7 +249,8 @@ def main(): feeder = FileLineFeeder(args.FILE) try: while not feeder.empty(): - evaluation = Evaluation(shell.definitions, out_callback=shell.out_callback, catch_interrupt=False) + evaluation = Evaluation( + shell.definitions, callbacks=Callbacks(out=shell.out_callback), catch_interrupt=False) query = evaluation.parse_feeder(feeder) if query is None: continue @@ -270,7 +271,7 @@ def main(): while True: try: - evaluation = Evaluation(shell.definitions, out_callback=shell.out_callback) + evaluation = Evaluation(shell.definitions, callbacks=Callbacks(out=shell.out_callback)) query = evaluation.parse_feeder(shell) if query is None: continue diff --git a/mathics/settings.py b/mathics/settings.py index c7cc79300a..d49b0f4730 100644 --- a/mathics/settings.py +++ b/mathics/settings.py @@ -27,6 +27,10 @@ MAX_RECURSION_DEPTH = 512 +# max pickle.dumps() size for storing results in DB +# historically 10000 was used on public mathics servers +MAX_STORED_SIZE = 10000 + ADMINS = ( ('Admin', 'mail@test.com'), ) From 9313c18fdce8a0da9d6fa707177f21ec2d281ca8 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Sat, 13 Aug 2016 07:17:51 +0200 Subject: [PATCH 2/8] undo change in files.py; to be addressed in separate PR --- mathics/builtin/files.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mathics/builtin/files.py b/mathics/builtin/files.py index 32df1e4d7f..63418f52fa 100644 --- a/mathics/builtin/files.py +++ b/mathics/builtin/files.py @@ -795,7 +795,8 @@ def apply(self, channel, expr, evaluation): expr = expr.get_sequence() expr = Expression('Row', Expression('List', *expr)) - text = evaluation.format_output(from_python(expr), 'text') + evaluation.format = 'text' + text = evaluation.format_output(from_python(expr)) stream.write(six.text_type(text) + '\n') return Symbol('Null') From 221c62ac1e202e4d2c16cd1d4da3482d265e74b1 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Mon, 15 Aug 2016 13:45:27 +0200 Subject: [PATCH 3/8] changed Callbacks to Output; added methods for svg/img xml production --- mathics/builtin/graphics.py | 10 ++------ mathics/core/evaluation.py | 30 +++++++++++++++--------- mathics/main.py | 16 +++++++++---- mathics/web/views.py | 46 +++++++++++++++++++++++++++++++++++-- 4 files changed, 77 insertions(+), 25 deletions(-) diff --git a/mathics/builtin/graphics.py b/mathics/builtin/graphics.py index b48512075b..2452e66264 100644 --- a/mathics/builtin/graphics.py +++ b/mathics/builtin/graphics.py @@ -1453,14 +1453,8 @@ def boxes_to_xml(self, leaves, **options): w += 2 h += 2 - xml = ( - '%s') % ( - width, height, xmin, ymin, w, h, svg) - - xml = """%s""" % xml - return xml + return options['evaluation'].output.svg_xml( + data=svg, width=width, height=height, viewbox=[xmin, ymin, w, h]) def axis_ticks(self, xmin, xmax): def round_to_zero(value): diff --git a/mathics/core/evaluation.py b/mathics/core/evaluation.py index d877689564..21dae79ff7 100644 --- a/mathics/core/evaluation.py +++ b/mathics/core/evaluation.py @@ -144,16 +144,26 @@ def get_data(self): } -class Callbacks(object): - def __init__(self, out=None, clear_output=None, display_data=None): - self.out = out - self.clear_output = clear_output - self.display_data = display_data +class Output(object): + def out(self, out): + pass + + def clear_output(wait=False): + raise NotImplementedError + + def display_data(self, result): + raise NotImplementedError + + def svg_xml(self, data, width, height, viewbox): + raise NotImplementedError + + def img_xml(self, data, width, height): + raise NotImplementedError class Evaluation(object): def __init__(self, definitions=None, - callbacks=None, format='text', catch_interrupt=True): + output=None, format='text', catch_interrupt=True): from mathics.core.definitions import Definitions if definitions is None: @@ -163,7 +173,7 @@ def __init__(self, definitions=None, self.timeout = False self.stopped = False self.out = [] - self.callbacks = callbacks if callbacks else Callbacks() + self.output = output if output else Output() self.listeners = {} self.options = None self.predetermined_out = None @@ -382,8 +392,7 @@ def message(self, symbol, tag, *args): 'StringForm', text, *(from_python(arg) for arg in args)), 'text') self.out.append(Message(symbol_shortname, tag, text)) - if self.callbacks.out: - self.callbacks.out(self.out[-1]) + self.output.out(self.out[-1]) def print_out(self, text): from mathics.core.expression import from_python @@ -391,8 +400,7 @@ def print_out(self, text): text = self.format_output(from_python(text), 'text') self.out.append(Print(text)) - if self.callbacks.out: - self.callbacks.out(self.out[-1]) + self.output.out(self.out[-1]) if settings.DEBUG_PRINT: print('OUT: ' + text) diff --git a/mathics/main.py b/mathics/main.py index f5f20acb35..e21e4f735e 100644 --- a/mathics/main.py +++ b/mathics/main.py @@ -13,7 +13,7 @@ from mathics.core.definitions import Definitions from mathics.core.expression import strip_context -from mathics.core.evaluation import Evaluation, Callbacks +from mathics.core.evaluation import Evaluation, Output from mathics.core.parser import LineFeeder, FileLineFeeder from mathics import version_string, license_string, __version__ from mathics import settings @@ -176,6 +176,14 @@ def empty(self): return False +class TerminalOutput(Output): + def __init__(self, shell): + self.shell = shell + + def out(self, out): + return self.shell.out_callback(out) + + def main(): argparser = argparse.ArgumentParser( prog='mathics', @@ -238,7 +246,7 @@ def main(): if args.execute: for expr in args.execute: print(shell.get_in_prompt() + expr) - evaluation = Evaluation(shell.definitions, callbacks=Callbacks(out=shell.out_callback)) + evaluation = Evaluation(shell.definitions, output=TerminalOutput(shell)) result = evaluation.parse_evaluate(expr, timeout=settings.TIMEOUT) shell.print_result(result) @@ -250,7 +258,7 @@ def main(): try: while not feeder.empty(): evaluation = Evaluation( - shell.definitions, callbacks=Callbacks(out=shell.out_callback), catch_interrupt=False) + shell.definitions, output=TerminalOutput(shell), catch_interrupt=False) query = evaluation.parse_feeder(feeder) if query is None: continue @@ -271,7 +279,7 @@ def main(): while True: try: - evaluation = Evaluation(shell.definitions, callbacks=Callbacks(out=shell.out_callback)) + evaluation = Evaluation(shell.definitions, output=TerminalOutput(shell)) query = evaluation.parse_feeder(shell) if query is None: continue diff --git a/mathics/web/views.py b/mathics/web/views.py index f6ee44a117..be6b5d8978 100644 --- a/mathics/web/views.py +++ b/mathics/web/views.py @@ -18,7 +18,7 @@ from django.contrib.auth.models import User from mathics.core.definitions import Definitions -from mathics.core.evaluation import Evaluation, Message, Result +from mathics.core.evaluation import Evaluation, Message, Result, Output from mathics.web.models import Query, Worksheet from mathics.web.forms import LoginForm, SaveForm @@ -26,6 +26,7 @@ from mathics.doc.doc import DocPart, DocChapter, DocSection import six from six.moves import range +from string import Template if settings.DEBUG: JSON_CONTENT_TYPE = 'text/html' @@ -44,6 +45,47 @@ def __init__(self, result={}): super(JsonResponse, self).__init__(response, content_type=JSON_CONTENT_TYPE) +class WebOutput(Output): + svg = Template(''' + + + + + $data + + + + + ''') + + img = Template(''' + + + + ''') + + def out(self, out): + pass + + def clear_output(wait=False): + raise NotImplementedError + + def display_data(self, result): + raise NotImplementedError + + def svg_xml(self, data, width, height, viewbox): + svg = self.svg.substitute( + data=data, width='%d' % width, height='%d' % height, + viewbox=' '.join(['%f' % t for t in viewbox])) + print(svg) + return svg + + def img_xml(self, data, width, height): + return self.img.substitue( + data=data, width=width, height=height) + + def require_ajax_login(func): def new_func(request, *args, **kwargs): if not request.user.is_authenticated(): @@ -102,7 +144,7 @@ def query(request): user_definitions = request.session.get('definitions') definitions.set_user_definitions(user_definitions) - evaluation = Evaluation(definitions, format='xml') + evaluation = Evaluation(definitions, format='xml', output=WebOutput()) feeder = MultiLineFeeder(input, '') results = [] try: From 0cab7eadb572c9e9a9764d4de6b9be278a4386ef Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Mon, 15 Aug 2016 13:46:34 +0200 Subject: [PATCH 4/8] removed debug code --- mathics/web/views.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mathics/web/views.py b/mathics/web/views.py index be6b5d8978..8dd27f9df1 100644 --- a/mathics/web/views.py +++ b/mathics/web/views.py @@ -75,11 +75,9 @@ def display_data(self, result): raise NotImplementedError def svg_xml(self, data, width, height, viewbox): - svg = self.svg.substitute( + return self.svg.substitute( data=data, width='%d' % width, height='%d' % height, viewbox=' '.join(['%f' % t for t in viewbox])) - print(svg) - return svg def img_xml(self, data, width, height): return self.img.substitue( From 40d8c720924089376fa570c74863f00accb9059e Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Tue, 16 Aug 2016 12:40:20 +0200 Subject: [PATCH 5/8] changed format_output to support default format --- mathics/core/evaluation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mathics/core/evaluation.py b/mathics/core/evaluation.py index 21dae79ff7..25f31531e5 100644 --- a/mathics/core/evaluation.py +++ b/mathics/core/evaluation.py @@ -314,7 +314,10 @@ def get_stored_result(self, result): def stop(self): self.stopped = True - def format_output(self, expr, format): + def format_output(self, expr, format=None): + if format is None: + format = self.format + if isinstance(format, dict): return dict((k, self.format_output(expr, f)) for k, f in format.items()) From 412304dcecff65720f162708ea573143fc1a19e4 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Tue, 16 Aug 2016 15:19:50 +0200 Subject: [PATCH 6/8] output for svgs and imgs now always relies on mglyphs --- mathics/builtin/graphics.py | 16 ++++++++++++++-- mathics/builtin/inout.py | 3 +-- mathics/core/evaluation.py | 6 ------ mathics/web/media/js/mathics.js | 28 ++++++++++++++++++++++++++++ mathics/web/views.py | 28 ---------------------------- 5 files changed, 43 insertions(+), 38 deletions(-) diff --git a/mathics/builtin/graphics.py b/mathics/builtin/graphics.py index 2452e66264..d93e6a173a 100644 --- a/mathics/builtin/graphics.py +++ b/mathics/builtin/graphics.py @@ -11,6 +11,7 @@ from math import floor, ceil, log10 import json +import base64 from six.moves import map from six.moves import range from six.moves import zip @@ -1453,8 +1454,19 @@ def boxes_to_xml(self, leaves, **options): w += 2 h += 2 - return options['evaluation'].output.svg_xml( - data=svg, width=width, height=height, viewbox=[xmin, ymin, w, h]) + svg_xml = ''' + + %s + + ''' % (' '.join('%f' % t for t in (xmin, ymin, w, h)), svg) + + return '' % ( + int(width), + int(height), + base64.b64encode(svg_xml.encode('utf8')).decode('utf8')) def axis_ticks(self, xmin, xmax): def round_to_zero(value): diff --git a/mathics/builtin/inout.py b/mathics/builtin/inout.py index debfecda2d..87e19c6c08 100644 --- a/mathics/builtin/inout.py +++ b/mathics/builtin/inout.py @@ -1732,8 +1732,7 @@ class MathMLForm(Builtin): = \u03bc #> MathMLForm[Graphics[Text["\u03bc"]]] - = \u03bc + = ## The should contain U+2062 INVISIBLE TIMES #> MathMLForm[MatrixForm[{{2*a, 0},{0,0}}]] diff --git a/mathics/core/evaluation.py b/mathics/core/evaluation.py index 25f31531e5..d28ecb928a 100644 --- a/mathics/core/evaluation.py +++ b/mathics/core/evaluation.py @@ -154,12 +154,6 @@ def clear_output(wait=False): def display_data(self, result): raise NotImplementedError - def svg_xml(self, data, width, height, viewbox): - raise NotImplementedError - - def img_xml(self, data, width, height): - raise NotImplementedError - class Evaluation(object): def __init__(self, definitions=None, diff --git a/mathics/web/media/js/mathics.js b/mathics/web/media/js/mathics.js index 45e34f3227..02bf3bd374 100644 --- a/mathics/web/media/js/mathics.js +++ b/mathics/web/media/js/mathics.js @@ -274,10 +274,38 @@ function translateDOMElement(element, svg) { return dom; } +function convertMathGlyphs(dom) { + // convert mglyphs to their classic representation ( or ), so the new mglyph logic does not make + // anything worse in the classic Mathics frontend for now. In the long run, this code should vanish. + + var MML = "http://www.w3.org/1998/Math/MathML"; + var glyphs = dom.getElementsByTagName("mglyph"); + for (var i = 0; i < glyphs.length; i++) { + var glyph = glyphs[i]; + var src = glyph.getAttribute('src'); + if (src.startsWith('data:image/svg+xml;base64,')) { + var svgText = atob(src.substring(src.indexOf(",") + 1)); + var mtable =document.createElementNS(MML, "mtable"); + mtable.innerHTML = '' + svgText + ''; + var svg = mtable.getElementsByTagNameNS("*", "svg")[0]; + svg.setAttribute('width', glyph.getAttribute('width')); + svg.setAttribute('height', glyph.getAttribute('height')); + glyph.parentNode.replaceChild(mtable, glyph); + } else if (src.startsWith('data:image/')) { + var img = document.createElement('img'); + img.setAttribute('src', src) + img.setAttribute('width', glyph.getAttribute('width')); + img.setAttribute('height', glyph.getAttribute('height')); + glyph.parentNode.replaceChild(img, glyph); + } + } +} + function createLine(value) { if (value.startsWith(' - - - - $data - - - - - ''') - - img = Template(''' - - - - ''') - def out(self, out): pass @@ -74,15 +55,6 @@ def clear_output(wait=False): def display_data(self, result): raise NotImplementedError - def svg_xml(self, data, width, height, viewbox): - return self.svg.substitute( - data=data, width='%d' % width, height='%d' % height, - viewbox=' '.join(['%f' % t for t in viewbox])) - - def img_xml(self, data, width, height): - return self.img.substitue( - data=data, width=width, height=height) - def require_ajax_login(func): def new_func(request, *args, **kwargs): From 511fd7b6fc156e005a9745d3c52539e89afbd508 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Wed, 17 Aug 2016 10:17:52 +0200 Subject: [PATCH 7/8] added Output.max_stored_size() --- mathics/core/evaluation.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mathics/core/evaluation.py b/mathics/core/evaluation.py index d28ecb928a..b7be3b1971 100644 --- a/mathics/core/evaluation.py +++ b/mathics/core/evaluation.py @@ -145,6 +145,9 @@ def get_data(self): class Output(object): + def max_stored_size(self, settings): + return settings.MAX_STORED_SIZE + def out(self, out): pass @@ -299,9 +302,10 @@ def get_stored_result(self, result): # Prevent too large results from being stored, as this can exceed the # DB's max_allowed_packet size - if settings.MAX_STORED_SIZE is not None: + max_stored_size = self.output.max_stored_size(settings) + if max_stored_size is not None: data = pickle.dumps(result) - if len(data) > settings.MAX_STORED_SIZE: + if len(data) > max_stored_size: return Symbol('Null') return result From 73fc5010a8d22c22dd7f3bf0e9a55a3798c9a858 Mon Sep 17 00:00:00 2001 From: Bernhard Liebl Date: Wed, 17 Aug 2016 13:06:01 +0200 Subject: [PATCH 8/8] changed Output interface --- mathics/core/evaluation.py | 4 ++-- mathics/web/views.py | 9 +-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/mathics/core/evaluation.py b/mathics/core/evaluation.py index b7be3b1971..fe56a650f0 100644 --- a/mathics/core/evaluation.py +++ b/mathics/core/evaluation.py @@ -151,10 +151,10 @@ def max_stored_size(self, settings): def out(self, out): pass - def clear_output(wait=False): + def clear(self, wait): raise NotImplementedError - def display_data(self, result): + def display(self, data, metadata): raise NotImplementedError diff --git a/mathics/web/views.py b/mathics/web/views.py index 8676e9dea5..876d1a48af 100644 --- a/mathics/web/views.py +++ b/mathics/web/views.py @@ -46,14 +46,7 @@ def __init__(self, result={}): class WebOutput(Output): - def out(self, out): - pass - - def clear_output(wait=False): - raise NotImplementedError - - def display_data(self, result): - raise NotImplementedError + pass def require_ajax_login(func):