source: trunk/LatexTool.py @ 3072

Revision 3072, 21.9 KB checked in by jukka, 9 years ago (diff)

Working with exercise editing

Line 
1
2# This file is composed from code originally from ZWiki's LatexWiki-plugin:
3#
4#LatexWiki - a patch to ZWiki for rendering embedded LaTeX code
5#Copyright (C) 2001 Open Software Services <info@OpenSoftwareServices.com>
6#Copyright (C) 2003,2004,2005 Bob McElrath <bob+latexwiki@mcelrath.org>
7#Copyright (C) 2006 Simon Michael and Zwiki contributors <http://zwiki.org>
8#All rights reserved, all disclaimers apply, etc.
9#
10#This program is free software; you can redistribute it and/or
11#modify it under the terms of the GNU General Public License
12#as published by the Free Software Foundation; either version 2
13#of the License, or (at your option) any later version.
14#
15#This program is distributed in the hope that it will be useful,
16#but WITHOUT ANY WARRANTY; without even the implied warranty of
17#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18#GNU General Public License for more details.
19#
20#You should have received a copy of the GNU General Public License
21#along with this program; if not, write to the Free Software
22#Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
23
24
25from Globals import InitializeClass
26from OFS.SimpleItem import SimpleItem
27from OFS.PropertyManager import PropertyManager
28from Products.CMFCore.utils import UniqueObject
29from AccessControl import ClassSecurityInfo
30from DocumentTemplate.html_quote import html_quote
31from string import strip, join, replace
32import os, sys, re, zLOG, string, math, popen2, select
33from struct import pack, unpack
34from cgi import escape
35from config import LATEX_IMAGES_STORAGE_PATH, EXTERNAL_LATEX_SERVICE
36from Products.CMFCore.utils import getToolByName
37
38#try:
39import fcntl
40from PIL import Image, ImageFile, ImageChops, PngImagePlugin
41#except ImportError:
42#    LATEX_IMAGES_STORAGE_PATH='' # No local latex for you
43
44
45# From config we need LATEX_IMAGES_STORAGE_PATH and EXTERNAL_LATEX_SERVICE
46# if LATEX_IMAGES_STORAGE_PATH is set, then we assume that the server is capable of rendering latex by its own power.
47# if EXTERNAL_LATEX_SERVICE is set, then we assume it is a service like ???,
48# where you ask for a image and image url contains the latex code.
49# If LeMill-site becomes too popular, this will be quite a task for service provider
50# If both are set, first we try local, then external.
51# If none is set, we just give the latex code back
52
53latexpath='/usr/texbin'
54
55latexRemoveDelim = re.compile(r'^(?:(\$\$|\$(?!\$))|\\\(|\\begin{[^}]*}|\\\[)(.*)(?:\\\]|\\end{[^}]*}|\\\)|\1)$', re.MULTILINE|re.DOTALL)
56
57errorMessage = """\n<hr/><font size="-1" color="red">
58Some or all expressions may not have rendered properly,
59because Latex returned the following error:<br/><pre>%s</pre></font>"""
60
61# Actual location of images
62workingDir = '%s/%s' % (sys.modules['__builtin__'].CLIENT_HOME,  LATEX_IMAGES_STORAGE_PATH)
63# Default character size, if the user doesn't specify
64defaultcharsizepx = 18
65
66imageExtension = '.png'
67
68class LatexSyntaxError(Exception): pass
69class LatexRenderError(Exception): pass
70class GhostscriptError(Exception): pass
71class AlignError(Exception): pass
72
73
74def findinpath(exe):
75    paths = [exe]
76    paths.extend( \
77        map(lambda x: os.path.join(x,exe), re.split(':', os.getenv('PATH'))))   # latexpath in possible paths is missing! 
78    for path in paths:
79        if os.access(path, os.X_OK): break
80        path = None
81    return path
82
83# find our external programs
84dvipngpath = findinpath('dvipng')
85gspath     = findinpath('gs')
86dvipspath  = findinpath('dvips') or '/usr/texbin/dvips'
87latexpath  = findinpath('latex') or '/usr/texbin/latex'
88print dvipngpath
89print gspath
90print latexpath
91print dvipspath
92
93charsizept = 10
94# dvipng and tex use 72.27 points per inch, internally and thus generate the
95# best-looking images.  Postscript uses 72 points per inch.  So if we have to
96# use ghostscript and go through a postscript conversion, there is a resolution
97# mismatch which puts nibs on the tops of letters for many choices of
98# charheightpx.
99if dvipngpath is not None:
100    ptperinch = 72.27
101else:
102    ptperinch = 72
103# Adjust the centerline by this many pixels, key is character height in px
104# This list was determined experimentally.  If anyone has a better algorithm
105# to align images, please contact me.
106centerfudge = dict({ # positive to move up, negative to move down
107    10:0,  11:+1, 12:0,  13:0,  14:0,  15:+2, 16:0, 17:0,  18:0,  19:0,  20:0,
108    21:0,  22:0,  23:+2, 24:0,  25:0,  26:0,  27:0, 28:0,  29:0,  30:0,  31:+1,
109    32:0,  33:+1, 34:0,  35:0,  36:0,  37:0,  38:0, 39:+1, 40:0,  41:+1, 42:-1,
110    43:0,  44:0,  45:0,  46:0,  47:+1, 48:0,  49:0, 50:0,  51:0,  52:-1, 53:0,
111    54:-1, 55:+1, 56:-1, 57:+1, 58:+3, 59:-1, 60:0, 61:-1, 62:-1, 63:+1, 64:-1
112    })
113
114latexInlinePattern = r'^(\$(?!\$)|\\\()$'
115
116# This is only used if your wiki does not have a node LatexTemplate.
117defaultLatexTemplate = r"""
118\documentclass[%dpt,notitlepage]{article}
119\usepackage{amsmath}
120\usepackage{amsfonts}
121\usepackage[all]{xy}
122\newenvironment{latex}{}{}
123\oddsidemargin -86pt
124\headheight 0pt
125\topmargin -96pt
126\nofiles
127\begin{document}
128\pagestyle{empty}
129%%s
130\end{document}
131"""  % (charsizept)
132# \topmargin -96pt
133
134
135class LatexTool(PropertyManager, UniqueObject, SimpleItem):
136    """ Tool for LaTex math support """
137
138    id = 'latex_tool'
139    meta_type = 'LatexTool'
140    security = ClassSecurityInfo()
141    plone_tool = 1
142    toolicon = 'skins/lemill/tool.gif'
143    __allow_access_to_unprotected_subobjects__ = 1
144
145
146
147
148
149
150    # This is the method you should call.
151
152    def getImageFor(self, latexCode, charheightpx=17):
153        if LATEX_IMAGES_STORAGE_PATH:
154            try:
155            #print latexCode
156                return self.getLocalImageFor(latexCode, charheightpx)
157            except (IOError, LatexSyntaxError), data:
158            #    print data
159                errors = str(data)
160                self.log(errors, 'LatexSyntaxError')
161        if EXTERNAL_LATEX_SERVICE:
162            return self.getRemoteImageFor(latexCode)           
163        else:
164            return latexCode           
165
166    def getLocalImageFor(self, latexCode, charheightpx=17):   
167        print 'using local latex'
168        portal_url = getToolByName(self, 'portal_url')
169        preamble, postamble = '', ''
170        width, height = '', ''
171        imageFile = self.fileNameFor(latexCode, charheightpx, '.png')
172        if not os.path.exists(os.path.join(workingDir, imageFile)):
173            errors = self.renderNonexistingImages([latexCode], charheightpx,  0.0, 1.03) # alignfudge orig. 0.0
174        imageUrl = '%s/%s/%s' % (portal_url(), LATEX_IMAGES_STORAGE_PATH, imageFile)
175        width, height = self.getPngSize(os.path.join(workingDir, imageFile))
176        alt = html_quote(latexRemoveDelim.match(latexCode).group(2))
177        return '%s<img alt="%s" class="equation" src="%s" width="%s" height="%s"/>%s' %(preamble,
178                                                                alt,
179                                                                imageUrl,
180                                                                width,
181                                                                height,
182                                                                postamble)
183
184    def getRemoteImageFor(self, latexCode):
185        print 'using remote latex'
186        alt= html_quote(latexRemoveDelim.match(latexCode).group(2))
187        if latexCode.startswith('\\('):
188            latexCode=latexCode[2:-2] # remove ( ) from '(code)'
189        src=''.join((EXTERNAL_LATEX_SERVICE,latexCode))
190        return '<img alt="%s" class="equation" src="%s" border="0" align="middle" />' % (alt, src)
191
192   
193    def fileNameFor(self, latexCode, size, extension=''):
194        return '%s-%spx%s' %(abs(hash(latexCode)), size, extension)
195   
196    def getPngSize(self, fname,
197                   magicBytes=pack('!BBBBBBBB', 137, 80, 78, 71, 13, 10, 26, 10)):
198        f = file(fname, 'r')
199        buf = f.read(24)
200        f.close()
201        assert buf[:8] == magicBytes, 'in getPngSize, file not a PNG!'
202        return tuple(map(int, unpack('!LL', buf[16:24])))
203   
204    def log(self, message,summary='',severity=0):
205            zLOG.LOG('LatexWikiDebugLog',severity,summary,message)
206   
207    # Make our file descriptors nonblocking so that reading doesn't hang.
208    def makeNonBlocking(self, f):
209        fl = fcntl.fcntl(f.fileno(), fcntl.F_GETFL)
210        fcntl.fcntl(f.fileno(), fcntl.F_SETFL, fl | os.O_NONBLOCK)
211       
212    def runCommand(self, cmdLine, input=None):
213        program = popen2.Popen3('cd %s; '%(workingDir) + cmdLine, 1)
214        if input:
215            program.tochild.write(input)
216        program.tochild.close()
217        self.makeNonBlocking(program.fromchild)
218        self.makeNonBlocking(program.childerr)
219        stderr = []
220        stdout = []
221        erreof = False
222        outeof = False
223        while(not (erreof and outeof)):
224            readme, writme, xme = select.select([program.fromchild, program.childerr], [], [])
225            for output in readme:
226                if(output == program.fromchild):
227                    text = program.fromchild.read()
228                    if(text == ''): outeof = True
229                    else: stdout.append(text)
230                elif(output == program.childerr):
231                    text = program.childerr.read()
232                    if(text == ''): erreof = True
233                    else: stderr.append(text)
234        status = program.wait()
235        error = os.WEXITSTATUS(status) or not os.WIFEXITED(status)
236        return error, string.join(stdout, ''), string.join(stderr, '')
237           
238    # methods from latexWrapper.py begin here
239
240    def imageDoesNotExist(self, code, charheightpx):
241        return not os.path.exists(os.path.join(workingDir,
242            self.fileNameFor(code, charheightpx, imageExtension)))
243   
244    def renderNonexistingImages(self, latexCodeList, charheightpx, alignfudge, resfudge, **kw):
245        """ take a list of strings of latex code, render the
246        images that don't already exist.
247        """
248        print 'LATEX: rendering images'
249        latexTemplate = (kw.get('latexTemplate', defaultLatexTemplate) or
250                         defaultLatexTemplate)
251        m = re.search(r'\\documentclass\[[^\]]*?(\d+)pt[^\]]*?\]', \
252            latexTemplate)
253        if m:
254            charsizept = int(m.group(1))
255        else:
256            charsizept = 10
257        res = charheightpx*ptperinch/charsizept*resfudge
258        errors = ""
259        latexCodeList=list(set(latexCodeList)) # removes duplicates
260        codeToRender = filter(lambda x: self.imageDoesNotExist(x, charheightpx), latexCodeList)
261        if (not codeToRender): return
262        #unifiedCode = re.sub(r'^(\$|\\\()', r'\1|~ ', codeToRender[0])
263        #for code in codeToRender[1:len(codeToRender)]:
264        #    unifiedCode = unifiedCode + '\n\\newpage\n' + re.sub(r'^(\$|\\\()', r'\1|~ ', code)
265        ### PATCH FOR LEMILL: JUST RENDER THE FIRST SNIPPET, THERE SHOULD NOT BE MORE
266        unifiedCode=codeToRender[0]
267        try:
268            self.runLatex(unifiedCode, res, charheightpx, latexTemplate)
269        except LatexSyntaxError, data:
270           errors = str(data)
271           self.log(errors, 'LatexSyntaxError')
272           # FIXME translate latex line number to source line number
273           return escape(errors)
274   
275        fName = self.fileNameFor(unifiedCode, charheightpx)
276        self.dviPng(fName, res)
277        for code, i in map(None, codeToRender, range(0, len(codeToRender))):
278            newFileName = self.fileNameFor(code, charheightpx, imageExtension)
279            imname = '%s-%03d.png'%(fName,i+1)
280
281            # The next clause causes problems, so I make it always False.
282            if False and re.match(r'^(?:\$|\\\()', code): # FIXME make dvipng do the alpha properly
283                im = Image.open(os.path.join(workingDir, imname))
284                try:
285                    im = self.align(im, charheightpx, alignfudge) # returns an RGBA image
286                except (AlignError, ValueError), data:
287                    raise LatexRenderError(str(data) + '\nThe code was:\n' + \
288                        code+ '\nin the file %s'%(os.path.join(workingDir, imname)))
289                if im.mode != 'RGBA':
290                    alpha = ImageChops.invert(im.convert('L'))
291                    i = im.putalpha(alpha)
292                im.save(os.path.join(workingDir, newFileName), "PNG")
293            else:
294                os.rename(os.path.join(workingDir, imname), os.path.join(workingDir, newFileName))
295        os.system('cd %s; rm -f *.log *.aux *.tex *.pdf *.dvi *.ps %s-???.png'%(workingDir, fName))
296        return escape(errors)
297   
298    def runLatex(self, code, res, charheightpx, latexTemplate):
299        def ensureWorkingDirectory(path):
300            """Ensure this directory exists and is writable."""
301            if not os.access(path,os.F_OK): os.mkdir(path)
302            if not os.access(path,os.W_OK): os.system('chmod u+rwx %s' % path)
303   
304        texfileName = self.fileNameFor(code, charheightpx, '.tex')
305        dvifileName = self.fileNameFor(code, charheightpx, '.dvi')
306        psfileName = self.fileNameFor(code, charheightpx, '.ps')
307        cmdLine = '%s %s' %(latexpath, texfileName)
308   
309        ensureWorkingDirectory(workingDir)
310        file = open(os.path.join(workingDir, texfileName), 'w')
311        file.write(latexTemplate %(code,))
312        file.close()
313        err, stdout, stderr = self.runCommand(cmdLine)
314       
315        if err:
316            out = stderr + '\n' + stdout+'\n'+ cmdLine
317            err = re.search('!.*\?', out, re.MULTILINE+re.DOTALL)
318            if err:
319                out = err.group(0)
320    # FIXME translate latex line numbers to source line numbers
321            raise LatexSyntaxError(out)
322   
323    def dviPng(self, fName, res):
324        input, output = fName+'.dvi', fName+'-%03d.png'
325        gspngfname = fName+'-gs-%03d.png'
326        psfname = fName+'-gs'; i=1
327        # '--truecolor -bg Transparent' generates RGB images with transparent pixel
328        # (not alpha channel) but it's close...
329        ppopt = ''
330        if dvipngpath is not None:
331            #cmdLine = '%s --truecolor -bg Transparent -picky -D %f -Ttight -v* -o %s %s'%\
332            #    (dvipngpath, res, output % i, input)
333            cmdLine = '%s -bg Transparent -picky -D %f -T tight -v -o %s %s'%\
334                (dvipngpath, res, output % i, input)
335            err, stdout, stderr = self.runCommand(cmdLine)
336            ppredo = []           
337            if not err: return
338            #print err
339            #print stdout
340            #print stderr
341            # dvipng -picky will give the following message on pages it cannot render
342            # (usually due to the use of postscript specials).  For that we fall
343            # through to ghostscript
344            matcher = re.finditer(r'\[(\d+) not rendered\]', stdout)
345            for m in matcher:
346                if ppredo: ppredo += ','
347                ppredo.append(m.group(1))           
348            if ppredo:
349                ppopt = '-pp ' + string.join(ppredo,',')
350        cmdLine = '%s %s -R -D %f -o %s %s'%(dvipspath, ppopt, res, psfname+'.ps', input)
351        #print cmdLine
352        err, stdout, stderr = self.runCommand(cmdLine)
353        if err:
354            self.log('%s\n%s\n%s\n'%(err, stdout, stderr), 'DVIPSError')
355            raise 'DVIPSError %s' % stderr+'\n'+stdout
356        if not ppopt:
357            ppredo = range(1,len(re.findall(r'\[\d+\]', stderr))+1)
358        err = self.runGhostscript(psfname, res, 'pngalpha')
359        self.center(psfname, res)
360        for page in ppredo:
361            oldfname = os.path.join(workingDir, gspngfname%i)
362            newfname = os.path.join(workingDir, output%int(page))
363            os.rename(oldfname, newfname)
364            i += 1
365   
366    def runGhostscript(self, fName, res, device):
367        input, output = fName+'.ps', fName+'-%03d.png'
368        cmdLine = '%s -dDOINTERPOLATE -dTextAlphaBits=4 '%gspath + \
369                  '-dGraphicsAlphaBits=4 -r%f -sDEVICE=%s ' + \
370                  '-dBATCH -dNOPAUSE -dQUIT -sOutputFile=%s %s '
371        cmdLine = cmdLine %(res, device, output, input)
372        err, stdout, stderr = self.runCommand(cmdLine)
373        if err:
374            log('%s\n%s\n%s\n'%(err, stdout, stderr), 'GhostscriptError')
375            raise GhostscriptError(stderr+'\n'+stdout)
376        return stderr # when using bbox, BoundingBox is on stderr
377   
378    # assumes png's already created
379    def center(self, fName, res):
380        bboxes = re.split('\n', self.runGhostscript(fName, res, 'bbox'))
381        pngfname = fName+'-%03d.png'
382        for i in range(0, len(bboxes)/2):
383            file = pngfname%(i+1)
384            start_x, start_y, end_x, end_y = map(float,
385                re.match(r'%%HiResBoundingBox: ([0-9\.]+) ([0-9\.]+) ([0-9\.]+) ([0-9\.]+)',
386                    bboxes[2*i+1]).groups())
387            xsize = int(round(((end_x - start_x) * res)/ptperinch))
388            ysize = int(round(((end_y - start_y) * res)/ptperinch))
389            if (xsize <= 0): xsize = 1
390            if (ysize <= 0): ysize = 1
391            start_x = int(round(start_x*res/ptperinch))
392            start_y = int(round(start_y*res/ptperinch))
393            im = Image.open(os.path.join(workingDir, file))
394            cropdim = (start_x, im.size[1]-start_y-ysize, start_x+xsize, im.size[1]-start_y)
395            cropdim = map(int, map(round, cropdim))
396            im = im.crop(cropdim)
397            im2 = Image.new('RGBA', im.size, (255,255,255))
398            im2.paste(im, (0, 0))
399            if im.mode != 'RGBA':
400                alpha = ImageChops.invert(im2.convert('L'))  # Image should already have an alpha
401                im3 = Image.new('RGBA', im.size, (0,0,0))
402                im3.putalpha(alpha)
403                im2 = im3
404            im2.save(os.path.join(workingDir, file), "PNG")
405   
406    def align(self, im, charheightpx=0, alignfudge=0):
407        dotstartx = -1; dotendx = -1; dotstarty = -1; dotendy = -1
408        widentop = 0; widenbottom = 0; letterstartx = -1; chopx = 0
409        if im.mode == 'P':
410            white = 0
411        elif im.mode == 'RGB':
412            white = (255,255,255)
413        elif im.mode == 'RGBA':
414            white = (254,254,254,0) # as output by ghostscript pngalpha device
415        elif im.mode == 'L':
416            white = 255 # FIXME I think
417        for x in range(0,im.size[0]):  # Try to find the leading dot
418            if(dotendy < 0) :
419                for y in range(0, im.size[1]):
420                    pixel=im.getpixel((x,y))
421                    if(dotstarty >= 0 and dotendy < 0):
422                        if(pixel == white):
423                            dotendy = y
424                            break
425                    if(dotstarty < 0 and pixel != white):
426                        dotstartx = x
427                        dotstarty = y
428                else:
429                    if dotstarty >= 0 and dotendy < 0:
430                        dotendy = im.size[1]
431            elif(dotendx < 0):
432                maybedotendx = x
433                for y in range(dotstarty, dotendy):
434                    pixel=im.getpixel((x,y))
435                    if pixel != white:
436                        maybedotendx = -1
437                if maybedotendx > 0:
438                    dotendx = x
439            else:               
440                for y in range(0,im.size[1]):
441                    pixel=im.getpixel((x,y))
442                    if pixel != white:
443                        letterstartx = x
444                        break
445                if letterstartx>0: break
446        else: # failed to find letterstartx
447            #print 'dotstartx=%d, dotendx=%d, dotstarty=%d, dotendy=%d, letterstartx=%d\n' % (dotstartx, dotendx, dotstarty, dotendy, letterstartx)
448            self.log('dotstartx=%d, dotendx=%d, dotstarty=%d, dotendy=%d, letterstartx=%d\n'
449                %(dotstartx, dotendx, dotstarty, dotendy, letterstartx))
450            self.log('Unable to find dot. (size=%dx%d)\n'%(im.size[0],im.size[1]), 'renderNonExistingImages')
451            raise AlignError('Image appears to be blank or not have an alignment dot.')
452        centerline = (dotendy-dotstarty)/2.0    # increase centerline to move char up WRT text
453        dotcenter = (dotendy-dotstarty)*7.0/144.0
454        centerline += dotcenter
455        if centerfudge.has_key(charheightpx):
456            centerline += centerfudge[charheightpx]/2.0
457        # if dot is not pixel-aligned, take that into account
458        # sum pixels above and below (dotendy-dotstarty)/2
459        dottophalf = 0
460        dotlinesize = dotendx-dotstartx
461        dottoplines = 0
462        for y in range(dotstarty, int(math.ceil(dotstarty+(dotendy-1-dotstarty)/2.0))):
463            dottoplines += 1
464            for x in range(dotstartx, dotendx):
465                dottophalf += self.cabs(im.getpixel((x,y)))
466            break
467        else:
468            dottophalf = 1 # dot was 1px high
469        dotbottomhalf = 0
470        for y in range(dotendy-1, dotstarty+(dotendy-1-dotstarty)/2,-1):
471            for x in range(dotstartx, dotendx):
472                dotbottomhalf += self.cabs(im.getpixel((x,y)))
473            break
474        else:
475            dotbottomhalf = 1 # dot was 1px high
476        if(dottophalf != 0.0 and dotbottomhalf != 0):
477            dotpixmiss = float(dottophalf-dotbottomhalf)/(dottophalf+dotbottomhalf)
478        else:
479            dotpixmiss = 0.0
480        centerline += dotpixmiss
481        centerline += alignfudge # user parameter -- FIXME remove?
482        bottomsize = im.size[1]-centerline               # pixels below midline
483        topsize = centerline                             # pixels above midline
484        if(topsize > bottomsize):
485            newheight = 2*topsize
486            widenbottom = topsize - topsize
487        else:
488            newheight = 2*bottomsize
489            widentop = bottomsize - topsize
490        chopx = letterstartx-1
491        newheight= int(newheight)
492        widentop = 0 #int(widentop) #SKWM broken
493        im2 = Image.new('RGBA', (im.size[0]-chopx,newheight), (255,255,255))
494        im2.paste(im,(-chopx,widentop,im.size[0]-chopx,im.size[1]-widentop))
495        return im2
496   
497    def cabs(self, A):
498        sq = 0.0
499        if type(A) == type(()):
500            for i in range(0,3):
501                sq += A[i]*A[i]
502            if len(A) == 4:
503                return math.sqrt(sq)*(A[3]/255.0)
504            else:
505                return math.sqrt(sq)
506        else:
507            return A
508   
509InitializeClass(LatexTool)
Note: See TracBrowser for help on using the repository browser.