scripts.dtx

% \section{Included Python scripts}
% \label{sec:included-scripts}
%
% Here we describe the Python code for \ST scripts, for running
% Sage only if necessary, substituting in Sage outputs to produce
% a ``static'' file, and extracting all Sage code from a |.tex| file.
%
% \subsection{\texttt{sagetex-run}}
% \label{sec:sagetex-run}
% \iffalse
%<*ifnecessaryscript>
% \fi
%
% When working on a document that uses \ST, running Sage every time you
% typeset your document may take too long, especially since it often
% is not necessary. This script is a drop-in replacement for Sage:
% instead of
% \begin{center}
% |sage document.sagetex.sage|
% \end{center}
% you can do
% \begin{center}
% |sagetex-run document.sagetex.sage|
% \end{center}
% and it will use the MD5 mechanism described in the |endofdoc| macro
% (page~{\pageref{macro:endofdoc}). With this, you can set up your editor
% (\TeX Shop, \TeX Works, etc) to typeset your document with a script
% that does
% \begin{quote}
%   |pdflatex $1|\\
%   |sagetex-run $1|
% \end{quote}
% which will only, of course, run Sage when necessary.
%    \begin{macrocode}
"""
Given a filename f, examines f.sagetex.sage and f.sagetex.sout and
runs Sage if necessary.
"""

import hashlib
import sys
import os
import re
import subprocess
import shutil
import argparse

def argparser():
    p = argparse.ArgumentParser(description=__doc__.strip())
    p.add_argument('--sage', action='store', default=find_sage(),
                   help="Location of the Sage executable")
    p.add_argument('src', help="Input file name (basename or .sagetex.sage)")
    return p

def find_sage():
    return shutil.which('sage') or 'sage'

def run(args):
    src = args.src
    path_to_sage = args.sage

    if src.endswith('.sagetex.sage'):
        src = src[:-13]
    else:
        src = os.path.splitext(src)[0]

    # Ensure results are output in the same directory as the source files
    os.chdir(os.path.dirname(src))
    src = os.path.basename(src)

    usepackage = r'usepackage(\[.*\])?{sagetex}'
    uses_sagetex = False

    # If it does not use sagetex, obviously running sage is unnecessary.
    if os.path.isfile(src + '.tex'):
        with open(src + '.tex') as texf:
            for line in texf:
                if line.strip().startswith(r'\usepackage') and re.search(usepackage, line):
                    uses_sagetex = True
                    break
    else:
        # The .tex file might not exist if LaTeX output was put to a different
        # directory, so in that case just assume we need to build.
        uses_sagetex = True

    if not uses_sagetex:
        print(src + ".tex doesn't seem to use SageTeX, exiting.", file=sys.stderr)
        sys.exit(1)

    # if something goes wrong, assume we need to run Sage
    run_sage = True
    ignore = r"^( _st_.goboom|print\('SageT| ?_st_.current_tex_line)"

    try:
        with open(src + '.sagetex.sage', 'r') as sagef:
            h = hashlib.md5()
            for line in sagef:
                if not re.search(ignore, line):
                    h.update(bytearray(line,'utf8'))
    except IOError:
        print('{0}.sagetex.sage not found, I think you need to typeset {0}.tex first.'
              ''.format(src), file=sys.stderr)
        sys.exit(1)

    try:
        with open(src + '.sagetex.sout', 'r') as outf:
            for line in outf:
                m = re.match('%([0-9a-f]+)% md5sum', line)
                if m:
                    print('computed md5:', h.hexdigest())
                    print('sagetex.sout md5:', m.group(1))
                    if h.hexdigest() == m.group(1):
                        run_sage = False
                        break
    except IOError:
        pass

    if run_sage:
        print('Need to run Sage on {0}.'.format(src))
        sys.exit(subprocess.call([path_to_sage, src + '.sagetex.sage']))
    else:
        print('Not necessary to run Sage on {0}.'.format(src))

if __name__ == "__main__":
    run(argparser().parse_args())
%    \end{macrocode}
%
% \subsection{\texttt{sagetex-makestatic}}
% \label{sec:sagetex-makestatic}
% \iffalse
%</ifnecessaryscript>
%<*staticscript>
% \fi
%
% Now the |sagetex-makestatic|:
%
%    \begin{macrocode}
"""
Removes SageTeX macros from `inputfile' and replaces them with the
Sage-computed results to make a "static" file. You'll need to have run
Sage on `inputfile' already.

`inputfile' can include the .tex extension or not. If you provide
`outputfile', the results will be written to a file of that name.
Specify `-o' or `--overwrite' to overwrite the file if it exists.

See the SageTeX documentation for more details.
"""
import sys
import time
import os.path
import argparse

from sagetexparse import DeSageTex

def argparser():
    p = argparse.ArgumentParser(description=__doc__.strip())
    p.add_argument('inputfile', help="Input file name (basename or .tex)")
    p.add_argument('outputfile', nargs='?', default=None, help="Output file name")
    p.add_argument('-o', '--overwrite', action="store_true", default=False,
                   help="Overwrite output file if it exists")
    p.add_argument('-s', '--sout', action="store", default=None,
                   help="Location of the .sagetex.sout file")
    return p

def run(args):
    src, dst, overwrite = args.inputfile, args.outputfile, args.overwrite

    if dst is not None and (os.path.exists(dst) and not overwrite):
        print('Error: %s exists and overwrite option not specified.' % dst,
              file=sys.stderr)
        sys.exit(1)

    src, ext = os.path.splitext(src)
    texfn = src + '.tex'
    soutfn = args.sout if args.sout is not None else src + '.sagetex.sout'
%    \end{macrocode}
% All the real work gets done in the line below. Sorry it's not more
% exciting-looking.
%    \begin{macrocode}
    desagetexed = DeSageTex(texfn, soutfn)
%    \end{macrocode}
% This part is cool: we need double percent signs at the beginning of
% the line because Python needs them (so they get turned into single
% percent signs) \emph{and} because \textsf{Docstrip} needs them (so the
% line gets passed into the generated file). It's perfect!
%    \begin{macrocode}
    header = ("%% SageTeX commands have been automatically removed from this file and\n"
              "%% replaced with plain LaTeX. Processed %s.\n"
              "" % time.strftime('%a %d %b %Y %H:%M:%S', time.localtime()))

    if dst is not None:
        dest = open(dst, 'w')
    else:
        dest = sys.stdout

    dest.write(header)
    dest.write(desagetexed.result)

if __name__ == "__main__":
    run(argparser().parse_args())
%    \end{macrocode}
%
% \iffalse
%</staticscript>
%<*extractscript>
% \fi
%
% \subsection{\texttt{sagetex-extract}}
%
% Same idea as |sagetex-makestatic|, except this does basically the opposite
% thing.
%    \begin{macrocode}
"""
Extracts Sage code from `inputfile'.

`inputfile' can include the .tex extension or not. If you provide
`outputfile', the results will be written to a file of that name,
otherwise the result will be printed to stdout.

Specify `-o' or `--overwrite' to overwrite the file if it exists.

See the SageTeX documentation for more details.
"""
import sys
import time
import os.path
import argparse

from sagetexparse import SageCodeExtractor

def argparser():
    p = argparse.ArgumentParser(description=__doc__.strip())
    p.add_argument('inputfile', help="Input file name (basename or .tex)")
    p.add_argument('outputfile', nargs='?', default=None, help="Output file name")
    p.add_argument('-o', '--overwrite', action="store_true", default=False,
                   help="Overwrite output file if it exists")
    p.add_argument('--no-inline', action="store_true", default=False,
                   help="Extract code only from Sage environments")
    return p

def run(args):
    src, dst, overwrite = args.inputfile, args.outputfile, args.overwrite

    if dst is not None and (os.path.exists(dst) and not overwrite):
        print('Error: %s exists and overwrite option not specified.' % dst,
              file=sys.stderr)
        sys.exit(1)

    src, ext = os.path.splitext(src)
    sagecode = SageCodeExtractor(src + '.tex', inline=not args.no_inline)
    header = ("#> This file contains Sage code extracted from %s%s.\n"
              "#> Processed %s.\n"
              "" % (src, ext, time.strftime('%a %d %b %Y %H:%M:%S', time.localtime())))

    if dst is not None:
        dest = open(dst, 'w')
    else:
        dest = sys.stdout

    dest.write(header)
    dest.write(sagecode.result)

if __name__ == "__main__":
    run(argparser().parse_args())
%    \end{macrocode}
%
% \iffalse
%</extractscript>
%<*parsermod>
% \fi
%
% \subsection{The parser module}
% \changes{v2.2}{2009/06/17}{Update parser module to handle pause/unpause}
%
% Here's the module that does the actual parsing and replacing. It's
% really quite simple, thanks to the awesome
% \href{http://pyparsing.wikispaces.com}{Pyparsing module}. The parsing
% code below is nearly self-documenting! Compare that to fancy regular
% expressions, which sometimes look like someone sneezed punctuation all
% over the screen.
%    \begin{macrocode}
import sys
import os
import textwrap
from pyparsing import *
%    \end{macrocode}
% First, we define this very helpful parser: it finds the matching
% bracket, and doesn't parse any of the intervening text. It's basically
% like hitting the percent sign in Vim. This is useful for parsing \LTX
% stuff, when you want to just grab everything enclosed by matching
% brackets.
%    \begin{macrocode}
def skipToMatching(opener, closer):
  nest = nestedExpr(opener, closer)
  return originalTextFor(nest)

curlybrackets = skipToMatching('{', '}')
squarebrackets = skipToMatching('[', ']')
%    \end{macrocode}
% Next, parser for |\sage|, |\sageplot|, and pause/unpause calls:
%    \begin{macrocode}
sagemacroparser = r'\sage' + curlybrackets('code')
sagestrmacroparser = r'\sagestr' + curlybrackets('code')
sageplotparser = (r'\sageplot'
                 + Optional(squarebrackets)('opts')
                 + Optional(squarebrackets)('format')
                 + curlybrackets('code'))
sagetexpause = Literal(r'\sagetexpause')
sagetexunpause = Literal(r'\sagetexunpause')
%    \end{macrocode}
%
% With those defined, let's move on to our classes.
%
% \begin{macro}{SoutParser}
% Here's the parser for the generated |.sout| file. The code below does
% all the parsing of the |.sout| file and puts the results into a
% list. Notice that it's on the order of 10 lines of code---hooray
% for Pyparsing!
%    \begin{macrocode}
class SoutParser():
  def __init__(self, fn):
    self.label = []
%    \end{macrocode}
% A label line looks like
% \begin{quote}
%  |\newlabel{@sageinline|\meta{integer}|}{|\marg{bunch of \LTX code}|{}{}{}{}}|
% \end{quote}
% which makes the parser definition below pretty obvious. We assign some
% names to the interesting bits so the |newlabel| method can make the
% \meta{integer} and \meta{bunch of \LTX code} into the keys and values
% of a dictionary. The |DeSageTeX| class then uses that dictionary to
% replace bits in the |.tex| file with their Sage-computed results.
%    \begin{macrocode}
    parselabel = (r'\newlabel{@sageinline'
                 + Word(nums)('num')
                 + '}{'
                 + curlybrackets('result')
                 + '{}{}{}{}}')
%    \end{macrocode}
% We tell it to ignore comments, and hook up the list-making method.
%    \begin{macrocode}
    parselabel.ignore('%' + restOfLine)
    parselabel.setParseAction(self.newlabel)
%    \end{macrocode}
% A |.sout| file consists of one or more such lines. Now go parse the
% file we were given.
%    \begin{macrocode}
    try:
      OneOrMore(parselabel).parseFile(fn)
    except IOError:
      print('Error accessing {}; exiting. Does your .sout file exist?'.format(fn))
      sys.exit(1)
%    \end{macrocode}
% Pyparser's parse actions get called with three arguments: the string
% that matched, the location of the beginning, and the resulting parse
% object. Here we just add a new key-value pair to the dictionary,
% remembering to strip off the enclosing brackets from the ``result''
% bit.
%    \begin{macrocode}
  def newlabel(self, s, l, t):
    self.label.append(t.result[1:-1])
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{DeSageTeX}
% Now we define a parser for \LTX files that use \ST commands. We assume
% that the provided |fn| is just a basename.
%    \begin{macrocode}
class DeSageTex():
  def __init__(self, texfn, soutfn):
    self.sagen = 0
    self.plotn = 0
    self.fn = os.path.basename(texfn)
    self.sout = SoutParser(soutfn)
%    \end{macrocode}
% Parse |\sage| macros. We just need to pull in the result from the
% |.sout| file and increment the counter---that's what |self.sage| does.
%    \begin{macrocode}
    strmacro = sagestrmacroparser
    smacro = sagemacroparser
    smacro.setParseAction(self.sage)
    strmacro.setParseAction(self.sage)
%    \end{macrocode}
% Parse the |\usepackage{sagetex}| line. Right now we don't support
% comma-separated lists of packages.
%    \begin{macrocode}
    usepackage = (r'\usepackage'
                 + Optional(squarebrackets)
                 + '{sagetex}')
    usepackage.setParseAction(replaceWith(r"""% "\usepackage{sagetex}" line was here:
\RequirePackage{verbatim}
\RequirePackage{graphicx}
\newcommand{\sagetexpause}{\relax}
\newcommand{\sagetexunpause}{\relax}"""))
%    \end{macrocode}
% Parse |\sageplot| macros.
%    \begin{macrocode}
    splot = sageplotparser
    splot.setParseAction(self.plot)
%    \end{macrocode}
% The printed environments (|sageblock| and |sageverbatim|) get turned
% into |verbatim| environments.
%    \begin{macrocode}
    beginorend = oneOf('begin end')
    blockorverb = 'sage' + oneOf('block verbatim')
    blockorverb.setParseAction(replaceWith('verbatim'))
    senv = '\\' + beginorend + '{' + blockorverb + '}'
%    \end{macrocode}
% The non-printed |sagesilent| environment gets commented out. We could
% remove all the text, but this works and makes going back to \ST
% commands (de-de-\ST{}ing?) easier.
%    \begin{macrocode}
    silent = Literal('sagesilent')
    silent.setParseAction(replaceWith('comment'))
    ssilent = '\\' + beginorend + '{' + silent + '}'
%    \end{macrocode}
% The |\sagetexindent| macro is no longer relevant, so remove it from
% the output (``suppress'', in Pyparsing terms).
%    \begin{macrocode}
    stexindent = Suppress(r'\setlength{\sagetexindent}' + curlybrackets)
%    \end{macrocode}
% Now we define the parser that actually goes through the file. It just
% looks for any one of the above bits, while ignoring anything that
% should be ignored.
%    \begin{macrocode}
    doit = smacro | senv | ssilent | usepackage | splot | stexindent |strmacro
    doit.ignore('%' + restOfLine)
    doit.ignore(r'\begin{verbatim}' + SkipTo(r'\end{verbatim}'))
    doit.ignore(r'\begin{comment}' + SkipTo(r'\end{comment}'))
    doit.ignore(r'\sagetexpause' + SkipTo(r'\sagetexunpause'))
%    \end{macrocode}
% We can't use the |parseFile| method, because that expects a ``complete
% grammar'' in which everything falls into some piece of the parser.
% Instead we suck in the whole file as a single string, and run
% |transformString| on it, since that will just pick out the interesting
% bits and munge them according to the above definitions.
%    \begin{macrocode}
    str = ''.join(open(texfn, 'r').readlines())
    self.result = doit.transformString(str)
%    \end{macrocode}
% That's the end of the class constructor, and it's all we need to do
% here. You access the results of parsing via the |result| string.
%
% We do have two methods to define. The first does the same thing that
% |\ref| does in your \LTX file: returns the content of the label and
% increments a counter.
%    \begin{macrocode}
  def sage(self, s, l, t):
    self.sagen += 1
    return self.sout.label[self.sagen - 1]
%    \end{macrocode}
% The second method returns the appropriate |\includegraphics| command.
% It does need to account for the default argument.
%    \begin{macrocode}
  def plot(self, s, l, t):
    self.plotn += 1
    if len(t.opts) == 0:
      opts = r'[width=.75\textwidth]'
    else:
      opts = t.opts[0]
    return (r'\includegraphics%s{sage-plots-for-%s.tex/plot-%s}' %
      (opts, self.fn, self.plotn - 1))
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{SageCodeExtractor}
% This class does the opposite of the first: instead of removing Sage
% stuff and leaving only \LTX, this removes all the \LTX and leaves only
% Sage.
%    \begin{macrocode}
class SageCodeExtractor():
  def __init__(self, texfn, inline=True):
    smacro = sagemacroparser
    smacro.setParseAction(self.macroout)

    splot = sageplotparser
    splot.setParseAction(self.plotout)
%    \end{macrocode}
% Above, we used the general parsers for |\sage| and |\sageplot|. We
% have to redo the environment parsers because it seems too hard to
% define one parser object that will do both things we want: above, we
% just wanted to change the environment name, and here we want to suck
% out the code. Here, it's important that we find matching begin/end
% pairs; above it wasn't. At any rate, it's not a big deal to redo this
% parser.
%    \begin{macrocode}
    env_names = oneOf('sageblock sageverbatim sagesilent')
    senv = r'\begin{' + env_names('env') + '}' + SkipTo(
           r'\end{' + matchPreviousExpr(env_names) + '}')('code')
    senv.leaveWhitespace()
    senv.setParseAction(self.envout)

    spause = sagetexpause
    spause.setParseAction(self.pause)

    sunpause = sagetexunpause
    sunpause.setParseAction(self.unpause)

    if inline:
        doit = smacro | splot | senv | spause | sunpause
    else:
        doit = senv | spause | sunpause
    doit.ignore('%' + restOfLine)

    str = ''.join(open(texfn, 'r').readlines())
    self.result = ''

    doit.transformString(str)

  def macroout(self, s, l, t):
    self.result += '#> \\sage{} from line %s\n' % lineno(l, s)
    self.result += textwrap.dedent(t.code[1:-1]) + '\n\n'

  def plotout(self, s, l, t):
    self.result += '#> \\sageplot{} from line %s:\n' % lineno(l, s)
    if t.format != '':
      self.result += '#> format: %s' % t.format[0][1:-1] + '\n'
    self.result += textwrap.dedent(t.code[1:-1]) + '\n\n'

  def envout(self, s, l, t):
    self.result += '#> %s environment from line %s:' % (t.env,
      lineno(l, s))
    self.result += textwrap.dedent(''.join(t.code)) + '\n'

  def pause(self, s, l, t):
    self.result += ('#> SageTeX (probably) paused on input line %s.\n\n' %
                    (lineno(l, s)))

  def unpause(self, s, l, t):
    self.result += ('#> SageTeX (probably) unpaused on input line %s.\n\n' %
                    (lineno(l, s)))
%    \end{macrocode}
% \end{macro}

% \endinput
%</parsermod>
% Local Variables:
% mode: doctex
% TeX-master: "sagetex"
% End: