/* -*- mode: java; c-basic-offset: 2; indent-tabs-mode: nil -*- */

/*

PdePreprocessor - wrapper for default ANTLR-generated parser

Part of the Processing project - http://processing.org

Copyright (c) 2004-15 Ben Fry and Casey Reas

Copyright (c) 2001-04 Massachusetts Institute of Technology

ANTLR-generated parser and several supporting classes written

by Dan Mosedale via funding from the Interaction Institute IVREA.

This program is free software; you can redistribute it and/or modify

it under the terms of the GNU General Public License as published by

the Free Software Foundation; either version 2 of the License, or

(at your option) any later version.

This program is distributed in the hope that it will be useful,

but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

GNU General Public License for more details.

You should have received a copy of the GNU General Public License

along with this program; if not, write to the Free Software Foundation,

Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

*/

package processing.mode.java.preproc;

import java.io.*;

import java.util.*;

import java.util.regex.MatchResult;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import processing.app.Messages;

import processing.app.Preferences;

import processing.app.SketchException;

import processing.core.PApplet;

import processing.data.StringList;

import antlr.*;

import antlr.collections.AST;

/**

* Class that orchestrates preprocessing p5 syntax into straight Java.

* <P/>

* <B>Current Preprocessor Subsitutions:</B>

* <UL>

* <LI>any function not specified as being protected or private will

* be made 'public'. this means that <TT>void setup()</TT> becomes

* <TT>public void setup()</TT>. This is important to note when

* coding with core.jar outside of the PDE.

* <LI><TT>compiler.substitute_floats</TT> (currently "substitute_f")

* treat doubles as floats, i.e. 12.3 becomes 12.3f so that people

* don't have to add f after their numbers all the time since it's

* confusing for beginners.

* <LI><TT>compiler.enhanced_casting</TT> byte(), char(), int(), float()

* works for casting. this is basic in the current implementation, but

* should be expanded as described above. color() works similarly to int(),

* however there is also a *function* called color(r, g, b) in p5.

* <LI><TT>compiler.color_datatype</TT> 'color' is aliased to 'int'

* as a datatype to represent ARGB packed into a single int, commonly

* used in p5 for pixels[] and other color operations. this is just a

* search/replace type thing, and it can be used interchangeably with int.

* <LI><TT>compiler.web_colors</TT> (currently "inline_web_colors")

* color c = #cc0080; should unpack to 0xffcc0080 (the ff at the top is

* so that the color is opaque), which is just an int.

* </UL>

* <B>Other preprocessor functionality</B>

* <UL>

* <LI>detects what 'mode' the program is in: static (no function

* brackets at all, just assumes everything is in draw), active

* (setup plus draw or loop), and java mode (full java support).

* http://processing.org/reference/environment/

* </UL>

* <P/>

* The PDE Preprocessor is based on the Java Grammar that comes with

* ANTLR 2.7.2. Moving it forward to a new version of the grammar

* shouldn't be too difficult.

* <P/>

* Here's some info about the various files in this directory:

* <P/>

* <TT>java.g:</TT> this is the ANTLR grammar for Java 1.3/1.4 from the

* ANTLR distribution. It is in the public domain. The only change to

* this file from the original this file is the uncommenting of the

* clauses required to support assert().

* <P/>

* <TT>java.tree.g:</TT> this describes the Abstract Syntax Tree (AST)

* generated by java.g. It is only here as a reference for coders hacking

* on the preprocessor, it is not built or used at all. Note that pde.g

* overrides some of the java.g rules so that in PDE ASTs, there are a

* few minor differences. Also in the public domain.

* <P/>

* <TT>pde.g:</TT> this is the grammar and lexer for the PDE language

* itself. It subclasses the java.g grammar and lexer. There are a couple

* of overrides to java.g that I hope to convince the ANTLR folks to fold

* back into their grammar, but most of this file is highly specific to

* PDE itself.

* <TT>PdeEmitter.java:</TT> this class traverses the AST generated by

* the PDE Recognizer, and emits it as Java code, doing any necessary

* transformations along the way. It is based on JavaEmitter.java,

* available from antlr.org, written by Andy Tripp <atripp@comcast.net>,

* who has given permission for it to be distributed under the GPL.

* <P/>

* <TT>ExtendedCommonASTWithHiddenTokens.java:</TT> this adds a necessary

* initialize() method, as well as a number of methods to allow for XML

* serialization of the parse tree in a such a way that the hidden tokens

* are visible. Much of the code is taken from the original

* CommonASTWithHiddenTokens class. I hope to convince the ANTLR folks

* to fold these changes back into that class so that this file will be

* unnecessary.

* <P/>

* <TT>TokenStreamCopyingHiddenTokenFilter.java:</TT> this class provides

* TokenStreamHiddenTokenFilters with the concept of tokens which can be

* copied so that they are seen by both the hidden token stream as well

* as the parser itself. This is useful when one wants to use an

* existing parser (like the Java parser included with ANTLR) that throws

* away some tokens to create a parse tree which can be used to spit out

* a copy of the code with only minor modifications. Partially derived

* from ANTLR code. I hope to convince the ANTLR folks to fold this

* functionality back into ANTLR proper as well.

* <P/>

* <TT>whitespace_test.pde:</TT> a torture test to ensure that the

* preprocessor is correctly preserving whitespace, comments, and other

* hidden tokens correctly. See the comments in the code for details about

* how to run the test.

* <P/>

* All other files in this directory are generated at build time by ANTLR

* itself. The ANTLR manual goes into a fair amount of detail about the

* what each type of file is for.

* <P/>

*/

public class PdePreprocessor {

protected static final String UNICODE_ESCAPES = "0123456789abcdefABCDEF";

// used for calling the ASTFactory to get the root node

private static final int ROOT_ID = 0;

protected final String indent;

private final String name;

public enum Mode {

STATIC, ACTIVE, JAVA

}

private TokenStreamCopyingHiddenTokenFilter filter;

private String advClassName = "";

protected Mode mode;

Set<String> foundMethods;

SurfaceInfo sizeInfo;

/**

* Regular expression for parsing the size() method. This should match

* against any uses of the size() function, whether numbers or variables

* or whatever. This way, no warning is shown if size() isn't actually used

* in the sketch, which is the case especially for anyone who is cutting

* and pasting from the reference.

*/

// public static final String SIZE_REGEX =

// "(?:^|\\s|;)size\\s*\\(\\s*([^\\s,]+)\\s*,\\s*([^\\s,\\)]+)\\s*,?\\s*([^\\)]*)\\s*\\)\\s*\\;";

// static private final String SIZE_CONTENTS_REGEX =

// "(?:^|\\s|;)size\\s*\\(([^\\)]+)\\)\\s*\\;";

// static private final String FULL_SCREEN_CONTENTS_REGEX =

// "(?:^|\\s|;)fullScreen\\s*\\(([^\\)]+)\\)\\s*\\;";

// /** Test whether there's a void somewhere (the program has functions). */

// static private final String VOID_REGEX =

// "(?:^|\\s|;)void\\s";

/** Used to grab the start of setup() so we can mine it for size() */

static private final Pattern VOID_SETUP_REGEX =

Pattern.compile("(?:^|\\s|;)void\\s+setup\\s*\\(", Pattern.MULTILINE);

// Can't only match any 'public class', needs to be a PApplet

// http://code.google.com/p/processing/issues/detail?id=551

static private final Pattern PUBLIC_CLASS =

Pattern.compile("(^|;)\\s*public\\s+class\\s+\\S+\\s+extends\\s+PApplet", Pattern.MULTILINE);

static private final Pattern FUNCTION_DECL =

Pattern.compile("(^|;)\\s*((public|private|protected|final|static)\\s+)*" +

"(void|int|float|double|String|char|byte|boolean)" +

"(\\s*\\[\\s*\\])?\\s+[a-zA-Z0-9]+\\s*\\(",

Pattern.MULTILINE);

static private final Pattern CLOSING_BRACE = Pattern.compile("\\}");

public PdePreprocessor(final String sketchName) {

this(sketchName, Preferences.getInteger("editor.tabs.size"));

}

public PdePreprocessor(final String sketchName, final int tabSize) {

this.name = sketchName;

final char[] indentChars = new char[tabSize];

Arrays.fill(indentChars, ' ');

indent = new String(indentChars);

}

public SurfaceInfo initSketchSize(String code,

boolean sizeWarning) throws SketchException {

sizeInfo = parseSketchSize(code, sizeWarning);

return sizeInfo;

}

/**

* Break on commas, except those inside quotes,

* e.g.: size(300, 200, PDF, "output,weirdname.pdf");

* No special handling implemented for escaped (\") quotes.

*/

static private StringList breakCommas(String contents) {

StringList outgoing = new StringList();

boolean insideQuote = false;

// The current word being read

StringBuilder current = new StringBuilder();

char[] chars = contents.toCharArray();

for (int i = 0; i < chars.length; i++) {

char c = chars[i];

if (insideQuote) {

current.append(c);

if (c == '\"') {

insideQuote = false;

}

} else {

if (c == ',') {

if (current.length() != 0) {

outgoing.append(current.toString());

current.setLength(0);

}

} else {

current.append(c);

if (c == '\"') {

insideQuote = true;

}

}

}

}

if (current.length() != 0) {

outgoing.append(current.toString());

}

return outgoing;

}

/**

* Parse a chunk of code and extract the size() command and its contents.

* Also goes after fullScreen(), smooth(), and noSmooth().

* @param code The code from the main tab in the sketch

* @param fussy true if it should show an error message if bad size()

* @return null if there was an error, otherwise an array (might contain some/all nulls)

*/

static public SurfaceInfo parseSketchSize(String code,

boolean fussy) throws SketchException {

// This matches against any uses of the size() function, whether numbers

// or variables or whatever. This way, no warning is shown if size() isn't

// actually used in the applet, which is the case especially for anyone

// who is cutting/pasting from the reference.

// String scrubbed = scrubComments(sketch.getCode(0).getProgram());

// String[] matches = PApplet.match(scrubbed, SIZE_REGEX);

// String[] matches = PApplet.match(scrubComments(code), SIZE_REGEX);

/*

1. no size() or fullScreen() method at all

will use the non-overridden settings() method in PApplet

2. size() or fullScreen() found inside setup() (static mode sketch or otherwise)

make sure that it uses numbers (or displayWidth/Height), copy into settings

3. size() or fullScreen() already in settings()

don't mess with the sketch, don't insert any defaults

really only need to deal with situation #2.. nothing to be done for 1 and 3

*/

// if static mode sketch, all we need is regex

// easy proxy for static in this case is whether [^\s]void\s is present

String uncommented = scrubComments(code);

Mode mode = parseMode(uncommented);

String searchArea = null;

switch (mode) {

case JAVA:

// it's up to the user

searchArea = null;

break;

case ACTIVE:

// active mode, limit scope to setup

// Find setup() in global scope

MatchResult setupMatch = findInCurrentScope(VOID_SETUP_REGEX, uncommented);

if (setupMatch != null) {

int start = uncommented.indexOf("{", setupMatch.end());

if (start >= 0) {

// Find a closing brace

MatchResult match = findInCurrentScope(CLOSING_BRACE, uncommented, start);

if (match != null) {

searchArea = uncommented.substring(start + 1, match.end() - 1);

} else {

throw new SketchException("Found a { that's missing a matching }", false);

}

}

}

break;

case STATIC:

// static mode, look everywhere

searchArea = uncommented;

break;

}

if (searchArea == null) {

return new SurfaceInfo();

}

StringList extraStatements = new StringList();

// First look for noSmooth() or smooth(N) so we can hoist it into settings.

String[] smoothContents = matchMethod("smooth", searchArea);

if (smoothContents != null) {

extraStatements.append(smoothContents[0]);

}

String[] noContents = matchMethod("noSmooth", searchArea);

if (noContents != null) {

if (extraStatements.size() != 0) {

throw new SketchException("smooth() and noSmooth() cannot be used in the same sketch");

} else {

extraStatements.append(noContents[0]);

}

}

String[] pixelDensityContents = matchMethod("pixelDensity", searchArea);

if (pixelDensityContents != null) {

extraStatements.append(pixelDensityContents[0]);

} else {

pixelDensityContents = matchDensityMess(searchArea);

if (pixelDensityContents != null) {

extraStatements.append(pixelDensityContents[0]);

}

}

String[] sizeContents = matchMethod("size", searchArea);

String[] fullContents = matchMethod("fullScreen", searchArea);

// First check and make sure they aren't both being used, otherwise it'll

// throw a confusing state exception error that one "can't be used here".

if (sizeContents != null && fullContents != null) {

throw new SketchException("size() and fullScreen() cannot be used in the same sketch", false);

}

// Get everything inside the parens for the size() method

//String[] contents = PApplet.match(searchArea, SIZE_CONTENTS_REGEX);

if (sizeContents != null) {

StringList args = breakCommas(sizeContents[1]);

SurfaceInfo info = new SurfaceInfo();

// info.statement = sizeContents[0];

info.addStatement(sizeContents[0]);

info.width = args.get(0).trim();

info.height = args.get(1).trim();

info.renderer = (args.size() >= 3) ? args.get(2).trim() : null;

info.path = (args.size() >= 4) ? args.get(3).trim() : null;

// Trying to remember why we wanted to allow people to use displayWidth

// as the height or displayHeight as the width, but maybe it's for

// making a square sketch window? Not going to

if (info.hasOldSyntax()) {

// return null;

throw new SketchException("Please update your code to continue.", false);

}

if (info.hasBadSize() && fussy) {

// found a reference to size, but it didn't seem to contain numbers

final String message =

"The size of this sketch could not be determined from your code.\n" +

"Use only numbers (not variables) for the size() command.\n" +

"Read the size() reference for more details.";

Messages.showWarning("Could not find sketch size", message, null);

// new Exception().printStackTrace(System.out);

// return null;

throw new SketchException("Please fix the size() line to continue.", false);

}

info.addStatements(extraStatements);

info.checkEmpty();

return info;

//return new String[] { contents[0], width, height, renderer, path };

}

// if no size() found, check for fullScreen()

//contents = PApplet.match(searchArea, FULL_SCREEN_CONTENTS_REGEX);

if (fullContents != null) {

SurfaceInfo info = new SurfaceInfo();

// info.statement = fullContents[0];

info.addStatement(fullContents[0]);

StringList args = breakCommas(fullContents[1]);

if (args.size() > 0) { // might have no args

String args0 = args.get(0).trim();

if (args.size() == 1) {

// could be either fullScreen(1) or fullScreen(P2D), figure out which

if (args0.equals("SPAN") || PApplet.parseInt(args0, -1) != -1) {

// it's the display parameter, not the renderer

info.display = args0;

} else {

info.renderer = args0;

}

} else if (args.size() == 2) {

info.renderer = args0;

info.display = args.get(1).trim();

} else {

throw new SketchException("That's too many parameters for fullScreen()");

}

}

info.width = "displayWidth";

info.height = "displayHeight";

// if (extraStatements.size() != 0) {

// info.statement += extraStatements.join(" ");

// }

info.addStatements(extraStatements);

info.checkEmpty();

return info;

}

// Made it this far, but no size() or fullScreen(), and still

// need to pull out the noSmooth() and smooth(N) methods.

if (extraStatements.size() != 0) {

SurfaceInfo info = new SurfaceInfo();

// info.statement = extraStatements.join(" ");

info.addStatements(extraStatements);

return info;

}

// not an error, just no size() specified

//return new String[] { null, null, null, null, null };

return new SurfaceInfo();

}

/*

static String readSingleQuote(char[] c, int i) {

StringBuilder sb = new StringBuilder();

try {

sb.append(c[i++]); // add the quote

if (c[i] == '\\') {

sb.append(c[i++]); // add the escape

if (c[i] == 'u') {

// grabs uNNN and the fourth N will be added below

for (int j = 0; j < 4; j++) {

sb.append(c[i++]);

}

}

}

sb.append(c[i++]); // get the char, escapee, or last unicode digit

sb.append(c[i++]); // get the closing quote

} catch (ArrayIndexOutOfBoundsException ignored) {

// this means they have bigger problems with their code

}

return sb.toString();

}

static String readDoubleQuote(char[] c, int i) {

StringBuilder sb = new StringBuilder();

try {

sb.append(c[i++]); // add the quote

while (i < c.length) {

if (c[i] == '\\') {

sb.append(c[i++]); // add the escape

sb.append(c[i++]); // add whatever was escaped

} else if (c[i] == '\"') {

sb.append(c[i++]);

break;

} else {

sb.append(c[i++]);

}

}

} catch (ArrayIndexOutOfBoundsException ignored) {

// this means they have bigger problems with their code

}

return sb.toString();

}

*/

/**

* Parses the code and determines the mode of the sketch.

*

* @param code code without comments

* @return determined mode

*/

static public Mode parseMode(CharSequence code) {

// See if we can find any function in the global scope

if (findInCurrentScope(FUNCTION_DECL, code) != null) {

return Mode.ACTIVE;

}

// See if we can find any public class extending PApplet

if (findInCurrentScope(PUBLIC_CLASS, code) != null) {

return Mode.JAVA;

}

return Mode.STATIC;

}

/**

* Calls {@link #findInScope(Pattern, String, int, int, int, int) findInScope}

* on the whole string with min and max target scopes set to zero.

*/

static protected MatchResult findInCurrentScope(Pattern pattern, CharSequence code) {

return findInScope(pattern, code, 0, code.length(), 0, 0);

}

/**

* Calls {@link #findInScope(Pattern, String, int, int, int, int) findInScope}

* starting at start char with min and max target scopes set to zero.

*/

static protected MatchResult findInCurrentScope(Pattern pattern, CharSequence code,

int start) {

return findInScope(pattern, code, start, code.length(), 0, 0);

}

/**

* Looks for the pattern at a specified target scope depth relative

* to the scope depth of the starting position.

*

* Example: Calling this with starting position inside a method body

* and target depth 0 would search only in the method body, while

* using target depth -1 would look only in the body of the enclosing class

* (but not in any methods of the class or outside of the class).

*

* By using a scope range, you can e.g. search in the whole class including

* bodies of methods and inner classes.

*

* @param pattern matching is realized by find() method of this pattern

* @param code Java code without comments

* @param start starting position in the code String (inclusive)

* @param stop ending position in the code Sting (exclusive)

* @param minTargetScopeDepth desired min scope depth of the match relative to the

* scope of the starting position

* @param maxTargetScopeDepth desired max scope depth of the match relative to the

* scope of the starting position

* @return first match at a desired relative scope depth,

* null if there isn't one

*/

static protected MatchResult findInScope(Pattern pattern, CharSequence code,

int start, int stop,

int minTargetScopeDepth,

int maxTargetScopeDepth) {

if (minTargetScopeDepth > maxTargetScopeDepth) {

int temp = minTargetScopeDepth;

minTargetScopeDepth = maxTargetScopeDepth;

maxTargetScopeDepth = temp;

}

Matcher m = pattern.matcher(code);

m.region(start, stop);

int depth = 0;

int position = start;

// We should not escape the enclosing scope. It can be either the original

// scope, or the min target scope, whichever is more out there (lower depth)

int minScopeDepth = PApplet.min(depth, minTargetScopeDepth);

while (m.find()) {

int newPosition = m.end();

int depthDiff = scopeDepthDiff(code, position, newPosition);

// Process this match only if it is not in string or char literal

if (depthDiff != Integer.MAX_VALUE) {

depth += depthDiff;

if (depth < minScopeDepth) break; // out of scope!

if (depth >= minTargetScopeDepth &&

depth <= maxTargetScopeDepth) {

return m.toMatchResult(); // jackpot

}

position = newPosition;

}

}

return null;

}

/**

* Walks the specified region (not including stop) and determines difference

* in scope depth. Adds one to depth on opening curly brace, subtracts one

* from depth on closing curly brace. Ignores string and char literals.

*

* @param code code without comments

* @param start start of the region, must not be in string literal,

* char literal or second char of escaped sequence

* @param stop end of the region (exclusive)

*

* @return scope depth difference between start and stop,

* Integer.MAX_VALUE if end is in string literal,

* char literal or second char of escaped sequence

*/

static protected int scopeDepthDiff(CharSequence code, int start, int stop) {

boolean insideString = false;

boolean insideChar = false;

boolean escapedChar = false;

int depth = 0;

for (int i = start; i < stop; i++) {

if (!escapedChar) {

char ch = code.charAt(i);

switch (ch) {

case '\\':

escapedChar = true;

break;

case '{':

if (!insideChar && !insideString) depth++;

break;

case '}':

if (!insideChar && !insideString) depth--;

break;

case '\"':

if (!insideChar) insideString = !insideString;

break;

case '\'':

if (!insideString) insideChar = !insideChar;

break;

}

} else {

escapedChar = false;

}

}

if (insideChar || insideString || escapedChar) {

return Integer.MAX_VALUE; // signal invalid location

}

return depth;

}

/**

* Looks for the specified method in the base scope of the search area.

*/

static protected String[] matchMethod(String methodName, String searchArea) {

final String left = "(?:^|\\s|;)";

// doesn't match empty pairs of parens

//final String right = "\\s*\\(([^\\)]+)\\)\\s*;";

final String right = "\\s*\\(([^\\)]*)\\)\\s*;";

String regexp = left + methodName + right;

Pattern p = matchPatterns.get(regexp);

if (p == null) {

p = Pattern.compile(regexp, Pattern.MULTILINE | Pattern.DOTALL);

matchPatterns.put(regexp, p);

}

MatchResult match = findInCurrentScope(p, searchArea);

if (match != null) {

int count = match.groupCount() + 1;

String[] groups = new String[count];

for (int i = 0; i < count; i++) {

groups[i] = match.group(i);

}

return groups;

}

return null;

}

static protected LinkedHashMap<String, Pattern> matchPatterns =

new LinkedHashMap<String, Pattern>(16, 0.75f, true) {

@Override

protected boolean removeEldestEntry(Map.Entry<String, Pattern> eldest) {

// Limit the number of match patterns at 10 most recently used

return size() == 10;

}

};

static protected String[] matchDensityMess(String searchArea) {

final String regexp =

"(?:^|\\s|;)pixelDensity\\s*\\(\\s*displayDensity\\s*\\([^\\)]*\\)\\s*\\)\\s*\\;";

return PApplet.match(searchArea, regexp);

}

/**

* Replace all commented portions of a given String as spaces.

* Utility function used here and in the preprocessor.

*/

static public String scrubComments(String what) {

char p[] = what.toCharArray();

// Track quotes to avoid problems with code like: String t = "*/*";

// http://code.google.com/p/processing/issues/detail?id=1435

boolean insideQuote = false;

int index = 0;

while (index < p.length) {

// for any double slash comments, ignore until the end of the line

if (!insideQuote &&

(p[index] == '/') &&

(index < p.length - 1) &&

(p[index+1] == '/')) {

p[index++] = ' ';

p[index++] = ' ';

while ((index < p.length) &&

(p[index] != '\n')) {

p[index++] = ' ';

}

// check to see if this is the start of a new multiline comment.

// if it is, then make sure it's actually terminated somewhere.

} else if (!insideQuote &&

(p[index] == '/') &&

(index < p.length - 1) &&

(p[index+1] == '*')) {

p[index++] = ' ';

p[index++] = ' ';

boolean endOfRainbow = false;

while (index < p.length - 1) {

if ((p[index] == '*') && (p[index+1] == '/')) {

p[index++] = ' ';

p[index++] = ' ';

endOfRainbow = true;

break;

} else {

// continue blanking this area

p[index++] = ' ';

}

}

if (!endOfRainbow) {

throw new RuntimeException("Missing the */ from the end of a " +

"/* comment */");

}

} else if (p[index] == '"' && index > 0 && p[index-1] != '\\') {

insideQuote = !insideQuote;

index++;

} else { // any old character, move along

index++;

}

}

return new String(p);

}

public void addMethod(String methodName) {

foundMethods.add(methodName);

}

public boolean hasMethod(String methodName) {

return foundMethods.contains(methodName);

}

// public void setFoundMain(boolean foundMain) {

// this.foundMain = foundMain;

// }

// public boolean getFoundMain() {

// return foundMain;

// }

public void setAdvClassName(final String advClassName) {

this.advClassName = advClassName;

}

public void setMode(final Mode mode) {

//System.err.println("Setting mode to " + mode);

this.mode = mode;

}

CommonHiddenStreamToken getHiddenAfter(final CommonHiddenStreamToken t) {

return filter.getHiddenAfter(t);

}

CommonHiddenStreamToken getInitialHiddenToken() {

return filter.getInitialHiddenToken();

}

private static int countNewlines(final String s) {

int count = 0;

for (int pos = s.indexOf('\n', 0); pos >= 0; pos = s.indexOf('\n', pos + 1))

count++;

return count;

}

private static void checkForUnterminatedMultilineComment(final String program)

throws SketchException {

final int length = program.length();

for (int i = 0; i < length; i++) {

// for any double slash comments, ignore until the end of the line

if ((program.charAt(i) == '/') && (i < length - 1)

&& (program.charAt(i + 1) == '/')) {

i += 2;

while ((i < length) && (program.charAt(i) != '\n')) {

i++;

}

// check to see if this is the start of a new multiline comment.

// if it is, then make sure it's actually terminated somewhere.

} else if ((program.charAt(i) == '/') && (i < length - 1)

&& (program.charAt(i + 1) == '*')) {

final int startOfComment = i;

i += 2;

boolean terminated = false;

while (i < length - 1) {

if ((program.charAt(i) == '*') && (program.charAt(i + 1) == '/')) {

i += 2;

terminated = true;

break;

} else {

i++;

}

}

if (!terminated) {

throw new SketchException("Unclosed /* comment */", 0,

countNewlines(program.substring(0,

startOfComment)));

}

} else if (program.charAt(i) == '"') {

final int stringStart = i;

boolean terminated = false;

for (i++; i < length; i++) {

final char c = program.charAt(i);

if (c == '"') {

terminated = true;

break;

} else if (c == '\\') {

if (i == length - 1) {

break;

}

i++;

} else if (c == '\n') {

break;

}

}

if (!terminated) {

throw new SketchException("Unterminated string constant", 0,

countNewlines(program.substring(0,

stringStart)));

}

} else if (program.charAt(i) == '\'') {

i++; // step over the initial quote

if (i >= length) {

throw new SketchException("Unterminated character constant (after initial quote)", 0,

countNewlines(program.substring(0, i)));

}

boolean escaped = false;

if (program.charAt(i) == '\\') {

i++; // step over the backslash

escaped = true;

}

if (i >= length) {

throw new SketchException("Unterminated character constant (after backslash)", 0,

countNewlines(program.substring(0, i)));

}

if (escaped && program.charAt(i) == 'u') { // unicode escape sequence?

i++; // step over the u

//i += 4; // and the four digit unicode constant

for (int j = 0; j < 4; j++) {

if (UNICODE_ESCAPES.indexOf(program.charAt(i)) == -1) {

throw new SketchException("Bad or unfinished \\uXXXX sequence " +

"(malformed Unicode character constant)", 0,

countNewlines(program.substring(0, i)));

}

i++;

}

} else {

i++; // step over a single character

}

if (i >= length) {

throw new SketchException("Unterminated character constant", 0,

countNewlines(program.substring(0, i)));

}

if (program.charAt(i) != '\'') {

throw new SketchException("Badly formed character constant " +

"(expecting quote, got " + program.charAt(i) + ")", 0,

countNewlines(program.substring(0, i)));

}

}

}

}

public PreprocessorResult write(final Writer out, String program)

throws SketchException, RecognitionException, TokenStreamException {

return write(out, program, null);

}

public PreprocessorResult write(Writer out, String program,

StringList codeFolderPackages)

throws SketchException, RecognitionException, TokenStreamException {

// these ones have the .* at the end, since a class name might be at the end

// instead of .* which would make trouble other classes using this can lop

// off the . and anything after it to produce a package name consistently.

final ArrayList<String> programImports = new ArrayList<String>();

// imports just from the code folder, treated differently

// than the others, since the imports are auto-generated.

final ArrayList<String> codeFolderImports = new ArrayList<String>();

// need to reset whether or not this has a main()

// foundMain = false;

foundMethods = new HashSet<String>();

// http://processing.org/bugs/bugzilla/5.html

if (!program.endsWith("\n")) {

program += "\n";

}

checkForUnterminatedMultilineComment(program);

if (Preferences.getBoolean("preproc.substitute_unicode")) {

program = substituteUnicode(program);

}

// For 0215, adding } as a legitimate prefix to the import (along with

// newline and semicolon) for cases where a tab ends with } and an import

// statement starts the next tab.

final String importRegexp =

"((?:^|;|\\})\\s*)(import\\s+)((?:static\\s+)?\\S+)(\\s*;)";

final Pattern importPattern = Pattern.compile(importRegexp);

String scrubbed = scrubComments(program);

Matcher m = null;

int offset = 0;

boolean found = false;

do {

m = importPattern.matcher(scrubbed);

found = m.find(offset);

if (found) {

// System.out.println("found " + m.groupCount() + " groups");

String before = m.group(1);

String piece = m.group(2) + m.group(3) + m.group(4);

// int len = piece.length(); // how much to trim out

if (!ignoreImport(m.group(3))) {

programImports.add(m.group(3)); // the package name

}

// find index of this import in the program

int start = m.start() + before.length();

int stop = start + piece.length();

// System.out.println(start + " " + stop + " " + piece);

//System.out.println("found " + m.group(3));

// System.out.println("removing '" + program.substring(start, stop) + "'");

// Remove the import from the main program

program = program.substring(0, start) + program.substring(stop);

scrubbed = scrubbed.substring(0, start) + scrubbed.substring(stop);

// Set the offset to start, because everything between

// start and stop has been deleted.

offset = m.start();

}

} while (found);

// System.out.println("program now:");

// System.out.println(program);

if (codeFolderPackages != null) {

for (String item : codeFolderPackages) {

codeFolderImports.add(item + ".*");

}

}

final PrintWriter stream = new PrintWriter(out);

final int headerOffset =

writeImports(stream, programImports, codeFolderImports);

return new PreprocessorResult(mode, headerOffset + 2,

write(program, stream), programImports);

}

static String substituteUnicode(String program) {

// check for non-ascii chars (these will be/must be in unicode format)

char p[] = program.toCharArray();

int unicodeCount = 0;

for (int i = 0; i < p.length; i++) {

if (p[i] > 127)

unicodeCount++;

}

if (unicodeCount == 0)

return program;

// if non-ascii chars are in there, convert to unicode escapes

// add unicodeCount * 5.. replacing each unicode char

// with six digit uXXXX sequence (xxxx is in hex)

// (except for nbsp chars which will be a replaced with a space)

int index = 0;

char p2[] = new char[p.length + unicodeCount * 5];

for (int i = 0; i < p.length; i++) {

if (p[i] < 128) {

p2[index++] = p[i];

} else if (p[i] == 160) { // unicode for non-breaking space

p2[index++] = ' ';