/* -*- mode: java; c-basic-offset: 2; indent-tabs-mode: nil -*- */

/*
	PdePreprocessor - wrapper for default ANTLR-generated parser
	Part of the Processing project - http://processing.org

	Copyright (c) 2012-19 The Processing Foundation
	Copyright (c) 2004-12 Ben Fry and Casey Reas
	Copyright (c) 2001-04 Massachusetts Institute of Technology

	ANTLR-generated parser and several supporting classes written
	by Dan Mosedale via funding from the Interaction Institute IVREA.

	This program is free software; you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation; either version 2 of the License, or
	(at your option) any later version.

	This program is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with this program; if not, write to the Free Software Foundation,
	Inc., 59 Temple Place, Suite 330, Boston, MA	02111-1307	USA
*/

package processing.mode.java.preproc;

import java.awt.EventQueue;
import java.io.*;
import java.util.*;
import java.util.regex.MatchResult;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import processing.app.Messages;
import processing.app.Preferences;
import processing.app.SketchException;
import processing.core.PApplet;
import processing.data.StringList;

import antlr.*;
import antlr.collections.AST;


/**
 * Class that orchestrates preprocessing p5 syntax into straight Java.
 * <P/>
 * <B>Current Preprocessor Subsitutions:</B>
 * <UL>
 * <LI>any function not specified as being protected or private will
 * be made 'public'. this means that <TT>void setup()</TT> becomes
 * <TT>public void setup()</TT>. This is important to note when
 * coding with core.jar outside of the PDE.
 * <LI><TT>compiler.substitute_floats</TT> (currently "substitute_f")
 * treat doubles as floats, i.e. 12.3 becomes 12.3f so that people
 * don't have to add f after their numbers all the time since it's
 * confusing for beginners.
 * <LI><TT>compiler.enhanced_casting</TT> byte(), char(), int(), float()
 * works for casting. this is basic in the current implementation, but
 * should be expanded as described above. color() works similarly to int(),
 * however there is also a *function* called color(r, g, b) in p5.
 * <LI><TT>compiler.color_datatype</TT> 'color' is aliased to 'int'
 * as a datatype to represent ARGB packed into a single int, commonly
 * used in p5 for pixels[] and other color operations. this is just a
 * search/replace type thing, and it can be used interchangeably with int.
 * <LI><TT>compiler.web_colors</TT> (currently "inline_web_colors")
 * color c = #cc0080; should unpack to 0xffcc0080 (the ff at the top is
 * so that the color is opaque), which is just an int.
 * </UL>
 * <B>Other preprocessor functionality</B>
 * <UL>
 * <LI>detects what 'mode' the program is in: static (no function
 * brackets at all, just assumes everything is in draw), active
 * (setup plus draw or loop), and java mode (full java support).
 * http://processing.org/reference/environment/
 * </UL>
 * <P/>
 * The PDE Preprocessor is based on the Java Grammar that comes with
 * ANTLR 2.7.2.	Moving it forward to a new version of the grammar
 * shouldn't be too difficult.
 * <P/>
 * Here's some info about the various files in this directory:
 * <P/>
 * <TT>java.g:</TT> this is the ANTLR grammar for Java 1.3/1.4 from the
 * ANTLR distribution.	It is in the public domain.	The only change to
 * this file from the original this file is the uncommenting of the
 * clauses required to support assert().
 * <P/>
 * <TT>java.tree.g:</TT> this describes the Abstract Syntax Tree (AST)
 * generated by java.g.	It is only here as a reference for coders hacking
 * on the preprocessor, it is not built or used at all.	Note that pde.g
 * overrides some of the java.g rules so that in PDE ASTs, there are a
 * few minor differences.	Also in the public domain.
 * <P/>
 * <TT>pde.g:</TT> this is the grammar and lexer for the PDE language
 * itself. It subclasses the java.g grammar and lexer.	There are a couple
 * of overrides to java.g that I hope to convince the ANTLR folks to fold
 * back into their grammar, but most of this file is highly specific to
 * PDE itself.
 * <TT>PdeEmitter.java:</TT> this class traverses the AST generated by
 * the PDE Recognizer, and emits it as Java code, doing any necessary
 * transformations along the way.	It is based on JavaEmitter.java,
 * available from antlr.org, written by Andy Tripp <atripp@comcast.net>,
 * who has given permission for it to be distributed under the GPL.
 * <P/>
 * <TT>ExtendedCommonASTWithHiddenTokens.java:</TT> this adds a necessary
 * initialize() method, as well as a number of methods to allow for XML
 * serialization of the parse tree in a such a way that the hidden tokens
 * are visible.	Much of the code is taken from the original
 * CommonASTWithHiddenTokens class.	I hope to convince the ANTLR folks
 * to fold these changes back into that class so that this file will be
 * unnecessary.
 * <P/>
 * <TT>TokenStreamCopyingHiddenTokenFilter.java:</TT> this class provides
 * TokenStreamHiddenTokenFilters with the concept of tokens which can be
 * copied so that they are seen by both the hidden token stream as well
 * as the parser itself.	This is useful when one wants to use an
 * existing parser (like the Java parser included with ANTLR) that throws
 * away some tokens to create a parse tree which can be used to spit out
 * a copy of the code with only minor modifications.	Partially derived
 * from ANTLR code.	I hope to convince the ANTLR folks to fold this
 * functionality back into ANTLR proper as well.
 * <P/>
 * <TT>whitespace_test.pde:</TT> a torture test to ensure that the
 * preprocessor is correctly preserving whitespace, comments, and other
 * hidden tokens correctly.	See the comments in the code for details about
 * how to run the test.
 * <P/>
 * All other files in this directory are generated at build time by ANTLR
 * itself.	The ANTLR manual goes into a fair amount of detail about the
 * what each type of file is for.
 * <P/>
 */
public class PdePreprocessor {
	protected static final String UNICODE_ESCAPES = "0123456789abcdefABCDEF";

	// used for calling the ASTFactory to get the root node
	private static final int ROOT_ID = 0;

	protected final String indent;
	private final String name;

	public enum Mode {
		STATIC, ACTIVE, JAVA
	}

	private TokenStreamCopyingHiddenTokenFilter filter;

	private String advClassName = "";
	protected Mode mode;
	Set<String> foundMethods;

	SurfaceInfo sizeInfo;
	boolean settingsMethod;


	/**
	 * Regular expression for parsing the size() method. This should match
	 * against any uses of the size() function, whether numbers or variables
	 * or whatever. This way, no warning is shown if size() isn't actually used
	 * in the sketch, which is the case especially for anyone who is cutting
	 * and pasting from the reference.
	 */
//	public static final String SIZE_REGEX =
//		"(?:^|\\s|;)size\\s*\\(\\s*([^\\s,]+)\\s*,\\s*([^\\s,\\)]+)\\s*,?\\s*([^\\)]*)\\s*\\)\\s*\\;";
//	static private final String SIZE_CONTENTS_REGEX =
//		"(?:^|\\s|;)size\\s*\\(([^\\)]+)\\)\\s*\\;";
//	static private final String FULL_SCREEN_CONTENTS_REGEX =
//		"(?:^|\\s|;)fullScreen\\s*\\(([^\\)]+)\\)\\s*\\;";
//	/** Test whether there's a void somewhere (the program has functions). */
//	static private final String VOID_REGEX =
//		"(?:^|\\s|;)void\\s";
	/** Used to grab the start of setup() so we can mine it for size() */
	static private final Pattern VOID_SETUP_REGEX =
		Pattern.compile("(?:^|\\s|;)void\\s+setup\\s*\\(", Pattern.MULTILINE);

	static private final Pattern VOID_SETTINGS_REGEX =
		Pattern.compile("(?:^|\\s|;)void\\s+settings\\s*\\(", Pattern.MULTILINE);

	// Can't only match any 'public class', needs to be a PApplet
	// http://code.google.com/p/processing/issues/detail?id=551
	static private final Pattern PUBLIC_CLASS =
		Pattern.compile("(^|;)\\s*public\\s+class\\s+\\S+\\s+extends\\s+PApplet", Pattern.MULTILINE);


	static private final Pattern FUNCTION_DECL =
		Pattern.compile("(^|;)\\s*((public|private|protected|final|static)\\s+)*" +
										"(void|int|float|double|String|char|byte|boolean)" +
										"(\\s*\\[\\s*\\])?\\s+[a-zA-Z0-9]+\\s*\\(",
										Pattern.MULTILINE);

	static private final Pattern CLOSING_BRACE = Pattern.compile("\\}");


	public PdePreprocessor(final String sketchName) {
		this(sketchName, Preferences.getInteger("editor.tabs.size"));
	}


	public PdePreprocessor(final String sketchName, final int tabSize) {
		this.name = sketchName;
		final char[] indentChars = new char[tabSize];
		Arrays.fill(indentChars, ' ');
		indent = new String(indentChars);
	}


	/** Parse the sketch size and set the internal sizeInfo variable */
	public SurfaceInfo initSketchSize(String code,
																		boolean sizeWarning) throws SketchException {
		sizeInfo = parseSketchSize(code, sizeWarning);
		return sizeInfo;
	}


	/**
	 * Break on commas, except those inside quotes,
	 * e.g.: size(300, 200, PDF, "output,weirdname.pdf");
	 * No special handling implemented for escaped (\") quotes.
	 */
	static private StringList breakCommas(String contents) {
		StringList outgoing = new StringList();

		boolean insideQuote = false;
		// The current word being read
		StringBuilder current = new StringBuilder();
		char[] chars = contents.toCharArray();
		for (int i = 0; i < chars.length; i++) {
			char c = chars[i];
			if (insideQuote) {
				current.append(c);
				if (c == '\"') {
					insideQuote = false;
				}
			} else {
				if (c == ',') {
					if (current.length() != 0) {
						outgoing.append(current.toString());
						current.setLength(0);
					}
				} else {
					current.append(c);
					if (c == '\"') {
						insideQuote = true;
					}
				}
			}
		}
		if (current.length() != 0) {
			outgoing.append(current.toString());
		}
		return outgoing;
	}


	// if there's a settings() method, we do less moving things around
	static public boolean hasSettingsMethod(String code) {
		final String uncommented = scrubComments(code);
		return findInCurrentScope(VOID_SETTINGS_REGEX, uncommented) != null;
	}


	/**
	 * Parse a chunk of code and extract the size() command and its contents.
	 * Also goes after fullScreen(), smooth(), and noSmooth().
	 * @param code The code from the main tab in the sketch
	 * @param fussy true if it should show an error message if bad size()
	 * @return null if there was an error, otherwise an array (might contain some/all nulls)
	 */
	static public SurfaceInfo parseSketchSize(String code,
																						boolean fussy) throws SketchException {
		// This matches against any uses of the size() function, whether numbers
		// or variables or whatever. This way, no warning is shown if size() isn't
		// actually used in the applet, which is the case especially for anyone
		// who is cutting/pasting from the reference.

		// 1. no size() or fullScreen() method at all
		//		will use the non-overridden settings() method in PApplet
		// 2. size() or fullScreen() found inside setup() (static mode sketch or otherwise)
		//		make sure that it uses numbers (or displayWidth/Height), copy into settings
		// 3. size() or fullScreen() already in settings()
		//		don't mess with the sketch, don't insert any defaults
		//
		// really only need to deal with situation #2.. nothing to be done for 1 and 3

		// if static mode sketch, all we need is regex
		// easy proxy for static in this case is whether [^\s]void\s is present

		String uncommented = scrubComments(code);

		Mode mode = parseMode(uncommented);

		String searchArea = null;

		if (mode == Mode.JAVA) {
			// it's up to the user
			searchArea = null;

		} else if (mode == Mode.ACTIVE) {
			// active mode, limit scope to setup

			// Find setup() in global scope
			MatchResult setupMatch = findInCurrentScope(VOID_SETUP_REGEX, uncommented);
			if (setupMatch != null) {
				int start = uncommented.indexOf("{", setupMatch.end());
				if (start >= 0) {
					// Find a closing brace
					MatchResult match = findInCurrentScope(CLOSING_BRACE, uncommented, start);
					if (match != null) {
						searchArea = uncommented.substring(start + 1, match.end() - 1);
					} else {
						throw new SketchException("Found a { that's missing a matching }", false);
					}
				}
			}
		} else if (mode == Mode.STATIC) {
			// static mode, look everywhere
			searchArea = uncommented;
		}

		if (searchArea == null) {
			return new SurfaceInfo();
		}

		StringList extraStatements = new StringList();

		// First look for noSmooth() or smooth(N) so we can hoist it into settings.
		String[] smoothContents = matchMethod("smooth", searchArea);
		if (smoothContents != null) {
			extraStatements.append(smoothContents[0]);
		}
		String[] noContents = matchMethod("noSmooth", searchArea);
		if (noContents != null) {
			if (extraStatements.size() != 0) {
				throw new SketchException("smooth() and noSmooth() cannot be used in the same sketch");
			} else {
				extraStatements.append(noContents[0]);
			}
		}
		String[] pixelDensityContents = matchMethod("pixelDensity", searchArea);
		if (pixelDensityContents != null) {
			extraStatements.append(pixelDensityContents[0]);
		} else {
			pixelDensityContents = matchDensityMess(searchArea);
			if (pixelDensityContents != null) {
				extraStatements.append(pixelDensityContents[0]);
			}
		}

		String[] sizeContents = matchMethod("size", searchArea);
		String[] fullContents = matchMethod("fullScreen", searchArea);
		// First check and make sure they aren't both being used, otherwise it'll
		// throw a confusing state exception error that one "can't be used here".
		if (sizeContents != null && fullContents != null) {
			throw new SketchException("size() and fullScreen() cannot be used in the same sketch", false);
		}

		// Get everything inside the parens for the size() method
		//String[] contents = PApplet.match(searchArea, SIZE_CONTENTS_REGEX);
		if (sizeContents != null) {
			StringList args = breakCommas(sizeContents[1]);
			SurfaceInfo info = new SurfaceInfo();
//			info.statement = sizeContents[0];
			info.addStatement(sizeContents[0]);
			info.width = args.get(0).trim();
			info.height = args.get(1).trim();
			info.renderer = (args.size() >= 3) ? args.get(2).trim() : null;
			info.path = (args.size() >= 4) ? args.get(3).trim() : null;

			// Trying to remember why we wanted to allow people to use displayWidth
			// as the height or displayHeight as the width, but maybe it's for
			// making a square sketch window? Not going to

			if (info.hasOldSyntax()) {
//				return null;
				throw new SketchException("Please update your code to continue.", false);
			}

			if (info.hasBadSize() && fussy) {
				// found a reference to size, but it didn't seem to contain numbers
				final String message =
					"The size of this sketch could not be determined from your code.\n" +
					"Use only numbers (not variables) for the size() command.\n" +
					"Read the size() reference for more details.";
				EventQueue.invokeLater(() -> {
					Messages.showWarning("Could not find sketch size", message, null);
				});
//				new Exception().printStackTrace(System.out);
//				return null;
				throw new SketchException("Please fix the size() line to continue.", false);
			}

			info.addStatements(extraStatements);
			info.checkEmpty();
			return info;
			//return new String[] { contents[0], width, height, renderer, path };
		}
		// if no size() found, check for fullScreen()
		//contents = PApplet.match(searchArea, FULL_SCREEN_CONTENTS_REGEX);
		if (fullContents != null) {
			SurfaceInfo info = new SurfaceInfo();
//			info.statement = fullContents[0];
			info.addStatement(fullContents[0]);
			StringList args = breakCommas(fullContents[1]);
			if (args.size() > 0) {	// might have no args
				String args0 = args.get(0).trim();
				if (args.size() == 1) {
					// could be either fullScreen(1) or fullScreen(P2D), figure out which
					if (args0.equals("SPAN") || PApplet.parseInt(args0, -1) != -1) {
						// it's the display parameter, not the renderer
						info.display = args0;
					} else {
						info.renderer = args0;
					}
				} else if (args.size() == 2) {
					info.renderer = args0;
					info.display = args.get(1).trim();
				} else {
					throw new SketchException("That's too many parameters for fullScreen()");
				}
			}
			info.width = "displayWidth";
			info.height = "displayHeight";
			info.addStatements(extraStatements);
			info.checkEmpty();
			return info;
		}

		// Made it this far, but no size() or fullScreen(), and still
		// need to pull out the noSmooth() and smooth(N) methods.
		if (extraStatements.size() != 0) {
			SurfaceInfo info = new SurfaceInfo();
			info.addStatements(extraStatements);
			return info;
		}

		// not an error, just no size() specified
		return new SurfaceInfo();
	}


	/**
	 * Parses the code and determines the mode of the sketch.
	 * @param code code without comments
	 * @return determined mode
	 */
	static public Mode parseMode(CharSequence code) {
		// See if we can find any function in the global scope
		if (findInCurrentScope(FUNCTION_DECL, code) != null) {
			return Mode.ACTIVE;
		}

		// See if we can find any public class extending PApplet
		if (findInCurrentScope(PUBLIC_CLASS, code) != null) {
			return Mode.JAVA;
		}

		return Mode.STATIC;
	}


	/**
	 * Calls {@link #findInScope(Pattern, String, int, int, int, int) findInScope}
	 * on the whole string with min and max target scopes set to zero.
	 */
	static protected MatchResult findInCurrentScope(Pattern pattern, CharSequence code) {
		return findInScope(pattern, code, 0, code.length(), 0, 0);
	}


	/**
	 * Calls {@link #findInScope(Pattern, String, int, int, int, int) findInScope}
	 * starting at start char with min and max target scopes set to zero.
	 */
	static protected MatchResult findInCurrentScope(Pattern pattern, CharSequence code,
																									int start) {
		return findInScope(pattern, code, start, code.length(), 0, 0);
	}


	/**
	 * Looks for the pattern at a specified target scope depth relative
	 * to the scope depth of the starting position.
	 *
	 * Example: Calling this with starting position inside a method body
	 * and target depth 0 would search only in the method body, while
	 * using target depth -1 would look only in the body of the enclosing class
	 * (but not in any methods of the class or outside of the class).
	 *
	 * By using a scope range, you can e.g. search in the whole class including
	 * bodies of methods and inner classes.
	 *
	 * @param pattern matching is realized by find() method of this pattern
	 * @param code Java code without comments
	 * @param start starting position in the code String (inclusive)
	 * @param stop ending position in the code Sting (exclusive)
	 * @param minTargetScopeDepth desired min scope depth of the match relative to the
	 *														scope of the starting position
	 * @param maxTargetScopeDepth desired max scope depth of the match relative to the
	 *														scope of the starting position
	 * @return first match at a desired relative scope depth,
	 *				 null if there isn't one
	 */
	static protected MatchResult findInScope(Pattern pattern, CharSequence code,
																					 int start, int stop,
																					 int minTargetScopeDepth,
																					 int maxTargetScopeDepth) {
		if (minTargetScopeDepth > maxTargetScopeDepth) {
			int temp = minTargetScopeDepth;
			minTargetScopeDepth = maxTargetScopeDepth;
			maxTargetScopeDepth = temp;
		}

		Matcher m = pattern.matcher(code);
		m.region(start, stop);
		int depth = 0;
		int position = start;

		// We should not escape the enclosing scope. It can be either the original
		// scope, or the min target scope, whichever is more out there (lower depth)
		int minScopeDepth = PApplet.min(depth, minTargetScopeDepth);

		while (m.find()) {
			int newPosition = m.end();
			int depthDiff = scopeDepthDiff(code, position, newPosition);
			// Process this match only if it is not in string or char literal
			if (depthDiff != Integer.MAX_VALUE) {
				depth += depthDiff;
				if (depth < minScopeDepth) break; // out of scope!
				if (depth >= minTargetScopeDepth &&
						depth <= maxTargetScopeDepth) {
					return m.toMatchResult(); // jackpot
				}
				position = newPosition;
			}
		}
		return null;
	}


	/**
	 * Walks the specified region (not including stop) and determines difference
	 * in scope depth. Adds one to depth on opening curly brace, subtracts one
	 * from depth on closing curly brace. Ignores string and char literals.
	 *
	 * @param code code without comments
	 * @param start start of the region, must not be in string literal,
	 *							char literal or second char of escaped sequence
	 * @param stop end of the region (exclusive)
	 *
	 * @return scope depth difference between start and stop,
	 *				 Integer.MAX_VALUE if end is in string literal,
	 *				 char literal or second char of escaped sequence
	 */
	static protected int scopeDepthDiff(CharSequence code, int start, int stop) {
		boolean insideString = false;
		boolean insideChar = false;
		boolean escapedChar = false;
		int depth = 0;
		for (int i = start; i < stop; i++) {
			if (!escapedChar) {
				char ch = code.charAt(i);
				switch (ch) {
					case '\\':
						escapedChar = true;
						break;
					case '{':
						if (!insideChar && !insideString) depth++;
						break;
					case '}':
						if (!insideChar && !insideString) depth--;
						break;
					case '\"':
						if (!insideChar) insideString = !insideString;
						break;
					case '\'':
						if (!insideString) insideChar = !insideChar;
						break;
				}
			} else {
				escapedChar = false;
			}
		}
		if (insideChar || insideString || escapedChar) {
			return Integer.MAX_VALUE; // signal invalid location
		}
		return depth;
	}


	/**
	 * Looks for the specified method in the base scope of the search area.
	 */
	static protected String[] matchMethod(String methodName, String searchArea) {
		final String left = "(?:^|\\s|;)";
		// doesn't match empty pairs of parens
		//final String right = "\\s*\\(([^\\)]+)\\)\\s*;";
		final String right = "\\s*\\(([^\\)]*)\\)\\s*;";
		String regexp = left + methodName + right;
		Pattern p = matchPatterns.get(regexp);
		if (p == null) {
			p = Pattern.compile(regexp, Pattern.MULTILINE | Pattern.DOTALL);
			matchPatterns.put(regexp, p);
		}
		MatchResult match = findInCurrentScope(p, searchArea);
		if (match != null) {
			int count = match.groupCount() + 1;
			String[] groups = new String[count];
			for (int i = 0; i < count; i++) {
				groups[i] = match.group(i);
			}
			return groups;
		}
		return null;
	}


	static protected LinkedHashMap<String, Pattern> matchPatterns =
			new LinkedHashMap<String, Pattern>(16, 0.75f, true) {
		@Override
		protected boolean removeEldestEntry(Map.Entry<String, Pattern> eldest) {
			// Limit the number of match patterns at 10 most recently used
			return size() == 10;
		}
	};


	static protected String[] matchDensityMess(String searchArea) {
		final String regexp =
			"(?:^|\\s|;)pixelDensity\\s*\\(\\s*displayDensity\\s*\\([^\\)]*\\)\\s*\\)\\s*\\;";
		return PApplet.match(searchArea, regexp);
	}


	/**
	 * Replace all commented portions of a given String as spaces.
	 * Utility function used here and in the preprocessor.
	 */
	static public String scrubComments(String what) {
		char p[] = what.toCharArray();
		// Track quotes to avoid problems with code like: String t = "*/*";
		// http://code.google.com/p/processing/issues/detail?id=1435
		boolean insideQuote = false;

		int index = 0;
		while (index < p.length) {
			// for any double slash comments, ignore until the end of the line
			if (!insideQuote &&
					(p[index] == '/') &&
					(index < p.length - 1) &&
					(p[index+1] == '/')) {
				p[index++] = ' ';
				p[index++] = ' ';
				while ((index < p.length) &&
							 (p[index] != '\n')) {
					p[index++] = ' ';
				}

				// check to see if this is the start of a new multiline comment.
				// if it is, then make sure it's actually terminated somewhere.
			} else if (!insideQuote &&
								 (p[index] == '/') &&
								 (index < p.length - 1) &&
								 (p[index+1] == '*')) {
				p[index++] = ' ';
				p[index++] = ' ';
				boolean endOfRainbow = false;
				while (index < p.length - 1) {
					if ((p[index] == '*') && (p[index+1] == '/')) {
						p[index++] = ' ';
						p[index++] = ' ';
						endOfRainbow = true;
						break;

					} else {
						// continue blanking this area
						p[index++] = ' ';
					}
				}
				if (!endOfRainbow) {
					throw new RuntimeException("Missing the */ from the end of a " +
																		 "/* comment */");
				}

				// switch in/out of quoted region
			} else if (p[index] == '"') {
				insideQuote = !insideQuote;
				index++;

				// skip the escaped char
			} else if (insideQuote && p[index] == '\\') {
				index += 2;

			} else {	// any old character, move along
				index++;
			}
		}
		return new String(p);
	}


	public void addMethod(String methodName) {
		foundMethods.add(methodName);
	}


	public boolean hasMethod(String methodName) {
		return foundMethods.contains(methodName);
	}


	public void setAdvClassName(final String advClassName) {
		this.advClassName = advClassName;
	}


	public void setMode(final Mode mode) {
		this.mode = mode;
	}


	CommonHiddenStreamToken getHiddenAfter(final CommonHiddenStreamToken t) {
		return filter.getHiddenAfter(t);
	}


	CommonHiddenStreamToken getInitialHiddenToken() {
		return filter.getInitialHiddenToken();
	}


	private static int countNewlines(final String s) {
		int count = 0;
		for (int pos = s.indexOf('\n', 0); pos >= 0; pos = s.indexOf('\n', pos + 1))
			count++;
		return count;
	}


	private static void checkForUnterminatedMultilineComment(final String program)
			throws SketchException {
		final int length = program.length();
		for (int i = 0; i < length; i++) {
			// for any double slash comments, ignore until the end of the line
			if ((program.charAt(i) == '/') && (i < length - 1)
					&& (program.charAt(i + 1) == '/')) {
				i += 2;
				while ((i < length) && (program.charAt(i) != '\n')) {
					i++;
				}
				// check to see if this is the start of a new multiline comment.
				// if it is, then make sure it's actually terminated somewhere.
			} else if ((program.charAt(i) == '/') && (i < length - 1)
					&& (program.charAt(i + 1) == '*')) {
				final int startOfComment = i;
				i += 2;
				boolean terminated = false;
				while (i < length - 1) {
					if ((program.charAt(i) == '*') && (program.charAt(i + 1) == '/')) {
						i++; // advance to the ending '/'
						terminated = true;
						break;
					} else {
						i++;
					}
				}
				if (!terminated) {
					throw new SketchException("Unclosed /* comment */", 0,
																		countNewlines(program.substring(0,
																			startOfComment)));
				}
			} else if (program.charAt(i) == '"') {
				final int stringStart = i;
				boolean terminated = false;
				for (i++; i < length; i++) {
					final char c = program.charAt(i);
					if (c == '"') {
						terminated = true;
						break;
					} else if (c == '\\') {
						if (i == length - 1) {
							break;
						}
						i++;
					} else if (c == '\n') {
						break;
					}
				}
				if (!terminated) {
					throw new SketchException("Unterminated string constant", 0,
																		countNewlines(program.substring(0,
																			stringStart)));
				}
			} else if (program.charAt(i) == '\'') {
				i++;	// step over the initial quote
				if (i >= length) {
					throw new SketchException("Unterminated character constant (after initial quote)", 0,
																		countNewlines(program.substring(0, i)));
				}
				boolean escaped = false;
				if (program.charAt(i) == '\\') {
					i++;	// step over the backslash
					escaped = true;
				}
				if (i >= length) {
					throw new SketchException("Unterminated character constant (after backslash)", 0,
																		countNewlines(program.substring(0, i)));
				}
				if (escaped && program.charAt(i) == 'u') {	// unicode escape sequence?
					i++;	// step over the u
					//i += 4;	// and the four digit unicode constant
					for (int j = 0; j < 4; j++) {
						if (UNICODE_ESCAPES.indexOf(program.charAt(i)) == -1) {
							throw new SketchException("Bad or unfinished \\uXXXX sequence " +
																				"(malformed Unicode character constant)", 0,
																				countNewlines(program.substring(0, i)));
						}
						i++;
					}
				} else {
					i++;	// step over a single character
				}
				if (i >= length) {
					throw new SketchException("Unterminated character constant", 0,
																		countNewlines(program.substring(0, i)));
				}
				if (program.charAt(i) != '\'') {
					throw new SketchException("Badly formed character constant " +
																		"(expecting quote, got " + program.charAt(i) + ")", 0,
																		countNewlines(program.substring(0, i)));
				}
			}
		}
	}


	public PreprocessorResult write(final Writer out, final String program)
			throws SketchException, RecognitionException, TokenStreamException {
		return write(out, program, null);
	}


	public PreprocessorResult write(Writer out, String program,
																	StringList codeFolderPackages)
			throws SketchException, RecognitionException, TokenStreamException {

		// these ones have the .* at the end, since a class name might be at the end
		// instead of .* which would make trouble other classes using this can lop
		// off the . and anything after it to produce a package name consistently.
		final ArrayList<String> programImports = new ArrayList<>();

		// imports just from the code folder, treated differently
		// than the others, since the imports are auto-generated.
		final ArrayList<String> codeFolderImports = new ArrayList<>();

		// need to reset whether or not this has a main()
//		foundMain = false;
		foundMethods = new HashSet<>();

		// http://processing.org/bugs/bugzilla/5.html
		if (!program.endsWith("\n")) {
			program += "\n";
		}

		checkForUnterminatedMultilineComment(program);

		// For 0215, adding } as a legitimate prefix to the import (along with
		// newline and semicolon) for cases where a tab ends with } and an import
		// statement starts the next tab.
		final String importRegexp =
			"((?:^|;|\\})\\s*)(import\\s+)((?:static\\s+)?\\S+)(\\s*;)";
		final Pattern importPattern = Pattern.compile(importRegexp);
		String scrubbed = scrubComments(program);
		Matcher m = null;
		int offset = 0;
		boolean found = false;
		do {
			m = importPattern.matcher(scrubbed);
			found = m.find(offset);
			if (found) {
				String before = m.group(1);
				String piece = m.group(2) + m.group(3) + m.group(4);

				if (!ignoreImport(m.group(3))) {
					programImports.add(m.group(3)); // the package name
				}

				// find index of this import in the program
				int start = m.start() + before.length();
				int stop = start + piece.length();

				// Remove the import from the main program
				program = program.substring(0, start) + program.substring(stop);
				scrubbed = scrubbed.substring(0, start) + scrubbed.substring(stop);
				// Set the offset to start, because everything between
				// start and stop has been deleted.
				offset = m.start();
			}
		} while (found);

		if (codeFolderPackages != null) {
			for (String item : codeFolderPackages) {
				codeFolderImports.add(item + ".*");
			}
		}

		final PrintWriter stream = new PrintWriter(out);
		final int headerOffset =
			writeImports(stream, programImports, codeFolderImports);
		return new PreprocessorResult(mode, headerOffset + 2,
																	write(program, stream), programImports);
	}


	/**
	 * preprocesses a pde file and writes out a java file
	 * @return the class name of the exported Java
	 */
	private String write(final String program, final PrintWriter stream)
			throws SketchException, RecognitionException, TokenStreamException {

		// Match on the uncommented version, otherwise code inside comments used
		// http://code.google.com/p/processing/issues/detail?id=1404
		String uncomment = scrubComments(program);
		PdeRecognizer parser = createParser(program);
		Mode mode = parseMode(uncomment);

		if (mode == Mode.JAVA) {
			try {
				final PrintStream saved = System.err;
				try {
					// throw away stderr for this tentative parse
					System.setErr(new PrintStream(new ByteArrayOutputStream()));
					parser.javaProgram();
				} finally {
					System.setErr(saved);
				}
				setMode(Mode.JAVA);
			} catch (Exception e) {
				// I can't figure out any other way of resetting the parser.
				parser = createParser(program);
				parser.pdeProgram();
			}
		} else if (mode == Mode.ACTIVE) {
			setMode(Mode.ACTIVE);
			parser.activeProgram();

		} else if (mode == Mode.STATIC) {
			parser.pdeProgram();
		}

		// set up the AST for traversal by PdeEmitter
		ASTFactory factory = new ASTFactory();
		AST parserAST = parser.getAST();
		AST rootNode = factory.create(ROOT_ID, "AST ROOT");
		rootNode.setFirstChild(parserAST);

		makeSimpleMethodsPublic(rootNode);

		// unclear if this actually works, but it's worth a shot
		//((CommonAST)parserAST).setVerboseStringConversion(
		//	true, parser.getTokenNames());
		// (made to use the static version because of jikes 1.22 warning)
		BaseAST.setVerboseStringConversion(true, parser.getTokenNames());

		final String className;
		if (mode == Mode.JAVA) {
			// in this mode, the class name is already defined.
			className = getFirstClassName(parserAST);
		} else {
			className = this.name;
		}

		// if 'null' was passed in for the name, but this isn't
		// a 'java' mode class, then there's a problem, so punt.
		if (className == null)
			return null;

		// debug
		if (false) {
			final StringWriter buf = new StringWriter();
			final PrintWriter bufout = new PrintWriter(buf);
			writeDeclaration(bufout, className);
			new PdeEmitter(this, bufout).print(rootNode);
			writeFooter(bufout, className);
			debugAST(rootNode, true);
			System.err.println(buf.toString());
		}

		writeDeclaration(stream, className);
		new PdeEmitter(this, stream).print(rootNode);
		writeFooter(stream, className);

		// if desired, serialize the parse tree to an XML file.	can
		// be viewed usefully with Mozilla or IE
		if (Preferences.getBoolean("preproc.output_parse_tree")) {
			writeParseTree("parseTree.xml", parserAST);
		}

		return className;
	}


	private PdeRecognizer createParser(final String program) {
		// create a lexer with the stream reader, and tell it to handle
		// hidden tokens (eg whitespace, comments) since we want to pass these
		// through so that the line numbers when the compiler reports errors
		// match those that will be highlighted in the PDE IDE
		//
		PdeLexer lexer = new PdeLexer(new StringReader(program));
		lexer.setTokenObjectClass("antlr.CommonHiddenStreamToken");

		// create the filter for hidden tokens and specify which tokens to
		// hide and which to copy to the hidden text
		//
		filter = new TokenStreamCopyingHiddenTokenFilter(lexer);
		filter.hide(PdePartialTokenTypes.SL_COMMENT);
		filter.hide(PdePartialTokenTypes.ML_COMMENT);
		filter.hide(PdePartialTokenTypes.WS);
		filter.copy(PdePartialTokenTypes.SEMI);
		filter.copy(PdePartialTokenTypes.LPAREN);
		filter.copy(PdePartialTokenTypes.RPAREN);
		filter.copy(PdePartialTokenTypes.LCURLY);
		filter.copy(PdePartialTokenTypes.RCURLY);
		filter.copy(PdePartialTokenTypes.COMMA);
		filter.copy(PdePartialTokenTypes.RBRACK);
		filter.copy(PdePartialTokenTypes.LBRACK);
		filter.copy(PdePartialTokenTypes.COLON);
		filter.copy(PdePartialTokenTypes.TRIPLE_DOT);

		// Because the meanings of < and > are overloaded to support
		// type arguments and type parameters, we have to treat them
		// as copyable to hidden text (or else the following syntax,
		// such as (); and what not gets lost under certain circumstances)
		// -- jdf
		filter.copy(PdePartialTokenTypes.LT);
		filter.copy(PdePartialTokenTypes.GT);
		filter.copy(PdePartialTokenTypes.SR);
		filter.copy(PdePartialTokenTypes.BSR);

		// create a parser and set what sort of AST should be generated
		//
		final PdeRecognizer parser = new PdeRecognizer(this, filter);

		// use our extended AST class
		//
		parser.setASTNodeClass("antlr.ExtendedCommonASTWithHiddenTokens");
		return parser;
	}

	/**
	 * Walk the tree looking for METHOD_DEFs. Any simple METHOD_DEF (one
	 * without TYPE_PARAMETERS) lacking an
	 * access specifier is given public access.
	 * @param node
	 */
	private void makeSimpleMethodsPublic(final AST node) {
		if (node.getType() == PdeTokenTypes.METHOD_DEF) {
			final AST mods = node.getFirstChild();
			final AST oldFirstMod = mods.getFirstChild();
			for (AST mod = oldFirstMod; mod != null; mod = mod.getNextSibling()) {
				final int t = mod.getType();
				if (t == PdeTokenTypes.LITERAL_private ||
						t == PdeTokenTypes.LITERAL_protected ||
						t == PdeTokenTypes.LITERAL_public) {
					return;
				}
			}
			if (mods.getNextSibling().getType() == PdeTokenTypes.TYPE_PARAMETERS) {
				return;
			}
			final CommonHiddenStreamToken publicToken =
				new CommonHiddenStreamToken(PdeTokenTypes.LITERAL_public, "public") {
				{
					setHiddenAfter(new CommonHiddenStreamToken(PdeTokenTypes.WS, " "));
				}
			};
			final AST publicNode = new CommonASTWithHiddenTokens(publicToken);
			publicNode.setNextSibling(oldFirstMod);
			mods.setFirstChild(publicNode);
		} else {
			for (AST kid = node.getFirstChild(); kid != null; kid = kid
					.getNextSibling())
				makeSimpleMethodsPublic(kid);
		}
	}

	protected void writeParseTree(String filename, AST ast) {
		try {
			PrintStream stream = new PrintStream(new FileOutputStream(filename));
			stream.println("<?xml version=\"1.0\"?>");
			stream.println("<document>");
			OutputStreamWriter writer = new OutputStreamWriter(stream);
			if (ast != null) {
				((CommonAST) ast).xmlSerialize(writer);
			}
			writer.flush();
			stream.println("</document>");
			writer.close();
		} catch (IOException e) {

		}
	}

	/**
	 *
	 * @param out
	 * @param programImports
	 * @param codeFolderImports
	 * @return the header offset
	 */
	protected int writeImports(final PrintWriter out,
														 final List<String> programImports,
														 final List<String> codeFolderImports) {
		int count = writeImportList(out, getCoreImports());
		count += writeImportList(out, programImports);
		count += writeImportList(out, codeFolderImports);
		count += writeImportList(out, getDefaultImports());
		return count;
	}

	protected int writeImportList(PrintWriter out, List<String> imports) {
		return writeImportList(out, imports.toArray(new String[0]));
	}

	protected int writeImportList(PrintWriter out, String[] imports) {
		int count = 0;
		if (imports != null && imports.length != 0) {
			for (String item : imports) {
				out.println("import " + item + "; ");
				count++;
			}
			out.println();
			count++;
		}
		return count;
	}

	/**
	 * Write any required header material (eg imports, class decl stuff)
	 *
	 * @param out								 PrintStream to write it to.
	 * @param exporting					 Is this being exported from PDE?
	 * @param className					 Name of the class being created.
	 */
	protected void writeDeclaration(PrintWriter out, String className) {
		if (mode == Mode.JAVA) {
			// Print two blank lines so that the offset doesn't change
			out.println();
			out.println();

		} else if (mode == Mode.ACTIVE) {
			// Print an extra blank line so the offset is identical to the others
			out.println("public class " + className + " extends PApplet {");
			out.println();

		} else if (mode == Mode.STATIC) {
			out.println("public class " + className + " extends PApplet {");
			out.println(indent + "public void setup() {");
		}
	}

	/**
	 * Write any necessary closing text.
	 *
	 * @param out PrintStream to write it to.
	 */
	protected void writeFooter(PrintWriter out, String className) {
		if (mode == Mode.STATIC) {
			// close off setup() definition
			out.println(indent + indent + "noLoop();");
			out.println(indent + "}");
			out.println();
		}

		if ((mode == Mode.STATIC) || (mode == Mode.ACTIVE)) {
			// doesn't remove the original size() method,
			// but calling size() again in setup() is harmless.
			if (!hasMethod("settings") && sizeInfo.hasSettings()) {
				out.println(indent + "public void settings() { " + sizeInfo.getSettings() + " }");
			}

			if (!hasMethod("main")) {
				out.println(indent + "static public void main(String[] passedArgs) {");
				out.print(indent + indent + "String[] appletArgs = new String[] { ");

				if (Preferences.getBoolean("export.application.present")) {
					out.print("\"" + PApplet.ARGS_PRESENT + "\", ");

					String farbe = Preferences.get("run.present.bgcolor");
					out.print("\"" + PApplet.ARGS_WINDOW_COLOR + "=" + farbe + "\", ");

					if (Preferences.getBoolean("export.application.stop")) {
						farbe = Preferences.get("run.present.stop.color");
						out.print("\"" + PApplet.ARGS_STOP_COLOR + "=" + farbe + "\", ");
					} else {
						out.print("\"" + PApplet.ARGS_HIDE_STOP + "\", ");
					}
				}
				out.println("\"" + className + "\" };");

				out.println(indent + indent + "if (passedArgs != null) {");
				out.println(indent + indent + "	PApplet.main(concat(appletArgs, passedArgs));");
				out.println(indent + indent + "} else {");
				out.println(indent + indent + "	PApplet.main(appletArgs);");
				out.println(indent + indent + "}");

				out.println(indent + "}");
			}

			// close off the class definition
			out.println("}");
		}
	}

	public String[] getCoreImports() {
		return new String[] {
			"processing.core.*",
			"processing.data.*",
			"processing.event.*",
			"processing.opengl.*"
		};
	}

	public String[] getDefaultImports() {
		// These may change in-between (if the prefs panel adds this option)
		//String prefsLine = Preferences.get("preproc.imports");
		//return PApplet.splitTokens(prefsLine, ", ");
		return new String[] {
			"java.util.HashMap",
			"java.util.ArrayList",
			"java.io.File",
			"java.io.BufferedReader",
			"java.io.PrintWriter",
			"java.io.InputStream",
			"java.io.OutputStream",
			"java.io.IOException"
		};
	}

	/**
	 * Return true if this import should be removed from the code. This is used
	 * for packages like processing.xml which no longer exist.
	 * @param pkg something like processing.xml.XMLElement or processing.xml.*
	 * @return true if this shouldn't be added to the final code
	 */
	public boolean ignoreImport(String pkg) {
		return false;
//		return pkg.startsWith("processing.xml.");
	}


	// . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .


	/**
	 * Find the first CLASS_DEF node in the tree, and return the name of the
	 * class in question.
	 *
	 * TODO [dmose] right now, we're using a little hack to the grammar to get
	 * this info.	In fact, we should be descending the AST passed in.
	 */
	String getFirstClassName(AST ast) {
		String t = advClassName;
		advClassName = "";
		return t;
	}


	public void debugAST(final AST ast, final boolean includeHidden) {
		System.err.println("------------------");
		debugAST(ast, includeHidden, 0);
	}


	private void debugAST(final AST ast, final boolean includeHidden,
												final int indent) {
		for (int i = 0; i < indent; i++)
			System.err.print("		");
		if (includeHidden) {
			System.err.print(debugHiddenBefore(ast));
		}
		if (ast.getType() > 0 && !ast.getText().equals(TokenUtil.nameOf(ast))) {
			System.err.print(TokenUtil.nameOf(ast) + "/");
		}
		System.err.print(ast.getText().replace("\n", "\\n"));
		if (includeHidden) {
			System.err.print(debugHiddenAfter(ast));
		}
		System.err.println();
		for (AST kid = ast.getFirstChild(); kid != null; kid = kid.getNextSibling())
			debugAST(kid, includeHidden, indent + 1);
	}


	private String debugHiddenAfter(AST ast) {
		return (ast instanceof antlr.CommonASTWithHiddenTokens) ?
			debugHiddenTokens(((antlr.CommonASTWithHiddenTokens) ast).getHiddenAfter()) : "";
	}

	private String debugHiddenBefore(AST ast) {
		if (!(ast instanceof antlr.CommonASTWithHiddenTokens)) {
			return "";
		}
		antlr.CommonHiddenStreamToken parent =
			((antlr.CommonASTWithHiddenTokens) ast).getHiddenBefore();

		if (parent == null) {
			return "";
		}

		antlr.CommonHiddenStreamToken child = null;
		do {
			child = parent;
			parent = child.getHiddenBefore();
		} while (parent != null);

		return debugHiddenTokens(child);
	}


	private String debugHiddenTokens(antlr.CommonHiddenStreamToken t) {
		final StringBuilder sb = new StringBuilder();
		for (; t != null; t = filter.getHiddenAfter(t)) {
			if (sb.length() == 0) {
				sb.append("[");
			}
			sb.append(t.getText().replace("\n", "\\n"));
		}
		if (sb.length() > 0) {
			sb.append("]");
		}
		return sb.toString();
	}
}
