001    package org.crsh.cmdline.matcher.tokenizer;
002    
003    import java.util.ArrayList;
004    import java.util.Iterator;
005    import java.util.NoSuchElementException;
006    
007    /**
008     * @author <a href="mailto:julien.viet@exoplatform.com">Julien Viet</a>
009     */
010    public class Tokenizer implements Iterator<Token> {
011    
012      /** . */
013      private final CharSequence s;
014    
015      /** . */
016      private int index;
017    
018      /** . */
019      private ArrayList<Token> stack;
020    
021      /** . */
022      private int ptr;
023    
024      /** . */
025      private Termination termination;
026    
027      public Tokenizer(CharSequence s) {
028        this.s = s;
029        this.stack = new ArrayList<Token>();
030        this.index = 0;
031        this.termination = null;
032      }
033    
034      public boolean hasNext() {
035        if (ptr < stack.size()) {
036          return true;
037        } else {
038          Token next = parse();
039          if (next != null) {
040            stack.add(next);
041          }
042          return next != null;
043        }
044      }
045    
046      private Token parse() {
047        Token token = null;
048        if (index < s.length()) {
049          char c = s.charAt(index);
050          int from = index;
051          while (true) {
052            if (Character.isWhitespace(c)) {
053              index++;
054              if (index < s.length()) {
055                c = s.charAt(index);
056              } else {
057                break;
058              }
059            } else {
060              break;
061            }
062          }
063          if (index > from) {
064            token = new Token.Whitespace(from, s.subSequence(from, index).toString());
065          } else {
066            State state = new State();
067            while (true) {
068              if (Character.isWhitespace(c) && state.escape == Escape.NONE) {
069                break;
070              } else {
071                index++;
072                state.push(c);
073                if (index < s.length()) {
074                  c = s.charAt(index);
075                } else {
076                  break;
077                }
078              }
079            }
080            if (index > from) {
081              switch (state.status) {
082                case INIT: {
083                  token = new Token.Literal.Word(from, s.subSequence(from, index).toString(), state.buffer.toString());
084                  break;
085                }
086                case WORD: {
087                  token = new Token.Literal.Word(from, s.subSequence(from, index).toString(), state.buffer.toString());
088                  break;
089                }
090                case SHORT_OPTION: {
091                  token = new Token.Literal.Option.Short(from, s.subSequence(from, index).toString(), state.buffer.toString());
092                  break;
093                }
094                case LONG_OPTION: {
095                  token = new Token.Literal.Option.Long(from, s.subSequence(from, index).toString(), state.buffer.toString());
096                  break;
097                }
098                default:
099                  throw new AssertionError(state.status);
100              }
101              termination = state.escape.termination;
102              return token;
103            }
104          }
105        }
106        return token;
107      }
108    
109      public Token next() {
110        if (hasNext()) {
111          return stack.get(ptr++);
112        } else {
113          throw new NoSuchElementException();
114        }
115      }
116    
117      public void remove() {
118        throw new UnsupportedOperationException();
119      }
120    
121      public int getIndex() {
122        Token peek = peek();
123        if (peek != null) {
124          return peek.getFrom();
125        } else {
126          return index;
127        }
128      }
129    
130      public void pushBack() {
131        pushBack(1);
132      }
133    
134      public void pushBack(int count) {
135        if (count < 0) {
136          throw new IllegalArgumentException();
137        }
138        if (ptr - count < 0) {
139          throw new IllegalStateException("Trying to push back too many tokens");
140        } else {
141          ptr -= count;
142        }
143      }
144    
145      public Token peek() {
146        if (hasNext()) {
147          return stack.get(ptr);
148        } else {
149          return null;
150        }
151      }
152    
153      public Termination getTermination() {
154        return termination;
155      }
156    }