base/src/com/google/idea/blaze/base/lang/projectview/lexer/ProjectViewLexerBase.java

/*
 * Copyright 2016 The Bazel Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.idea.blaze.base.lang.projectview.lexer;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
import com.google.idea.blaze.base.lang.projectview.language.ProjectViewKeywords;
import java.util.List;

/** Lexer for project view files. */
public class ProjectViewLexerBase {

  @VisibleForTesting
  static class Token {
    final ProjectViewTokenType type;
    final int left;
    final int right;

    private Token(ProjectViewTokenType type, int left, int right) {
      this.type = type;
      this.left = left;
      this.right = right;
    }
  }

  private final List<Token> tokens;

  // Input buffer and position
  private final char[] buffer;
  private int pos;

  private int identifierStart = -1;
  private boolean lineHasPrecedingNonWhitespaceChar = false;

  public ProjectViewLexerBase(CharSequence input) {
    this.buffer = input.toString().toCharArray();
    this.tokens = Lists.newArrayList();
    this.pos = 0;
    tokenize();
  }

  public List<Token> getTokens() {
    return tokens;
  }

  /** Performs tokenization of the character buffer of file contents provided to the constructor. */
  private void tokenize() {
    while (pos < buffer.length) {
      char c = buffer[pos];
      pos++;
      switch (c) {
        case '\n':
          addPrecedingIdentifier(pos - 1);
          tokens.add(new Token(ProjectViewTokenType.NEWLINE, pos - 1, pos));
          lineHasPrecedingNonWhitespaceChar = false;
          break;
        case ' ':
        case '\t':
        case '\r':
          addPrecedingIdentifier(pos - 1);
          handleWhitespace();
          break;
        case ':':
          addPrecedingIdentifier(pos - 1);
          tokens.add(new Token(ProjectViewTokenType.COLON, pos - 1, pos));
          break;
        case '#':
          if (!lineHasPrecedingNonWhitespaceChar) {
            addPrecedingIdentifier(pos - 1);
            addCommentLine(pos - 1);
            break;
          }
          // otherwise '#' treated as part of the identifier; intentional fall-through
        default:
          lineHasPrecedingNonWhitespaceChar = true;
          // all other characters combined into an 'identifier' lexical token
          if (identifierStart == -1) {
            identifierStart = pos - 1;
          }
      }
    }
    addPrecedingIdentifier(pos);
  }

  private void addPrecedingIdentifier(int end) {
    if (identifierStart != -1) {
      tokens.add(new Token(getIdentifierToken(identifierStart, end), identifierStart, end));
      identifierStart = -1;
    }
  }

  private void addCommentLine(int start) {
    while (pos < buffer.length) {
      char c = buffer[pos];
      if (c == '\n') {
        break;
      }
      pos++;
    }
    tokens.add(new Token(ProjectViewTokenType.COMMENT, start, pos));
  }

  /**
   * If the whitespace is followed by an end-of-line comment or a newline, it's combined with those
   * tokens.
   */
  private void handleWhitespace() {
    int oldPos = pos - 1;
    while (pos < buffer.length) {
      char c = buffer[pos];
      switch (c) {
        case ' ':
        case '\t':
        case '\r':
          pos++;
          break;
        default:
          if (lineHasPrecedingNonWhitespaceChar || c == '#' || c == '\n') {
            tokens.add(new Token(ProjectViewTokenType.WHITESPACE, oldPos, pos));
          } else {
            tokens.add(new Token(ProjectViewTokenType.INDENT, oldPos, pos));
          }
          return;
      }
    }
    tokens.add(new Token(ProjectViewTokenType.WHITESPACE, oldPos, pos));
  }

  private ProjectViewTokenType getIdentifierToken(int start, int end) {
    String string = bufferSlice(start, end);
    if (ProjectViewKeywords.LIST_KEYWORD_MAP.keySet().contains(string)) {
      return ProjectViewTokenType.LIST_KEYWORD;
    }
    if (ProjectViewKeywords.SCALAR_KEYWORD_MAP.keySet().contains(string)) {
      return ProjectViewTokenType.SCALAR_KEYWORD;
    }
    return ProjectViewTokenType.IDENTIFIER;
  }

  private String bufferSlice(int start, int end) {
    return new String(this.buffer, start, end - start);
  }
}