src/tools/singlejar/token_stream.h

// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#ifndef THIRD_PARTY_BAZEL_SRC_TOOLS_SINGLEJAR_TOKEN_STREAM_H_
#define THIRD_PARTY_BAZEL_SRC_TOOLS_SINGLEJAR_TOKEN_STREAM_H_ 1

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <memory>
#include <string>
#include <utility>
#include <vector>

#include "src/tools/singlejar/diag.h"

/*
 * Tokenize command line containing indirect command line arguments.
 * An '@' at the beginning of a command line argument indicates that
 * the rest of the argument is the name of the file which should be
 * read and tokenized as Bash does it: tokens are separated by the
 * whitespace, quotes and double quotes can be used to have whitespace
 * and the other quote inside the token, and backslash followed by
 * newline is treated as empty string.
 */

class ArgTokenStream {
  /* This class is used as follows:
   *
   *  int main(int argc, char* argv[]) {
   *    ArgTokenStream tokens(argc-1, argv+1);
   *    while (!tokens.AtEnd()) {
   *       if (tokens.MatchAndSet("--opt1", ...) ||
   *           tokens.MatchAndSet("--opt2", ...) ||
   *           ...) {
   *         continue;
   *       }
   *       // Process non-option argument or report an error.
   *       // ArgTokenStream::token() returns the current token.
   *    }
   *  }
   */

 private:
  // Internal class to handle indirect command files.
  class FileTokenStream {
   public:
    FileTokenStream(const char *filename) {
      // TODO(laszlocsomor): use the fopen and related file handling API
      // implementations from ProtoBuf, in order to support long paths:
      // https://github.com/google/protobuf/blob/
      //   47b7d2c7cadf74ceec90fc5042232819cd0dd557/
      //   src/google/protobuf/stubs/io_win32.cc
      // Best would be to extract that library to a common location and use
      // here, in ProtoBuf, and in Bazel itself.
      if (!(fp_ = fopen(filename, "r"))) {
        diag_err(1, "%s", filename);
      }
      filename_ = filename;
      next_char();
    }

    ~FileTokenStream() { close(); }

    // Assign next token to TOKEN, return true on success, false on EOF.
    bool next_token(std::string *token) {
      if (!fp_) {
        return false;
      }
      *token = "";
      while (current_char_ != EOF && isspace(current_char_)) {
        next_char();
      }
      if (current_char_ == EOF) {
        close();
        return false;
      }
      for (;;) {
        if (current_char_ == '\'' || current_char_ == '"') {
          process_quoted(token);
          if (isspace(current_char_)) {
            next_char();
            return true;
          } else {
            next_char();
          }
        } else if (current_char_ == '\\') {
          next_char();
          if ((current_char_ != EOF)) {
            token->push_back(current_char_);
            next_char();
          } else {
            diag_errx(1, "Expected character after \\, got EOF in %s",
                      filename_.c_str());
          }
        } else if (current_char_ == EOF || isspace(current_char_)) {
          next_char();
          return true;
        } else {
          token->push_back(current_char_);
          next_char();
        }
      }
    }

   private:
    void close() {
      if (fp_) {
        fclose(fp_);
        fp_ = nullptr;
      }
      filename_.clear();
    }

    // Append the quoted string to the TOKEN. The quote character (which can be
    // single or double quote) is in the current character. Everything up to the
    // matching quote character is appended.
    void process_quoted(std::string *token) {
      char quote = current_char_;
      next_char();
      while (current_char_ != quote) {
        if (current_char_ == '\\' && quote == '"') {
          // In the "-quoted token, \" stands for ", and \x
          // is copied literally for any other x.
          next_char();
          if (current_char_ == '"') {
            token->push_back('"');
            next_char();
          } else if (current_char_ != EOF) {
            token->push_back('\\');
            token->push_back(current_char_);
            next_char();
          } else {
            diag_errx(1, "No closing %c in %s", quote, filename_.c_str());
          }
        } else if (current_char_ != EOF) {
          token->push_back(current_char_);
          next_char();
        } else {
          diag_errx(1, "No closing %c in %s", quote, filename_.c_str());
        }
      }
    }

    // Get the next character from the input stream. Skip backslash followed
    // by the newline.
    void next_char() {
      if (feof(fp_)) {
        current_char_ = EOF;
        return;
      }
      current_char_ = getc(fp_);
      // Eat "\\\n" sequence.
      while (current_char_ == '\\') {
        int c = getc(fp_);
        if (c == '\n') {
          current_char_ = getc(fp_);
        } else {
          if (c != EOF) {
            ungetc(c, fp_);
          }
          break;
        }
      }
    }

    FILE *fp_;
    std::string filename_;
    int current_char_;
  };

 public:
  // Constructor. Automatically reads the first token.
  ArgTokenStream(int argc, const char *const *argv)
      : argv_(argv), argv_end_(argv + argc) {
    next();
  }

  // Process --OPTION
  // If the current token is --OPTION, set given FLAG to true, proceed to next
  // token and return true
  bool MatchAndSet(const char *option, bool *flag) {
    if (token_.compare(option) != 0) {
      return false;
    }
    *flag = true;
    next();
    return true;
  }

  // Process --OPTION OPTARG
  // If the current token is --OPTION, set OPTARG to the next token, proceed to
  // the next token after it and return true.
  bool MatchAndSet(const char *option, std::string *optarg) {
    if (token_.compare(option) != 0) {
      return false;
    }
    next();
    if (AtEnd()) {
      diag_errx(1, "%s requires argument", option);
    }
    *optarg = token_;
    next();
    return true;
  }

  // Process --OPTION OPTARG1 OPTARG2 ...
  // If a current token is --OPTION, push_back all subsequent tokens up to the
  // next option to the OPTARGS array, proceed to the next option and return
  // true.
  bool MatchAndSet(const char *option, std::vector<std::string> *optargs) {
    if (token_.compare(option) != 0) {
      return false;
    }
    next();
    while (!AtEnd() && '-' != token_.at(0)) {
      optargs->push_back(token_);
      next();
    }
    return true;
  }

  // Process --OPTION OPTARG1,OPTSUFF1 OPTARG2,OPTSUFF2 ...
  // If a current token is --OPTION, push_back all subsequent tokens up to the
  // next option to the OPTARGS array, splitting the OPTARG,OPTSUFF by a comma,
  // proceed to the next option and return true.
  bool MatchAndSet(const char *option,
                   std::vector<std::pair<std::string, std::string> > *optargs) {
    if (token_.compare(option) != 0) {
      return false;
    }
    next();
    while (!AtEnd() && '-' != token_.at(0)) {
      size_t commapos = token_.find(',');
      if (commapos == std::string::npos) {
        optargs->push_back(std::pair<std::string, std::string>(token_, ""));
      } else {
        std::string first = token_.substr(0, commapos);
        token_.erase(0, commapos + 1);
        optargs->push_back(std::pair<std::string, std::string>(first, token_));
      }

      next();
    }
    return true;
  }

  // Current token.
  const std::string &token() const { return token_; }

  // Read the next token.
  void next() {
    if (AtEnd()) {
      return;
    }
    if (file_token_stream_.get() && token_from_file()) {
      return;
    }
    while (argv_ < argv_end_) {
      if (**argv_ != '@') {
        token_ = *argv_++;
        return;
      }
      file_token_stream_.reset(new FileTokenStream(*(argv_++) + 1));
      if (token_from_file()) {
        return;
      }
    }
    argv_++;
  }

  // True if there are no more tokens.
  bool AtEnd() const { return argv_ > argv_end_; }

 private:
  bool token_from_file() {
    if (file_token_stream_->next_token(&token_)) {
      return true;
    }
    file_token_stream_.reset(nullptr);
    return false;
  }
  std::unique_ptr<FileTokenStream> file_token_stream_;
  const char *const *argv_;
  const char *const *argv_end_;
  std::string token_;
};

#endif  //  THIRD_PARTY_BAZEL_SRC_TOOLS_SINGLEJAR_TOKEN_STREAM_H_