cexpr/token.rs
1// (C) Copyright 2016 Jethro G. Beekman
2//
3// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
4// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
5// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
6// option. This file may not be copied, modified, or distributed
7// except according to those terms.
8//! Representation of a C token
9//!
10//! This is designed to map onto a libclang CXToken.
11
12#[derive(Debug, Copy, Clone, PartialEq, Eq)]
13#[allow(missing_docs)]
14pub enum Kind {
15 Punctuation,
16 Keyword,
17 Identifier,
18 Literal,
19 Comment,
20}
21
22/// A single token in a C expression.
23#[derive(Debug, Clone, PartialEq, Eq)]
24pub struct Token {
25 /// The type of this token.
26 pub kind: Kind,
27 /// The bytes that make up the token.
28 pub raw: Box<[u8]>,
29}
30
31impl<'a> From<(Kind, &'a [u8])> for Token {
32 fn from((kind, value): (Kind, &'a [u8])) -> Token {
33 Token {
34 kind,
35 raw: value.to_owned().into_boxed_slice(),
36 }
37 }
38}
39
40/// Remove all comment tokens from a vector of tokens
41pub fn remove_comments(v: &mut Vec<Token>) -> &mut Vec<Token> {
42 v.retain(|t| t.kind != Kind::Comment);
43 v
44}