feature: tokenizer (#4)

* tokenizer
* remove unused fields
* tell compiler that this branch is not reachable

Author: justjavac

Cargo.lock

@@ -10,6 +10,8 @@ license = "MIT"
 keywords = ["css", "syntax", "postcss", "parser", "ast"]
 
 [dependencies]
+memchr = "2.4"
+once_cell = "1.8.0"
 
 [dev-dependencies]
 criterion = { version = "0.3", features = ["html_reports"] }

Cargo.toml

@@ -1 +1,4 @@
+pub mod node;
+pub mod ref_ring;
+pub mod tokenizer;
 pub mod unit;

src/lib.rs

@@ -0,0 +1,14 @@
+pub mod node;
+pub mod ref_ring;
+pub mod tokenizer;
+pub mod unit;
+
+use tokenizer::Tokenizer;
+
+fn main() {
+  let value = "abc";
+  let processor = Tokenizer::new(value);
+  while !processor.end_of_file() {
+    println!("{:?}", processor.next_token());
+  }
+}

src/main.rs

@@ -0,0 +1,59 @@
+use std::borrow::Cow;
+
+pub struct Word<'a> {
+  pub source_index: u32,
+  pub value: Cow<'a, str>,
+}
+
+pub struct Space<'a> {
+  pub source_index: u32,
+  pub value: Cow<'a, str>,
+}
+
+pub struct Comment<'a> {
+  pub source_index: u32,
+  pub value: Cow<'a, str>,
+}
+
+pub struct UnicodeRange<'a> {
+  pub source_index: u32,
+  pub value: Cow<'a, str>,
+}
+
+pub struct Div<'a> {
+  pub source_index: u32,
+  pub value: Cow<'a, str>,
+}
+
+pub struct String<'a> {
+  pub source_index: u32,
+  pub value: Cow<'a, str>,
+}
+
+pub struct Function<'a> {
+  pub source_index: u32,
+  pub value: Cow<'a, str>,
+  pub nodes: Vec<Node<'a>>,
+}
+
+pub enum Node<'a> {
+  Word(Word<'a>),
+  Space(Space<'a>),
+  Comment(Comment<'a>),
+  UnicodeRange(UnicodeRange<'a>),
+  Div(Div<'a>),
+  String(String<'a>),
+  Function(Function<'a>),
+}
+
+pub trait ClosableNode {
+  /// Whether the parsed CSS value ended before the node was properly closed
+  fn unclosed(&self) -> bool;
+}
+
+pub trait AdjacentAwareNode<'a> {
+  /// The token at the start of the node
+  fn before(&self) -> Cow<'a, str>;
+  /// The token at the end of the node
+  fn after(&self) -> Cow<'a, str>;
+}

src/node.rs

@@ -0,0 +1,39 @@
+use std::mem;
+const BUFFER_SIZE: usize = u8::MAX as usize + 1;
+
+#[derive(Debug)]
+pub struct RefRing<'a> {
+  buffer: [Option<&'a str>; BUFFER_SIZE],
+  index: u8,
+}
+
+impl<'a> Default for RefRing<'a> {
+  fn default() -> Self {
+    RefRing {
+      // SAFETY: SINCE NONE IS ZERO IN MEMORY REPRESENTATION,
+      // WE COULD JUST ZEROED OUT THE WHOLE BUFFER TO INITIALIZE IT.
+      buffer: unsafe { mem::zeroed() },
+      index: 0,
+    }
+  }
+}
+
+impl<'a> RefRing<'a> {
+  #[inline]
+  pub fn push(&mut self, e: &'a str) {
+    let index = self.index as usize;
+    // The assert let the compiler optimize out the bounds checks.
+    assert!(index < BUFFER_SIZE);
+    self.buffer[index].replace(e);
+    self.index = self.index.wrapping_add(1);
+  }
+
+  pub fn pop(&mut self) -> Option<&'a str> {
+    self.index = self.index.wrapping_sub(1);
+    let index = self.index as usize;
+    // The assert let the compiler optimize out the bounds checks.
+    assert!(index < BUFFER_SIZE);
+
+    self.buffer[index].take()
+  }
+}