tokenizer.h

Bu dosyanın dokümantasyonuna git.

 
#ifndef TOKENIZER_H
#define TOKENIZER_H
 
 
#include "lexer.h"
 
#include "map.h"
 
#include <stddef.h>
#include <stdint.h>
 
 
static const size_t TK_NOTOKEN = INT16_MAX - 1;
 
static const size_t TK_INT = INT16_MAX - 2;
 
static const size_t TK_FLOAT = INT16_MAX - 3;
 
static const size_t TK_IDENT = INT16_MAX - 4;
 
 
struct tokenizer {
        struct map ident_map;
        size_t last_id;
 
        struct map keywords;
        struct map punctuations;
 
        struct lexeme current_lexeme;
};
 
union seminfo {
        double num_float;
        intmax_t num_int;
        size_t ident_id;
};
 
struct token {
        union seminfo seminfo;
 
        size_t id;
};
 
 
void tokenizer_init(struct tokenizer *tokenizer);
 
void tokenizer_destroy(struct tokenizer *tokenizer);
 
void tokenizer_add_keyword(struct tokenizer *tokenizer,
                           const char *keyword, size_t id);
 
void tokenizer_add_punctuation(struct tokenizer *tokenizer,
                               const char *punctuation, size_t id);
 
size_t tokenizer_ident_id2(struct tokenizer *tokenizer,
                           const char *ident, size_t ident_len);
 
size_t tokenizer_ident_id(struct tokenizer *tokenizer, const char *ident);
 
void tokenizer_feed(struct tokenizer *tokenizer, struct lexeme lexeme);
 
struct token tokenizer_next(struct tokenizer *tokenzier);
 
 
#endif