convert tokenizer to generator

Author: takev

README.md

@@ -38,13 +38,15 @@ The second integer type is `long`, this integer dynamically scales in size
 and is allocated on the heap, and includes SIO (Short Integer Optimization).
 
 ## Units system
+Real, rational and decimal types can be tagged with a unit, which is used
+for dimensional analysis.
 
 ```
 let speed = 100.0 (km/h)
 let duration = 15.0 min
 let distance = speed * duration // 25.0 km
 
-fn convert(length : real (m), dpi : real (px/in)) -> real (px)
+fn convert(length : real #iso.length, dpi : real #(screen.length/iso.length)) -> real #screen.length
 {
     return length * dpi 
 }

src/hkc/repository.cpp

@@ -58,13 +58,13 @@ void repository::scan_prologues(bool force)
             continue;
         }
 
-        auto cursor = file_cursor(module_path);
-        auto it = 
-
-        m.errors.clear();
-        if (auto r = parse_module(cursor, m.errors, true)) {
-
-        }
+        //auto cursor = file_cursor(module_path);
+        //auto it = 
+//
+        //m.errors.clear();
+        //if (auto r = parse_module(cursor, m.errors, true)) {
+//
+        //}
     }
 
     untouch(true);

src/parser/parse_module.cpp

@@ -4,6 +4,7 @@
 #include "consume.hpp"
 #include "ast/nodes.hpp"
 #include "tokenizer/tokenizer.hpp"
+#include "utility/lazy_vector.hpp"
 
 namespace hk {
 
@@ -45,32 +46,11 @@ namespace hk {
 
 [[nodiscard]] ast::module_node_ptr parse_module(hk::file_cursor& c, bool only_prologue)
 {
-    class delegate_type : public tokenize_delegate {
-    public:
-        delegate_type(std::vector<token> &tokens) : _tokens(std::addressof(tokens)) {}
+    
+    auto token_generator = hk::tokenize(c);
+    auto lazy_tokens = lazy_vector{token_generator.cbegin(), token_generator.cend()};
 
-        void on_token(token t) override
-        {
-            if (t == '\0') {
-                // If the token is an end-of-file token, we append extra tokens to make sure
-                // the parser can always look ahead without checking for end-of-file.
-                for (auto i = 0uz; i != 8; ++i) {
-                    _tokens->push_back(t);
-                }
-            }
-
-            _tokens->push_back(std::move(t));
-        }
-
-    private:
-        std::vector<token>* _tokens;
-    };
-
-    auto tokens = std::vector<token>{};
-    auto delegate = delegate_type(tokens);
-    hk::tokenize(c, delegate);
-
-    auto it = tokens.cbegin();
+    auto it = lazy_tokens.cbegin();
     auto m = parse_module(it, only_prologue);
     m->upstream_paths = c.upstream_paths();
     return m;

src/parser/parsers.hpp

@@ -5,13 +5,16 @@
 #include "tokenizer/token.hpp"
 #include "error/error_list.hpp"
 #include "utility/file_cursor.hpp"
+#include "utility/lazy_vector.hpp"
+#include "utility/generator.hpp"
 #include <vector>
 #include <memory>
 #include <expected>
 
 namespace hk {
 
-using token_iterator = std::vector<token>::const_iterator;
+using token_vector = lazy_vector<generator<token>::const_iterator, std::default_sentinel_t, token>;
+using token_iterator = token_vector::const_iterator;
 
 template<typename T>
 using parse_result = std::expected<T, error_code>;