Skip to content

Commit 3931619

Browse files
committed
recursive repository scan finished
1 parent 5b53c2e commit 3931619

File tree

10 files changed

+145
-105
lines changed

10 files changed

+145
-105
lines changed

.github/workflows/build-on-windows.yml

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,12 @@ jobs:
2727
runs-on: windows-2022
2828

2929
env:
30-
DEBUG_BUILD_DIR: ${{github.workspace}}\out\build\x64-windows\Debug
31-
RELEASE_BUILD_DIR: ${{github.workspace}}\out\build\x64-windows\Release
32-
INSTALL_DIR: ${{github.workspace}}\out\install
33-
VCPKG_ROOT: ${{github.workspace}}\vcpkg
34-
VCPKG_DEFAULT_BINARY_CACHE: ${{github.workspace}}\vcpkg\binary-sources
35-
VCPKG_BINARY_SOURCES: clear;file,${{github.workspace}}\vcpkg\binary-sources,readwrite
30+
DEBUG_BUILD_DIR: ${{github.workspace}}/out/build/x64-windows/Debug
31+
RELEASE_BUILD_DIR: ${{github.workspace}}/out/build/x64-windows/Release
32+
INSTALL_DIR: ${{github.workspace}}/out/install
33+
VCPKG_ROOT: ${{github.workspace}}/vcpkg
34+
VCPKG_DEFAULT_BINARY_CACHE: ${{github.workspace}}/vcpkg/binary-sources
35+
VCPKG_BINARY_SOURCES: clear;file,${{github.workspace}}/vcpkg/binary-sources,readwrite
3636
VCPKG_FEATURE_FLAGS: binarycaching
3737

3838
defaults:
@@ -63,8 +63,8 @@ jobs:
6363
id: restore-vcpkg-cache
6464
uses: actions/cache/restore@v4
6565
with:
66-
path: vcpkg\binary-sources\**\*.zip
67-
key: vcpkg-cache-${{hashFiles('vcpkg.json', 'vcpkg\commit.txt')}}
66+
path: vcpkg/binary-sources/**/*.zip
67+
key: vcpkg-cache-${{hashFiles('vcpkg.json', 'vcpkg/commit.txt')}}
6868

6969
- name: CMake - Configure
7070
run: cmake --preset=x64-windows .
@@ -74,7 +74,7 @@ jobs:
7474
id: save-vcpkg-cache
7575
if: always() && steps.restore-vcpkg-cache.outputs.cache-hit != 'true'
7676
with:
77-
path: vcpkg\binary-sources\**\*.zip
77+
path: vcpkg/binary-sources/**/*.zip
7878
key: ${{steps.restore-vcpkg-cache.outputs.cache-primary-key}}
7979

8080
- name: CMake - Build (Debug)
@@ -96,15 +96,15 @@ jobs:
9696
if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
9797
with:
9898
files: |
99-
${{env.DEBUG_BUILD_DIR}}\hktests-dbg.xml
100-
${{env.RELEASE_BUILD_DIR}}\hktests-rel.xml
99+
${{env.DEBUG_BUILD_DIR}}/hktests-dbg.xml
100+
${{env.RELEASE_BUILD_DIR}}/hktests-rel.xml
101101
102102
- name: Publish Test Results to Codecov
103103
if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
104104
uses: codecov/test-results-action@v1
105105
with:
106106
token: ${{secrets.CODECOV_TOKEN}}
107-
files: ${{env.DEBUG_BUILD_DIR}}\hktests-dbg.xml
107+
files: ${{env.DEBUG_BUILD_DIR}}/hktests-dbg.xml
108108

109109
- name: Generate CodeCoverage Report
110110
if: github.event_name == 'push' && github.ref == 'refs/heads/main'

CMakeLists.txt

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
cmake_minimum_required(VERSION 3.25)
22
project(hikolang LANGUAGES C CXX)
3+
configure_file("src/test_utilities/paths.cpp.in" "src/test_utilities/paths.cpp" @ONLY)
4+
35
include(FetchContent)
46

57
# disable adding CTest build targets like "NightlyStart" (before ctest include)
@@ -172,10 +174,10 @@ if(BUILD_TESTING)
172174
"${CMAKE_CURRENT_SOURCE_DIR}/src/repository/repository_tests.cpp"
173175
"${CMAKE_CURRENT_SOURCE_DIR}/src/tokenizer/tokenizer_tests.cpp"
174176
"${CMAKE_CURRENT_SOURCE_DIR}/src/utility/git_tests.cpp"
175-
"${CMAKE_CURRENT_SOURCE_DIR}/src/utility/unit_test.cpp"
176-
"${CMAKE_CURRENT_SOURCE_DIR}/src/utility/unit_test.hpp"
177177
"${CMAKE_CURRENT_SOURCE_DIR}/src/utility/strings_tests.cpp"
178178
"${CMAKE_CURRENT_SOURCE_DIR}/src/utility/unicode_tests.cpp"
179+
"${CMAKE_CURRENT_BINARY_DIR}/src/test_utilities/paths.cpp"
180+
"${CMAKE_CURRENT_SOURCE_DIR}/src/test_utilities/paths.hpp"
179181
)
180182
#target_compile_definitions(hktests PRIVATE ${LLVM_DEFINITIONS})
181183
#target_link_libraries(hktests PRIVATE ${llvm_libs})
@@ -199,12 +201,5 @@ if(BUILD_TESTING)
199201
target_link_options(hktests PRIVATE -fsanitize=address)
200202
endif()
201203

202-
add_custom_command(
203-
TARGET hktests POST_BUILD
204-
COMMAND ${CMAKE_COMMAND} -E copy_directory
205-
${CMAKE_SOURCE_DIR}/test_data
206-
${CMAKE_BINARY_DIR}/test_data
207-
)
208-
209204
add_test(NAME hktests COMMAND hktests)
210205
endif()

src/repository/repository.cpp

Lines changed: 48 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,15 @@
1212

1313
namespace hk {
1414

15-
repository::repository(std::filesystem::path path) : _path(std::move(path))
15+
repository::repository(std::filesystem::path path, repository_url remote) : remote(remote), path(std::move(path))
1616
{
17-
assert(std::filesystem::canonical(_path) == _path);
1817
}
1918

2019
void repository::scan_prologues(repository_flags flags)
2120
{
2221
untouch(false);
2322

24-
auto first = std::filesystem::recursive_directory_iterator{_path};
23+
auto first = std::filesystem::recursive_directory_iterator{path};
2524
auto last = std::filesystem::recursive_directory_iterator{};
2625

2726
auto visited = vector_set<std::filesystem::path>{};
@@ -71,54 +70,57 @@ void repository::scan_prologues(repository_flags flags)
7170
untouch(true);
7271
}
7372

74-
error_code repository::recursive_scan_prologues(repository_flags flags)
73+
void repository::recursive_scan_prologues(repository_flags flags)
7574
{
76-
auto todo = std::map<repository_url, error_location>{};
77-
auto done = std::map<repository_url, error_location>{};
75+
auto todo = std::vector<std::pair<repository_url, error_location>>{};
7876

7977
scan_prologues(flags);
8078
for (auto item : remote_repositories()) {
81-
todo.insert(std::move(item));
79+
todo.push_back(std::move(item));
8280
}
8381

84-
auto hkdeps_path = _path / "_hkdeps";
82+
auto hkdeps_path = path / "_hkdeps";
8583

8684
auto ec = std::error_code{};
8785
if (not std::filesystem::create_directory(hkdeps_path, ec) and ec) {
8886
std::println(stderr, "Error: could not create directory '{}': {}.", hkdeps_path.string(), ec.message());
8987
std::terminate();
9088
}
9189

90+
// The mark will be used to see if a child repository was already processed.
91+
for (auto &repo : child_repositories()) {
92+
repo->mark = false;
93+
}
94+
9295
while (not todo.empty()) {
93-
auto child_repo_url_node = todo.extract(todo.begin());
94-
auto [it, inserted, _] = done.insert(std::move(child_repo_url_node));
95-
if (not inserted) {
96+
auto [child_remote, import_errors] = std::move(todo.back());
97+
todo.pop_back();
98+
99+
auto child_local_path = hkdeps_path / child_remote.directory();
100+
auto &child_repo = get_child_repository(child_remote, child_local_path);
101+
if (child_repo.mark) {
96102
continue;
97103
}
98104

99-
assert(it != done.end());
100-
auto child_repo_path = hkdeps_path / it->first.directory();
101-
auto &child_repo = get_child_repository(it->first);
102-
if (not child_repo.repository) {
103-
if (auto r = git_checkout_or_clone(it->first, child_repo_path, flags); r != git_error::ok) {
104-
auto short_hkdeps = std::format("_hkdeps/{}", it->first.directory());
105-
return it->second.add(error::could_not_clone_repository, it->first.url(), it->first.rev(), short_hkdeps, r).error();
106-
}
105+
child_repo.mark = true;
106+
if (auto r = git_checkout_or_clone(child_remote, child_local_path, flags); r != git_error::ok) {
107+
auto short_hkdeps = std::format("_hkdeps/{}", child_remote.directory());
108+
// TODO #1 All failing import statements of a single repository should be marked with an error.
109+
import_errors.add(error::could_not_clone_repository, child_remote.url(), child_remote.rev(), short_hkdeps, r);
110+
erase_child_repository(child_remote);
111+
continue;
112+
}
107113

108-
child_repo.repository = std::make_unique<repository>(child_repo_path);
109-
child_repo.repository->scan_prologues(flags);
110-
for (auto item : child_repo.repository->remote_repositories()) {
111-
todo.insert(std::move(item));
112-
}
114+
child_repo.scan_prologues(flags);
115+
for (auto item : child_repo.remote_repositories()) {
116+
todo.push_back(std::move(item));
113117
}
114118
}
115119

116120
// Remove internal repositories not in 'done'.
117121
std::erase_if(_child_repositories, [&](auto const& item) {
118-
return not done.contains(item.url);
122+
return not item->mark;
119123
});
120-
121-
return hk::error_code{};
122124
}
123125

124126
void repository::untouch(bool remove)
@@ -145,7 +147,7 @@ void repository::untouch(bool remove)
145147

146148
[[nodiscard]] repository::module_type& repository::get_module(std::filesystem::path const& module_path)
147149
{
148-
assert(is_subpath(module_path, _path));
150+
assert(is_subpath(module_path, path));
149151

150152
auto it = std::lower_bound(_modules.begin(), _modules.end(), module_path, [](auto const& item, auto const& key) {
151153
return item.path < key;
@@ -156,16 +158,28 @@ void repository::untouch(bool remove)
156158
return *it;
157159
}
158160

159-
[[nodiscard]] repository::child_repository_type& repository::get_child_repository(repository_url const& url)
161+
[[nodiscard]] repository& repository::get_child_repository(repository_url const& remote, std::filesystem::path child_path)
160162
{
161163
auto it =
162-
std::lower_bound(_child_repositories.begin(), _child_repositories.end(), url, [](auto const& item, auto const& key) {
163-
return item.url < key;
164+
std::lower_bound(_child_repositories.begin(), _child_repositories.end(), remote, [](auto const& item, auto const& key) {
165+
return item->remote < key;
164166
});
165-
if (it == _child_repositories.end() or it->url != url) {
166-
it = _child_repositories.insert(it, child_repository_type{url});
167+
if (it == _child_repositories.end() or (*it)->remote != remote) {
168+
it = _child_repositories.insert(it, std::make_unique<repository>(child_path, remote));
169+
}
170+
return **it;
171+
}
172+
173+
void repository::erase_child_repository(repository_url const& remote)
174+
{
175+
auto it =
176+
std::lower_bound(_child_repositories.begin(), _child_repositories.end(), remote, [](auto const& item, auto const& key) {
177+
return item->remote < key;
178+
});
179+
180+
if (it != _child_repositories.end() and (*it)->remote == remote) {
181+
_child_repositories.erase(it);
167182
}
168-
return *it;
169183
}
170184

171185
repository::module_type::module_type(std::filesystem::path path) : path(std::move(path))

src/repository/repository.hpp

Lines changed: 41 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,24 @@ namespace hk {
1515

1616
class repository {
1717
public:
18-
repository(std::filesystem::path path);
18+
/** The remote url to the repository.
19+
*/
20+
repository_url remote = {};
21+
22+
/** The path to the repository.
23+
*/
24+
std::filesystem::path path = {};
25+
26+
/** A flag for algorithms to check if a repository was visited.
27+
*/
28+
bool mark = false;
29+
30+
/** Construct a repository.
31+
*
32+
* @param path The disk location of the repository.
33+
* @param remote The remote location of the repository.
34+
*/
35+
repository(std::filesystem::path path, repository_url remote = repository_url{});
1936

2037
/** Scan the prologue of each *.hkm in the repository.
2138
*
@@ -27,10 +44,15 @@ class repository {
2744
*
2845
* @param force Force scanning even on files that were already parsed.
2946
*/
30-
error_code recursive_scan_prologues(repository_flags flags);
47+
void recursive_scan_prologues(repository_flags flags);
3148

3249
[[nodiscard]] generator<std::pair<repository_url, error_location>> remote_repositories() const;
3350

51+
[[nodiscard]] std::vector<std::unique_ptr<repository>> const& child_repositories() const noexcept
52+
{
53+
return _child_repositories;
54+
}
55+
3456
private:
3557
struct module_type {
3658
std::filesystem::path path;
@@ -49,22 +71,17 @@ class repository {
4971
module_type(std::filesystem::path path);
5072
};
5173

52-
struct child_repository_type {
53-
repository_url url;
54-
std::unique_ptr<repository> repository;
55-
};
56-
57-
/** The path to the repository.
58-
*/
59-
std::filesystem::path _path;
60-
61-
/** modules, sorted by path.
74+
/** modules.
75+
*
76+
* @note sorted by path.
6277
*/
6378
std::vector<module_type> _modules;
6479

6580
/** The root repository also has a list of child repositories.
81+
*
82+
* @note sorted by url.
6683
*/
67-
std::vector<child_repository_type> _child_repositories;
84+
std::vector<std::unique_ptr<repository>> _child_repositories;
6885

6986
/** Unset the touch flag on all modules.
7087
*
@@ -80,13 +97,19 @@ class repository {
8097
*/
8198
[[nodiscard]] module_type &get_module(std::filesystem::path const& path);
8299

83-
/** Get or make a child repository based on the repository_url.
100+
/** Get or make a child repository based on the remote.
84101
*
85-
* @note It is UNDEFINED BEHAVIOR if @a path is not canonical or is not
86-
* inside the repository.
87-
* @param path The path the module.
102+
* @param remote URL of the remote repository.
103+
* @param child_path The local path to the repository.
104+
* @return A reference to the repository object.
105+
*/
106+
[[nodiscard]] repository& get_child_repository(repository_url const& remote, std::filesystem::path child_path);
107+
108+
/** Remove the child repository based on the remote.
109+
*
110+
* @param remote URL of the remote repository.
88111
*/
89-
[[nodiscard]] child_repository_type &get_child_repository(repository_url const& url);
112+
void erase_child_repository(repository_url const& remote);
90113

91114
};
92115

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#include "repository.hpp"
2-
#include "utility/unit_test.hpp"
2+
#include "test_utilities/paths.hpp"
33
#include <hikotest/hikotest.hpp>
44
#include <ranges>
55
#include <filesystem>
@@ -9,7 +9,7 @@ TEST_SUITE(repository_suite)
99

1010
TEST_CASE(single_repository_scan)
1111
{
12-
auto test_data_path = hk::test_data_path();
12+
auto test_data_path = test::test_data_path();
1313
auto repository_path = std::filesystem::canonical(test_data_path / "single_repository_scan");
1414
auto repository = hk::repository{repository_path};
1515
repository.scan_prologues(hk::repository_flags{});
@@ -22,14 +22,16 @@ TEST_CASE(single_repository_scan)
2222

2323
TEST_CASE(recursive_repository_scan)
2424
{
25-
auto test_data_path = hk::test_data_path();
26-
auto repository_path = std::filesystem::canonical(test_data_path / "recursive_repository_scan");
25+
auto source_path = std::filesystem::canonical(test::test_data_path() / "recursive_repository_scan");
26+
auto repository_path = hk::scoped_temporary_directory("recursive_repository_scan");
27+
std::filesystem::copy(source_path, repository_path);
28+
2729
auto repository = hk::repository{repository_path};
30+
repository.recursive_scan_prologues(hk::repository_flags{});
2831

29-
std::filesystem::remove_all(repository_path / "_hkdeps");
30-
auto error = repository.recursive_scan_prologues(hk::repository_flags{});
31-
REQUIRE(error.kind == '\0');
32-
REQUIRE(error.code == 0);
32+
REQUIRE(repository.child_repositories().size() == 2);
33+
REQUIRE(repository.child_repositories()[0]->remote.url() == "https://github.com/hikogui/hikolang-test-a.git");
34+
REQUIRE(repository.child_repositories()[1]->remote.url() == "https://github.com/hikogui/hikolang-test-b.git");
3335
}
3436

3537
};

src/test_utilities/paths.cpp.in

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
2+
#include "test_utilities/paths.hpp"
3+
4+
namespace test {
5+
6+
[[nodiscard]] std::filesystem::path source_path()
7+
{
8+
auto source_path_ = std::filesystem::path("@CMAKE_CURRENT_SOURCE_DIR@");
9+
return std::filesystem::canonical(source_path_);
10+
}
11+
12+
[[nodiscard]] std::filesystem::path test_data_path()
13+
{
14+
return source_path() / "test_data";
15+
}
16+
17+
}
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
#pragma once
55

6-
namespace hk {
6+
namespace test {
77

88
[[nodiscard]] std::filesystem::path test_data_path();
99

0 commit comments

Comments
 (0)