Merge pull request #161 from liulanzheng/filecache

optimizations for vcpu and bump version v0.6.0

Author: liulanzheng

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.15)
 
 project(
   overlaybd
-  VERSION 0.5.3
+  VERSION 0.6.0
   LANGUAGES C CXX
 )
 enable_language(C)

README.md

@@ -147,7 +147,7 @@ Default configure file `overlaybd.json` is installed to `/etc/overlaybd/`.
 | logLevel            | DEBUG 0, INFO  1, WARN  2, ERROR 3                                                                    |
 | ioEngine            | IO engine used to open local files: psync 0, libaio 1, posix aio 2.                                   |
 | logPath             | The path for log file, `/var/log/overlaybd.log` is the default value.                                 |
-| cacheConfig.cacheType   | Cache type used, `file` and `ocf` are supported.                                                  |
+| cacheConfig.cacheType   | Cache type used, `file`, `ocf` and `download` are supported.                                      |
 | cacheConfig.cacheDir    | The cache directory for remote image data.                                                        |
 | cacheConfig.cacheSizeGB | The max size of cache, in GB.                                                                     |
 | cacheConfig.refillSize  | The refill size from source, in byte. `262144` is default (256 KB).                               |

src/bk_download.cpp

@@ -13,6 +13,7 @@
    See the License for the specific language governing permissions and
    limitations under the License.
 */
+#include "bk_download.h"
 #include <errno.h>
 #include <list>
 #include <set>
@@ -24,13 +25,11 @@
 #include <photon/fs/localfs.h>
 #include <photon/fs/throttled-file.h>
 #include <photon/thread/thread.h>
-#include <photon/common/event-loop.h>
-#include <photon/io/fd-events.h>
-#include "bk_download.h"
-
 #include <openssl/sha.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include "switch_file.h"
+
 using namespace photon::fs;
 
 static constexpr size_t ALIGNMENT = 4096;
@@ -90,57 +89,10 @@ bool check_downloaded(const std::string &dir) {
 
 static std::set<std::string> lock_files;
 
-ssize_t filecopy(IFile *infile, IFile *outfile, size_t bs, int retry_limit, int &running) {
-    if (bs == 0)
-        LOG_ERROR_RETURN(EINVAL, -1, "bs should not be 0");
-    void *buff = nullptr;
-
-    // buffer allocate, with 4K alignment
-    ::posix_memalign(&buff, ALIGNMENT, bs);
-    if (buff == nullptr)
-        LOG_ERROR_RETURN(ENOMEM, -1, "Fail to allocate buffer with ", VALUE(bs));
-    DEFER(free(buff));
-    off_t offset = 0;
-    ssize_t count = bs;
-    while (count == (ssize_t)bs) {
-        if (running != 1) {
-            LOG_INFO("image file exit when background downloading");
-            return -1;
-        }
-
-        int retry = retry_limit;
-    again_read:
-        if (!(retry--))
-            LOG_ERROR_RETURN(EIO, -1, "Fail to read at ", VALUE(offset), VALUE(count));
-        auto rlen = infile->pread(buff, bs, offset);
-        if (rlen < 0) {
-            LOG_DEBUG("Fail to read at ", VALUE(offset), VALUE(count), " retry...");
-            goto again_read;
-        }
-        retry = retry_limit;
-    again_write:
-        if (!(retry--))
-            LOG_ERROR_RETURN(EIO, -1, "Fail to write at ", VALUE(offset), VALUE(count));
-        // cause it might write into file with O_DIRECT
-        // keep write length as bs
-        auto wlen = outfile->pwrite(buff, bs, offset);
-        // but once write lenth larger than read length treats as OK
-        if (wlen < rlen) {
-            LOG_DEBUG("Fail to write at ", VALUE(offset), VALUE(count), " retry...");
-            goto again_write;
-        }
-        count = rlen;
-        offset += count;
-    }
-    // truncate after write, for O_DIRECT
-    outfile->ftruncate(offset);
-    return offset;
-}
-
 void BkDownload::switch_to_local_file() {
     std::string path = dir + "/" + COMMIT_FILE_NAME;
     ((ISwitchFile *)sw_file)->set_switch_file(path.c_str());
-    LOG_DEBUG("set switch tag done. (localpath: `)", path);
+    LOG_DEBUG("set switch done. (localpath: `)", path);
 }
 
 bool BkDownload::download_done() {
@@ -166,6 +118,7 @@ bool BkDownload::download_done() {
     photon::thread_usleep(-1UL);
     if (shares != digest) {
         LOG_ERROR("verify checksum ` failed (expect: `, got: `)", old_name, digest, shares);
+        force_download = true; // force redownload next time
         return false;
     }
 
@@ -224,13 +177,59 @@ bool BkDownload::download_blob(int &running) {
 
     auto dst = open_localfile_adaptor(dl_file_path.c_str(), O_RDWR | O_CREAT, 0644);
     if (dst == nullptr) {
-        LOG_ERRNO_RETURN(0, -1, "failed to open dst file `", dl_file_path.c_str());
+        LOG_ERRNO_RETURN(0, false, "failed to open dst file `", dl_file_path.c_str());
     }
     DEFER(delete dst;);
+    dst->ftruncate(file_size);
 
-    auto res = filecopy(src, dst, 1024UL * 1024, 1, running);
-    if (res < 0)
-        return false;
+    size_t bs = 256 * 1024;
+    off_t offset = 0;
+    void *buff = nullptr;
+    // buffer allocate, with 4K alignment
+    ::posix_memalign(&buff, ALIGNMENT, bs);
+    if (buff == nullptr)
+        LOG_ERRNO_RETURN(0, false, "failed to allocate buffer with ", VALUE(bs));
+    DEFER(free(buff));
+
+    while (offset < file_size) {
+        if (running != 1) {
+            LOG_INFO("image file exit when background downloading");
+            return false;
+        }
+        if (!force_download) {
+            // check aleady downloaded.
+            auto hole_pos = dst->lseek(offset, SEEK_HOLE);
+            if (hole_pos >= offset + bs) {
+                // alread downloaded
+                offset += bs;
+                continue;
+            }
+        }
+
+        int retry = 2;
+        auto count = bs;
+        if (offset + count > file_size)
+            count = file_size - offset;
+    again_read:
+        if (!(retry--))
+            LOG_ERROR_RETURN(EIO, false, "failed to read at ", VALUE(offset), VALUE(count));
+        auto rlen = src->pread(buff, bs, offset);
+        if (rlen < 0) {
+            LOG_WARN("failed to read at ", VALUE(offset), VALUE(count), VALUE(errno), " retry...");
+            goto again_read;
+        }
+        retry = 2;
+    again_write:
+        if (!(retry--))
+            LOG_ERROR_RETURN(EIO, false, "failed to write at ", VALUE(offset), VALUE(count));
+        auto wlen = dst->pwrite(buff, count, offset);
+        // but once write lenth larger than read length treats as OK
+        if (wlen < rlen) {
+            LOG_WARN("failed to write at ", VALUE(offset), VALUE(count), VALUE(errno), " retry...");
+            goto again_write;
+        }
+        offset += count;
+    }
     return true;
 }
 

src/bk_download.h

@@ -16,18 +16,14 @@
 #pragma once
 #include <list>
 #include <string>
-#include "switch_file.h"
+#include <photon/fs/filesystem.h>
 
 class ImageFile;
+class ISwitchFile;
 
-namespace photon {
-    namespace fs {
-        class IFile;
-    }
-}
 namespace BKDL {
 
-static std::string DOWNLOAD_TMP_NAME = "overlaybd.download";
+static std::string DOWNLOAD_TMP_NAME = ".download";
 static std::string COMMIT_FILE_NAME = "overlaybd.commit";
 
 bool check_downloaded(const std::string &dir);
@@ -46,10 +42,11 @@ class BkDownload {
         unlock_file();
         delete src_file;
     }
-    BkDownload(ISwitchFile *sw_file, photon::fs::IFile *src_file, const std::string dir,
-               int32_t limit_MB_ps, int32_t try_cnt, ImageFile *image_file, std::string digest)
-        : sw_file(sw_file), src_file(src_file), dir(dir), limit_MB_ps(limit_MB_ps),
-          try_cnt(try_cnt), image_file(image_file), digest(digest) {
+    BkDownload(ISwitchFile *sw_file, photon::fs::IFile *src_file, size_t file_size,
+               const std::string dir, int32_t limit_MB_ps, int32_t try_cnt, ImageFile *image_file,
+               std::string digest)
+        : sw_file(sw_file), src_file(src_file), file_size(file_size), dir(dir),
+          limit_MB_ps(limit_MB_ps), try_cnt(try_cnt), image_file(image_file), digest(digest) {
     }
 
 private:
@@ -62,6 +59,8 @@ class BkDownload {
     int32_t limit_MB_ps;
     ImageFile *image_file;
     std::string digest;
+    size_t file_size;
+    bool force_download = false;
 };
 
 void bk_download_proc(std::list<BKDL::BkDownload *> &, uint64_t, int &);

src/image_file.cpp

@@ -37,6 +37,9 @@
 #define PARALLEL_LOAD_INDEX 32
 using namespace photon::fs;
 
+#define SET_LOCAL_DIR 118
+#define SET_SIZE 119
+
 IFile *ImageFile::__open_ro_file(const std::string &path) {
     int flags = O_RDONLY;
 
@@ -82,7 +85,7 @@ IFile *ImageFile::__open_ro_file(const std::string &path) {
 }
 
 IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &digest,
-                                               const uint64_t size, int layer_index) {
+                                   const uint64_t size, int layer_index) {
     std::string url;
     int64_t extra_range, rand_wait;
 
@@ -118,6 +121,8 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di
         set_failed(err_msg);
         LOG_ERRNO_RETURN(0, nullptr, err_msg);
     }
+    remote_file->ioctl(SET_SIZE, size);
+    remote_file->ioctl(SET_LOCAL_DIR, dir);
 
     ISwitchFile *switch_file = new_switch_file(remote_file);
     if (!switch_file) {
@@ -145,7 +150,7 @@ IFile *ImageFile::__open_ro_remote(const std::string &dir, const std::string &di
             LOG_WARN("failed to open source file, ignore download");
         } else {
             BKDL::BkDownload *obj =
-                new BKDL::BkDownload(switch_file, srcfile, dir, conf.download().maxMBps(),
+                new BKDL::BkDownload(switch_file, srcfile, size, dir, conf.download().maxMBps(),
                                      conf.download().tryCnt(), this, digest);
             LOG_DEBUG("add to download list for `", dir);
             dl_list.push_back(obj);

src/image_service.cpp

@@ -22,13 +22,14 @@
 #include <photon/fs/localfs.h>
 #include <photon/net/curl.h>
 #include <photon/thread/thread.h>
+#include <photon/common/alog.h>
+#include <photon/net/curl.h>
+#include <photon/fs/path.h>
 #include "overlaybd/cache/cache.h"
 #include "overlaybd/registryfs/registryfs.h"
 #include "overlaybd/tar_file.h"
 #include "overlaybd/zfile/zfile.h"
 #include "overlaybd/base64.h"
-#include "photon/common/alog.h"
-#include "photon/net/curl.h"
 #include <errno.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -254,6 +255,10 @@ void ImageService::set_result_file(std::string &filename, std::string &data) {
               data.c_str());
 }
 
+static std::string cache_fn_trans_sha256(std::string_view path) {
+    return std::string(photon::fs::Path(path).basename());
+}
+
 int ImageService::init() {
     if (read_global_config_and_set() < 0) {
         return -1;
@@ -273,8 +278,8 @@ int ImageService::init() {
     refill_size = global_conf.cacheConfig().refillSize();
     block_size = global_conf.cacheConfig().blockSize();
 
-    if (cache_type != "file" && cache_type != "ocf") {
-        LOG_ERROR_RETURN(0, -1, "unknown cache type `", cache_type);
+    if (cache_type != "file" && cache_type != "ocf" && cache_type != "download") {
+        LOG_ERROR_RETURN(0, -1, "unknown cache type: `", cache_type);
     }
     LOG_INFO("cache config: ", VALUE(cache_type), VALUE(cache_dir),
                                VALUE(cache_size_GB), VALUE(refill_size));
@@ -302,11 +307,6 @@ int ImageService::init() {
             LOG_ERROR_RETURN(0, -1, "create registryfs failed.");
         }
 
-        auto tar_fs = new_tar_fs_adaptor(registry_fs);
-        if (tar_fs == nullptr) {
-            delete registry_fs;
-            LOG_ERROR_RETURN(0, -1, "create tar_fs failed.");
-        }
         global_fs.srcfs = registry_fs;
 
         if (global_conf.p2pConfig().enable() == true) {
@@ -320,14 +320,13 @@ int ImageService::init() {
         if (cache_type == "file") {
             auto registry_cache_fs = new_localfs_adaptor(cache_dir.c_str());
             if (registry_cache_fs == nullptr) {
-                delete tar_fs;
+                delete global_fs.srcfs;
                 LOG_ERROR_RETURN(0, -1, "new_localfs_adaptor for ` failed", cache_dir.c_str());
             }
             // file cache will delete its src_fs automatically when destructed
             global_fs.remote_fs = FileSystem::new_full_file_cached_fs(
-                tar_fs, registry_cache_fs, refill_size /* refill unit 256KB */,
-                cache_size_GB /*GB*/, 10000000,
-                (uint64_t)1048576 * 4096, nullptr);
+                global_fs.srcfs, registry_cache_fs, refill_size, cache_size_GB, 10000000,
+                (uint64_t)1048576 * 4096, nullptr, cache_fn_trans_sha256);
 
         } else if (cache_type == "ocf") {
             auto namespace_dir = std::string(cache_dir + "/namespace");
@@ -358,8 +357,12 @@ int ImageService::init() {
             }
             global_fs.media_file = media_file;
 
-            global_fs.remote_fs = FileSystem::new_ocf_cached_fs(tar_fs, namespace_fs, block_size, refill_size,
+            global_fs.remote_fs = FileSystem::new_ocf_cached_fs(global_fs.srcfs, namespace_fs, block_size, refill_size,
                                                                 media_file, reload_media, io_alloc);
+        } else if (cache_type == "download") {
+            auto io_alloc = new IOAlloc;
+            global_fs.io_alloc = io_alloc;
+            global_fs.remote_fs = FileSystem::new_download_cached_fs(global_fs.srcfs, 4096, refill_size, io_alloc);
         }
 
         if (global_fs.remote_fs == nullptr) {

src/overlaybd/cache/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(frontend)
 add_subdirectory(full_file_cache)
 add_subdirectory(ocf_cache)
+add_subdirectory(download_cache)
 
 file(GLOB SRC_CACHE "*.cpp")
 
@@ -10,6 +11,7 @@ target_link_libraries(cache_lib
     cache_frontend_lib
     full_file_cache_lib
     ocf_cache_lib
+    download_cache_lib
 )
 target_include_directories(cache_lib PUBLIC
     ${PHOTON_INCLUDE_DIR}

src/overlaybd/cache/cache.cpp

@@ -23,6 +23,8 @@
 #include "full_file_cache/cache_pool.h"
 
 namespace FileSystem {
+using namespace photon::fs;
+
 ICacheStore::try_preadv_result ICacheStore::try_preadv(const struct iovec *iov, int iovcnt,
                                                        off_t offset) {
     try_preadv_result rst;
@@ -60,7 +62,7 @@ ssize_t ICacheStore::pwritev_mutable(struct iovec *iov, int iovcnt, off_t offset
 ICachedFileSystem *new_full_file_cached_fs(IFileSystem *srcFs, IFileSystem *mediaFs,
                                            uint64_t refillUnit, uint64_t capacityInGB,
                                            uint64_t periodInUs, uint64_t diskAvailInBytes,
-                                           IOAlloc *allocator) {
+                                           IOAlloc *allocator, Fn_trans_func name_trans) {
     if (refillUnit % 4096 != 0) {
         LOG_ERROR_RETURN(EINVAL, nullptr, "refill Unit need to be aligned to 4KB")
     }
@@ -69,7 +71,7 @@ ICachedFileSystem *new_full_file_cached_fs(IFileSystem *srcFs, IFileSystem *medi
     }
     Cache::FileCachePool *pool = nullptr;
     pool =
-        new ::Cache::FileCachePool(mediaFs, capacityInGB, periodInUs, diskAvailInBytes, refillUnit);
+        new ::Cache::FileCachePool(mediaFs, capacityInGB, periodInUs, diskAvailInBytes, refillUnit, name_trans);
     pool->Init();
     return new_cached_fs(srcFs, pool, 4096, refillUnit, allocator);
 }