Refactor SkyplaneConfig and TransferConfig for less redundancy

docs/_api/skyplane.api.rst

@@ -9,9 +9,9 @@
    :toctree:
 
    skyplane.api.client
-   skyplane.api.config
    skyplane.api.dataplane
    skyplane.api.provisioner
    skyplane.api.tracker
+   skyplane.api.transfer_config
    skyplane.api.transfer_job
 

docs/_api/skyplane.api.config.rst → docs/_api/skyplane.api.transfer_config.rst

@@ -1,7 +1,7 @@
-skyplane.api.config
+skyplane.api.transfer_config
 ===================
 
-.. automodule:: skyplane.api.config
+.. automodule:: skyplane.api.transfer_config
 
 
 

skyplane/__init__.py

@@ -2,7 +2,7 @@
 
 from skyplane import compute, exceptions
 from skyplane.api.client import SkyplaneClient
-from skyplane.api.config import TransferConfig, AWSConfig, AzureConfig, GCPConfig
+from skyplane.api.transfer_config import TransferConfig, AWSConfig, AzureConfig, GCPConfig
 from skyplane.api.dataplane import Dataplane
 from skyplane.api.pipeline import Pipeline
 from skyplane.api.tracker import TransferHook

skyplane/api/client.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 from skyplane.api.dataplane import Dataplane
 from skyplane.api.provisioner import Provisioner
 from skyplane.api.obj_store import ObjectStore
@@ -18,7 +18,7 @@
 from skyplane.api.pipeline import Pipeline
 
 if TYPE_CHECKING:
-    from skyplane.api.config import AWSConfig, AzureConfig, GCPConfig, TransferConfig, IBMCloudConfig
+    from skyplane.api.transfer_config import AWSConfig, AzureConfig, GCPConfig, TransferConfig, IBMCloudConfig
 
 
 class SkyplaneClient:

skyplane/api/dataplane.py

@@ -16,7 +16,7 @@
 from skyplane.exceptions import GatewayContainerStartException
 from skyplane.api.tracker import TransferProgressTracker, TransferHook
 from skyplane.api.transfer_job import CopyJob, SyncJob, TransferJob
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 from skyplane.planner.topology import TopologyPlan, TopologyPlanGateway
 from skyplane.utils import logger
 from skyplane.utils.definitions import gateway_docker_image, tmp_log_dir

skyplane/api/pipeline.py

@@ -15,7 +15,7 @@
 from skyplane import compute
 from skyplane.api.tracker import TransferProgressTracker, TransferHook
 from skyplane.api.transfer_job import CopyJob, SyncJob, TransferJob
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 
 from skyplane.planner.planner import MulticastDirectPlanner, DirectPlannerSourceOneSided, DirectPlannerDestOneSided
 from skyplane.planner.topology import TopologyPlanGateway

skyplane/api/tracker.py

@@ -12,7 +12,7 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from skyplane import exceptions
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 from skyplane.chunk import ChunkState, Chunk
 from skyplane.utils import logger, imports
 from skyplane.utils.fn import do_parallel

skyplane/api/config.py → skyplane/api/transfer_config.py

@@ -85,7 +85,7 @@ class TransferConfig:
 
     aws_instance_class: str = "m5.8xlarge"
     azure_instance_class: str = "Standard_D2_v5"
-    gcp_instance_class: str = "n2-standard-16"
+    gcp_instance_class: str = "n2-standard-32"
     ibmcloud_instance_class: str = "bx2-2x8"
     gcp_use_premium_network: bool = True
 
@@ -98,4 +98,4 @@ class TransferConfig:
     multipart_enabled: bool = True
     multipart_threshold_mb: int = 128
     multipart_chunk_size_mb: int = 64
-    multipart_max_chunks: int = 10000
+    multipart_max_chunks: int = 9990

skyplane/api/transfer_job.py

@@ -23,7 +23,7 @@
 from functools import partial
 
 from skyplane import exceptions
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 from skyplane.chunk import Chunk, ChunkRequest
 from skyplane.obj_store.storage_interface import StorageInterface
 from skyplane.obj_store.object_store_interface import ObjectStoreObject, ObjectStoreInterface

skyplane/cli/cli_transfer.py

@@ -9,7 +9,7 @@
 from rich.progress import Progress, TextColumn, SpinnerColumn
 
 import skyplane
-from skyplane.api.config import TransferConfig, AWSConfig, GCPConfig, AzureConfig, IBMCloudConfig
+from skyplane.api.transfer_config import TransferConfig, AWSConfig, GCPConfig, AzureConfig, IBMCloudConfig
 from skyplane.api.transfer_job import CopyJob, SyncJob, TransferJob
 from skyplane.cli.impl.cp_replicate_fallback import (
     replicate_onprem_cp_cmd,
@@ -63,6 +63,7 @@ def __init__(self, src_region_tag: str, dst_region_tag: str, args: Dict[str, Any
             cloud_config.set_flag("multipart_enabled", str(self.args["multipart"]))
 
         self.transfer_config = self.make_transfer_config(skyplane_config or cloud_config)
+
         self.client = skyplane.SkyplaneClient(
             aws_config=self.aws_config,
             azure_config=self.azure_config,
@@ -103,26 +104,10 @@ def to_api_config(self, config: SkyplaneConfig):
         return aws_config, azure_config, gcp_config, ibmcloud_config
 
     def make_transfer_config(self, config: SkyplaneConfig) -> TransferConfig:
-        intraregion = self.src_region_tag == self.dst_region_tag
         return TransferConfig(
             autoterminate_minutes=config.get_flag("autoshutdown_minutes"),
             requester_pays=config.get_flag("requester_pays"),
-            use_bbr=config.get_flag("bbr"),
-            use_compression=config.get_flag("compress") if not intraregion else False,
-            use_e2ee=config.get_flag("encrypt_e2e") if not intraregion else False,
-            use_socket_tls=config.get_flag("encrypt_socket_tls") if not intraregion else False,
-            aws_use_spot_instances=config.get_flag("aws_use_spot_instances"),
-            azure_use_spot_instances=config.get_flag("azure_use_spot_instances"),
-            gcp_use_spot_instances=config.get_flag("gcp_use_spot_instances"),
-            aws_instance_class=config.get_flag("aws_instance_class"),
-            azure_instance_class=config.get_flag("azure_instance_class"),
-            gcp_instance_class=config.get_flag("gcp_instance_class"),
-            ibmcloud_instance_class=config.get_flag("ibmcloud_instance_class"),
-            gcp_use_premium_network=config.get_flag("gcp_use_premium_network"),
             multipart_enabled=config.get_flag("multipart_enabled"),
-            multipart_threshold_mb=config.get_flag("multipart_min_threshold_mb"),
-            multipart_chunk_size_mb=config.get_flag("multipart_chunk_size_mb"),
-            multipart_max_chunks=config.get_flag("multipart_max_chunks"),
         )
 
     def check_config(self) -> bool:

skyplane/config.py

@@ -10,31 +10,15 @@
 
 _FLAG_TYPES = {
     "autoconfirm": bool,
-    "bbr": bool,
-    "compress": bool,
-    "encrypt_e2e": bool,
-    "encrypt_socket_tls": bool,
+    "autoshutdown_minutes": int,
     "verify_checksums": bool,
     "multipart_enabled": bool,
-    "multipart_min_threshold_mb": int,
-    "multipart_min_size_mb": int,
-    "multipart_chunk_size_mb": int,
-    "multipart_max_chunks": int,
     "num_connections": int,
     "max_instances": int,
-    "autoshutdown_minutes": int,
-    "aws_use_spot_instances": bool,
-    "azure_use_spot_instances": bool,
-    "gcp_use_spot_instances": bool,
-    "aws_instance_class": str,
-    "azure_instance_class": str,
-    "gcp_instance_class": str,
-    "ibmcloud_instance_class": str,
     "aws_default_region": str,
     "azure_default_region": str,
     "gcp_default_region": str,
     "ibmcloud_default_region": str,
-    "gcp_use_premium_network": bool,
     "usage_stats": bool,
     "gcp_service_account_name": str,
     "requester_pays": bool,
@@ -44,32 +28,15 @@
 
 _DEFAULT_FLAGS = {
     "autoconfirm": False,
-    "bbr": True,
-    "compress": True,
-    "encrypt_e2e": True,
-    "encrypt_socket_tls": False,
+    "autoshutdown_minutes": 15,
     "verify_checksums": True,
     "multipart_enabled": True,
-    "multipart_min_threshold_mb": 128,
-    "multipart_min_size_mb": 5,  # 5 MiB minimum size for multipart uploads
-    "multipart_chunk_size_mb": 64,
-    "multipart_max_chunks": 9990,  # AWS limit is 10k chunks
     "num_connections": 32,
     "max_instances": 1,
-    "autoshutdown_minutes": 15,
-    "aws_use_spot_instances": False,
-    "azure_use_spot_instances": False,
-    "gcp_use_spot_instances": False,
-    "ibmcloud_use_spot_instances": False,
-    "aws_instance_class": "m5.8xlarge",
-    "azure_instance_class": "Standard_D32_v5",
-    "gcp_instance_class": "n2-standard-32",
-    "ibmcloud_instance_class": "bx2-2x8",
     "aws_default_region": "us-east-1",
     "azure_default_region": "eastus",
     "gcp_default_region": "us-central1-a",
     "ibmcloud_default_region": "us-east",
-    "gcp_use_premium_network": True,
     "usage_stats": True,
     "gcp_service_account_name": "skyplane-manual",
     "requester_pays": False,

skyplane/planner/planner.py

@@ -5,7 +5,7 @@
 import csv
 
 from skyplane import compute
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 from skyplane.utils import logger
 
 from skyplane.planner.topology import TopologyPlan

tests/integration/test_cp.py

@@ -1,5 +1,5 @@
 import pytest
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 from skyplane.utils import logger
 import time
 from skyplane.api.client import SkyplaneClient

tests/unit_nocloud/test_fall_back.py

@@ -2,7 +2,7 @@
 import os
 
 import pytest
-from skyplane.api.config import TransferConfig
+from skyplane.api.transfer_config import TransferConfig
 from skyplane.planner.planner import MulticastDirectPlanner