Merge pull request #1514 from FedML-AI/test/v0.7.0

Test/v0.7.0

Author: fedml-alex

python/fedml/__init__.py

@@ -34,7 +34,7 @@
 _global_training_type = None
 _global_comm_backend = None
 
-__version__ = "0.8.9a2"
+__version__ = "0.8.9a3"
 
 
 # This is the deployment environment used for different roles (RD/PM/BD/Public Developers). Potential VALUE: local, dev, test, release

python/fedml/computing/scheduler/master/server_runner.py

@@ -23,6 +23,7 @@
 
 import requests
 
+import fedml
 from ..scheduler_core.scheduler_matcher import SchedulerMatcher
 from ..comm_utils.constants import SchedulerConstants
 from ..comm_utils.job_utils import JobRunnerUtils
@@ -1058,7 +1059,7 @@ def ota_upgrade(self, payload, request_json):
             pass
 
         if force_ota and ota_version is not None:
-            should_upgrade = True
+            should_upgrade = True if ota_version != fedml.__version__ else False
             upgrade_version = ota_version
         else:
             try:

python/fedml/computing/scheduler/model_scheduler/device_client_runner.py

@@ -18,6 +18,8 @@
 from urllib.parse import urlparse
 
 import requests
+
+import fedml
 from fedml import mlops
 from fedml.computing.scheduler.model_scheduler.device_model_msg_object import FedMLModelMsgObject
 from fedml.core.distributed.communication.s3.remote_storage import S3Storage
@@ -348,6 +350,7 @@ def run_impl(self):
 
         # download model net and load into the torch model
         model_from_open = None
+        self.model_is_from_open = None
         if self.model_is_from_open:
             logging.info("process the model net from open...")
             self.check_runner_stop_event()
@@ -562,7 +565,7 @@ def ota_upgrade(self, payload, request_json):
             pass
 
         if force_ota and ota_version is not None:
-            should_upgrade = True
+            should_upgrade = True if ota_version != fedml.__version__ else False
             upgrade_version = ota_version
         else:
             try:

python/fedml/computing/scheduler/model_scheduler/device_model_deployment.py

@@ -174,7 +174,7 @@ def start_deployment(end_point_id, end_point_name, model_id, model_version,
             logging.info("LLM model loaded from the open")
         else:
             raise Exception("Unsupported inference engine type: {}".format(inference_engine))
-    elif model_is_from_open == False:
+    elif model_is_from_open == False or model_is_from_open is None:
         model_location = os.path.join(model_storage_local_path, "fedml_model.bin")
         try:
             model = torch.jit.load(model_location)
@@ -188,6 +188,8 @@ def start_deployment(end_point_id, end_point_name, model_id, model_version,
                 config = yaml.safe_load(file)
                 # Resource related
                 use_gpu = config.get('use_gpu', False)
+                usr_indicated_wait_time = config.get('deploy_timeout', 100)
+                usr_indicated_retry_cnt = max(int(usr_indicated_wait_time) // 10, 1)
                 inference_image_name = config.get('inference_image_name',
                                                   ClientConstants.INFERENCE_SERVER_CUSTOME_IMAGE)
                 # Source code dir, bootstrap dir, data cache dir
@@ -374,7 +376,8 @@ def start_deployment(end_point_id, end_point_name, model_id, model_version,
         # Logging the info from the container
         log_deployment_result(end_point_id, model_id, default_server_container_name,
                               ClientConstants.CMD_TYPE_RUN_DEFAULT_SERVER,
-                              running_model_name, inference_engine, inference_http_port, inference_type="default")
+                              running_model_name, inference_engine, inference_http_port, inference_type="default",
+                              retry_interval=10, deploy_attempt_threshold=usr_indicated_retry_cnt)
 
         # Check if the inference server is ready
         inference_output_url, running_model_version, ret_model_metadata, ret_model_config = \
@@ -542,10 +545,9 @@ def should_exit_logs(end_point_id, model_id, cmd_type, model_name, inference_eng
 
 def log_deployment_result(end_point_id, model_id, cmd_container_name, cmd_type,
                           inference_model_name, inference_engine,
-                          inference_http_port, inference_type="default"):
+                          inference_http_port, inference_type="default",
+                          retry_interval=10, deploy_attempt_threshold=10):
     deploy_attempt = 0
-    retry_interval = 10
-    deploy_attempt_threshold = 10
     last_out_logs = ""
     last_err_logs = ""
 

python/fedml/computing/scheduler/model_scheduler/device_server_runner.py

@@ -21,6 +21,7 @@
 import requests
 import torch
 
+import fedml
 from ..comm_utils import sys_utils
 from .device_server_data_interface import FedMLServerDataInterface
 from ....core.mlops.mlops_runtime_log import MLOpsRuntimeLog
@@ -693,7 +694,7 @@ def ota_upgrade(self, payload, request_json):
             pass
 
         if force_ota and ota_version is not None:
-            should_upgrade = True
+            should_upgrade = True if ota_version != fedml.__version__ else False
             upgrade_version = ota_version
         else:
             try:

python/fedml/computing/scheduler/slave/client_runner.py

@@ -832,7 +832,7 @@ def ota_upgrade(self, payload, request_json):
             pass
 
         if force_ota and ota_version is not None:
-            should_upgrade = True
+            should_upgrade = True if ota_version != fedml.__version__ else False
             upgrade_version = ota_version
         else:
             try:

python/fedml/core/distributed/communication/s3/remote_storage.py

@@ -308,11 +308,15 @@ def read_model_net_progress(bytes_transferred):
         )
 
         unpickle_start_time = time.time()
+        model = None
         try:
             model = torch.jit.load(temp_file_path)
         except Exception as e:
             logging.info("jit.load failed")
-            model = torch.load(temp_file_path, pickle_module=dill)
+            try:
+                model = torch.load(temp_file_path, pickle_module=dill)
+            except Exception as e:
+                logging.info("torch.load failed")
         os.remove(temp_file_path)
         MLOpsProfilerEvent.log_to_wandb(
             {"UnpickleTime": time.time() - unpickle_start_time}

python/setup.py

@@ -94,7 +94,7 @@ def finalize_options(self):
 
 setup(
     name="fedml",
-    version="0.8.9a2",
+    version="0.8.9a3",
     author="FedML Team",
     author_email="[email protected]",
     description="A research and production integrated edge-cloud library for "