silence destroy_proces_group() warning (#1387)

Author: msaroufim

distributed/tensor_parallelism/fsdp_tp_example.py

@@ -173,3 +173,6 @@
     rank_log(_rank, logger, f"2D iter {i} complete")
 
 rank_log(_rank, logger, "2D training successfully completed!")
+
+if dist.is_initialized():
+    dist.destroy_process_group()

distributed/tensor_parallelism/sequence_parallel_example.py

@@ -22,6 +22,7 @@
 import torch
 import torch.nn as nn
 
+import torch.distributed as dist
 from torch.distributed._tensor import Shard
 
 from torch.distributed.tensor.parallel import (
@@ -107,3 +108,6 @@ def forward(self, x):
     rank_log(_rank, logger, f"Sequence Parallel iter {i} completed")
 
 rank_log(_rank, logger, "Sequence Parallel training completed!")
+
+if dist.is_initialized():
+    dist.destroy_process_group()

distributed/tensor_parallelism/tensor_parallel_example.py

@@ -122,3 +122,6 @@ def forward(self, x):
     rank_log(_rank, logger, f"Tensor Parallel iter {i} completed")
 
 rank_log(_rank, logger, "Tensor Parallel training completed!")
+
+if dist.is_initialized():
+    dist.destroy_process_group()