EP installed through docker on p6 hangs

[YangZhou1997]

Possible because of the inability to detect SM100 in Docker.

[YangZhou1997]

But the docker version runs on p5en

[YangZhou1997]

It gets stuck in the cudaLaunchKernel of all_gather_object:

(base) ubuntu@ip-10-1-18-53:~/uccl_yang/ep$ sudo env "PATH=$PATH" py-spy dump --pid 47553 --native
Process 47553: /fsx/ubuntu/miniconda3/bin/python3.13 -u test_internode.py --num-tokens=4096 --hidden=7168 --num-topk=8 --num-experts=288 --test-ll-compatibility
Python v3.13.9 (/fsx/ubuntu/miniconda3/bin/python3.13)

Thread 47553 (active): "MainThread"
    0x784304e61bae (libcuda.so.580.95.05)
    0x784304c1f183 (libcuda.so.580.95.05)
    0x784304c5de3b (libcuda.so.580.95.05)
    0x78430599d137 (libcuda.so.580.95.05)
    0x78430599d585 (libcuda.so.580.95.05)
    0x784304c2b9f4 (libcuda.so.580.95.05)
    0x78430599b749 (libcuda.so.580.95.05)
    0x784304cfe418 (libcuda.so.580.95.05)
    0x784304cbff08 (libcuda.so.580.95.05)
    0x784304cc0275 (libcuda.so.580.95.05)
    0x784304d3f1cf (libcuda.so.580.95.05)
    cuLibraryGetModule (libcuda.so.580.95.05)
    0x78443563d10a (libcudart.so.12)
    0x78443563d8a8 (libcudart.so.12)
    0x78443563d936 (libcudart.so.12)
    0x78443564086a (libcudart.so.12)
    0x7844356141a1 (libcudart.so.12)
    cudaLaunchKernel (libcudart.so.12)
    at::native::gpu_kernel_impl_nocast<at::native::(anonymous namespace)::CompareFunctor<long> > (libtorch_cuda.so)
    at::native::compare_kernel_impl<long> (libtorch_cuda.so)
    at::(anonymous namespace)::wrapper_CUDA_gt_Tensor (libtorch_cuda.so)
    c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<at::Tensor(at::Tensor const&, at::Tensor const&), &at::(anonymous namespace)::wrapper_CUDA_gt_Tensor(at::Tensor const&, at::Tensor const&)>, at::Tensor, c10::guts::typelist::typelist<at::Tensor const&, at::Tensor const&> >, at::Tensor(at::Tensor const&, at::Tensor const&)>::call (libtorch_cuda.so)
    at::_ops::gt_Tensor::redispatch (libtorch_cpu.so)
    torch::autograd::VariableType::(anonymous namespace)::gt_Tensor (libtorch_cpu.so)
    c10::impl::wrap_kernel_functor_unboxed_<c10::impl::detail::WrapFunctionIntoFunctor_<c10::CompileTimeFunctionPointer<at::Tensor(c10::DispatchKeySet, at::Tensor const&, at::Tensor const&), &torch::autograd::VariableType::(anonymous namespace)::gt_Tensor(c10::DispatchKeySet, at::Tensor const&, at::Tensor const&)>, at::Tensor, c10::guts::typelist::typelist<c10::DispatchKeySet, at::Tensor const&, at::Tensor const&> >, at::Tensor(c10::DispatchKeySet, at::Tensor const&, at::Tensor const&)>::call (libtorch_cpu.so)
    at::_ops::gt_Tensor::call (libtorch_cpu.so)
    torch::autograd::THPVariable_gt (libtorch_python.so)
    __torch_function__ (torch/utils/_device.py:103)
    torch::handle_torch_function_no_python_arg_parser (libtorch_python.so)
    torch::handle_torch_function (libtorch_python.so)
    torch::autograd::THPVariable_gt (libtorch_python.so)
    torch::autograd::TypeError_to_NotImplemented_<&torch::autograd::THPVariable_gt(_object*, _object*, _object*)> (libtorch_python.so)
    all_gather_object (torch/distributed/distributed_c10d.py:3180)
    wrapper (torch/distributed/c10d_logger.py:81)
    get_cpu_proxies_meta (utils.py:121)
    initialize_uccl (utils.py:475)
    __init__ (buffer.py:74)
    test_loop (test_internode.py:469)
    <module> (test_internode.py:548)
    0x784436429d90 (libc.so.6)