[CloudTest] Make TCP port wait period configurable (#39)

* [CloudTest] Make TCP port wait period configurable

* PR feedback

* reduce startupDelay on tests

* Bump minio_jll

Author: andrebsguedes

Project.toml

@@ -1,7 +1,7 @@
 name = "CloudBase"
 uuid = "85eb1798-d7c4-4918-bb13-c944d38e27ed"
 authors = ["quinnj <[email protected]>"]
-version = "1.4.8"
+version = "1.4.9"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -31,7 +31,7 @@ SHA = "0, 1, 2"
 URIs = "1"
 azurite_jll = "3"
 julia = "1.6"
-minio_jll = "1"
+minio_jll = "2"
 
 [extras]
 JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"

src/CloudTest.jl

@@ -136,7 +136,7 @@ using minio_jll
 import ..Config, ..findOpenPorts, ...AWS, .._cmd, .._wait_for_port
 
 """
-    Minio.with(f; dir, bucket, public, startupDelay, debug)
+    Minio.with(f; dir, bucket, public, startupDelay, debug, waitForPortTimeout)
 
 Starts a minio server on a random open port, and passes a
 [`CloudTest.Config`](@ref) to `f`, which contains the credentials
@@ -152,6 +152,7 @@ keyword arguments include:
     can be useful on slower systems to allow time for the server
     to fully startup
   * `debug`: whether to turn on minio debug logging, defaults to `false`
+  * `waitForPortTimeout`: Time to wait in seconds for the TCP port of Minio to be ready for connections
 """
 function with(f; dir=nothing, kw...)
     config, proc = run(; dir, kw...)
@@ -198,7 +199,7 @@ publicPolicy(bucket) = """
 # use `with`, not `run`! if you `run`, it returns `conf, p`, where `p` is the server process
 # note that existing the Julia process *will not* stop the server process, which can easily
 # lead to "dangling" server processes. You can `kill(p)` to stop the server process manually
-function run(; dir=nothing, bucket=nothing, public=false, startupDelay=0.25, debug=false, bindIP="127.0.0.1")
+function run(; dir=nothing, bucket=nothing, public=false, startupDelay=0.25, debug=false, bindIP="127.0.0.1", waitForPortTimeout=30)
     if dir === nothing
         dir = mktempdir()
     elseif !isdir(dir)
@@ -208,8 +209,8 @@ function run(; dir=nothing, bucket=nothing, public=false, startupDelay=0.25, deb
         port, cport = ports
         cmd = _cmd(`$(minio_jll.minio()) server $dir --address $(bindIP):$(port) --console-address $(bindIP):$(cport)`)
         p = debug ? Base.run(cmd, devnull, stderr, stderr; wait=false) : Base.run(cmd; wait=false)
-        # Wait for the port to be open for up to 10 seconds
-        _wait_for_port("127.0.0.1", port, 10)
+        # Wait for the port to be open
+        _wait_for_port("127.0.0.1", port, waitForPortTimeout)
         return p, port
     end
     credentials = AWS.Credentials("minioadmin", "minioadmin")
@@ -243,7 +244,7 @@ using NodeJS_16_jll, azurite_jll, Dates
 import ..Config, ..findOpenPorts, ...Azure, .._cmd, .._wait_for_port
 
 """
-    Azurite.with(f; dir, bucket, public, startupDelay, debug)
+    Azurite.with(f; dir, bucket, public, startupDelay, debug, waitForPortTimeout)
 
 Starts an azurite server on a random open port, and passes a
 [`CloudTest.Config`](@ref) to `f`, which contains the credentials
@@ -259,6 +260,7 @@ keyword arguments include:
     can be useful on slower systems to allow time for the server
     to fully startup
   * `debug`: whether to turn on minio debug logging, defaults to `false`
+  * `waitForPortTimeout`: Time to wait in seconds for the TCP port of Azurite to be ready for connections
 """
 function with(f; dir=nothing, debug::Bool=false, debugLog::Union{Nothing, Ref{String}}=nothing, kw...)
     config, proc = run(; dir, debug, kw...)
@@ -307,7 +309,7 @@ publicPolicy() = """
 # use `with`, not `run`! if you `run`, it returns `conf, p`, where `p` is the server process
 # note that existing the Julia process *will not* stop the server process, which can easily
 # lead to "dangling" server processes. You can `kill(p)` to stop the server process manually
-function run(; dir=nothing, container=nothing, public=false, startupDelay=3, debug=false, use_ssl=true, skipApiVersionCheck=false)
+function run(; dir=nothing, container=nothing, public=false, startupDelay=0.5, debug=false, use_ssl=true, skipApiVersionCheck=false, waitForPortTimeout=30)
     if dir === nothing
         dir = mktempdir()
     elseif !isdir(dir)
@@ -325,8 +327,8 @@ function run(; dir=nothing, container=nothing, public=false, startupDelay=3, deb
         skipApiVersionCheck && push!(cmd_args, "--skipApiVersionCheck")
         cmd = _cmd(`$(node()) $(azurite) $(cmd_args)`)
         p = debug ? Base.run(cmd, devnull, stderr, stderr; wait=false) : Base.run(cmd; wait=false)
-        # Wait for the port to be open for up to 10 seconds
-        _wait_for_port("127.0.0.1", port, 10)
+        # Wait for the port to be open
+        _wait_for_port("127.0.0.1", port, waitForPortTimeout)
         return p, port
     end
     acct = "devstoreaccount1"
@@ -335,6 +337,8 @@ function run(; dir=nothing, container=nothing, public=false, startupDelay=3, deb
     key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
     credentials = Azure.Credentials(acct, key)
     headers = public ? ["x-ms-blob-public-access" => "container"] : []
+    # Small delay to ensure the HTTP server is ready even though the TCP port is open
+    sleep(startupDelay)
     resp = Azure.put("$(cont.baseurl)?restype=container", headers; credentials, status_exception=false)
     if resp.status != 201
         @error resp

test/runtests.jl

@@ -38,7 +38,7 @@ end
 
 @time @testset "AWS" begin
     config = Ref{Any}()
-    Minio.with(bindIP="127.0.0.1", startupDelay=3) do conf
+    Minio.with(bindIP="127.0.0.1", startupDelay=0.5, waitForPortTimeout=10) do conf
         config[] = conf
         credentials, bucket = conf
         csv = "a,b,c\n1,2,3\n4,5,$(rand())"
@@ -49,7 +49,7 @@ end
     @test !isdir(config[].dir)
     @test success(config[].process)
     # test public access
-    Minio.with(bindIP="127.0.0.1", startupDelay=3, public=true) do conf
+    Minio.with(bindIP="127.0.0.1", startupDelay=0.5, public=true) do conf
         credentials, bucket = conf
         csv = "a,b,c\n1,2,3\n4,5,$(rand())"
         AWS.put("$(bucket.baseurl)test.csv", [], csv; service="s3")
@@ -67,7 +67,7 @@ end
 if !x32bit
 @time @testset "Azure" begin
     config = Ref{Any}()
-    Azurite.with(startupDelay=3) do conf
+    Azurite.with(startupDelay=0.5, waitForPortTimeout=10) do conf
         config[] = conf
         credentials, container = conf
         csv = "a,b,c\n1,2,3\n4,5,$(rand())"
@@ -97,7 +97,7 @@ if !x32bit
     @test !isdir(config[].dir)
     @test success(config[].process)
     # test public access
-    Azurite.with(startupDelay=3, public=true) do conf
+    Azurite.with(startupDelay=0.5, public=true) do conf
         credentials, container = conf
         csv = "a,b,c\n1,2,3\n4,5,$(rand())"
         # have to supply credentials for put since "public" is only for get
@@ -117,7 +117,7 @@ end
     mconfigs = Vector{Any}(undef, 10)
     aconfigs = Vector{Any}(undef, 10)
     @sync for i = 1:10
-        @async Minio.with(bindIP="127.0.0.1", startupDelay=3) do conf
+        @async Minio.with(bindIP="127.0.0.1", startupDelay=0.5) do conf
             mconfigs[i] = conf
             credentials, bucket = conf
             csv = "a,b,c\n1,2,3\n4,5,$(rand())"
@@ -126,7 +126,7 @@ end
             @test String(resp.body) == csv
         end
         if !x32bit
-            @async Azurite.with(startupDelay=3) do conf
+            @async Azurite.with(startupDelay=0.5) do conf
                 aconfigs[i] = conf
                 credentials, container = conf
                 csv = "a,b,c\n1,2,3\n4,5,$(rand())"
@@ -212,7 +212,7 @@ end
 
 # metrics hooks
 @time @testset "Cloud metrics hooks" begin
-    Minio.with(bindIP="127.0.0.1", startupDelay=3) do conf
+    Minio.with(bindIP="127.0.0.1", startupDelay=0.5) do conf
         credentials, bucket = conf
         prereq_ref = Ref(0)
         metrics_ref = Ref{Any}()