portdbapi: aux_get retry for unexpected process returncode

Retry for an intermittent unexpected bash returncode which
annoyingly occurs in CI runs with forkserver (bug 965132).

Bug: https://bugs.gentoo.org/965132
Signed-off-by: Zac Medico <[email protected]>

Author: zmedico

lib/portage/dbapi/porttree.py

@@ -752,25 +752,51 @@ async def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None
 
         mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation)
 
-        if mydata is not None:
-            future = loop.create_future()
-            self._aux_get_return(
-                future,
-                mycpv,
-                mylist,
-                myebuild,
-                ebuild_hash,
-                mydata,
-                mylocation,
-                cache_me,
-                None,
-            )
-            return future.result()
+        proc = None
+        if mydata is None:
+            if myebuild in self._broken_ebuilds:
+                raise PortageKeyError(mycpv)
+
+            # Retry for an intermittent unexpected bash returncode which
+            # annoyingly occurs in CI runs with forkserver (bug 965132).
+            tries = 3
+            while tries > 0:
+                tries -= 1
+                proc = await self._run_metadata_phase(mycpv, mylocation, ebuild_hash)
+
+                if proc.returncode != os.EX_OK:
+                    if proc.returncode != 1:
+                        writemsg(
+                            _(
+                                "!!! aux_get(): metadata phase for package '%(pkg)s' failed with unexpected returncode %(returncode)s\n"
+                            )
+                            % {"pkg": mycpv, "returncode": proc.returncode},
+                            noiselevel=-1,
+                        )
+                        # Only retry for an unexpected returncode.
+                        if tries > 0:
+                            continue
+                    self._broken_ebuilds.add(myebuild)
+                    raise PortageKeyError(mycpv)
 
-        if myebuild in self._broken_ebuilds:
-            raise PortageKeyError(mycpv)
+                mydata = proc.metadata
+                break
 
-        proc = None
+        return self._aux_get_return(
+            mycpv,
+            mylist,
+            myebuild,
+            ebuild_hash,
+            mydata,
+            mylocation,
+            cache_me,
+        )
+
+    async def _run_metadata_phase(
+        self, mycpv, mylocation, ebuild_hash
+    ) -> EbuildMetadataPhase:
+
+        loop = self._event_loop
         deallocate_config = None
         async with contextlib.AsyncExitStack() as stack:
             try:
@@ -800,21 +826,6 @@ async def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None
                     deallocate_config=deallocate_config,
                 )
 
-                future = loop.create_future()
-                proc.addExitListener(
-                    functools.partial(
-                        self._aux_get_return,
-                        future,
-                        mycpv,
-                        mylist,
-                        myebuild,
-                        ebuild_hash,
-                        mydata,
-                        mylocation,
-                        cache_me,
-                    )
-                )
-                future.add_done_callback(functools.partial(self._aux_get_cancel, proc))
                 proc.start()
 
             finally:
@@ -824,46 +835,32 @@ async def async_aux_get(self, mycpv, mylist, mytree=None, myrepo=None, loop=None
                     if proc is None or not proc.isAlive():
                         deallocate_config.done() or deallocate_config.cancel()
                     else:
-                        await deallocate_config
+                        try:
+                            await deallocate_config
+                        except asyncio.CancelledError:
+                            proc.cancel()
+                            raise
 
         # After deallocate_config is done, release self._doebuild_settings_lock
-        # by leaving the stack context, and wait for proc to finish and
-        # trigger a call to self._aux_get_return.
-        return await future
-
-    @staticmethod
-    def _aux_get_cancel(proc, future):
-        if future.cancelled() and proc.returncode is None:
+        # by leaving the stack context, and wait for proc to finish.
+        try:
+            await proc.async_wait()
+        except asyncio.CancelledError:
             proc.cancel()
+            raise
+
+        return proc
 
     def _aux_get_return(
         self,
-        future,
         mycpv,
         mylist,
         myebuild,
         ebuild_hash,
         mydata,
         mylocation,
         cache_me,
-        proc,
     ):
-        if future.cancelled():
-            return
-        if proc is not None:
-            if proc.returncode != os.EX_OK:
-                if proc.returncode != 1:
-                    writemsg(
-                        _(
-                            "!!! aux_get(): metadata phase for package '%(pkg)s' failed with unexpected returncode %(returncode)s\n"
-                        )
-                        % {"pkg": mycpv, "returncode": proc.returncode},
-                        noiselevel=-1,
-                    )
-                self._broken_ebuilds.add(myebuild)
-                future.set_exception(PortageKeyError(mycpv))
-                return
-            mydata = proc.metadata
         mydata["repository"] = self.repositories.get_name_for_location(mylocation)
         mydata["_mtime_"] = ebuild_hash.mtime
         eapi = mydata.get("EAPI")
@@ -882,7 +879,7 @@ def _aux_get_return(
                 aux_cache[x] = mydata.get(x, "")
             self._aux_cache[mycpv] = aux_cache
 
-        future.set_result(returnme)
+        return returnme
 
     def getFetchMap(self, mypkg, useflags=None, mytree=None):
         """